vulkan python bindings on windows fixes

Signed-off-by: Andriy Mulyar <andriy.mulyar@gmail.com>

.circleci/config.yml

@@ -12,6 +12,7 @@ workflows:
           config-path: .circleci/continue_config.yml
           mapping: |
             gpt4all-bindings/python/.* run-python-workflow true
+            gpt4all-bindings/typescript/.* run-ts-workflow true
             gpt4all-bindings/csharp/.* run-csharp-workflow true
             gpt4all-backend/.* run-chat-workflow true
             gpt4all-chat/.* run-chat-workflow true

.circleci/continue_config.yml

@@ -2,6 +2,7 @@ version: 2.1
 orbs:
   win: circleci/windows@5.0
   python: circleci/python@1.2
+  node: circleci/node@5.1
 
 parameters:
   run-default-workflow:
@@ -13,6 +14,9 @@ parameters:
   run-chat-workflow:
     type: boolean
     default: false
+  run-ts-workflow:
+    type: boolean
+    default: false
   run-csharp-workflow:
     type: boolean
     default: false
@@ -37,10 +41,12 @@ jobs:
       - restore_cache:  # this is the new step to restore cache
           keys:
             - linux-qt-cache
-      - run: 
+      - run:
           name: Setup Linux and Dependencies
           command: |
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk
       - run:
           name: Installing Qt
           command: |
@@ -88,12 +94,18 @@ jobs:
           key: windows-qt-cache
           paths:
             - C:\Qt
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
       - run:
           name: Build
           command: |
             $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
             $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
             $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
+            $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
@@ -113,6 +125,7 @@ jobs:
               "-DCMAKE_BUILD_TYPE=Release" `
               "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
               "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
+              "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
               "-S ..\gpt4all-chat" `
               "-B ."
             & "C:\Qt\Tools\Ninja\ninja.exe"
@@ -156,12 +169,29 @@ jobs:
               -S ../gpt4all-chat \
               -B .
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
-
+  build-ts-docs: 
+    docker: 
+      - image: cimg/base:stable
+    steps:
+      - checkout
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - node/install-packages:
+          pkg-manager: yarn
+          app-dir: gpt4all-bindings/typescript
+          override-ci-command: yarn install
+      - run:
+          name: build docs ts yo
+          command: |
+            cd gpt4all-bindings/typescript
+            yarn docs:build
   build-py-docs:
     docker:
       - image: circleci/python:3.8
     steps:
-      - checkout
+      - checkout 
       - run:
           name: Install dependencies
           command: |
@@ -184,15 +214,20 @@ jobs:
           command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"
 
   build-py-linux:
-    docker:
-      - image: circleci/python:3.8
+    machine:
+      image: ubuntu-2204:2023.04.2
     steps:
       - checkout
+      - run:
+          name: Set Python Version
+          command: pyenv global 3.11.2
       - run:
           name: Install dependencies
           command: |
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential
+            sudo apt-get install -y cmake build-essential vulkan-sdk
             pip install setuptools wheel cmake
       - run:
           name: Build C library
@@ -256,9 +291,15 @@ jobs:
       - run:
           name: Add MinGW64 to PATH
           command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
       - run:
           name: Install dependencies
-          command: choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+          command:
+            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
       - run:
           name: Install Python dependencies
           command: pip install setuptools wheel cmake
@@ -270,7 +311,8 @@ jobs:
             cd gpt4all-backend
             mkdir build
             cd build
-            cmake -G "MinGW Makefiles" .. 
+            $env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            cmake -G "MinGW Makefiles" .. -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
             cmake --build . --parallel
       - run:
           name: Build wheel
@@ -322,8 +364,10 @@ jobs:
       - run:
           name: Install dependencies
           command: |
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list            
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential
+            sudo apt-get install -y cmake build-essential vulkan-sdk
       - run:
           name: Build Libraries
           command: |
@@ -386,6 +430,11 @@ jobs:
       - run:
           name: Install MinGW64
           command: choco install -y mingw --force --no-progress
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
       - run:
           name: Install dependencies
           command: |
@@ -396,10 +445,11 @@ jobs:
             $MinGWBin = "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
             $Env:Path += ";$MinGwBin"
             $Env:Path += ";C:\Program Files\CMake\bin"
+            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
             cd gpt4all-backend
             mkdir runtimes/win-x64
             cd runtimes/win-x64
-            cmake -G "MinGW Makefiles" ../..
+            cmake -G "MinGW Makefiles" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON ../..
             cmake --build . --parallel --config Release
             cp "$MinGWBin\libgcc*.dll" .
             cp "$MinGWBin\libstdc++*.dll" .
@@ -422,6 +472,11 @@ jobs:
           command: |
             git submodule sync
             git submodule update --init --recursive
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
       - run:
           name: Install dependencies
           command: |
@@ -430,10 +485,11 @@ jobs:
           name: Build Libraries
           command: |
             $Env:Path += ";C:\Program Files\CMake\bin"
+            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
             cd gpt4all-backend
             mkdir runtimes/win-x64_msvc
             cd runtimes/win-x64_msvc
-            cmake -G "Visual Studio 17 2022" -A X64 ../..
+            cmake -G "Visual Studio 17 2022" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -A X64 ../..
             cmake --build . --parallel --config Release
             cp bin/Release/*.dll .
       - persist_to_workspace:
@@ -445,50 +501,47 @@ jobs:
     docker:
       - image: mcr.microsoft.com/dotnet/sdk:7.0-jammy # Ubuntu 22.04
     steps:
-      - when:
-          condition: << pipeline.parameters.run-csharp-workflow >>
-          steps:
-            - checkout
-            - attach_workspace:
-                at: /tmp/workspace
-            - run:
-                name: "Prepare Native Libs"
-                command: |
-                  cd gpt4all-bindings/csharp
-                  mkdir -p runtimes/linux-x64/native
-                  cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
-                  ls -R runtimes
-            - restore_cache:
-                keys:
-                  - gpt4all-csharp-nuget-packages-nix
-            - run:
-                name: "Install project dependencies"
-                command: |
-                  cd gpt4all-bindings/csharp
-                  dotnet restore Gpt4All
-            - save_cache:
-                paths:
-                  - ~/.nuget/packages
-                key: gpt4all-csharp-nuget-packages-nix
-            - run:
-                name: Build C# Project
-                command: |
-                  cd gpt4all-bindings/csharp
-                  dotnet build Gpt4All --configuration Release --nologo
-            - run:
-                name: "Run C# Tests"
-                command: |
-                  cd gpt4all-bindings/csharp
-                  dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
-            - run:
-                name: Test results
-                command: |
-                    cd gpt4all-bindings/csharp/Gpt4All.Tests
-                    dotnet tool install -g trx2junit
-                    export PATH="$PATH:$HOME/.dotnet/tools"
-                    trx2junit TestResults/*.trx
-            - store_test_results:
-                path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+      - checkout
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: "Prepare Native Libs"
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes/linux-x64/native
+            cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
+            ls -R runtimes
+      - restore_cache:
+          keys:
+            - gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: "Install project dependencies"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet restore Gpt4All
+      - save_cache:
+          paths:
+            - ~/.nuget/packages
+          key: gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: Build C# Project
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet build Gpt4All --configuration Release --nologo
+      - run:
+          name: "Run C# Tests"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+      - run:
+          name: Test results
+          command: |
+              cd gpt4all-bindings/csharp/Gpt4All.Tests
+              dotnet tool install -g trx2junit
+              export PATH="$PATH:$HOME/.dotnet/tools"
+              trx2junit TestResults/*.trx
+      - store_test_results:
+          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
 
   build-csharp-windows:
     executor:
@@ -496,104 +549,98 @@ jobs:
       size: large
       shell: powershell.exe -ExecutionPolicy Bypass
     steps:
-      - when:
-          condition: << pipeline.parameters.run-csharp-workflow >>
-          steps:
-            - checkout
-            - restore_cache:
-                keys:
-                  - gpt4all-csharp-nuget-packages-win
-            - attach_workspace:
-                at: C:\Users\circleci\workspace
-            - run:
-                name: "Prepare Native Libs"
-                command: |
-                  cd gpt4all-bindings/csharp
-                  mkdir -p runtimes\win-x64\native
-                  cp C:\Users\circleci\workspace\runtimes\win-x64\*.dll runtimes\win-x64\native\
-                  ls -R runtimes
-            - run:
-                name: "Install project dependencies"
-                command: |
-                  cd gpt4all-bindings/csharp
-                  dotnet.exe restore Gpt4All
-            - save_cache:
-                paths:
-                  - C:\Users\circleci\.nuget\packages
-                key: gpt4all-csharp-nuget-packages-win
-            - run:
-                name: Build C# Project
-                command: |
-                  cd gpt4all-bindings/csharp
-                  dotnet.exe build Gpt4All --configuration Release --nologo
-            - run:
-                name: "Run C# Tests"
-                command: |
-                  cd gpt4all-bindings/csharp
-                  dotnet.exe test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
-            - run:
-                name: Test results
-                command: |
-                    cd gpt4all-bindings/csharp/Gpt4All.Tests
-                    dotnet tool install -g trx2junit
-                    $Env:Path += ";$Env:USERPROFILE\.dotnet\tools"
-                    trx2junit TestResults/*.trx
-            - store_test_results:
-                path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+      - checkout
+      - restore_cache:
+          keys:
+            - gpt4all-csharp-nuget-packages-win
+      - attach_workspace:
+          at: C:\Users\circleci\workspace
+      - run:
+          name: "Prepare Native Libs"
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes\win-x64\native
+            cp C:\Users\circleci\workspace\runtimes\win-x64\*.dll runtimes\win-x64\native\
+            ls -R runtimes
+      - run:
+          name: "Install project dependencies"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet.exe restore Gpt4All
+      - save_cache:
+          paths:
+            - C:\Users\circleci\.nuget\packages
+          key: gpt4all-csharp-nuget-packages-win
+      - run:
+          name: Build C# Project
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet.exe build Gpt4All --configuration Release --nologo
+      - run:
+          name: "Run C# Tests"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet.exe test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+      - run:
+          name: Test results
+          command: |
+              cd gpt4all-bindings/csharp/Gpt4All.Tests
+              dotnet tool install -g trx2junit
+              $Env:Path += ";$Env:USERPROFILE\.dotnet\tools"
+              trx2junit TestResults/*.trx
+      - store_test_results:
+          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
 
   build-csharp-macos:
     macos:
       xcode: "14.0.0"
     steps:
-      - when:
-          condition: << pipeline.parameters.run-csharp-workflow >>
-          steps:
-            - checkout
-            - restore_cache:
-                keys:
-                  - gpt4all-csharp-nuget-packages-nix
-            - run:
-                name: Install dependencies
-                command: |
-                  brew install --cask dotnet-sdk
-            - attach_workspace:
-                at: /tmp/workspace
-            - run:
-                name: "Prepare Native Libs"
-                command: |
-                  cd gpt4all-bindings/csharp
-                  mkdir -p runtimes/osx/native
-                  cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
-                  cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
-                  ls -R runtimes
-            - run:
-                name: "Install project dependencies"
-                command: |
-                  cd gpt4all-bindings/csharp
-                  dotnet restore Gpt4All
-            - save_cache:
-                paths:
-                  - ~/.nuget/packages
-                key: gpt4all-csharp-nuget-packages-nix
-            - run:
-                name: Build C# Project
-                command: |
-                  cd gpt4all-bindings/csharp
-                  dotnet build Gpt4All --configuration Release --nologo
-            - run:
-                name: "Run C# Tests"
-                command: |
-                  cd gpt4all-bindings/csharp
-                  dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
-            - run:
-                name: Test results
-                command: |
-                    cd gpt4all-bindings/csharp/Gpt4All.Tests
-                    dotnet tool install -g trx2junit
-                    export PATH="$PATH:$HOME/.dotnet/tools"
-                    trx2junit TestResults/*.trx
-            - store_test_results:
-                path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+      - checkout
+      - restore_cache:
+          keys:
+            - gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: Install dependencies
+          command: |
+            brew install --cask dotnet-sdk
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: "Prepare Native Libs"
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes/osx/native
+            cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
+            cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
+            ls -R runtimes
+      - run:
+          name: "Install project dependencies"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet restore Gpt4All
+      - save_cache:
+          paths:
+            - ~/.nuget/packages
+          key: gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: Build C# Project
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet build Gpt4All --configuration Release --nologo
+      - run:
+          name: "Run C# Tests"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+      - run:
+          name: Test results
+          command: |
+              cd gpt4all-bindings/csharp/Gpt4All.Tests
+              dotnet tool install -g trx2junit
+              export PATH="$PATH:$HOME/.dotnet/tools"
+              trx2junit TestResults/*.trx
+      - store_test_results:
+          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
 
   store-and-upload-nupkgs:
     docker:
@@ -621,6 +668,160 @@ jobs:
       - store_artifacts:
           path: gpt4all-bindings/csharp/Gpt4All/bin/Release
 
+  build-nodejs-linux: 
+    docker: 
+      - image: cimg/base:stable
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - node/install-packages:
+          app-dir: gpt4all-bindings/typescript
+          pkg-manager: yarn
+      - run:
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn prebuildify -t 18.16.0 --napi
+      - run: 
+          command: |
+            mkdir -p gpt4all-backend/prebuilds/linux-x64
+            mkdir -p gpt4all-backend/runtimes/linux-x64
+            cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so gpt4all-backend/runtimes/linux-x64
+            cp gpt4all-bindings/typescript/prebuilds/linux-x64/*.node gpt4all-backend/prebuilds/linux-x64
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - prebuilds/linux-x64/*.node 
+            - runtimes/linux-x64/*-*.so
+  build-nodejs-macos: 
+    macos:
+      xcode: "14.0.0"
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - node/install-packages:
+          app-dir: gpt4all-bindings/typescript
+          pkg-manager: yarn
+      - run:
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn prebuildify -t 18.16.0 --napi
+      - run: 
+          name: "Persisting all necessary things to workspace"
+          command: |  
+            mkdir -p gpt4all-backend/prebuilds/darwin-x64
+            mkdir -p gpt4all-backend/runtimes/darwin-x64
+            cp /tmp/gpt4all-backend/runtimes/osx-x64/*-*.* gpt4all-backend/runtimes/darwin-x64
+            cp gpt4all-bindings/typescript/prebuilds/darwin-x64/*.node gpt4all-backend/prebuilds/darwin-x64
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - prebuilds/darwin-x64/*.node
+            - runtimes/darwin-x64/*-*.*
+
+  build-nodejs-windows: 
+    executor:
+      name: win/default
+      size: large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - run: choco install wget -y
+      - run:
+         command: wget https://nodejs.org/dist/v18.16.0/node-v18.16.0-x86.msi -P C:\Users\circleci\Downloads\
+         shell: cmd.exe
+      - run: MsiExec.exe /i C:\Users\circleci\Downloads\node-v18.16.0-x86.msi /qn
+      - run: 
+          command: |
+              Start-Process powershell -verb runAs -Args "-start GeneralProfile"
+              nvm install 18.16.0
+              nvm use 18.16.0
+      - run: node --version 
+      - run:           
+          command: |
+            npm install -g yarn
+            cd gpt4all-bindings/typescript
+            yarn install
+      - run:
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn prebuildify -t 18.16.0 --napi 
+      - run: 
+          command: |
+            mkdir -p gpt4all-backend/prebuilds/win32-x64
+            mkdir -p gpt4all-backend/runtimes/win32-x64
+            cp /tmp/gpt4all-backend/runtimes/win-x64_msvc/*-*.dll gpt4all-backend/runtimes/win32-x64
+            cp gpt4all-bindings/typescript/prebuilds/win32-x64/*.node gpt4all-backend/prebuilds/win32-x64
+        
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - prebuilds/win32-x64/*.node
+            - runtimes/win32-x64/*-*.dll
+
+  prepare-npm-pkg:
+    docker: 
+      - image: cimg/base:stable
+    steps:
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - checkout
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - run: 
+          command: |
+            cd gpt4all-bindings/typescript
+            # excluding llmodel. nodejs bindings dont need llmodel.dll
+            mkdir -p runtimes/win32-x64/native
+            mkdir -p prebuilds/win32-x64/
+            cp /tmp/gpt4all-backend/runtimes/win-x64_msvc/*-*.dll runtimes/win32-x64/native/ 
+            cp /tmp/gpt4all-backend/prebuilds/win32-x64/*.node prebuilds/win32-x64/    
+
+            mkdir -p runtimes/linux-x64/native 
+            mkdir -p prebuilds/linux-x64/
+            cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so runtimes/linux-x64/native/ 
+            cp /tmp/gpt4all-backend/prebuilds/linux-x64/*.node prebuilds/linux-x64/    
+
+            mkdir -p runtimes/darwin-x64/native
+            mkdir -p prebuilds/darwin-x64/
+            cp /tmp/gpt4all-backend/runtimes/darwin-x64/*-*.* runtimes/darwin-x64/native/
+            cp /tmp/gpt4all-backend/prebuilds/darwin-x64/*.node prebuilds/darwin-x64/    
+            
+            # Fallback build if user is not on above prebuilds
+            mv -f binding.ci.gyp binding.gyp
+
+            mkdir gpt4all-backend
+            cd ../../gpt4all-backend
+            mv llmodel.h llmodel.cpp llmodel_c.cpp llmodel_c.h sysinfo.h dlhandle.h ../gpt4all-bindings/typescript/gpt4all-backend/
+            
+      # Test install
+      - node/install-packages:
+          app-dir: gpt4all-bindings/typescript
+          pkg-manager: yarn
+          override-ci-command: yarn install
+      - run: 
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn run test
+      - run:
+          command: |
+            cd gpt4all-bindings/typescript
+            npm set //registry.npmjs.org/:_authToken=$NPM_TOKEN
+            npm publish --access public --tag alpha
+
 workflows:
   version: 2
   default:
@@ -644,6 +845,11 @@ workflows:
   deploy-docs:
     when: << pipeline.parameters.run-python-workflow >>
     jobs:
+      - build-ts-docs:
+          filters:
+            branches:
+              only:
+                - main
       - build-py-docs:
           filters:
             branches:
@@ -688,11 +894,14 @@ workflows:
         or:
          - << pipeline.parameters.run-python-workflow >>
          - << pipeline.parameters.run-csharp-workflow >>
+         - <<   pipeline.parameters.run-ts-workflow   >>
     jobs:
       - hold:
           type: approval
       - nuget-hold:
           type: approval
+      - npm-hold:
+          type: approval
       - build-bindings-backend-linux:
           filters:
             branches:
@@ -717,24 +926,61 @@ workflows:
               only:
           requires:
             - hold
+
+      # NodeJs Jobs 
+      - prepare-npm-pkg: 
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-nodejs-linux
+            - build-nodejs-windows
+            - build-nodejs-macos
+      - build-nodejs-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-bindings-backend-linux
+      - build-nodejs-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-bindings-backend-windows-msvc
+      - build-nodejs-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-bindings-backend-macos
+
+
       # CSharp Jobs
       - build-csharp-linux:
           filters:
             branches:
               only:
           requires:
+            - nuget-hold
             - build-bindings-backend-linux
       - build-csharp-windows:
           filters:
             branches:
               only:
           requires:
+            - nuget-hold
             - build-bindings-backend-windows
       - build-csharp-macos:
           filters:
             branches:
               only:
           requires:
+            - nuget-hold
             - build-bindings-backend-macos
       - store-and-upload-nupkgs:
           filters:
@@ -745,4 +991,3 @@ workflows:
             - build-csharp-windows
             - build-csharp-linux
             - build-csharp-macos
-        

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
-ignore-words-list = blong, belong, afterall
+ignore-words-list = blong, belong, afterall, som
 skip = .git,*.pdf,*.svg,*.lock

LICENSE_SOM.txt

@@ -0,0 +1,30 @@
+Software for Open Models License (SOM)
+Version 1.0 dated August 30th, 2023
+
+This license governs use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
+
+This license is intended to encourage open release of models created, modified, processed, or otherwise used via the Software under open licensing terms, and should be interpreted in light of that intent.
+
+1. Definitions
+The “Licensor” is the person or entity who is making the Software available under this license. “Software” is the software made available by Licensor under this license.
+A “Model” is the output of a machine learning algorithm, and excludes the Software.
+“Model Source Materials” must include the Model and model weights, and may include any input data, input data descriptions, documentation or training descriptions for the Model.
+“Open Licensing Terms” means: (a) any open source license approved by the Open Source Initiative, or (b) any other terms that make the Model Source Materials publicly available free of charge, and allow recipients to use, modify and distribute the Model Source Materials. Terms described in (b) may include reasonable restrictions such as non-commercial or non-production limitations, or require use in compliance with law.
+
+2. Grant of Rights. Subject to the conditions and limitations in section 3:
+(A) Copyright Grant. Licensor grants you a non-exclusive, worldwide, royalty-free copyright license to copy, modify, and distribute the Software and any modifications of the Software you create under this license. The foregoing license includes without limitation the right to create, modify, and use Models using this Software.
+
+(B) Patent Grant. Licensor grants you a non-exclusive, worldwide, royalty-free license, under any patents owned or controlled by Licensor, to make, have made, use, sell, offer for sale, import, or otherwise exploit the Software.  No license is granted to patent rights that are not embodied in the operation of the Software in the form provided by Licensor.
+
+3. Conditions and Limitations
+(A) Model Licensing and Access. If you use the Software to create, modify, process, or otherwise use any Model, including usage to create inferences with a Model, whether or not you make the Model available to others, you must make that Model Source Materials publicly available under Open Licensing Terms. 
+
+(B) No Re-Licensing. If you redistribute the Software, or modifications to the Software made under the license granted above, you must make it available only under the terms of this license. You may offer additional terms such as warranties, maintenance and support, but You, and not Licensor, are responsible for performing such terms.
+
+(C) No Trademark License. This license does not grant you rights to use the Licensor’s name, logo, or trademarks.
+
+(D) If you assert in writing a claim against any person or entity alleging that the use of the Software infringes any patent, all of your licenses to the Software under Section 2 end automatically as of the date you asserted the claim.
+
+(E) If you distribute any portion of the Software, you must retain all copyright, patent, trademark, and attribution notices that are present in the Software, and you must include a copy of this license.
+
+(F) The Software is licensed “as-is.” You bear the entire risk of using it. Licensor gives You no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws that this license cannot change. To the extent permitted under your local laws, the Licensor disclaims and excludes the implied warranties of merchantability, fitness for a particular purpose and non-infringement. To the extent this disclaimer is unlawful, you, and not Licensor, are responsible for any liability.

README.md

@@ -1,4 +1,5 @@
 <h1 align="center">GPT4All</h1>
+
 <p align="center">Open-source assistant-style large language models that run locally on your CPU</p>
 
 <p align="center">
@@ -29,7 +30,7 @@ Run on an M1 macOS Device (not sped up!)
 </p>
 
 ## GPT4All: An ecosystem of open-source on-edge large language models.
-GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs.
+GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
 
 Learn more in the [documentation](https://docs.gpt4all.io).
 
@@ -57,12 +58,15 @@ Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io
 
 ### Bindings
 
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a>
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/java">:computer: Official Java Bindings</a>
 
+### Integrations
+
+* 🗃️ [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
 
 ## Contributing
 GPT4All welcomes contributions, involvement, and discussion from the open source community!

gpt4all-api/.isort.cfg

@@ -0,0 +1,7 @@
+[settings]
+known_third_party=geopy,nltk,np,numpy,pandas,pysbd,fire,torch
+
+line_length=120
+include_trailing_comma=True
+multi_line_output=3
+use_parentheses=True

gpt4all-api/README.md

@@ -4,9 +4,13 @@ for serving inference from GPT4All models. The API matches the OpenAI API spec.
 
 ## Tutorial
 
+The following tutorial assumes that you have checked out this repo and cd'd into it.
+
 ### Starting the app
 
-First build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
+First change your working directory to `gpt4all/gpt4all-api`.
+
+Now you can build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
 ```bash
 DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
 ```
@@ -17,6 +21,18 @@ Then, start the backend with:
 docker compose up --build
 ```
 
+This will run both the API and locally hosted GPU inference server. If you want to run the API without the GPU inference server, you can run:
+
+```bash
+docker compose up --build gpt4all_api
+```
+
+To run the API with the GPU inference server, you will need to include environment variables (like the `MODEL_ID`). Edit the `.env` file and run
+```bash
+docker compose --env-file .env up --build
+```
+
+
 #### Spinning up your app
 Run `docker compose up` to spin up the backend. Monitor the logs for errors in-case you forgot to set an environment variable above.
 

gpt4all-api/docker-compose.gpu.yaml

@@ -0,0 +1,24 @@
+version: "3.8"
+
+services:
+  gpt4all_gpu:
+    image: ghcr.io/huggingface/text-generation-inference:0.9.3
+    container_name: gpt4all_gpu
+    restart: always #restart on error (usually code compilation from save during bad state)
+    environment:
+      - HUGGING_FACE_HUB_TOKEN=token
+      - USE_FLASH_ATTENTION=false
+      - MODEL_ID=''
+      - NUM_SHARD=1
+    command: --model-id $MODEL_ID --num-shard $NUM_SHARD
+    volumes:
+      - ./:/data
+    ports:
+      - "8080:80"
+    shm_size: 1g
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: [gpu]

gpt4all-api/docker-compose.yaml

@@ -1,4 +1,4 @@
-version: "3.5"
+version: "3.8"
 
 services:
   gpt4all_api:
@@ -13,6 +13,7 @@ services:
       - LOGLEVEL=debug
       - PORT=4891
       - model=ggml-mpt-7b-chat.bin
+      - inference_mode=cpu
     volumes:
       - './gpt4all_api/app:/app'
     command: ["/start-reload.sh"]

gpt4all-api/gpt4all_api/app/api_v1/api.py

@@ -1,4 +1,4 @@
-from api_v1.routes import chat, completions, engines
+from api_v1.routes import chat, completions, engines, health
 from fastapi import APIRouter
 
 router = APIRouter()
@@ -6,3 +6,4 @@ router = APIRouter()
 router.include_router(chat.router)
 router.include_router(completions.router)
 router.include_router(engines.router)
+router.include_router(health.router)

gpt4all-api/gpt4all_api/app/api_v1/events.py

@@ -1,8 +1,10 @@
 import logging
+
+from api_v1.settings import settings
 from fastapi import HTTPException
 from fastapi.responses import JSONResponse
 from starlette.requests import Request
-from api_v1.settings import settings
+
 log = logging.getLogger(__name__)
 
 
@@ -19,8 +21,9 @@ async def on_startup(app):
     startup_msg = startup_msg_fmt.format(settings=settings)
     log.info(startup_msg)
 
+
 def startup_event_handler(app):
     async def start_app() -> None:
         await on_startup(app)
 
-    return start_app
+    return start_app

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -1,9 +1,10 @@
-from fastapi import APIRouter, Depends, Response, Security, status
-from pydantic import BaseModel, Field
-from typing import List, Dict
 import logging
 import time
+from typing import Dict, List
+
 from api_v1.settings import settings
+from fastapi import APIRouter, Depends, Response, Security, status
+from pydantic import BaseModel, Field
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -11,11 +12,11 @@ logger.setLevel(logging.DEBUG)
 ### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
 
 
-
 class ChatCompletionMessage(BaseModel):
     role: str
     content: str
 
+
 class ChatCompletionRequest(BaseModel):
     model: str = Field(..., description='The model to generate a completion from.')
     messages: List[ChatCompletionMessage] = Field(..., description='The model to generate a completion from.')
@@ -26,11 +27,13 @@ class ChatCompletionChoice(BaseModel):
     index: int
     finish_reason: str
 
+
 class ChatCompletionUsage(BaseModel):
     prompt_tokens: int
     completion_tokens: int
     total_tokens: int
 
+
 class ChatCompletionResponse(BaseModel):
     id: str
     object: str = 'text_completion'
@@ -42,6 +45,7 @@ class ChatCompletionResponse(BaseModel):
 
 router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
 
+
 @router.post("/completions", response_model=ChatCompletionResponse)
 async def chat_completion(request: ChatCompletionRequest):
     '''
@@ -53,11 +57,5 @@ async def chat_completion(request: ChatCompletionRequest):
         created=time.time(),
         model=request.model,
         choices=[{}],
-        usage={
-            'prompt_tokens': 0,
-            'completion_tokens': 0,
-            'total_tokens': 0
-        }
+        usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
     )
-
-

gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py

@@ -1,14 +1,16 @@
 import json
-
-from fastapi import APIRouter, Depends, Response, Security, status
-from fastapi.responses import StreamingResponse
-from pydantic import BaseModel, Field
 from typing import List, Dict, Iterable, AsyncIterable
 import logging
-from uuid import uuid4
-from api_v1.settings import settings
-from gpt4all import GPT4All
 import time
+from typing import Dict, List, Union, Optional
+from uuid import uuid4
+import aiohttp
+import asyncio
+from api_v1.settings import settings
+from fastapi import APIRouter, Depends, Response, Security, status, HTTPException
+from fastapi.responses import StreamingResponse
+from gpt4all import GPT4All
+from pydantic import BaseModel, Field
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -16,14 +18,17 @@ logger.setLevel(logging.DEBUG)
 
 ### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
 
+
 class CompletionRequest(BaseModel):
-    model: str = Field(..., description='The model to generate a completion from.')
-    prompt: str = Field(..., description='The prompt to begin completing from.')
-    max_tokens: int = Field(7, description='Max tokens to generate')
-    temperature: float = Field(0, description='Model temperature')
-    top_p: float = Field(1.0, description='top_p')
-    n: int = Field(1, description='')
+    model: str = Field(settings.model, description='The model to generate a completion from.')
+    prompt: Union[List[str], str] = Field(..., description='The prompt to begin completing from.')
+    max_tokens: int = Field(None, description='Max tokens to generate')
+    temperature: float = Field(settings.temp, description='Model temperature')
+    top_p: Optional[float] = Field(settings.top_p, description='top_p')
+    top_k: Optional[int] = Field(settings.top_k, description='top_k')
+    n: int = Field(1, description='How many completions to generate for each prompt')
     stream: bool = Field(False, description='Stream responses')
+    repeat_penalty: float = Field(settings.repeat_penalty, description='Repeat penalty')
 
 
 class CompletionChoice(BaseModel):
@@ -58,7 +63,6 @@ class CompletionStreamResponse(BaseModel):
 
 router = APIRouter(prefix="/completions", tags=["Completion Endpoints"])
 
-
 def stream_completion(output: Iterable, base_response: CompletionStreamResponse):
     """
     Streams a GPT4All output to the client.
@@ -80,49 +84,132 @@ def stream_completion(output: Iterable, base_response: CompletionStreamResponse)
         ))]
         yield f"data: {json.dumps(dict(chunk))}\n\n"
 
+async def gpu_infer(payload, header):
+    async with aiohttp.ClientSession() as session:
+        try:
+            async with session.post(
+                settings.hf_inference_server_host, headers=header, data=json.dumps(payload)
+            ) as response:
+                resp = await response.json()
+            return resp
+
+        except aiohttp.ClientError as e:
+            # Handle client-side errors (e.g., connection error, invalid URL)
+            logger.error(f"Client error: {e}")
+        except aiohttp.ServerError as e:
+            # Handle server-side errors (e.g., internal server error)
+            logger.error(f"Server error: {e}")
+        except json.JSONDecodeError as e:
+            # Handle JSON decoding errors
+            logger.error(f"JSON decoding error: {e}")
+        except Exception as e:
+            # Handle other unexpected exceptions
+            logger.error(f"Unexpected error: {e}")
 
 @router.post("/", response_model=CompletionResponse)
 async def completions(request: CompletionRequest):
     '''
     Completes a GPT4All model response.
     '''
+    if settings.inference_mode == "gpu":
+        params = request.dict(exclude={'model', 'prompt', 'max_tokens', 'n'})
+        params["max_new_tokens"] = request.max_tokens
+        params["num_return_sequences"] = request.n
 
-    model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+        header = {"Content-Type": "application/json"}
+        if isinstance(request.prompt, list):
+            tasks = []
+            for prompt in request.prompt:
+                payload = {"parameters": params}
+                payload["inputs"] = prompt
+                task = gpu_infer(payload, header)
+                tasks.append(task)
+            results = await asyncio.gather(*tasks)
 
-    output = model.generate(prompt=request.prompt,
-                            n_predict=request.max_tokens,
-                            streaming=request.stream,
-                            top_k=20,
-                            top_p=request.top_p,
-                            temp=request.temperature,
-                            n_batch=1024,
-                            repeat_penalty=1.2,
-                            repeat_last_n=10)
+            choices = []
+            for response in results:
+                scores = response["scores"] if "scores" in response else -1.0
+                choices.append(
+                    dict(
+                        CompletionChoice(
+                            text=response["generated_text"], index=0, logprobs=scores, finish_reason='stop'
+                        )
+                    )
+                )
+
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=choices,
+                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+            )
+
+        else:
+            payload = {"parameters": params}
+            # If streaming, we need to return a StreamingResponse
+            payload["inputs"] = request.prompt
+
+            resp = await gpu_infer(payload, header)
+
+            output = resp["generated_text"]
+            # this returns all logprobs
+            scores = resp["scores"] if "scores" in resp else -1.0
+
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[dict(CompletionChoice(text=output, index=0, logprobs=scores, finish_reason='stop'))],
+                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+            )
 
-    # If streaming, we need to return a StreamingResponse
-    if request.stream:
-        base_chunk = CompletionStreamResponse(
-            id=str(uuid4()),
-            created=time.time(),
-            model=request.model,
-            choices=[]
-        )
-        return StreamingResponse((response for response in stream_completion(output, base_chunk)),
-                                 media_type="text/event-stream")
     else:
-        return CompletionResponse(
-            id=str(uuid4()),
-            created=time.time(),
-            model=request.model,
-            choices=[dict(CompletionChoice(
-                text=output,
-                index=0,
-                logprobs=-1,
-                finish_reason='stop'
-            ))],
-            usage={
-                'prompt_tokens': 0, #TODO how to compute this?
-                'completion_tokens': 0,
-                'total_tokens': 0
-            }
-        )
+
+        if request.model != settings.model:
+            raise HTTPException(status_code=400,
+                                detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
+
+        if isinstance(request.prompt, list):
+            if len(request.prompt) > 1:
+                raise HTTPException(status_code=400, detail="Can only infer one inference per request in CPU mode.")
+            else:
+                request.prompt = request.prompt[0]
+
+        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+        output = model.generate(prompt=request.prompt,
+                                max_tokens=request.max_tokens,
+                                streaming=request.stream,
+                                top_k=request.top_k,
+                                top_p=request.top_p,
+                                temp=request.temperature,
+                                )
+
+        # If streaming, we need to return a StreamingResponse
+        if request.stream:
+            base_chunk = CompletionStreamResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[]
+            )
+            return StreamingResponse((response for response in stream_completion(output, base_chunk)),
+                                     media_type="text/event-stream")
+        else:
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[dict(CompletionChoice(
+                    text=output,
+                    index=0,
+                    logprobs=-1,
+                    finish_reason='stop'
+                ))],
+                usage={
+                    'prompt_tokens': 0,  # TODO how to compute this?
+                    'completion_tokens': 0,
+                    'total_tokens': 0
+                }
+            )

gpt4all-api/gpt4all_api/app/api_v1/routes/embeddings.py

@@ -0,0 +1,65 @@
+from typing import List, Union
+from fastapi import APIRouter
+from api_v1.settings import settings
+from gpt4all import Embed4All
+from pydantic import BaseModel, Field
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+class EmbeddingRequest(BaseModel):
+    model: str = Field(
+        settings.model, description="The model to generate an embedding from."
+    )
+    input: Union[str, List[str], List[int], List[List[int]]] = Field(
+        ..., description="Input text to embed, encoded as a string or array of tokens."
+    )
+
+
+class EmbeddingUsage(BaseModel):
+    prompt_tokens: int = 0
+    total_tokens: int = 0
+
+
+class Embedding(BaseModel):
+    index: int = 0
+    object: str = "embedding"
+    embedding: List[float]
+
+
+class EmbeddingResponse(BaseModel):
+    object: str = "list"
+    model: str
+    data: List[Embedding]
+    usage: EmbeddingUsage
+
+
+router = APIRouter(prefix="/embeddings", tags=["Embedding Endpoints"])
+
+embedder = Embed4All()
+
+
+def get_embedding(data: EmbeddingRequest) -> EmbeddingResponse:
+    """
+    Calculates the embedding for the given input using a specified model.
+
+    Args:
+        data (EmbeddingRequest): An EmbeddingRequest object containing the input data
+        and model name.
+
+    Returns:
+        EmbeddingResponse: An EmbeddingResponse object encapsulating the calculated embedding,
+        usage info, and the model name.
+    """
+    embedding = embedder.embed(data.input)
+    return EmbeddingResponse(
+        data=[Embedding(embedding=embedding)], usage=EmbeddingUsage(), model=data.model
+    )
+
+
+@router.post("/", response_model=EmbeddingResponse)
+def embeddings(data: EmbeddingRequest):
+    """
+    Creates a GPT4All embedding
+    """
+    return get_embedding(data)

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -1,22 +1,27 @@
+import logging
+from typing import Dict, List
+
+from api_v1.settings import settings
 from fastapi import APIRouter, Depends, Response, Security, status
 from pydantic import BaseModel, Field
-from typing import List, Dict
-import logging
-from api_v1.settings import settings
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
 ### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
 
+
 class ListEnginesResponse(BaseModel):
     data: List[Dict] = Field(..., description="All available models.")
 
+
 class EngineResponse(BaseModel):
     data: List[Dict] = Field(..., description="All available models.")
 
+
 router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
 
+
 @router.get("/", response_model=ListEnginesResponse)
 async def list_engines():
     '''
@@ -29,10 +34,7 @@ async def list_engines():
 
 @router.get("/{engine_id}", response_model=EngineResponse)
 async def retrieve_engine(engine_id: str):
-    '''
-
-    '''
+    ''' '''
 
     raise NotImplementedError()
     return EngineResponse()
-

gpt4all-api/gpt4all_api/app/api_v1/routes/health.py

@@ -1,6 +1,7 @@
 import logging
 from fastapi import APIRouter
 from fastapi.responses import JSONResponse
+
 log = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/health", tags=["Health"])

gpt4all-api/gpt4all_api/app/api_v1/settings.py

@@ -5,6 +5,15 @@ class Settings(BaseSettings):
     app_environment = 'dev'
     model: str = 'ggml-mpt-7b-chat.bin'
     gpt4all_path: str = '/models'
+    inference_mode: str = "cpu"
+    hf_inference_server_host: str = "http://gpt4all_gpu:80/generate"
+    sentry_dns: str = None
+
+    temp: float = 0.18
+    top_p: float = 1.0
+    top_k: int = 50
+    repeat_penalty: float = 1.18
+
 
 
 settings = Settings()

gpt4all-api/gpt4all_api/app/main.py

@@ -1,19 +1,19 @@
-import os
-import docs
 import logging
-from fastapi import FastAPI, HTTPException, Request
-from starlette.middleware.cors import CORSMiddleware
-from fastapi.logger import logger as fastapi_logger
-from api_v1.settings import settings
-from api_v1.api import router as v1_router
-from api_v1 import events
 import os
 
+import docs
+from api_v1 import events
+from api_v1.api import router as v1_router
+from api_v1.settings import settings
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.logger import logger as fastapi_logger
+from starlette.middleware.cors import CORSMiddleware
+
 logger = logging.getLogger(__name__)
 
 app = FastAPI(title='GPT4All API', description=docs.desc)
 
-#CORS Configuration (in-case you want to deploy)
+# CORS Configuration (in-case you want to deploy)
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -29,20 +29,41 @@ app.include_router(v1_router, prefix='/v1')
 app.add_event_handler('startup', events.startup_event_handler(app))
 app.add_exception_handler(HTTPException, events.on_http_error)
 
+
 @app.on_event("startup")
 async def startup():
     global model
-    logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
-    from gpt4all import GPT4All
-    model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+    if settings.inference_mode == "cpu":
+        logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
+        from gpt4all import GPT4All
+
+        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
+
+    else:
+        # is it possible to do this once the server is up?
+        ## TODO block until HF inference server is up.
+        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
+
 
-    logger.info("GPT4All API is ready.")
 
 @app.on_event("shutdown")
 async def shutdown():
     logger.info("Shutting down API")
 
 
+if settings.sentry_dns is not None:
+    import sentry_sdk
+
+    def traces_sampler(sampling_context):
+        if 'health' in sampling_context['transaction_context']['name']:
+            return False
+
+    sentry_sdk.init(
+        dsn=settings.sentry_dns, traces_sample_rate=0.1, traces_sampler=traces_sampler, send_default_pii=False
+    )
+
 # This is needed to get logs to show up in the app
 if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
     gunicorn_error_logger = logging.getLogger("gunicorn.error")
@@ -57,5 +78,7 @@ if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
     uvicorn_logger.handlers = gunicorn_error_logger.handlers
 else:
     # https://github.com/tiangolo/fastapi/issues/2019
-    LOG_FORMAT2 = "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
-    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)
+    LOG_FORMAT2 = (
+        "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
+    )
+    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -1,31 +1,25 @@
 """
 Use the OpenAI python API to test gpt4all models.
 """
+from typing import List, get_args
+
 import openai
+
 openai.api_base = "http://localhost:4891/v1"
 
 openai.api_key = "not needed for a local LLM"
 
 
 def test_completion():
-    model = "gpt4all-j-v1.3-groovy"
+    model = "ggml-mpt-7b-chat.bin"
     prompt = "Who is Michael Jordan?"
     response = openai.Completion.create(
-        model=model,
-        prompt=prompt,
-        max_tokens=50,
-        temperature=0.28,
-        top_p=0.95,
-        n=1,
-        echo=True,
-        stream=False
+        model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
     )
     assert len(response['choices'][0]['text']) > len(prompt)
-    print(response)
-
 
 def test_streaming_completion():
-    model = "gpt4all-j-v1.3-groovy"
+    model = "ggml-mpt-7b-chat.bin"
     prompt = "Who is Michael Jordan?"
     tokens = []
     for resp in openai.Completion.create(
@@ -42,10 +36,24 @@ def test_streaming_completion():
     assert (len(tokens) > 0)
     assert (len("".join(tokens)) > len(prompt))
 
-# def test_chat_completions():
-#     model = "gpt4all-j-v1.3-groovy"
-#     prompt = "Who is Michael Jordan?"
-#     response = openai.ChatCompletion.create(
-#         model=model,
-#         messages=[]
-#     )
+
+def test_batched_completion():
+    model = "ggml-mpt-7b-chat.bin"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Completion.create(
+        model=model, prompt=[prompt] * 3, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
+    )
+    assert len(response['choices'][0]['text']) > len(prompt)
+    assert len(response['choices']) == 3
+
+
+def test_embedding():
+    model = "ggml-all-MiniLM-L6-v2-f16.bin"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Embedding.create(model=model, input=prompt)
+    output = response["data"][0]["embedding"]
+    args = get_args(List[float])
+
+    assert response["model"] == model
+    assert isinstance(output, list)
+    assert all(isinstance(x, args) for x in output)

gpt4all-api/gpt4all_api/requirements.txt

@@ -1,10 +1,12 @@
 aiohttp>=3.6.2
 aiofiles
-pydantic>=1.4.0
+pydantic>=1.4.0,<2.0.0
 requests>=2.24.0
 ujson>=2.0.2
 fastapi>=0.95.0
 Jinja2>=3.0
-gpt4all==1.0.1
+gpt4all>=1.0.0
 pytest
-openai
+openai
+black
+isort

gpt4all-api/makefile

@@ -1,22 +1,26 @@
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 APP_NAME:=gpt4all_api
 PYTHON:=python3.8
+SHELL := /bin/bash
 
 all: dependencies
 
 fresh: clean dependencies
 
 testenv: clean_testenv test_build
-	docker compose up --build
+	docker compose -f docker-compose.yaml up --build
+
+testenv_gpu: clean_testenv test_build
+	docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up --build
 
 testenv_d: clean_testenv test_build
 	docker compose up --build -d
 
 test:
-	docker compose exec gpt4all_api pytest -svv --disable-warnings -p no:cacheprovider /app/tests
+	docker compose exec $(APP_NAME) pytest -svv --disable-warnings -p no:cacheprovider /app/tests
 
 test_build:
-    DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
+    DOCKER_BUILDKIT=1 docker build -t $(APP_NAME) --progress plain -f $(APP_NAME)/Dockerfile.buildkit .
 
 clean_testenv:
 	docker compose down -v
@@ -27,7 +31,7 @@ venv:
 	if [ ! -d $(ROOT_DIR)/env ]; then $(PYTHON) -m venv $(ROOT_DIR)/env; fi
 
 dependencies: venv
-	source $(ROOT_DIR)/env/bin/activate; yes w | python -m pip install -r $(ROOT_DIR)/gpt4all_api/requirements.txt
+	source $(ROOT_DIR)/env/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
 
 clean: clean_testenv
 	# Remove existing environment
@@ -35,3 +39,8 @@ clean: clean_testenv
 	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
 
 
+black:
+	source $(ROOT_DIR)/env/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
+
+isort:
+	source $(ROOT_DIR)/env/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)

gpt4all-backend/CMakeLists.txt

@@ -20,7 +20,7 @@ endif()
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 
 set(LLMODEL_VERSION_MAJOR 0)
-set(LLMODEL_VERSION_MINOR 2)
+set(LLMODEL_VERSION_MINOR 4)
 set(LLMODEL_VERSION_PATCH 0)
 set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
 project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
@@ -39,6 +39,10 @@ else()
     message(STATUS "Interprocedural optimization support detected")
 endif()
 
+if(NOT APPLE)
+  set(LLAMA_KOMPUTE YES)
+endif()
+
 include(llama.cpp.cmake)
 
 set(BUILD_VARIANTS default avxonly)
@@ -69,11 +73,6 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     # Include GGML
     set(LLAMA_K_QUANTS YES)
     include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
-    if (NOT LLAMA_METAL)
-        set(LLAMA_K_QUANTS NO)
-        include_ggml(llama.cpp-230511 -230511-${BUILD_VARIANT} ON)
-        include_ggml(llama.cpp-230519 -230519-${BUILD_VARIANT} ON)
-    endif()
 
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
@@ -100,31 +99,33 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
 
     add_library(replit-mainline-${BUILD_VARIANT} SHARED
     replit.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+    target_compile_definitions(replit-mainline-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(replit-mainline llama-mainline)
 
     if (NOT LLAMA_METAL)
-        add_library(llamamodel-230519-${BUILD_VARIANT} SHARED
-            llamamodel.cpp llmodel_shared.cpp)
-        target_compile_definitions(llamamodel-230519-${BUILD_VARIANT} PRIVATE
-            LLAMA_VERSIONS===2 LLAMA_DATE=230519)
-        prepare_target(llamamodel-230519 llama-230519)
-        add_library(llamamodel-230511-${BUILD_VARIANT} SHARED
-            llamamodel.cpp llmodel_shared.cpp)
-        target_compile_definitions(llamamodel-230511-${BUILD_VARIANT} PRIVATE
-            LLAMA_VERSIONS=<=1 LLAMA_DATE=230511)
-        prepare_target(llamamodel-230511 llama-230511)
-
-        add_library(gptj-${BUILD_VARIANT} SHARED
-            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        prepare_target(gptj ggml-230511)
+# FIXME: These need to be forward ported to latest ggml
+#        add_library(gptj-${BUILD_VARIANT} SHARED
+#            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+#        prepare_target(gptj ggml-230511)
 
         add_library(falcon-${BUILD_VARIANT} SHARED
             falcon.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(falcon-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
         prepare_target(falcon llama-mainline)
+# FIXME: These need to be forward ported to latest ggml
+#        add_library(mpt-${BUILD_VARIANT} SHARED
+#            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+#        prepare_target(mpt ggml-230511)
 
-        add_library(mpt-${BUILD_VARIANT} SHARED
-            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        prepare_target(mpt ggml-230511)
+        add_library(bert-${BUILD_VARIANT} SHARED
+            bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+        prepare_target(bert llama-mainline)
+
+        add_library(starcoder-${BUILD_VARIANT} SHARED
+            starcoder.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(starcoder-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+        prepare_target(starcoder llama-mainline)
     endif()
 endforeach()
 

gpt4all-backend/bert_impl.h

@@ -0,0 +1,44 @@
+#ifndef BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of bert.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef BERT_H
+#define BERT_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <memory>
+#include "llmodel.h"
+
+struct BertPrivate;
+class Bert : public LLModel {
+public:
+    Bert();
+    ~Bert();
+
+    bool supportsEmbedding() const override { return true; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+    std::vector<float> embedding(const std::string &text) override;
+
+private:
+    std::unique_ptr<BertPrivate> d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // BERT_H

gpt4all-backend/dlhandle.h

@@ -75,7 +75,7 @@ public:
 
     Dlhandle() : chandle(nullptr) {}
     Dlhandle(const std::string& fpath) {
-        chandle = LoadLibraryA(fpath.c_str());
+        chandle = LoadLibraryExA(fpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
         if (!chandle) {
             throw Exception("dlopen(\""+fpath+"\"): Error");
         }

gpt4all-backend/falcon.cpp

@@ -1,3 +1,4 @@
+#include "ggml.h"
 #define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
 #include "falcon_impl.h"
 #include "llama.h"
@@ -64,6 +65,7 @@ struct falcon_model {
     std::map<std::string, struct ggml_tensor*> tensors;
 
     llm_buffer eval_buf;
+    llm_buffer work_buf;
     llm_buffer scr0_buf;
     llm_buffer scr1_buf;
 };
@@ -446,7 +448,7 @@ bool falcon_model_load(const std::string & fname, falcon_model & model, gpt_voca
 //   - embd_w:    the predicted logits for the next token
 //
 bool falcon_eval(
-        const falcon_model & model,
+        falcon_model & model,
         const int n_threads,
         const int n_past,
         const std::vector<gpt_vocab::id> & embd_inp,
@@ -473,7 +475,6 @@ bool falcon_eval(
 
     struct ggml_context * ctx0 = ggml_init(eval_ctx_params);
     struct ggml_cgraph gf = {};
-    gf.n_threads = n_threads;
 
     struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
     memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -546,8 +547,8 @@ bool falcon_eval(
                 head_dim * (n_head + n_head_kv) * sizeof_wtype);
 
             // using mode = 2 for neox mode
-            Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, head_dim, 2);
-            Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, head_dim, 2);
+            Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, head_dim, 2, n_ctx);
+            Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, head_dim, 2, n_ctx);
 
             // store key and value to memory
             {
@@ -678,7 +679,8 @@ bool falcon_eval(
 
     // run the computation
     ggml_build_forward_expand(&gf, inpL);
-    ggml_graph_compute       (ctx0, &gf);
+    ggml_graph_compute_g4a(model.work_buf, &gf, n_threads);
+  
 
     //if (n_past%100 == 0) {
     //    ggml_graph_print   (&gf);

gpt4all-backend/falcon_impl.h

@@ -16,6 +16,8 @@ public:
     Falcon();
     ~Falcon();
 
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath) override;

gpt4all-backend/gptj.cpp

@@ -961,6 +961,11 @@ DLL_EXPORT const char *get_build_variant() {
 DLL_EXPORT bool magic_match(std::istream& f) {
     uint32_t magic = 0;
     f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    gptj_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    if (!(hparams.n_vocab >= 50300 && hparams.n_vocab <= 50400)) {
+        return false; // not a gptj.
+    }
     return magic == 0x67676d6c;
 }
 

gpt4all-backend/gptj_impl.h

@@ -15,6 +15,8 @@ public:
     GPTJ();
     ~GPTJ();
 
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath) override;

gpt4all-backend/llama.cpp.cmake

@@ -1,3 +1,11 @@
+#
+# Copyright (c) 2023 Nomic, Inc. All rights reserved.
+#
+# This software is licensed under the terms of the Software for Open Models License (SOM),
+# version 1.0, as detailed in the LICENSE_SOM.txt file. A copy of this license should accompany
+# this software. Except as expressly granted in the SOM license, all rights are reserved by Nomic, Inc.
+#
+
 cmake_minimum_required(VERSION 3.12) # Don't bump this version for no reason
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@@ -145,6 +153,130 @@ if (LLAMA_OPENBLAS)
     endif()
 endif()
 
+if (LLAMA_KOMPUTE)
+    add_compile_definitions(VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
+    find_package(Vulkan COMPONENTS glslc REQUIRED)
+    find_program(glslc_executable NAMES glslc HINTS Vulkan::glslc)
+    if (NOT glslc_executable)
+        message(FATAL_ERROR "glslc not found")
+    endif()
+
+    set(LLAMA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp-mainline)
+
+    function(compile_shader)
+      set(options)
+      set(oneValueArgs)
+      set(multiValueArgs SOURCES)
+      cmake_parse_arguments(compile_shader "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+      foreach(source ${compile_shader_SOURCES})
+        get_filename_component(OP_FILE ${source} NAME)
+        set(spv_file ${CMAKE_CURRENT_BINARY_DIR}/${OP_FILE}.spv)
+        add_custom_command(
+            OUTPUT ${spv_file}
+            DEPENDS ${LLAMA_DIR}/${source}
+            COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
+            COMMENT "Compiling ${source} to ${source}.spv"
+        )
+
+        get_filename_component(RAW_FILE_NAME ${spv_file} NAME)
+        set(FILE_NAME "shader${RAW_FILE_NAME}")
+        string(REPLACE ".comp.spv" ".h" HEADER_FILE ${FILE_NAME})
+        string(TOUPPER ${HEADER_FILE} HEADER_FILE_DEFINE)
+        string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}")
+        set(OUTPUT_HEADER_FILE "${HEADER_FILE}")
+        message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}")
+        add_custom_command(
+          OUTPUT ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+          DEPENDS ${spv_file} xxd
+          COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
+        )
+      endforeach()
+    endfunction()
+
+    if (EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
+        message(STATUS "Kompute found")
+        add_subdirectory(${LLAMA_DIR}/kompute)
+
+        # Compile our shaders
+        compile_shader(SOURCES
+          kompute/op_scale.comp
+          kompute/op_add.comp
+          kompute/op_addrow.comp
+          kompute/op_mul.comp
+          kompute/op_mulrow.comp
+          kompute/op_silu.comp
+          kompute/op_relu.comp
+          kompute/op_gelu.comp
+          kompute/op_softmax.comp
+          kompute/op_norm.comp
+          kompute/op_rmsnorm.comp
+          kompute/op_diagmask.comp
+          kompute/op_mul_mat_f16.comp
+          kompute/op_mul_mat_q4_0.comp
+          kompute/op_mul_mat_q4_1.comp
+          kompute/op_getrows_f16.comp
+          kompute/op_getrows_q4_0.comp
+          kompute/op_getrows_q4_1.comp
+          kompute/op_rope.comp
+          kompute/op_cpy_f16_f16.comp
+          kompute/op_cpy_f16_f32.comp
+          kompute/op_cpy_f32_f16.comp
+          kompute/op_cpy_f32_f32.comp
+        )
+
+        # Create a custom target for our generated shaders
+        add_custom_target(generated_shaders DEPENDS
+          shaderop_scale.h
+          shaderop_add.h
+          shaderop_addrow.h
+          shaderop_mul.h
+          shaderop_mulrow.h
+          shaderop_silu.h
+          shaderop_relu.h
+          shaderop_gelu.h
+          shaderop_softmax.h
+          shaderop_norm.h
+          shaderop_rmsnorm.h
+          shaderop_diagmask.h
+          shaderop_mul_mat_f16.h
+          shaderop_mul_mat_q4_0.h
+          shaderop_mul_mat_q4_1.h
+          shaderop_getrows_f16.h
+          shaderop_getrows_q4_0.h
+          shaderop_getrows_q4_1.h
+          shaderop_rope.h
+          shaderop_cpy_f16_f16.h
+          shaderop_cpy_f16_f32.h
+          shaderop_cpy_f32_f16.h
+          shaderop_cpy_f32_f32.h
+        )
+
+        # Create a custom command that depends on the generated_shaders
+        add_custom_command(
+            OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp
+            COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp
+            DEPENDS generated_shaders
+            COMMENT "Ensuring shaders are generated before compiling ggml-vulkan.cpp"
+        )
+
+        # Add the stamp to the main sources to ensure dependency tracking
+        set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml-vulkan.cpp ${LLAMA_DIR}/ggml-vulkan.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp)
+        add_compile_definitions(GGML_USE_KOMPUTE)
+        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute)
+        set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CMAKE_BINARY_DIR})
+    else()
+        message(WARNING "Kompute not found")
+    endif()
+endif()
+
 if (LLAMA_ALL_WARNINGS)
     if (NOT MSVC)
         set(c_flags
@@ -296,10 +428,13 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
     add_library(ggml${SUFFIX} OBJECT
                 ${DIRECTORY}/ggml.c
                 ${DIRECTORY}/ggml.h
+                ${DIRECTORY}/ggml-alloc.c
+                ${DIRECTORY}/ggml-alloc.h
                 ${GGML_SOURCES_QUANT_K}
                 ${GGML_SOURCES_CUDA}
                 ${GGML_METAL_SOURCES}
-                ${GGML_OPENCL_SOURCES})
+                ${GGML_OPENCL_SOURCES}
+                ${GGML_SOURCES_KOMPUTE})
 
     if (LLAMA_K_QUANTS)
         target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_K_QUANTS)

gpt4all-backend/llamamodel.cpp

@@ -28,6 +28,9 @@
 #include <llama.h>
 #include <ggml.h>
 
+#ifdef GGML_USE_KOMPUTE
+#include "ggml-vulkan.h"
+#endif
 
 namespace {
 const char *modelType_ = "LLaMA";
@@ -155,6 +158,13 @@ bool LLamaModel::loadModel(const std::string &modelPath)
     // currently
     d_ptr->params.n_gpu_layers = 1;
 #endif
+#ifdef GGML_USE_KOMPUTE
+    if (ggml_vk_has_device()) {
+        // vulkan always runs the whole model if n_gpu_layers is not 0, at least
+        // currently
+        d_ptr->params.n_gpu_layers = 1;
+    }
+#endif
 
     d_ptr->ctx = llama_init_from_file(modelPath.c_str(), d_ptr->params);
     if (!d_ptr->ctx) {
@@ -162,6 +172,12 @@ bool LLamaModel::loadModel(const std::string &modelPath)
         return false;
     }
 
+#ifdef GGML_USE_KOMPUTE
+    if (ggml_vk_has_device()) {
+        std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
+    }
+#endif
+
     d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
     d_ptr->modelLoaded = true;
     fflush(stderr);
@@ -252,6 +268,75 @@ const std::vector<LLModel::Token> &LLamaModel::endTokens() const
     return fres;
 }
 
+#if defined(GGML_USE_KOMPUTE)
+#include "ggml-vulkan.h"
+#endif
+
+std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryRequired)
+{
+#if defined(GGML_USE_KOMPUTE)
+    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(memoryRequired);
+
+    std::vector<LLModel::GPUDevice> devices;
+    for(const auto& vkDevice : vkDevices) {
+        LLModel::GPUDevice device;
+        device.index = vkDevice.index;
+        device.type = vkDevice.type;
+        device.heapSize = vkDevice.heapSize;
+        device.name = vkDevice.name;
+        device.vendor = vkDevice.vendor;
+
+        devices.push_back(device);
+    }
+
+    return devices;
+#else
+    return std::vector<LLModel::GPUDevice>();
+#endif
+}
+
+bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string& device)
+{
+#if defined(GGML_USE_KOMPUTE)
+    return ggml_vk_init_device(memoryRequired, device);
+#else
+    return false;
+#endif
+}
+
+bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device)
+{
+#if defined(GGML_USE_KOMPUTE)
+    ggml_vk_device vkDevice;
+    vkDevice.index = device.index;
+    vkDevice.type = device.type;
+    vkDevice.heapSize = device.heapSize;
+    vkDevice.name = device.name;
+    vkDevice.vendor = device.vendor;
+    return ggml_vk_init_device(vkDevice);
+#else
+    return false;
+#endif
+}
+
+bool LLamaModel::initializeGPUDevice(int device)
+{
+#if defined(GGML_USE_KOMPUTE)
+    return ggml_vk_init_device(device);
+#else
+    return false;
+#endif
+}
+
+bool LLamaModel::hasGPUDevice()
+{
+#if defined(GGML_USE_KOMPUTE)
+    return ggml_vk_has_device();
+#else
+    return false;
+#endif
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else

gpt4all-backend/llamamodel_impl.h

@@ -15,6 +15,8 @@ public:
     LLamaModel();
     ~LLamaModel();
 
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath) override;
@@ -23,6 +25,11 @@ public:
     size_t restoreState(const uint8_t *src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
+    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) override;
+    bool initializeGPUDevice(size_t memoryRequired, const std::string& device) override;
+    bool initializeGPUDevice(const GPUDevice &device) override;
+    bool initializeGPUDevice(int device) override;
+    bool hasGPUDevice() override;
 
 private:
     LLamaPrivate *d_ptr;

gpt4all-backend/llmodel.cpp

@@ -11,18 +11,19 @@
 #include <cstdlib>
 #include <sstream>
 #ifdef _MSC_VER
-#include <windows.h>
-#include <processthreadsapi.h>
+#include <intrin.h>
 #endif
 
 std::string s_implementations_search_path = ".";
 
 static bool has_at_least_minimal_hardware() {
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(_M_X64)
     #ifndef _MSC_VER
         return __builtin_cpu_supports("avx");
     #else
-        return IsProcessorFeaturePresent(PF_AVX_INSTRUCTIONS_AVAILABLE);
+        int cpuInfo[4];
+        __cpuid(cpuInfo, 1);
+        return cpuInfo[2] & (1 << 28);
     #endif
 #else
     return true; // Don't know how to handle non-x86_64
@@ -30,52 +31,55 @@ static bool has_at_least_minimal_hardware() {
 }
 
 static bool requires_avxonly() {
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(_M_X64)
     #ifndef _MSC_VER
         return !__builtin_cpu_supports("avx2");
     #else
-        return !IsProcessorFeaturePresent(PF_AVX2_INSTRUCTIONS_AVAILABLE);
+        int cpuInfo[4];
+        __cpuidex(cpuInfo, 7, 0);
+        return !(cpuInfo[1] & (1 << 5));
     #endif
 #else
     return false; // Don't know how to handle non-x86_64
 #endif
 }
 
-LLModel::Implementation::Implementation(Dlhandle &&dlhandle_) : dlhandle(new Dlhandle(std::move(dlhandle_))) {
-    auto get_model_type = dlhandle->get<const char *()>("get_model_type");
+LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
+    : m_dlhandle(new Dlhandle(std::move(dlhandle_))) {
+    auto get_model_type = m_dlhandle->get<const char *()>("get_model_type");
     assert(get_model_type);
-    modelType = get_model_type();
-    auto get_build_variant = dlhandle->get<const char *()>("get_build_variant");
+    m_modelType = get_model_type();
+    auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
-    buildVariant = get_build_variant();
-    magicMatch = dlhandle->get<bool(std::ifstream&)>("magic_match");
-    assert(magicMatch);
-    construct_ = dlhandle->get<LLModel *()>("construct");
-    assert(construct_);
+    m_buildVariant = get_build_variant();
+    m_magicMatch = m_dlhandle->get<bool(std::ifstream&)>("magic_match");
+    assert(m_magicMatch);
+    m_construct = m_dlhandle->get<LLModel *()>("construct");
+    assert(m_construct);
 }
 
 LLModel::Implementation::Implementation(Implementation &&o)
-    : construct_(o.construct_)
-    , modelType(o.modelType)
-    , buildVariant(o.buildVariant)
-    , magicMatch(o.magicMatch)
-    , dlhandle(o.dlhandle) {
-    o.dlhandle = nullptr;
+    : m_magicMatch(o.m_magicMatch)
+    , m_construct(o.m_construct)
+    , m_modelType(o.m_modelType)
+    , m_buildVariant(o.m_buildVariant)
+    , m_dlhandle(o.m_dlhandle) {
+    o.m_dlhandle = nullptr;
 }
 
 LLModel::Implementation::~Implementation() {
-    if (dlhandle) delete dlhandle;
+    if (m_dlhandle) delete m_dlhandle;
 }
 
 bool LLModel::Implementation::isImplementation(const Dlhandle &dl) {
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
-const std::vector<LLModel::Implementation> &LLModel::implementationList() {
+const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
     // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
     // individual models without the cleanup of the static list interfering
-    static auto* libs = new std::vector<LLModel::Implementation>([] () {
-        std::vector<LLModel::Implementation> fres;
+    static auto* libs = new std::vector<Implementation>([] () {
+        std::vector<Implementation> fres;
 
         auto search_in_directory = [&](const std::string& paths) {
             std::stringstream ss(paths);
@@ -107,17 +111,17 @@ const std::vector<LLModel::Implementation> &LLModel::implementationList() {
     return *libs;
 }
 
-const LLModel::Implementation* LLModel::implementation(std::ifstream& f, const std::string& buildVariant) {
+const LLModel::Implementation* LLModel::Implementation::implementation(std::ifstream& f, const std::string& buildVariant) {
     for (const auto& i : implementationList()) {
         f.seekg(0);
-        if (!i.magicMatch(f)) continue;
-        if (buildVariant != i.buildVariant) continue;
+        if (!i.m_magicMatch(f)) continue;
+        if (buildVariant != i.m_buildVariant) continue;
         return &i;
     }
     return nullptr;
 }
 
-LLModel *LLModel::construct(const std::string &modelPath, std::string buildVariant) {
+LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant) {
 
     if (!has_at_least_minimal_hardware())
         return nullptr;
@@ -126,14 +130,15 @@ LLModel *LLModel::construct(const std::string &modelPath, std::string buildVaria
     std::ifstream f(modelPath, std::ios::binary);
     if (!f) return nullptr;
     // Get correct implementation
-    const LLModel::Implementation* impl = nullptr;
+    const Implementation* impl = nullptr;
 
     #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
         if (buildVariant == "auto") {
             size_t total_mem = getSystemTotalRAMInBytes();
             impl = implementation(f, "metal");
             if(impl) {
-                LLModel* metalimpl = impl->construct();
+                LLModel* metalimpl = impl->m_construct();
+                metalimpl->m_implementation = impl;
                 size_t req_mem = metalimpl->requiredMem(modelPath);
                 float req_to_total = (float) req_mem / (float) total_mem;
                 // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
@@ -160,14 +165,17 @@ LLModel *LLModel::construct(const std::string &modelPath, std::string buildVaria
         if (!impl) return nullptr;
     }
     f.close();
+
     // Construct and return llmodel implementation
-    return impl->construct();
+    auto fres = impl->m_construct();
+    fres->m_implementation = impl;
+    return fres;
 }
 
-void LLModel::setImplementationsSearchPath(const std::string& path) {
+void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
     s_implementations_search_path = path;
 }
 
-const std::string& LLModel::implementationsSearchPath() {
+const std::string& LLModel::Implementation::implementationsSearchPath() {
     return s_implementations_search_path;
 }

gpt4all-backend/llmodel.h

@@ -12,32 +12,34 @@
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
 class Dlhandle;
-
 class LLModel {
 public:
     using Token = int32_t;
-
     class Implementation {
-        LLModel *(*construct_)();
-
     public:
         Implementation(Dlhandle&&);
         Implementation(const Implementation&) = delete;
         Implementation(Implementation&&);
         ~Implementation();
 
+        std::string_view modelType() const { return m_modelType; }
+        std::string_view buildVariant() const { return m_buildVariant; }
+
         static bool isImplementation(const Dlhandle&);
+        static const std::vector<Implementation>& implementationList();
+        static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
+        static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
+        static void setImplementationsSearchPath(const std::string& path);
+        static const std::string& implementationsSearchPath();
 
-        std::string_view modelType, buildVariant;
-        bool (*magicMatch)(std::ifstream& f);
-        Dlhandle *dlhandle;
+    private:
+        bool (*m_magicMatch)(std::ifstream& f);
+        LLModel *(*m_construct)();
 
-        // The only way an implementation should be constructed
-        LLModel *construct() const {
-            auto fres = construct_();
-            fres->m_implementation = this;
-            return fres;
-        }
+    private:
+        std::string_view m_modelType;
+        std::string_view m_buildVariant;
+        Dlhandle *m_dlhandle;
     };
 
     struct PromptContext {
@@ -56,21 +58,36 @@ public:
             // window
     };
 
+    struct GPUDevice {
+        int index = 0;
+        int type = 0;
+        size_t heapSize = 0;
+        std::string name;
+        std::string vendor;
+    };
+
     explicit LLModel() {}
     virtual ~LLModel() {}
 
+    virtual bool supportsEmbedding() const = 0;
+    virtual bool supportsCompletion() const = 0;
     virtual bool loadModel(const std::string &modelPath) = 0;
     virtual bool isModelLoaded() const = 0;
     virtual size_t requiredMem(const std::string &modelPath) = 0;
     virtual size_t stateSize() const { return 0; }
     virtual size_t saveState(uint8_t */*dest*/) const { return 0; }
     virtual size_t restoreState(const uint8_t */*src*/) { return 0; }
+
+    // This method requires the model to return true from supportsCompletion otherwise it will throw
+    // an error
     virtual void prompt(const std::string &prompt,
                         std::function<bool(int32_t)> promptCallback,
                         std::function<bool(int32_t, const std::string&)> responseCallback,
                         std::function<bool(bool)> recalculateCallback,
                         PromptContext &ctx);
 
+    virtual std::vector<float> embedding(const std::string &text);
+
     virtual void setThreadCount(int32_t /*n_threads*/) {}
     virtual int32_t threadCount() const { return 1; }
 
@@ -78,12 +95,11 @@ public:
         return *m_implementation;
     }
 
-    static const std::vector<Implementation>& implementationList();
-    static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
-    static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
-
-    static void setImplementationsSearchPath(const std::string& path);
-    static const std::string& implementationsSearchPath();
+    virtual std::vector<GPUDevice> availableGPUDevices(size_t /*memoryRequired*/) { return std::vector<GPUDevice>(); }
+    virtual bool initializeGPUDevice(size_t /*memoryRequired*/, const std::string& /*device*/) { return false; }
+    virtual bool initializeGPUDevice(const GPUDevice &/*device*/) { return false; }
+    virtual bool initializeGPUDevice(int /*device*/) { return false; }
+    virtual bool hasGPUDevice() { return false; }
 
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
@@ -100,5 +116,9 @@ protected:
     void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate);
 
     const Implementation *m_implementation = nullptr;
+
+private:
+    friend class LLMImplementation;
 };
+
 #endif // LLMODEL_H

gpt4all-backend/llmodel_c.cpp

@@ -5,7 +5,6 @@
 #include <cerrno>
 #include <utility>
 
-
 struct LLModelWrapper {
     LLModel *llModel = nullptr;
     LLModel::PromptContext promptContext;
@@ -29,7 +28,7 @@ llmodel_model llmodel_model_create2(const char *model_path, const char *build_va
     int error_code = 0;
 
     try {
-        wrapper->llModel = LLModel::construct(model_path, build_variant);
+        wrapper->llModel = LLModel::Implementation::construct(model_path, build_variant);
     } catch (const std::exception& e) {
         error_code = EINVAL;
         last_error_message = e.what();
@@ -166,6 +165,29 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->context_erase = wrapper->promptContext.contextErase;
 }
 
+float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size)
+{
+    if (model == nullptr || text == nullptr || !strlen(text)) {
+        *embedding_size = 0;
+        return nullptr;
+    }
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    std::vector<float> embeddingVector = wrapper->llModel->embedding(text);
+    float *embedding = (float *)malloc(embeddingVector.size() * sizeof(float));
+    if (embedding == nullptr) {
+        *embedding_size = 0;
+        return nullptr;
+    }
+    std::copy(embeddingVector.begin(), embeddingVector.end(), embedding);
+    *embedding_size = embeddingVector.size();
+    return embedding;
+}
+
+void llmodel_free_embedding(float *ptr)
+{
+    free(ptr);
+}
+
 void llmodel_setThreadCount(llmodel_model model, int32_t n_threads)
 {
     LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
@@ -180,10 +202,64 @@ int32_t llmodel_threadCount(llmodel_model model)
 
 void llmodel_set_implementation_search_path(const char *path)
 {
-    LLModel::setImplementationsSearchPath(path);
+    LLModel::Implementation::setImplementationsSearchPath(path);
 }
 
 const char *llmodel_get_implementation_search_path()
 {
-    return LLModel::implementationsSearchPath().c_str();
+    return LLModel::Implementation::implementationsSearchPath().c_str();
+}
+
+struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    std::vector<LLModel::GPUDevice> devices = wrapper->llModel->availableGPUDevices(memoryRequired);
+
+    // Set the num_devices
+    *num_devices = devices.size();
+
+    if (*num_devices == 0) return nullptr;  // Return nullptr if no devices are found
+
+    // Allocate memory for the output array
+    struct llmodel_gpu_device* output = (struct llmodel_gpu_device*) malloc(*num_devices * sizeof(struct llmodel_gpu_device));
+
+    for (int i = 0; i < *num_devices; i++) {
+        output[i].index = devices[i].index;
+        output[i].type = devices[i].type;
+        output[i].heapSize = devices[i].heapSize;
+        output[i].name = strdup(devices[i].name.c_str());  // Convert std::string to char* and allocate memory
+        output[i].vendor = strdup(devices[i].vendor.c_str());  // Convert std::string to char* and allocate memory
+    }
+
+    return output;
+}
+
+bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(memoryRequired, std::string(device));
+}
+
+bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device)
+{
+    LLModel::GPUDevice d;
+    d.index = device->index;
+    d.type = device->type;
+    d.heapSize = device->heapSize;
+    d.name = device->name;
+    d.vendor = device->vendor;
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(d);
+}
+
+bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(device);
+}
+
+bool llmodel_has_gpu_device(llmodel_model model)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->hasGPUDevice();
 }

gpt4all-backend/llmodel_c.h

@@ -56,8 +56,18 @@ struct llmodel_prompt_context {
     int32_t repeat_last_n;  // last n tokens to penalize
     float context_erase;    // percent of context to erase if we exceed the context window
 };
+
+struct llmodel_gpu_device {
+    int index = 0;
+    int type = 0;           // same as VkPhysicalDeviceType
+    size_t heapSize = 0;
+    const char * name;
+    const char * vendor;
+};
+
 #ifndef __cplusplus
 typedef struct llmodel_prompt_context llmodel_prompt_context;
+typedef struct llmodel_gpu_device llmodel_gpu_device;
 #endif
 
 /**
@@ -171,6 +181,25 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
                     llmodel_recalculate_callback recalculate_callback,
                     llmodel_prompt_context *ctx);
 
+/**
+ * Generate an embedding using the model.
+ * NOTE: If given NULL pointers for the model or text, or an empty text, a NULL pointer will be
+ * returned. Bindings should signal an error when NULL is the return value.
+ * @param model A pointer to the llmodel_model instance.
+ * @param text A string representing the text to generate an embedding for.
+ * @param embedding_size A pointer to a size_t type that will be set by the call indicating the length
+ * of the returned floating point array.
+ * @return A pointer to an array of floating point values passed to the calling method which then will
+ * be responsible for lifetime of this memory.
+ */
+float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size);
+
+/**
+ * Frees the memory allocated by the llmodel_embedding function.
+ * @param ptr A pointer to the embedding as returned from llmodel_embedding.
+ */
+void llmodel_free_embedding(float *ptr);
+
 /**
  * Set the number of threads to be used by the model.
  * @param model A pointer to the llmodel_model instance.
@@ -199,6 +228,50 @@ void llmodel_set_implementation_search_path(const char *path);
  */
 const char *llmodel_get_implementation_search_path();
 
+/**
+ * Get a list of available GPU devices given the memory required.
+ * @return A pointer to an array of llmodel_gpu_device's whose number is given by num_devices.
+ */
+struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices);
+
+/**
+ * Initializes a GPU device based on a specified string criterion.
+ *
+ * This function initializes a GPU device based on a string identifier provided. The function
+ * allows initialization based on general device type ("gpu"), vendor name ("amd", "nvidia", "intel"),
+ * or any specific device name.
+ *
+ * @param memoryRequired The amount of memory (in bytes) required by the application or task
+ *                       that will utilize the GPU device.
+ * @param device A string specifying the desired criterion for GPU device selection. It can be:
+ *               - "gpu": To initialize the best available GPU.
+ *               - "amd", "nvidia", or "intel": To initialize the best available GPU from that vendor.
+ *               - A specific GPU device name: To initialize a GPU with that exact name.
+ *
+ * @return True if the GPU device is successfully initialized based on the provided string
+ *         criterion. Returns false if the desired GPU device could not be initialized.
+ */
+bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device);
+
+/**
+ * Initializes a GPU device by specifying a valid gpu device pointer.
+ * @param device A gpu device pointer.
+ * @return True if the GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device);
+
+/**
+ * Initializes a GPU device by its index.
+ * @param device An integer representing the index of the GPU device to be initialized.
+ * @return True if the GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
+
+/**
+ * @return True if a GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_has_gpu_device(llmodel_model model);
+
 #ifdef __cplusplus
 }
 #endif

gpt4all-backend/llmodel_shared.cpp

@@ -33,7 +33,14 @@ void LLModel::prompt(const std::string &prompt,
                      PromptContext &promptCtx)
 {
     if (!isModelLoaded()) {
-        std::cerr << implementation().modelType << " ERROR: prompt won't work with an unloaded model!\n";
+        std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
+        return;
+    }
+
+    if (!supportsCompletion()) {
+        std::string errorMessage = "ERROR: this model does not support text completion or chat!\n";
+        responseCallback(-1, errorMessage);
+        std::cerr << implementation().modelType() << errorMessage;
         return;
     }
 
@@ -45,8 +52,8 @@ void LLModel::prompt(const std::string &prompt,
 
     if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
         responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed.");
-        std::cerr << implementation().modelType << " ERROR: The prompt is" << embd_inp.size() <<
-            "tokens and the context window is" << promptCtx.n_ctx << "!\n";
+        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
+            " tokens and the context window is " << promptCtx.n_ctx << "!\n";
         return;
     }
 
@@ -64,7 +71,7 @@ void LLModel::prompt(const std::string &prompt,
         if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
             const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
             // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n";
+            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
             promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
             promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
@@ -72,7 +79,7 @@ void LLModel::prompt(const std::string &prompt,
         }
 
         if (!evalTokens(promptCtx, batch)) {
-            std::cerr << implementation().modelType << " ERROR: Failed to process prompt\n";
+            std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
             return;
         }
 
@@ -103,7 +110,7 @@ void LLModel::prompt(const std::string &prompt,
         if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
             const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
             // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n";
+            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
             promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
             promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
@@ -111,7 +118,7 @@ void LLModel::prompt(const std::string &prompt,
         }
 
         if (!evalTokens(promptCtx, { id })) {
-            std::cerr << implementation().modelType << " ERROR: Failed to predict next token\n";
+            std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
             return;
         }
 
@@ -158,3 +165,12 @@ void LLModel::prompt(const std::string &prompt,
         cachedTokens.clear();
     }
 }
+
+std::vector<float> LLModel::embedding(const std::string &/*text*/)
+{
+    if (!supportsCompletion()) {
+        std::string errorMessage = "ERROR: this model does not support generating embeddings!\n";
+        std::cerr << implementation().modelType() << errorMessage;
+    }
+    return std::vector<float>();
+}

gpt4all-backend/llmodel_shared.h

@@ -1,8 +1,52 @@
 #pragma once
 #include <cstdint>
 #include <cstddef>
+#include <vector>
 #include <ggml.h>
 
+#if defined(GGML_USE_KOMPUTE)
+#include "ggml-vulkan.h"
+struct llm_buffer {
+    uint8_t * addr = NULL;
+    size_t size = 0;
+    ggml_vk_memory memory;
+
+    llm_buffer() = default;
+
+    void resize(size_t size) {
+        free();
+
+        if (!ggml_vk_has_device()) {
+            this->addr = new uint8_t[size];
+            this->size = size;
+        } else {
+            this->memory = ggml_vk_allocate(size);
+            this->addr = (uint8_t*)memory.data;
+            this->size = size;
+        }
+    }
+
+    void free() {
+        if (!memory.primaryMemory) {
+            delete[] addr;
+        } else if (memory.data) {
+            ggml_vk_free_memory(memory);
+        }
+        this->addr = NULL;
+        this->size = 0;
+    }
+
+    ~llm_buffer() {
+        free();
+    }
+
+    // disable copy and move
+    llm_buffer(const llm_buffer&) = delete;
+    llm_buffer(llm_buffer&&) = delete;
+    llm_buffer& operator=(const llm_buffer&) = delete;
+    llm_buffer& operator=(llm_buffer&&) = delete;
+};
+#else
 struct llm_buffer {
     uint8_t * addr = NULL;
     size_t size = 0;
@@ -17,6 +61,7 @@ struct llm_buffer {
         delete[] addr;
     }
 };
+#endif
 
 struct llm_kv_cache {
     struct ggml_tensor * k;
@@ -34,3 +79,14 @@ struct llm_kv_cache {
         }
     }
 };
+
+#if LLAMA_DATE >= 230519
+inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads) {
+    struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
+    if (plan.work_size > 0) {
+        buf.resize(plan.work_size);
+        plan.work_data = buf.addr;
+    }
+    ggml_graph_compute(graph, &plan);
+}
+#endif

gpt4all-backend/mpt_impl.h

@@ -15,6 +15,8 @@ public:
     MPT();
     ~MPT();
 
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath) override;

gpt4all-backend/replit.cpp

@@ -196,6 +196,7 @@ struct replit_model {
 
     struct ggml_context * ctx;
     llm_buffer eval_buf;
+    llm_buffer work_buf;
     llm_buffer scr0_buf;
     llm_buffer scr1_buf;
     #ifdef GGML_USE_METAL
@@ -490,7 +491,7 @@ bool replit_model_load(const std::string & fname, std::istream &fin, replit_mode
    model.scr1_buf.resize(256u * 1024 * 1024);
 
 #ifdef GGML_USE_METAL
-    model.ctx_metal = ggml_metal_init();
+    model.ctx_metal = ggml_metal_init(1);
     void* data_ptr = ggml_get_mem_buffer(model.ctx);
     size_t data_size = ggml_get_mem_size(model.ctx);
     const size_t max_size = ggml_get_max_tensor_size(model.ctx);
@@ -534,7 +535,7 @@ bool replit_model_load(const std::string & fname, replit_model & model, replit_t
 //   - embd_inp:  the embeddings of the tokens in the context
 //   - embd_w:    the predicted logits for the next token
 //
-bool replit_eval(const replit_model & model, const int n_threads, const int n_past,
+bool replit_eval(replit_model & model, const int n_threads, const int n_past,
                  const std::vector<gpt_vocab::id> & embd_inp, std::vector<float> & embd_w, size_t & mem_per_token) {
     const int N = embd_inp.size();
 
@@ -552,7 +553,7 @@ bool replit_eval(const replit_model & model, const int n_threads, const int n_pa
         .no_alloc = false,
     };
     struct ggml_context * ctx0 = ggml_init(eval_ctx_params);
-    struct ggml_cgraph gf = {.n_threads = n_threads};
+    struct ggml_cgraph gf = {};
 
     struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
     memcpy(embd->data, embd_inp.data(), N * ggml_element_size(embd));
@@ -706,10 +707,10 @@ bool replit_eval(const replit_model & model, const int n_threads, const int n_pa
         ggml_metal_get_tensor(model.ctx_metal, model.kv_self.k);
         ggml_metal_get_tensor(model.ctx_metal, model.kv_self.v);
 
-        ggml_graph_compute(ctx0, &gf);
+        ggml_graph_compute_g4a(model.work_buf, &gf, n_threads);
     }
 #else
-    ggml_graph_compute(ctx0, &gf);
+    ggml_graph_compute_g4a(model.work_buf, &gf, n_threads);
 #endif
 
     // std::cout << "Qcur" << std::endl;

gpt4all-backend/replit_impl.h

@@ -17,6 +17,8 @@ public:
     Replit();
     ~Replit();
 
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string & modelPath) override;

gpt4all-backend/scripts/convert_bert_hf_to_ggml.py

@@ -0,0 +1,102 @@
+import sys
+import struct
+import json
+import torch
+import numpy as np
+
+from transformers import AutoModel, AutoTokenizer
+
+if len(sys.argv) < 3:
+    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = sys.argv[1]
+fname_out = sys.argv[1] + "/ggml-model.bin"
+
+with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f:
+    encoder = json.load(f)
+
+with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
+    hparams = json.load(f)
+
+with open(dir_model + "/vocab.txt", "r", encoding="utf-8") as f:
+    vocab = f.readlines()
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+    fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
+
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+model = AutoModel.from_pretrained(dir_model, low_cpu_mem_usage=True)
+print (model)
+
+print(tokenizer.encode('I believe the meaning of life is'))
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+fout = open(fname_out, "wb")
+
+print(hparams)
+
+fout.write(struct.pack("i", 0x62657274)) # magic: ggml in hex
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["max_position_embeddings"]))
+fout.write(struct.pack("i", hparams["hidden_size"]))
+fout.write(struct.pack("i", hparams["intermediate_size"]))
+fout.write(struct.pack("i", hparams["num_attention_heads"]))
+fout.write(struct.pack("i", hparams["num_hidden_layers"]))
+fout.write(struct.pack("i", ftype))
+
+for i in range(hparams["vocab_size"]):
+    text = vocab[i][:-1] # strips newline at the end
+    #print(f"{i}:{text}")
+    data = bytes(text, 'utf-8')
+    fout.write(struct.pack("i", len(data)))
+    fout.write(data)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
+        continue
+    print("Processing variable: " + name + " with shape: ", data.shape)
+
+    n_dims = len(data.shape);
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+            print("  Converting to float16")
+            data = data.astype(np.float16)
+            l_type = 1
+    else:
+        l_type = 0
+
+    # header
+    str = name.encode('utf-8')
+    fout.write(struct.pack("iii", n_dims, len(str), l_type))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str);
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/starcoder_impl.h

@@ -0,0 +1,42 @@
+#ifndef STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of starcoder.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef STARCODER_H
+#define STARCODER_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <memory>
+#include "llmodel.h"
+
+struct StarcoderPrivate;
+class Starcoder : public LLModel {
+public:
+    Starcoder();
+    ~Starcoder();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    std::unique_ptr<StarcoderPrivate> d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // STARCODER_H

gpt4all-bindings/java/README.md

@@ -12,12 +12,12 @@ You can add Java bindings into your Java project by adding the following depende
 <dependency>
     <groupId>com.hexadevlabs</groupId>
     <artifactId>gpt4all-java-binding</artifactId>
-    <version>1.1.3</version>
+    <version>1.1.5</version>
 </dependency>
 ```
 **Gradle**
 ```
-implementation 'com.hexadevlabs:gpt4all-java-binding:1.1.3'
+implementation 'com.hexadevlabs:gpt4all-java-binding:1.1.5'
 ```
 
 To add the library dependency for another build system see [Maven Central Java bindings](https://central.sonatype.com/artifact/com.hexadevlabs/gpt4all-java-binding/).
@@ -121,4 +121,6 @@ If this is the case you can easily download and install the latest x64 Microsoft
 3. Version **1.1.4**:
    - Java bindings is compatible with gpt4all version 2.4.11
    - Falcon model support included.
+4. Version **1.1.5**:
+   - Add a check for model file readability before loading model.
    

gpt4all-bindings/java/TODO.md

@@ -1,2 +1,6 @@
+## Needed
+1. Integrate with circleci build pipeline like the C# binding.
+
+## These are just ideas
 1. Better Chat completions function.
 2. Chat completion that returns result in OpenAI compatible format.

gpt4all-bindings/java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.hexadevlabs</groupId>
     <artifactId>gpt4all-java-binding</artifactId>
-    <version>1.1.4</version>
+    <version>1.1.5</version>
     <packaging>jar</packaging>
 
     <properties>

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java

@@ -184,11 +184,16 @@ public  class LLModel implements AutoCloseable {
             throw new IllegalStateException("Model file does not exist: " + modelPathAbs);
         }
 
+        // Check if file is Readable
+        if(!Files.isReadable(modelPath)){
+            throw new IllegalStateException("Model file cannot be read: " + modelPathAbs);
+        }
+
         // Create Model Struct. Will load dynamically the correct backend based on model type
         model = library.llmodel_model_create2(modelPathAbs, "auto", error);
 
         if(model == null) {
-            throw new IllegalStateException("Could not load gpt4all backend :" + error.message);
+            throw new IllegalStateException("Could not load, gpt4all backend returned error: " + error.message);
         }
         library.llmodel_loadModel(model, modelPathAbs);
 

gpt4all-bindings/python/README.md

@@ -25,7 +25,7 @@ cd gpt4all/gpt4all-backend/
 mkdir build
 cd build
 cmake ..
-cmake --build . --parallel
+cmake --build . --parallel  # optionally append: --config Release
 ```
 Confirm that `libllmodel.*` exists in `gpt4all-backend/build`.
 
@@ -47,6 +47,15 @@ output = model.generate("The capital of France is ", max_tokens=3)
 print(output)
 ```
 
+
+GPU Usage
+```python
+from gpt4all import GPT4All
+model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin", device='gpu') # device='amd', device='intel'
+output = model.generate("The capital of France is ", max_tokens=3)
+print(output)
+```
+
 ## Troubleshooting a Local Build
 - If you're on Windows and have compiled with a MinGW toolchain, you might run into an error like:
   ```

gpt4all-bindings/python/docs/gpt4all_faq.md

@@ -2,31 +2,42 @@
 
 ## What models are supported by the GPT4All ecosystem?
 
-Currently, there are five different model architectures that are supported:
+Currently, there are six different model architectures that are supported:
 
 1. GPT-J - Based off of the GPT-J architecture with examples found [here](https://huggingface.co/EleutherAI/gpt-j-6b)
 2. LLaMA - Based off of the LLaMA architecture with examples found [here](https://huggingface.co/models?sort=downloads&search=llama)
 3. MPT - Based off of Mosaic ML's MPT architecture with examples found [here](https://huggingface.co/mosaicml/mpt-7b)
 4. Replit - Based off of Replit Inc.'s Replit architecture with examples found [here](https://huggingface.co/replit/replit-code-v1-3b)
 5. Falcon - Based off of TII's Falcon architecture with examples found [here](https://huggingface.co/tiiuae/falcon-40b)
+6. StarCoder - Based off of BigCode's StarCoder architecture with examples found [here](https://huggingface.co/bigcode/starcoder)
 
 ## Why so many different architectures? What differentiates them?
 
-One of the major differences is license. Currently, the LLAMA based models are subject to a non-commercial license, whereas the GPTJ and MPT base models allow commercial usage. In the early advent of the recent explosion of activity in open source local models, the llama models have generally been seen as performing better, but that is changing quickly. Every week - even every day! - new models are released with some of the GPTJ and MPT models competitive in performance/quality with LLAMA. What's more, there are some very nice architectural innovations with the MPT models that could lead to new performance/quality gains.
+One of the major differences is license. Currently, the LLaMA based models are subject to a non-commercial license, whereas the GPTJ and MPT base
+models allow commercial usage. However, its successor [Llama 2 is commercially licensable](https://ai.meta.com/llama/license/), too. In the early
+advent of the recent explosion of activity in open source local models, the LLaMA models have generally been seen as performing better, but that is
+changing quickly. Every week - even every day! - new models are released with some of the GPTJ and MPT models competitive in performance/quality with
+LLaMA. What's more, there are some very nice architectural innovations with the MPT models that could lead to new performance/quality gains.
 
 ## How does GPT4All make these models available for CPU inference?
 
-By leveraging the ggml library written by Georgi Gerganov and a growing community of developers. There are currently multiple different versions of this library. The original github repo can be found [here](https://github.com/ggerganov/ggml), but the developer of the library has also created a LLAMA based version [here](https://github.com/ggerganov/llama.cpp). Currently, this backend is using the latter as a submodule.
+By leveraging the ggml library written by Georgi Gerganov and a growing community of developers. There are currently multiple different versions of
+this library. The original GitHub repo can be found [here](https://github.com/ggerganov/ggml), but the developer of the library has also created a
+LLaMA based version [here](https://github.com/ggerganov/llama.cpp). Currently, this backend is using the latter as a submodule.
 
 ## Does that mean GPT4All is compatible with all llama.cpp models and vice versa?
 
 Yes!
 
-The upstream [llama.cpp](https://github.com/ggerganov/llama.cpp) project has introduced several [compatibility breaking](https://github.com/ggerganov/llama.cpp/commit/b9fd7eee57df101d4a3e3eabc9fd6c2cb13c9ca1) quantization methods recently. This is a breaking change that renders all previous models (including the ones that GPT4All uses) inoperative with newer versions of llama.cpp since that change.
+The upstream [llama.cpp](https://github.com/ggerganov/llama.cpp) project has introduced several [compatibility breaking] quantization methods recently.
+This is a breaking change that renders all previous models (including the ones that GPT4All uses) inoperative with newer versions of llama.cpp since
+that change.
 
 Fortunately, we have engineered a submoduling system allowing us to dynamically load different versions of the underlying library so that
 GPT4All just works.
 
+[compatibility breaking]: https://github.com/ggerganov/llama.cpp/commit/b9fd7eee57df101d4a3e3eabc9fd6c2cb13c9ca1
+
 ## What are the system requirements?
 
 Your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) and you need enough RAM to load a model into memory.
@@ -35,8 +46,55 @@ Your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wi
 
 In newer versions of llama.cpp, there has been some added support for NVIDIA GPU's for inference. We're investigating how to incorporate this into our downloadable installers.
 
-## Ok, so bottom line... how do I make my model on huggingface compatible with GPT4All ecosystem right now?
+## Ok, so bottom line... how do I make my model on Hugging Face compatible with GPT4All ecosystem right now?
 
-1. Check to make sure the huggingface model is available in one of our three supported architectures
-2. If it is, then you can use the conversion script inside of our pinned llama.cpp submodule for GPTJ and LLAMA based models
+1. Check to make sure the Hugging Face model is available in one of our three supported architectures
+2. If it is, then you can use the conversion script inside of our pinned llama.cpp submodule for GPTJ and LLaMA based models
 3. Or if your model is an MPT model you can use the conversion script located directly in this backend directory under the scripts subdirectory 
+
+## Language Bindings
+
+#### There's a problem with the download
+
+Some bindings can download a model, if allowed to do so. For example, in Python or TypeScript if `allow_download=True`
+or `allowDownload=true` (default), a model is automatically downloaded into `.cache/gpt4all/` in the user's home folder,
+unless it already exists.
+
+In case of connection issues or errors during the download, you might want to manually verify the model file's MD5
+checksum by comparing it with the one listed in [models.json].
+
+As an alternative to the basic downloader built into the bindings, you can choose to download from the 
+<https://gpt4all.io/> website instead. Scroll down to 'Model Explorer' and pick your preferred model.
+
+[models.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json
+
+#### I need the chat GUI and bindings to behave the same
+
+The chat GUI and bindings are based on the same backend. You can make them behave the same way by following these steps:
+
+- First of all, ensure that all parameters in the chat GUI settings match those passed to the generating API, e.g.:
+
+    === "Python"
+        ``` py
+        from gpt4all import GPT4All
+        model = GPT4All(...)
+        model.generate("prompt text", temp=0, ...)  # adjust parameters
+        ```
+    === "TypeScript"
+        ``` ts
+        import { createCompletion, loadModel } from '../src/gpt4all.js'
+        const ll = await loadModel(...);
+        const messages = ...
+        const re = await createCompletion(ll, messages, { temp: 0, ... });  // adjust parameters
+        ```
+
+- To make comparing the output easier, set _Temperature_ in both to 0 for now. This will make the output deterministic.
+
+- Next you'll have to compare the templates, adjusting them as necessary, based on how you're using the bindings.
+    - Specifically, in Python:
+        - With simple `generate()` calls, the input has to be surrounded with system and prompt templates.
+        - When using a chat session, it depends on whether the bindings are allowed to download [models.json]. If yes,
+          and in the chat GUI the default templates are used, it'll be handled automatically. If no, use
+          `chat_session()` template parameters to customize them.
+
+- Once you're done, remember to reset _Temperature_ to its previous value in both chat GUI and your custom code.

gpt4all-bindings/python/docs/gpt4all_python.md

@@ -1,10 +1,9 @@
-# GPT4All Python API
+# GPT4All Python Generation API
 The `GPT4All` python package provides bindings to our C/C++ model backend libraries.
 The source code and local build instructions can be found [here](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python).
 
 
 ## Quickstart
-
 ```bash
 pip install gpt4all
 ```
@@ -21,8 +20,16 @@ pip install gpt4all
     1. Paris
     ```
 
+This will:
+
+- Instantiate `GPT4All`,  which is the primary public API to your large language model (LLM).
+- Automatically download the given model to `~/.cache/gpt4all/` if not already present.
+- Through `model.generate(...)` the model starts working on a response. There are various ways to
+  steer that process. Here, `max_tokens` sets an upper limit, i.e. a hard cut-off point to the output.
+
+
 ### Chatting with GPT4All
-Local LLMs can be optimized for chat conversions by reusing previous computational history.
+Local LLMs can be optimized for chat conversations by reusing previous computational history.
 
 Use the GPT4All `chat_session` context manager to hold chat conversations with the model.
 
@@ -30,9 +37,9 @@ Use the GPT4All `chat_session` context manager to hold chat conversations with t
     ``` py
     model = GPT4All(model_name='orca-mini-3b.ggmlv3.q4_0.bin')
     with model.chat_session():
-        response = model.generate(prompt='hello', top_k=1)
-        response = model.generate(prompt='write me a short poem', top_k=1)
-        response = model.generate(prompt='thank you', top_k=1)
+        response1 = model.generate(prompt='hello', temp=0)
+        response2 = model.generate(prompt='write me a short poem', temp=0)
+        response3 = model.generate(prompt='thank you', temp=0)
         print(model.current_chat_session)
     ```
 === "Output"
@@ -64,19 +71,20 @@ Use the GPT4All `chat_session` context manager to hold chat conversations with t
        }
     ]
     ```
-When using GPT4All models in the chat_session context:
 
-- The model is given a prompt template which makes it chatty.
-- Internal K/V caches are preserved from previous conversation history speeding up inference.
+When using GPT4All models in the `chat_session` context:
 
+- Consecutive chat exchanges are taken into account and not discarded until the session ends; as long as the model has capacity.
+- Internal K/V caches are preserved from previous conversation history, speeding up inference.
+- The model is given a system and prompt template which make it chatty. Depending on `allow_download=True` (default),
+  it will obtain the latest version of [models.json] from the repository, which contains specifically tailored templates
+  for models. Conversely, if it is not allowed to download, it falls back to default templates instead.
 
-### Generation Parameters
-
-::: gpt4all.gpt4all.GPT4All.generate
+[models.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json
 
 
 ### Streaming Generations
-To interact with GPT4All responses as the model generates, use the `streaming = True` flag during generation.
+To interact with GPT4All responses as the model generates, use the `streaming=True` flag during generation.
 
 === "GPT4All Streaming Example"
     ``` py
@@ -92,22 +100,338 @@ To interact with GPT4All responses as the model generates, use the `streaming =
     [' Paris', ' is', ' a', ' city', ' that', ' has', ' been', ' a', ' major', ' cultural', ' and', ' economic', ' center', ' for', ' over', ' ', '2', ',', '0', '0']
     ```
 
-#### Streaming and Chat Sessions
-When streaming tokens in a chat session, you must manually handle collection and updating of the chat history.
 
-```python
+### The Generate Method API
+::: gpt4all.gpt4all.GPT4All.generate
+
+
+## Examples & Explanations
+### Influencing Generation
+The three most influential parameters in generation are _Temperature_ (`temp`), _Top-p_ (`top_p`) and _Top-K_ (`top_k`).
+In a nutshell, during the process of selecting the next token, not just one or a few are considered, but every single
+token in the vocabulary is given a probability. The parameters can change the field of candidate tokens.
+
+- **Temperature** makes the process either more or less random. A _Temperature_ above 1 increasingly "levels the playing
+  field", while at a _Temperature_ between 0 and 1 the likelihood of the best token candidates grows even more. A
+  _Temperature_ of 0 results in selecting the best token, making the output deterministic. A _Temperature_ of 1
+  represents a neutral setting with regard to randomness in the process.
+
+- _Top-p_ and _Top-K_ both narrow the field:
+    - **Top-K** limits candidate tokens to a fixed number after sorting by probability. Setting it higher than the
+      vocabulary size deactivates this limit.
+    - **Top-p** selects tokens based on their total probabilities. For example, a value of 0.8 means "include the best
+      tokens, whose accumulated probabilities reach or just surpass 80%". Setting _Top-p_ to 1, which is 100%,
+      effectively disables it.
+
+The recommendation is to keep at least one of _Top-K_ and _Top-p_ active. Other parameters can also influence
+generation; be sure to review all their descriptions.
+
+
+### Specifying the Model Folder
+The model folder can be set with the `model_path` parameter when creating a `GPT4All` instance. The example below is
+is the same as if it weren't provided; that is, `~/.cache/gpt4all/` is the default folder.
+
+=== "GPT4All Model Folder Example"
+    ``` py
+    from pathlib import Path
+    from gpt4all import GPT4All
+    model = GPT4All(model_name='orca-mini-3b.ggmlv3.q4_0.bin',
+                    model_path=(Path.home() / '.cache' / 'gpt4all'),
+                    allow_download=False)
+    response = model.generate('my favorite 3 fruits are:', temp=0)
+    print(response)
+    ```
+=== "Output"
+    ```
+    My favorite three fruits are apples, bananas and oranges.
+    ```
+
+If you want to point it at the chat GUI's default folder, it should be:
+=== "macOS"
+    ``` py
+    from pathlib import Path
+    from gpt4all import GPT4All
+
+    model_name = 'orca-mini-3b.ggmlv3.q4_0.bin'
+    model_path = Path.home() / 'Library' / 'Application Support' / 'nomic.ai' / 'GPT4All'
+    model = GPT4All(model_name, model_path)
+    ```
+=== "Windows"
+    ``` py
+    from pathlib import Path
+    from gpt4all import GPT4All
+    import os
+    model_name = 'orca-mini-3b.ggmlv3.q4_0.bin'
+    model_path = Path(os.environ['LOCALAPPDATA']) / 'nomic.ai' / 'GPT4All'
+    model = GPT4All(model_name, model_path)
+    ```
+=== "Linux"
+    ``` py
+    from pathlib import Path
+    from gpt4all import GPT4All
+
+    model_name = 'orca-mini-3b.ggmlv3.q4_0.bin'
+    model_path = Path.home() / '.local' / 'share' / 'nomic.ai' / 'GPT4All'
+    model = GPT4All(model_name, model_path)
+    ```
+
+Alternatively, you could also change the module's default model directory:
+
+``` py
+from pathlib import Path
+import gpt4all.gpt4all
+gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = Path.home() / 'my' / 'models-directory'
 from gpt4all import GPT4All
-model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin")
-
-with model.chat_session():
-    tokens = list(model.generate(prompt='hello', top_k=1, streaming=True))
-    model.current_chat_session.append({'role': 'assistant', 'content': ''.join(tokens)})
-
-    tokens = list(model.generate(prompt='write me a poem about dogs', top_k=1, streaming=True))
-    model.current_chat_session.append({'role': 'assistant', 'content': ''.join(tokens)})
-
-    print(model.current_chat_session)
+model = GPT4All('orca-mini-3b.ggmlv3.q4_0.bin')
+...
 ```
 
 
+### Managing Templates
+Session templates can be customized when starting a `chat_session` context:
+
+=== "GPT4All Custom Session Templates Example"
+    ``` py
+    from gpt4all import GPT4All
+    model = GPT4All('ggml-Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin')
+    system_template = 'A chat between a curious user and an artificial intelligence assistant.'
+    # many models use triple hash '###' for keywords, Vicunas are simpler:
+    prompt_template = 'USER: {0}\nASSISTANT: '
+    with model.chat_session(system_template, prompt_template):
+        response1 = model.generate('why is the grass green?')
+        print(response1)
+        print()
+        response2 = model.generate('why is the sky blue?')
+        print(response2)
+    ```
+=== "Possible Output"
+    ```
+    The color of grass can be attributed to its chlorophyll content, which allows it
+    to absorb light energy from sunlight through photosynthesis. Chlorophyll absorbs
+    blue and red wavelengths of light while reflecting other colors such as yellow
+    and green. This is why the leaves appear green to our eyes.
+
+    The color of the sky appears blue due to a phenomenon called Rayleigh scattering,
+    which occurs when sunlight enters Earth's atmosphere and interacts with air
+    molecules such as nitrogen and oxygen. Blue light has shorter wavelength than
+    other colors in the visible spectrum, so it is scattered more easily by these
+    particles, making the sky appear blue to our eyes.
+    ```
+
+To do the same outside a session, the input has to be formatted manually. For example:
+
+=== "GPT4All Templates Outside a Session Example"
+    ``` py
+    model = GPT4All('ggml-Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin')
+    system_template = 'A chat between a curious user and an artificial intelligence assistant.'
+    prompt_template = 'USER: {0}\nASSISTANT: '
+    prompts = ['name 3 colors', 'now name 3 fruits', 'what were the 3 colors in your earlier response?']
+    first_input = system_template + prompt_template.format(prompts[0])
+    response = model.generate(first_input, temp=0)
+    print(response)
+    for prompt in prompts[1:]:
+        response = model.generate(prompt_template.format(prompt), temp=0)
+        print(response)
+    ```
+=== "Output"
+    ```
+    1) Red
+    2) Blue
+    3) Green
+
+    1. Apple
+    2. Banana
+    3. Orange
+
+    The colors in my previous response are blue, green and red.
+    ```
+
+Ultimately, the method `GPT4All._format_chat_prompt_template()` is responsible for formatting templates. It can be
+customized in a subclass. As an example:
+
+=== "Custom Subclass"
+    ``` py
+    from itertools import cycle
+    from gpt4all import GPT4All
+
+    class RotatingTemplateGPT4All(GPT4All):
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            self._templates = [
+                "Respond like a pirate.",
+                "Respond like a politician.",
+                "Respond like a philosopher.",
+                "Respond like a Klingon.",
+            ]
+            self._cycling_templates = cycle(self._templates)
+
+        def _format_chat_prompt_template(
+            self,
+            messages: list,
+            default_prompt_header: str = "",
+            default_prompt_footer: str = "",
+        ) -> str:
+            full_prompt = default_prompt_header + "\n\n" if default_prompt_header != "" else ""
+            for message in messages:
+                if message["role"] == "user":
+                    user_message = f"USER: {message['content']} {next(self._cycling_templates)}\n"
+                    full_prompt += user_message
+                if message["role"] == "assistant":
+                    assistant_message = f"ASSISTANT: {message['content']}\n"
+                    full_prompt += assistant_message
+            full_prompt += "\n\n" + default_prompt_footer if default_prompt_footer != "" else ""
+            print(full_prompt)
+            return full_prompt
+    ```
+=== "GPT4All Custom Subclass Example"
+    ``` py
+    model = RotatingTemplateGPT4All('ggml-Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin')
+    with model.chat_session():  # starting a session is optional in this example
+        response1 = model.generate("hi, who are you?")
+        print(response1)
+        print()
+        response2 = model.generate("what can you tell me about snakes?")
+        print(response2)
+        print()
+        response3 = model.generate("what's your opinion on Chess?")
+        print(response3)
+        print()
+        response4 = model.generate("tell me about ancient Rome.")
+        print(response4)
+    ```
+=== "Possible Output"
+    ```
+    USER: hi, who are you? Respond like a pirate.
+
+    Pirate: Ahoy there mateys! I be Cap'n Jack Sparrow of the Black Pearl.
+
+    USER: what can you tell me about snakes? Respond like a politician.
+
+    Politician: Snakes have been making headlines lately due to their ability to
+    slither into tight spaces and evade capture, much like myself during my last
+    election campaign. However, I believe that with proper education and
+    understanding of these creatures, we can work together towards creating a
+    safer environment for both humans and snakes alike.
+
+    USER: what's your opinion on Chess? Respond like a philosopher.
+
+    Philosopher: The game of chess is often used as an analogy to illustrate the
+    complexities of life and decision-making processes. However, I believe that it
+    can also be seen as a reflection of our own consciousness and subconscious mind.
+    Just as each piece on the board has its unique role to play in shaping the
+    outcome of the game, we too have different roles to fulfill in creating our own
+    personal narrative.
+
+    USER: tell me about ancient Rome. Respond like a Klingon.
+
+    Klingon: Ancient Rome was once a great empire that ruled over much of Europe and
+    the Mediterranean region. However, just as the Empire fell due to internal strife
+    and external threats, so too did my own house come crashing down when I failed to
+    protect our homeworld from invading forces.
+    ```
+
+
+### Introspection
+A less apparent feature is the capacity to log the final prompt that gets sent to the model. It relies on
+[Python's logging facilities][py-logging] implemented in the `pyllmodel` module at the `INFO` level. You can activate it
+for example with a `basicConfig`, which displays it on the standard error stream. It's worth mentioning that Python's
+logging infrastructure offers [many more customization options][py-logging-cookbook].
+
+[py-logging]: https://docs.python.org/3/howto/logging.html
+[py-logging-cookbook]: https://docs.python.org/3/howto/logging-cookbook.html
+
+=== "GPT4All Prompt Logging Example"
+    ``` py
+    import logging
+    from gpt4all import GPT4All
+    logging.basicConfig(level=logging.INFO)
+    model = GPT4All('nous-hermes-13b.ggmlv3.q4_0.bin')
+    with model.chat_session('You are a geography expert.\nBe terse.',
+                            '### Instruction:\n{0}\n### Response:\n'):
+        response = model.generate('who are you?', temp=0)
+        print(response)
+        response = model.generate('what are your favorite 3 mountains?', temp=0)
+        print(response)
+    ```
+=== "Output"
+    ```
+    INFO:gpt4all.pyllmodel:LLModel.prompt_model -- prompt:
+    You are a geography expert.
+    Be terse.
+
+    ### Instruction:
+    who are you?
+    ### Response:
+
+    ===/LLModel.prompt_model -- prompt/===
+    I am an AI-powered chatbot designed to assist users with their queries related to geographical information.
+    INFO:gpt4all.pyllmodel:LLModel.prompt_model -- prompt:
+    ### Instruction:
+    what are your favorite 3 mountains?
+    ### Response:
+
+    ===/LLModel.prompt_model -- prompt/===
+    1) Mount Everest - Located in the Himalayas, it is the highest mountain on Earth and a significant challenge for mountaineers.
+    2) Kangchenjunga - This mountain is located in the Himalayas and is the third-highest peak in the world after Mount Everest and K2.
+    3) Lhotse - Located in the Himalayas, it is the fourth highest mountain on Earth and offers a challenging climb for experienced mountaineers.
+    ```
+
+
+### Without Online Connectivity
+To prevent GPT4All from accessing online resources, instantiate it with `allow_download=False`. This will disable both
+downloading missing models and [models.json], which contains information about them. As a result, predefined templates
+are used instead of model-specific system and prompt templates:
+
+=== "GPT4All Default Templates Example"
+    ``` py
+    from gpt4all import GPT4All
+    model = GPT4All('ggml-mpt-7b-chat.bin', allow_download=False)
+    # when downloads are disabled, it will use the default templates:
+    print("default system template:", repr(model.config['systemPrompt']))
+    print("default prompt template:", repr(model.config['promptTemplate']))
+    print()
+    # even when inside a session:
+    with model.chat_session():
+        assert model.current_chat_session[0]['role'] == 'system'
+        print("session system template:", repr(model.current_chat_session[0]['content']))
+        print("session prompt template:", repr(model._current_prompt_template))
+    ```
+=== "Output"
+    ```
+    default system template: ''
+    default prompt template: '### Human: \n{0}\n### Assistant:\n'
+
+    session system template: ''
+    session prompt template: '### Human: \n{0}\n### Assistant:\n'
+    ```
+
+
+### Interrupting Generation
+The simplest way to stop generation is to set a fixed upper limit with the `max_tokens` parameter.
+
+If you know exactly when a model should stop responding, you can add a custom callback, like so:
+
+=== "GPT4All Custom Stop Callback"
+    ``` py
+    from gpt4all import GPT4All
+    model = GPT4All('orca-mini-3b.ggmlv3.q4_0.bin')
+
+    def stop_on_token_callback(token_id, token_string):
+        # one sentence is enough:
+        if '.' in token_string:
+            return False
+        else:
+            return True
+
+    response = model.generate('Blue Whales are the biggest animal to ever inhabit the Earth.',
+                              temp=0, callback=stop_on_token_callback)
+    print(response)
+    ```
+=== "Output"
+    ```
+     They can grow up to 100 feet (30 meters) long and weigh as much as 20 tons (18 metric tons).
+    ```
+
+
+## API Documentation
 ::: gpt4all.gpt4all.GPT4All

gpt4all-bindings/python/docs/gpt4all_python_embedding.md

@@ -0,0 +1,35 @@
+# Embeddings
+GPT4All supports generating high quality embeddings of arbitrary length documents of text using a CPU optimized contrastively trained [Sentence Transformer](https://www.sbert.net/). These embeddings are comparable in quality for many tasks with OpenAI.
+
+## Quickstart
+
+```bash
+pip install gpt4all
+```
+
+### Generating embeddings
+The embedding model will automatically be downloaded if not installed.
+
+=== "Embed4All Example"
+    ``` py
+    from gpt4all import GPT4All, Embed4All
+    text = 'The quick brown fox jumps over the lazy dog'
+    embedder = Embed4All()
+    output = embedder.embed(text)
+    print(output)
+    ```
+=== "Output"
+    ```
+    [0.034696947783231735, -0.07192722707986832, 0.06923297047615051, ...]
+    ```
+### Speed of embedding generation
+The following table lists the generation speed for text document captured on an Intel i913900HX CPU with DDR5 5600 running with 8 threads under stable load.
+
+| Tokens          | 128  | 512  | 2048 | 8129 | 16,384 |
+| --------------- | ---- | ---- | ---- | ---- | ---- |
+| Wall time (s)   | .02  | .08  | .24  | .96  | 1.9  |
+| Tokens / Second | 6508 | 6431 | 8622 | 8509 | 8369 |
+
+
+### API documentation
+::: gpt4all.gpt4all.Embed4All

gpt4all-bindings/python/docs/gpt4all_typescript.md

@@ -0,0 +1,731 @@
+# GPT4All Node.js API
+
+```sh
+yarn add gpt4all@alpha
+
+npm install gpt4all@alpha
+
+pnpm install gpt4all@alpha
+```
+
+The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
+
+*   New bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
+*   The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
+*   Everything should work out the box.
+*   See [API Reference](#api-reference)
+
+### Chat Completion (alpha)
+
+```js
+import { createCompletion, loadModel } from '../src/gpt4all.js'
+
+const model = await loadModel('ggml-vicuna-7b-1.1-q4_2', { verbose: true });
+
+const response = await createCompletion(model, [
+    { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
+    { role : 'user', content: 'What is 1 + 1?'  } 
+]);
+
+```
+
+### Embedding (alpha)
+
+```js
+import { createEmbedding, loadModel } from '../src/gpt4all.js'
+
+const model = await loadModel('ggml-all-MiniLM-L6-v2-f16', { verbose: true });
+
+const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional");
+```
+
+### Build Instructions
+
+*   binding.gyp is compile config
+*   Tested on Ubuntu. Everything seems to work fine
+*   Tested on Windows. Everything works fine.
+*   Sparse testing on mac os.
+*   MingW works as well to build the gpt4all-backend. **HOWEVER**, this package works only with MSVC built dlls.
+
+### Requirements
+
+*   git
+*   [node.js >= 18.0.0](https://nodejs.org/en)
+*   [yarn](https://yarnpkg.com/)
+*   [node-gyp](https://github.com/nodejs/node-gyp)
+    *   all of its requirements.
+*   (unix) gcc version 12
+*   (win) msvc version 143
+    *   Can be obtained with visual studio 2022 build tools
+*   python 3
+
+### Build (from source)
+
+```sh
+git clone https://github.com/nomic-ai/gpt4all.git
+cd gpt4all-bindings/typescript
+```
+
+*   The below shell commands assume the current working directory is `typescript`.
+
+*   To Build and Rebuild:
+
+```sh
+yarn
+```
+
+*   llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
+
+```sh
+git submodule update --init --depth 1 --recursive
+```
+
+**AS OF NEW BACKEND** to build the backend,
+
+```sh
+yarn build:backend
+```
+
+This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native The only current way to use them is to put them in the current working directory of your application. That is, **WHEREVER YOU RUN YOUR NODE APPLICATION**
+
+*   llama-xxxx.dll is required.
+*   According to whatever model you are using, you'll need to select the proper model loader.
+    *   For example, if you running an Mosaic MPT model, you will need to select the mpt-(buildvariant).(dynamiclibrary)
+
+### Test
+
+```sh
+yarn test
+```
+
+### Source Overview
+
+#### src/
+
+*   Extra functions to help aid devex
+*   Typings for the native node addon
+*   the javascript interface
+
+#### test/
+
+*   simple unit testings for some functions exported.
+*   more advanced ai testing is not handled
+
+#### spec/
+
+*   Average look and feel of the api
+*   Should work assuming a model and libraries are installed locally in working directory
+
+#### index.cc
+
+*   The bridge between nodejs and c. Where the bindings are.
+
+#### prompt.cc
+
+*   Handling prompting and inference of models in a threadsafe, asynchronous way.
+
+### Known Issues
+
+*   why your model may be spewing bull 💩
+    *   The downloaded model is broken (just reinstall or download from official site)
+    *   That's it so far
+
+### Roadmap
+
+This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
+
+*   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
+*   \[ ] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
+*   \[x] proper unit testing (integrate with circle ci)
+*   \[x] publish to npm under alpha tag `gpt4all@alpha`
+*   \[x] have more people test on other platforms (mac tester needed)
+*   \[x] switch to new pluggable backend
+*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
+    *   Should include prebuilds to avoid painful node-gyp errors
+*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
+
+### API Reference
+
+<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
+
+##### Table of Contents
+
+*   [ModelType](#modeltype)
+*   [ModelFile](#modelfile)
+    *   [gptj](#gptj)
+    *   [llama](#llama)
+    *   [mpt](#mpt)
+    *   [replit](#replit)
+*   [type](#type)
+*   [LLModel](#llmodel)
+    *   [constructor](#constructor)
+        *   [Parameters](#parameters)
+    *   [type](#type-1)
+    *   [name](#name)
+    *   [stateSize](#statesize)
+    *   [threadCount](#threadcount)
+    *   [setThreadCount](#setthreadcount)
+        *   [Parameters](#parameters-1)
+    *   [raw\_prompt](#raw_prompt)
+        *   [Parameters](#parameters-2)
+    *   [embed](#embed)
+        *   [Parameters](#parameters-3)
+    *   [isModelLoaded](#ismodelloaded)
+    *   [setLibraryPath](#setlibrarypath)
+        *   [Parameters](#parameters-4)
+    *   [getLibraryPath](#getlibrarypath)
+*   [loadModel](#loadmodel)
+    *   [Parameters](#parameters-5)
+*   [createCompletion](#createcompletion)
+    *   [Parameters](#parameters-6)
+*   [createEmbedding](#createembedding)
+    *   [Parameters](#parameters-7)
+*   [CompletionOptions](#completionoptions)
+    *   [verbose](#verbose)
+    *   [systemPromptTemplate](#systemprompttemplate)
+    *   [promptTemplate](#prompttemplate)
+    *   [promptHeader](#promptheader)
+    *   [promptFooter](#promptfooter)
+*   [PromptMessage](#promptmessage)
+    *   [role](#role)
+    *   [content](#content)
+*   [prompt\_tokens](#prompt_tokens)
+*   [completion\_tokens](#completion_tokens)
+*   [total\_tokens](#total_tokens)
+*   [CompletionReturn](#completionreturn)
+    *   [model](#model)
+    *   [usage](#usage)
+    *   [choices](#choices)
+*   [CompletionChoice](#completionchoice)
+    *   [message](#message)
+*   [LLModelPromptContext](#llmodelpromptcontext)
+    *   [logitsSize](#logitssize)
+    *   [tokensSize](#tokenssize)
+    *   [nPast](#npast)
+    *   [nCtx](#nctx)
+    *   [nPredict](#npredict)
+    *   [topK](#topk)
+    *   [topP](#topp)
+    *   [temp](#temp)
+    *   [nBatch](#nbatch)
+    *   [repeatPenalty](#repeatpenalty)
+    *   [repeatLastN](#repeatlastn)
+    *   [contextErase](#contexterase)
+*   [createTokenStream](#createtokenstream)
+    *   [Parameters](#parameters-8)
+*   [DEFAULT\_DIRECTORY](#default_directory)
+*   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
+*   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
+*   [DEFAULT\_PROMT\_CONTEXT](#default_promt_context)
+*   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
+*   [downloadModel](#downloadmodel)
+    *   [Parameters](#parameters-9)
+    *   [Examples](#examples)
+*   [DownloadModelOptions](#downloadmodeloptions)
+    *   [modelPath](#modelpath)
+    *   [verbose](#verbose-1)
+    *   [url](#url)
+    *   [md5sum](#md5sum)
+*   [DownloadController](#downloadcontroller)
+    *   [cancel](#cancel)
+    *   [promise](#promise)
+
+#### ModelType
+
+Type of the model
+
+Type: (`"gptj"` | `"llama"` | `"mpt"` | `"replit"`)
+
+#### ModelFile
+
+Full list of models available
+@deprecated These model names are outdated and this type will not be maintained, please use a string literal instead
+
+##### gptj
+
+List of GPT-J Models
+
+Type: (`"ggml-gpt4all-j-v1.3-groovy.bin"` | `"ggml-gpt4all-j-v1.2-jazzy.bin"` | `"ggml-gpt4all-j-v1.1-breezy.bin"` | `"ggml-gpt4all-j.bin"`)
+
+##### llama
+
+List Llama Models
+
+Type: (`"ggml-gpt4all-l13b-snoozy.bin"` | `"ggml-vicuna-7b-1.1-q4_2.bin"` | `"ggml-vicuna-13b-1.1-q4_2.bin"` | `"ggml-wizardLM-7B.q4_2.bin"` | `"ggml-stable-vicuna-13B.q4_2.bin"` | `"ggml-nous-gpt4-vicuna-13b.bin"` | `"ggml-v3-13b-hermes-q5_1.bin"`)
+
+##### mpt
+
+List of MPT Models
+
+Type: (`"ggml-mpt-7b-base.bin"` | `"ggml-mpt-7b-chat.bin"` | `"ggml-mpt-7b-instruct.bin"`)
+
+##### replit
+
+List of Replit Models
+
+Type: `"ggml-replit-code-v1-3b.bin"`
+
+#### type
+
+Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
+
+Type: [ModelType](#modeltype)
+
+#### LLModel
+
+LLModel class representing a language model.
+This is a base class that provides common functionality for different types of language models.
+
+##### constructor
+
+Initialize a new LLModel.
+
+###### Parameters
+
+*   `path` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Absolute path to the model file.
+
+<!---->
+
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model file does not exist.
+
+##### type
+
+either 'gpt', mpt', or 'llama' or undefined
+
+Returns **([ModelType](#modeltype) | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
+
+##### name
+
+The name of the model.
+
+Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+
+##### stateSize
+
+Get the size of the internal state of the model.
+NOTE: This state data is specific to the type of model you have created.
+
+Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the size in bytes of the internal state of the model
+
+##### threadCount
+
+Get the number of threads used for model inference.
+The default is the number of physical cores your computer has.
+
+Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The number of threads used for model inference.
+
+##### setThreadCount
+
+Set the number of threads used for model inference.
+
+###### Parameters
+
+*   `newNumber` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The new number of threads.
+
+Returns **void**&#x20;
+
+##### raw\_prompt
+
+Prompt the model with a given input and optional parameters.
+This is the raw output from model.
+Use the prompt function exported for a value
+
+###### Parameters
+
+*   `q` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
+*   `params` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
+*   `callback` **function (res: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): void**&#x20;
+
+Returns **void** The result of the model prompt.
+
+##### embed
+
+Embed text with the model. Keep in mind that
+not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
+Use the prompt function exported for a value
+
+###### Parameters
+
+*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+*   `q`  The prompt input.
+*   `params`  Optional parameters for the prompt context.
+
+Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The result of the model prompt.
+
+##### isModelLoaded
+
+Whether the model is loaded or not.
+
+Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)**&#x20;
+
+##### setLibraryPath
+
+Where to search for the pluggable backend libraries
+
+###### Parameters
+
+*   `s` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+
+Returns **void**&#x20;
+
+##### getLibraryPath
+
+Where to get the pluggable backend libraries
+
+Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+
+#### loadModel
+
+Loads a machine learning model with the specified name. The defacto way to create a model.
+By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
+
+##### Parameters
+
+*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The name of the model to load.
+*   `options` **(LoadModelOptions | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))?** (Optional) Additional options for loading the model.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<(InferenceModel | EmbeddingModel)>** A promise that resolves to an instance of the loaded LLModel.
+
+#### createCompletion
+
+The nodejs equivalent to python binding's chat\_completion
+
+##### Parameters
+
+*   `model` **InferenceModel** The language model object.
+*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
+*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
+
+Returns **[CompletionReturn](#completionreturn)** The completion result.
+
+#### createEmbedding
+
+The nodejs moral equivalent to python binding's Embed4All().embed()
+meow
+
+##### Parameters
+
+*   `model` **EmbeddingModel** The language model object.
+*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** text to embed
+
+Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The completion result.
+
+#### CompletionOptions
+
+**Extends Partial\<LLModelPromptContext>**
+
+The options for creating the completion.
+
+##### verbose
+
+Indicates if verbose logging is enabled.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### systemPromptTemplate
+
+Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
+Note that if this is not defined, system messages will not be included in the prompt.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### promptTemplate
+
+Template for user messages, with %1 being replaced by the message.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### promptHeader
+
+The initial instruction for the model, on top of the prompt
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### promptFooter
+
+The last instruction for the model, appended to the end of the prompt.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### PromptMessage
+
+A message in the conversation, identical to OpenAI's chat message.
+
+##### role
+
+The role of the message.
+
+Type: (`"system"` | `"assistant"` | `"user"`)
+
+##### content
+
+The message content.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### prompt\_tokens
+
+The number of tokens used in the prompt.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### completion\_tokens
+
+The number of tokens used in the completion.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### total\_tokens
+
+The total number of tokens used.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### CompletionReturn
+
+The result of the completion, similar to OpenAI's format.
+
+##### model
+
+The model used for the completion.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### usage
+
+Token usage report.
+
+Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
+
+##### choices
+
+The generated completions.
+
+Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[CompletionChoice](#completionchoice)>
+
+#### CompletionChoice
+
+A completion choice, similar to OpenAI's format.
+
+##### message
+
+Response message
+
+Type: [PromptMessage](#promptmessage)
+
+#### LLModelPromptContext
+
+Model inference arguments for generating completions.
+
+##### logitsSize
+
+The size of the raw logits vector.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### tokensSize
+
+The size of the raw tokens vector.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nPast
+
+The number of tokens in the past conversation.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nCtx
+
+The number of tokens possible in the context window.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nPredict
+
+The number of tokens to predict.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### topK
+
+The top-k logits to sample from.
+Top-K sampling selects the next token only from the top K most likely tokens predicted by the model.
+It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit
+the diversity of the output. A higher value for top-K (eg., 100) will consider more tokens and lead
+to more diverse text, while a lower value (eg., 10) will focus on the most probable tokens and generate
+more conservative text. 30 - 60 is a good range for most tasks.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### topP
+
+The nucleus sampling probability threshold.
+Top-P limits the selection of the next token to a subset of tokens with a cumulative probability
+above a threshold P. This method, also known as nucleus sampling, finds a balance between diversity
+and quality by considering both token probabilities and the number of tokens available for sampling.
+When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
+On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
+The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
+for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### temp
+
+The temperature to adjust the model's output distribution.
+Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
+(eg., 1.2) increase randomness, resulting in more imaginative and diverse text. Lower temperatures (eg., 0.5)
+make the output more focused, predictable, and conservative. When the temperature is set to 0, the output
+becomes completely deterministic, always selecting the most probable next token and producing identical results
+each time. A safe range would be around 0.6 - 0.85, but you are free to search what value fits best for you.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nBatch
+
+The number of predictions to generate in parallel.
+By splitting the prompt every N tokens, prompt-batch-size reduces RAM usage during processing. However,
+this can increase the processing time as a trade-off. If the N value is set too low (e.g., 10), long prompts
+with 500+ tokens will be most affected, requiring numerous processing runs to complete the prompt processing.
+To ensure optimal performance, setting the prompt-batch-size to 2048 allows processing of all tokens in a single run.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### repeatPenalty
+
+The penalty factor for repeated tokens.
+Repeat-penalty can help penalize tokens based on how frequently they occur in the text, including the input prompt.
+A token that has already appeared five times is penalized more heavily than a token that has appeared only one time.
+A value of 1 means that there is no penalty and values larger than 1 discourage repeated tokens.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### repeatLastN
+
+The number of last tokens to penalize.
+The repeat-penalty-tokens N option controls the number of tokens in the history to consider for penalizing repetition.
+A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only
+consider recent tokens.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### contextErase
+
+The percentage of context to erase if the context window is exceeded.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### createTokenStream
+
+TODO: Help wanted to implement this
+
+##### Parameters
+
+*   `llmodel` **[LLModel](#llmodel)**&#x20;
+*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>**&#x20;
+*   `options` **[CompletionOptions](#completionoptions)**&#x20;
+
+Returns **function (ll: [LLModel](#llmodel)): AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>**&#x20;
+
+#### DEFAULT\_DIRECTORY
+
+From python api:
+models will be stored in (homedir)/.cache/gpt4all/\`
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### DEFAULT\_LIBRARIES\_DIRECTORY
+
+From python api:
+The default path for dynamic libraries to be stored.
+You may separate paths by a semicolon to search in multiple areas.
+This searches DEFAULT\_DIRECTORY/libraries, cwd/libraries, and finally cwd.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### DEFAULT\_MODEL\_CONFIG
+
+Default model configuration.
+
+Type: ModelConfig
+
+#### DEFAULT\_PROMT\_CONTEXT
+
+Default prompt context.
+
+Type: [LLModelPromptContext](#llmodelpromptcontext)
+
+#### DEFAULT\_MODEL\_LIST\_URL
+
+Default model list url.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### downloadModel
+
+Initiates the download of a model file.
+By default this downloads without waiting. use the controller returned to alter this behavior.
+
+##### Parameters
+
+*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The model to be downloaded.
+*   `options` **DownloadOptions** to pass into the downloader. Default is { location: (cwd), verbose: false }.
+
+##### Examples
+
+```javascript
+const download = downloadModel('ggml-gpt4all-j-v1.3-groovy.bin')
+download.promise.then(() => console.log('Downloaded!'))
+```
+
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model already exists in the specified location.
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model cannot be found at the specified url.
+
+Returns **[DownloadController](#downloadcontroller)** object that allows controlling the download process.
+
+#### DownloadModelOptions
+
+Options for the model download process.
+
+##### modelPath
+
+location to download the model.
+Default is process.cwd(), or the current working directory
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### verbose
+
+Debug mode -- check how long it took to download in seconds
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### url
+
+Remote download url. Defaults to `https://gpt4all.io/models/<modelName>`
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### md5sum
+
+MD5 sum of the model file. If this is provided, the downloaded file will be checked against this sum.
+If the sums do not match, an error will be thrown and the file will be deleted.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### DownloadController
+
+Model download controller.
+
+##### cancel
+
+Cancel the request to download if this is called.
+
+Type: function (): void
+
+##### promise
+
+A promise resolving to the downloaded models config once the download is done
+
+Type: [Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<ModelConfig>

gpt4all-bindings/python/gpt4all/__init__.py

@@ -1,2 +1,2 @@
-from .gpt4all import GPT4All  # noqa
+from .gpt4all import Embed4All, GPT4All  # noqa
 from .pyllmodel import LLModel  # noqa

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -5,7 +5,7 @@ import os
 import time
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Dict, Iterable, List, Union, Optional
+from typing import Any, Dict, Iterable, List, Optional, Union
 
 import requests
 from tqdm import tqdm
@@ -15,6 +15,44 @@ from . import pyllmodel
 # TODO: move to config
 DEFAULT_MODEL_DIRECTORY = os.path.join(str(Path.home()), ".cache", "gpt4all").replace("\\", "\\\\")
 
+DEFAULT_MODEL_CONFIG = {
+    "systemPrompt": "",
+    "promptTemplate": "### Human: \n{0}\n### Assistant:\n",
+}
+
+ConfigType = Dict[str, str]
+MessageType = Dict[str, str]
+
+
+class Embed4All:
+    """
+    Python class that handles embeddings for GPT4All.
+    """
+
+    def __init__(
+        self,
+        n_threads: Optional[int] = None,
+    ):
+        """
+        Constructor
+
+        Args:
+            n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
+        """
+        self.gpt4all = GPT4All(model_name='ggml-all-MiniLM-L6-v2-f16.bin', n_threads=n_threads)
+
+    def embed(self, text: str) -> List[float]:
+        """
+        Generate an embedding.
+
+        Args:
+            text: The text document to generate an embedding for.
+
+        Returns:
+            An embedding of your document of text.
+        """
+        return self.gpt4all.model.generate_embedding(text)
+
 
 class GPT4All:
     """
@@ -28,6 +66,7 @@ class GPT4All:
         model_type: Optional[str] = None,
         allow_download: bool = True,
         n_threads: Optional[int] = None,
+        device: Optional[str] = "cpu",
     ):
         """
         Constructor
@@ -39,22 +78,34 @@ class GPT4All:
             model_type: Model architecture. This argument currently does not have any functionality and is just used as
                 descriptive identifier for user. Default is None.
             allow_download: Allow API to download models from gpt4all.io. Default is True.
-            n_threads: number of CPU threads used by GPT4All. Default is None, than the number of threads are determined automatically.
+            n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
+            device: The processing unit on which the GPT4All model will run. It can be set to:
+                - "cpu": Model will run on the central processing unit.
+                - "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
+                - "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
+                Alternatively, a specific GPU name can also be provided, and the model will run on the GPU that matches the name if it's available.
+                Default is "cpu".
+
+                Note: If a selected GPU device does not have sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the model.
         """
         self.model_type = model_type
         self.model = pyllmodel.LLModel()
         # Retrieve model and download if allowed
-        model_dest = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download)
-        self.model.load_model(model_dest)
+        self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download)
+        if device is not None:
+            if device != "cpu":
+                self.model.init_gpu(model_path=self.config["path"], device=device)
+        self.model.load_model(self.config["path"])
         # Set n_threads
         if n_threads is not None:
             self.model.set_thread_count(n_threads)
 
-        self._is_chat_session_activated = False
-        self.current_chat_session = []
+        self._is_chat_session_activated: bool = False
+        self.current_chat_session: List[MessageType] = empty_chat_session()
+        self._current_prompt_template: str = "{0}"
 
     @staticmethod
-    def list_models() -> Dict:
+    def list_models() -> List[ConfigType]:
         """
         Fetch model list from https://gpt4all.io/models/models.json.
 
@@ -65,8 +116,11 @@ class GPT4All:
 
     @staticmethod
     def retrieve_model(
-        model_name: str, model_path: Optional[str] = None, allow_download: bool = True, verbose: bool = True
-    ) -> str:
+        model_name: str,
+        model_path: Optional[str] = None,
+        allow_download: bool = True,
+        verbose: bool = True,
+    ) -> ConfigType:
         """
         Find model file, and if it doesn't exist, download the model.
 
@@ -78,11 +132,25 @@ class GPT4All:
             verbose: If True (default), print debug messages.
 
         Returns:
-            Model file destination.
+            Model config.
         """
 
         model_filename = append_bin_suffix_if_missing(model_name)
 
+        # get the config for the model
+        config: ConfigType = DEFAULT_MODEL_CONFIG
+        if allow_download:
+            available_models = GPT4All.list_models()
+
+            for m in available_models:
+                if model_filename == m["filename"]:
+                    config.update(m)
+                    config["systemPrompt"] = config["systemPrompt"].strip()
+                    config["promptTemplate"] = config["promptTemplate"].replace(
+                        "%1", "{0}", 1
+                    )  # change to Python-style formatting
+                    break
+
         # Validate download directory
         if model_path is None:
             try:
@@ -101,31 +169,28 @@ class GPT4All:
 
         model_dest = os.path.join(model_path, model_filename).replace("\\", "\\\\")
         if os.path.exists(model_dest):
+            config.pop("url", None)
+            config["path"] = model_dest
             if verbose:
                 print("Found model file at ", model_dest)
-            return model_dest
 
         # If model file does not exist, download
         elif allow_download:
-            # Make sure valid model filename before attempting download
-            available_models = GPT4All.list_models()
+            url = config.pop("url", None)
 
-            selected_model = None
-            for m in available_models:
-                if model_filename == m['filename']:
-                    selected_model = m
-                    break
-
-            if selected_model is None:
-                raise ValueError(f"Model filename not in model list: {model_filename}")
-            url = selected_model.pop('url', None)
-
-            return GPT4All.download_model(model_filename, model_path, verbose=verbose, url=url)
+            config["path"] = GPT4All.download_model(model_filename, model_path, verbose=verbose, url=url)
         else:
             raise ValueError("Failed to retrieve model")
 
+        return config
+
     @staticmethod
-    def download_model(model_filename: str, model_path: str, verbose: bool = True, url: Optional[str] = None) -> str:
+    def download_model(
+        model_filename: str,
+        model_path: str,
+        verbose: bool = True,
+        url: Optional[str] = None,
+    ) -> str:
         """
         Download model from https://gpt4all.io.
 
@@ -161,7 +226,7 @@ class GPT4All:
             except Exception:
                 if os.path.exists(download_path):
                     if verbose:
-                        print('Cleaning up the interrupted download...')
+                        print("Cleaning up the interrupted download...")
                     os.remove(download_path)
                 raise
 
@@ -182,13 +247,14 @@ class GPT4All:
         max_tokens: int = 200,
         temp: float = 0.7,
         top_k: int = 40,
-        top_p: float = 0.1,
+        top_p: float = 0.4,
         repeat_penalty: float = 1.18,
         repeat_last_n: int = 64,
         n_batch: int = 8,
         n_predict: Optional[int] = None,
         streaming: bool = False,
-    ) -> Union[str, Iterable]:
+        callback: pyllmodel.ResponseCallbackType = pyllmodel.empty_response_callback,
+    ) -> Union[str, Iterable[str]]:
         """
         Generate outputs from any GPT4All model.
 
@@ -203,12 +269,14 @@ class GPT4All:
             n_batch: Number of prompt tokens processed in parallel. Larger values decrease latency but increase resource requirements.
             n_predict: Equivalent to max_tokens, exists for backwards compatibility.
             streaming: If True, this method will instead return a generator that yields tokens as the model generates them.
+            callback: A function with arguments token_id:int and response:str, which receives the tokens from the model as they are generated and stops the generation by returning False.
 
         Returns:
             Either the entire completion or a generator that yields the completion token by token.
         """
-        generate_kwargs = dict(
-            prompt=prompt,
+
+        # Preparing the model request
+        generate_kwargs: Dict[str, Any] = dict(
             temp=temp,
             top_k=top_k,
             top_p=top_p,
@@ -219,42 +287,92 @@ class GPT4All:
         )
 
         if self._is_chat_session_activated:
+            # check if there is only one message, i.e. system prompt:
+            generate_kwargs["reset_context"] = len(self.current_chat_session) == 1
             self.current_chat_session.append({"role": "user", "content": prompt})
-            generate_kwargs['prompt'] = self._format_chat_prompt_template(messages=self.current_chat_session[-1:])
-            generate_kwargs['reset_context'] = len(self.current_chat_session) == 1
+
+            prompt = self._format_chat_prompt_template(
+                messages=self.current_chat_session[-1:],
+                default_prompt_header=self.current_chat_session[0]["content"]
+                if generate_kwargs["reset_context"]
+                else "",
+            )
         else:
-            generate_kwargs['reset_context'] = True
+            generate_kwargs["reset_context"] = True
 
-        if streaming:
-            return self.model.prompt_model_streaming(**generate_kwargs)
-
-        output = self.model.prompt_model(**generate_kwargs)
+        # Prepare the callback, process the model response
+        output_collector: List[MessageType]
+        output_collector = [
+            {"content": ""}
+        ]  # placeholder for the self.current_chat_session if chat session is not activated
 
         if self._is_chat_session_activated:
-            self.current_chat_session.append({"role": "assistant", "content": output})
+            self.current_chat_session.append({"role": "assistant", "content": ""})
+            output_collector = self.current_chat_session
 
-        return output
+        def _callback_wrapper(
+            callback: pyllmodel.ResponseCallbackType,
+            output_collector: List[MessageType],
+        ) -> pyllmodel.ResponseCallbackType:
+
+            def _callback(token_id: int, response: str) -> bool:
+                nonlocal callback, output_collector
+
+                output_collector[-1]["content"] += response
+
+                return callback(token_id, response)
+
+            return _callback
+
+        # Send the request to the model
+        if streaming:
+            return self.model.prompt_model_streaming(
+                prompt=prompt,
+                callback=_callback_wrapper(callback, output_collector),
+                **generate_kwargs,
+            )
+
+        self.model.prompt_model(
+            prompt=prompt,
+            callback=_callback_wrapper(callback, output_collector),
+            **generate_kwargs,
+        )
+
+        return output_collector[-1]["content"]
 
     @contextmanager
-    def chat_session(self):
-        '''
+    def chat_session(
+        self,
+        system_prompt: str = "",
+        prompt_template: str = "",
+    ):
+        """
         Context manager to hold an inference optimized chat session with a GPT4All model.
-        '''
+
+        Args:
+            system_prompt: An initial instruction for the model.
+            prompt_template: Template for the prompts with {0} being replaced by the user message.
+        """
         # Code to acquire resource, e.g.:
         self._is_chat_session_activated = True
-        self.current_chat_session = []
+        self.current_chat_session = empty_chat_session(system_prompt or self.config["systemPrompt"])
+        self._current_prompt_template = prompt_template or self.config["promptTemplate"]
         try:
             yield self
         finally:
             # Code to release resource, e.g.:
             self._is_chat_session_activated = False
-            self.current_chat_session = []
+            self.current_chat_session = empty_chat_session()
+            self._current_prompt_template = "{0}"
 
     def _format_chat_prompt_template(
-        self, messages: List[Dict], default_prompt_header=True, default_prompt_footer=True
+        self,
+        messages: List[MessageType],
+        default_prompt_header: str = "",
+        default_prompt_footer: str = "",
     ) -> str:
         """
-        Helper method for building a prompt using template from list of messages.
+        Helper method for building a prompt from list of messages using the self._current_prompt_template as a template for each message.
 
         Args:
             messages:  List of dictionaries. Each dictionary should have a "role" key
@@ -266,19 +384,44 @@ class GPT4All:
         Returns:
             Formatted prompt.
         """
-        full_prompt = ""
+
+        if isinstance(default_prompt_header, bool):
+            import warnings
+
+            warnings.warn(
+                "Using True/False for the 'default_prompt_header' is deprecated. Use a string instead.",
+                DeprecationWarning,
+            )
+            default_prompt_header = ""
+
+        if isinstance(default_prompt_footer, bool):
+            import warnings
+
+            warnings.warn(
+                "Using True/False for the 'default_prompt_footer' is deprecated. Use a string instead.",
+                DeprecationWarning,
+            )
+            default_prompt_footer = ""
+
+        full_prompt = default_prompt_header + "\n\n" if default_prompt_header != "" else ""
 
         for message in messages:
             if message["role"] == "user":
-                user_message = "### Human: \n" + message["content"] + "\n### Assistant:\n"
+                user_message = self._current_prompt_template.format(message["content"])
                 full_prompt += user_message
             if message["role"] == "assistant":
-                assistant_message = message["content"] + '\n'
+                assistant_message = message["content"] + "\n"
                 full_prompt += assistant_message
 
+        full_prompt += "\n\n" + default_prompt_footer if default_prompt_footer != "" else ""
+
         return full_prompt
 
 
+def empty_chat_session(system_prompt: str = "") -> List[MessageType]:
+    return [{"role": "system", "content": system_prompt}]
+
+
 def append_bin_suffix_if_missing(model_name):
     if not model_name.endswith(".bin"):
         model_name += ".bin"

gpt4all-bindings/python/gpt4all/pyllmodel.py

@@ -1,26 +1,17 @@
 import ctypes
+import logging
 import os
 import platform
-import queue
+from queue import Queue
 import re
 import subprocess
 import sys
 import threading
-from typing import Iterable
+from typing import Callable, Iterable, List
 
 import pkg_resources
 
-
-class DualStreamProcessor:
-    def __init__(self, stream=None):
-        self.stream = stream
-        self.output = ""
-
-    def write(self, text):
-        if self.stream is not None:
-            self.stream.write(text)
-            self.stream.flush()
-        self.output += text
+logger: logging.Logger = logging.getLogger(__name__)
 
 
 # TODO: provide a config file to make this more robust
@@ -43,9 +34,9 @@ def load_llmodel_library():
 
     c_lib_ext = get_c_shared_lib_extension()
 
-    llmodel_file = "libllmodel" + '.' + c_lib_ext
+    llmodel_file = "libllmodel" + "." + c_lib_ext
 
-    llmodel_dir = str(pkg_resources.resource_filename('gpt4all', os.path.join(LLMODEL_PATH, llmodel_file))).replace(
+    llmodel_dir = str(pkg_resources.resource_filename("gpt4all", os.path.join(LLMODEL_PATH, llmodel_file))).replace(
         "\\", "\\\\"
     )
 
@@ -79,6 +70,14 @@ class LLModelPromptContext(ctypes.Structure):
         ("context_erase", ctypes.c_float),
     ]
 
+class LLModelGPUDevice(ctypes.Structure):
+    _fields_ = [
+        ("index", ctypes.c_int32),
+        ("type", ctypes.c_int32),
+        ("heapSize", ctypes.c_size_t),
+        ("name", ctypes.c_char_p),
+        ("vendor", ctypes.c_char_p),
+    ]
 
 # Define C function signatures using ctypes
 llmodel.llmodel_model_create.argtypes = [ctypes.c_char_p]
@@ -112,6 +111,17 @@ llmodel.llmodel_prompt.argtypes = [
 
 llmodel.llmodel_prompt.restype = None
 
+llmodel.llmodel_embedding.argtypes = [
+    ctypes.c_void_p,
+    ctypes.c_char_p,
+    ctypes.POINTER(ctypes.c_size_t),
+]
+
+llmodel.llmodel_embedding.restype = ctypes.POINTER(ctypes.c_float)
+
+llmodel.llmodel_free_embedding.argtypes = [ctypes.POINTER(ctypes.c_float)]
+llmodel.llmodel_free_embedding.restype = None
+
 llmodel.llmodel_setThreadCount.argtypes = [ctypes.c_void_p, ctypes.c_int32]
 llmodel.llmodel_setThreadCount.restype = None
 
@@ -121,7 +131,29 @@ llmodel.llmodel_set_implementation_search_path.restype = None
 llmodel.llmodel_threadCount.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_threadCount.restype = ctypes.c_int32
 
-llmodel.llmodel_set_implementation_search_path(MODEL_LIB_PATH.encode('utf-8'))
+llmodel.llmodel_set_implementation_search_path(MODEL_LIB_PATH.encode("utf-8"))
+
+llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
+llmodel.llmodel_available_gpu_devices.restype = ctypes.POINTER(LLModelGPUDevice)
+
+llmodel.llmodel_gpu_init_gpu_device_by_string.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_char_p]
+llmodel.llmodel_gpu_init_gpu_device_by_string.restype = ctypes.c_bool
+
+llmodel.llmodel_gpu_init_gpu_device_by_struct.argtypes = [ctypes.c_void_p, ctypes.POINTER(LLModelGPUDevice)]
+llmodel.llmodel_gpu_init_gpu_device_by_struct.restype = ctypes.c_bool
+
+llmodel.llmodel_gpu_init_gpu_device_by_int.argtypes = [ctypes.c_void_p, ctypes.c_int32]
+llmodel.llmodel_gpu_init_gpu_device_by_int.restype = ctypes.c_bool
+
+llmodel.llmodel_has_gpu_device.argtypes = [ctypes.c_void_p]
+llmodel.llmodel_has_gpu_device.restype = ctypes.c_bool
+
+ResponseCallbackType = Callable[[int, str], bool]
+RawResponseCallbackType = Callable[[int, bytes], bool]
+
+
+def empty_response_callback(token_id: int, response: str) -> bool:
+    return True
 
 
 class LLModel:
@@ -141,10 +173,14 @@ class LLModel:
         self.model = None
         self.model_name = None
         self.context = None
+        self.llmodel_lib = llmodel
+
+        self.buffer = bytearray()
+        self.buff_expecting_cont_bytes: int = 0
 
     def __del__(self):
         if self.model is not None:
-            llmodel.llmodel_model_destroy(self.model)
+            self.llmodel_lib.llmodel_model_destroy(self.model)
 
     def memory_needed(self, model_path: str) -> int:
         model_path_enc = model_path.encode("utf-8")
@@ -155,6 +191,60 @@ class LLModel:
         else:
             raise ValueError("Unable to instantiate model")
 
+    def list_gpu(self, model_path: str) -> list:
+        """
+        Lists available GPU devices that satisfy the model's memory requirements.
+
+        Parameters
+        ----------
+        model_path : str
+            Path to the model.
+
+        Returns
+        -------
+        list
+            A list of LLModelGPUDevice structures representing available GPU devices.
+        """
+        if self.model is not None:
+            model_path_enc = model_path.encode("utf-8")
+            mem_required = llmodel.llmodel_required_mem(self.model, model_path_enc)
+        else:
+            mem_required = self.memory_needed(model_path)
+        num_devices = ctypes.c_int32(0)
+        devices_ptr = self.llmodel_lib.llmodel_available_gpu_devices(self.model, mem_required, ctypes.byref(num_devices))
+        if not devices_ptr:
+            raise ValueError("Unable to retrieve available GPU devices")
+        devices = [devices_ptr[i] for i in range(num_devices.value)]
+        return devices
+
+    def init_gpu(self, model_path: str, device: str):
+        if self.model is not None:
+            model_path_enc = model_path.encode("utf-8")
+            mem_required = llmodel.llmodel_required_mem(self.model, model_path_enc)
+        else:
+            mem_required = self.memory_needed(model_path)
+        device_enc = device.encode("utf-8")
+        success = self.llmodel_lib.llmodel_gpu_init_gpu_device_by_string(self.model, mem_required, device_enc)
+        if not success:
+            # Retrieve all GPUs without considering memory requirements.
+            num_devices = ctypes.c_int32(0)
+            all_devices_ptr = self.llmodel_lib.llmodel_available_gpu_devices(self.model, 0, ctypes.byref(num_devices))
+            if not all_devices_ptr:
+                raise ValueError("Unable to retrieve list of all GPU devices")
+            all_gpus = [all_devices_ptr[i].name.decode('utf-8') for i in range(num_devices.value)]
+
+            # Retrieve GPUs that meet the memory requirements using list_gpu
+            available_gpus = [device.name.decode('utf-8') for device in self.list_gpu(model_path)]
+
+            # Identify GPUs that are unavailable due to insufficient memory or features
+            unavailable_gpus = set(all_gpus) - set(available_gpus)
+
+            # Formulate the error message
+            error_msg = "Unable to initialize model on GPU: '{}'.".format(device)
+            error_msg += "\nAvailable GPUs: {}.".format(available_gpus)
+            error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}.".format(unavailable_gpus)
+            raise ValueError(error_msg)
+
     def load_model(self, model_path: str) -> bool:
         """
         Load model from a file.
@@ -233,9 +323,21 @@ class LLModel:
         self.context.repeat_last_n = repeat_last_n
         self.context.context_erase = context_erase
 
+    def generate_embedding(self, text: str) -> List[float]:
+        if not text:
+            raise ValueError("Text must not be None or empty")
+
+        embedding_size = ctypes.c_size_t()
+        c_text = ctypes.c_char_p(text.encode('utf-8'))
+        embedding_ptr = llmodel.llmodel_embedding(self.model, c_text, ctypes.byref(embedding_size))
+        embedding_array = [embedding_ptr[i] for i in range(embedding_size.value)]
+        llmodel.llmodel_free_embedding(embedding_ptr)
+        return list(embedding_array)
+
     def prompt_model(
         self,
         prompt: str,
+        callback: ResponseCallbackType,
         n_predict: int = 4096,
         top_k: int = 40,
         top_p: float = 0.9,
@@ -245,8 +347,7 @@ class LLModel:
         repeat_last_n: int = 10,
         context_erase: float = 0.75,
         reset_context: bool = False,
-        streaming=False,
-    ) -> str:
+    ):
         """
         Generate response from model from a prompt.
 
@@ -254,26 +355,27 @@ class LLModel:
         ----------
         prompt: str
             Question, task, or conversation for model to respond to
-        streaming: bool
-            Stream response to stdout
+        callback(token_id:int, response:str): bool
+            The model sends response tokens to callback
 
         Returns
         -------
-        Model response str
+        None
         """
 
-        prompt_bytes = prompt.encode('utf-8')
+        self.buffer.clear()
+        self.buff_expecting_cont_bytes = 0
+
+        logger.info(
+            "LLModel.prompt_model -- prompt:\n"
+            + "%s\n"
+            + "===/LLModel.prompt_model -- prompt/===",
+            prompt,
+        )
+
+        prompt_bytes = prompt.encode("utf-8")
         prompt_ptr = ctypes.c_char_p(prompt_bytes)
 
-        old_stdout = sys.stdout
-
-        stream_processor = DualStreamProcessor()
-
-        if streaming:
-            stream_processor.stream = sys.stdout
-
-        sys.stdout = stream_processor
-
         self._set_context(
             n_predict=n_predict,
             top_k=top_k,
@@ -290,70 +392,43 @@ class LLModel:
             self.model,
             prompt_ptr,
             PromptCallback(self._prompt_callback),
-            ResponseCallback(self._response_callback),
+            ResponseCallback(self._callback_decoder(callback)),
             RecalculateCallback(self._recalculate_callback),
             self.context,
         )
 
-        # Revert to old stdout
-        sys.stdout = old_stdout
-        # Force new line
-        return stream_processor.output
 
     def prompt_model_streaming(
-        self,
-        prompt: str,
-        n_predict: int = 4096,
-        top_k: int = 40,
-        top_p: float = 0.9,
-        temp: float = 0.1,
-        n_batch: int = 8,
-        repeat_penalty: float = 1.2,
-        repeat_last_n: int = 10,
-        context_erase: float = 0.75,
-        reset_context: bool = False,
-    ) -> Iterable:
+        self, prompt: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
+    ) -> Iterable[str]:
         # Symbol to terminate from generator
         TERMINATING_SYMBOL = object()
 
-        output_queue = queue.Queue()
-
-        prompt_bytes = prompt.encode('utf-8')
-        prompt_ptr = ctypes.c_char_p(prompt_bytes)
-
-        self._set_context(
-            n_predict=n_predict,
-            top_k=top_k,
-            top_p=top_p,
-            temp=temp,
-            n_batch=n_batch,
-            repeat_penalty=repeat_penalty,
-            repeat_last_n=repeat_last_n,
-            context_erase=context_erase,
-            reset_context=reset_context,
-        )
+        output_queue: Queue = Queue()
 
         # Put response tokens into an output queue
-        def _generator_response_callback(token_id, response):
-            output_queue.put(response.decode('utf-8', 'replace'))
-            return True
+        def _generator_callback_wrapper(callback: ResponseCallbackType) -> ResponseCallbackType:
+            def _generator_callback(token_id: int, response: str):
+                nonlocal callback
 
-        def run_llmodel_prompt(model, prompt, prompt_callback, response_callback, recalculate_callback, context):
-            llmodel.llmodel_prompt(model, prompt, prompt_callback, response_callback, recalculate_callback, context)
+                if callback(token_id, response):
+                    output_queue.put(response)
+                    return True
+
+                return False
+
+            return _generator_callback
+
+        def run_llmodel_prompt(prompt: str, callback: ResponseCallbackType, **kwargs):
+            self.prompt_model(prompt, callback, **kwargs)
             output_queue.put(TERMINATING_SYMBOL)
 
         # Kick off llmodel_prompt in separate thread so we can return generator
         # immediately
         thread = threading.Thread(
             target=run_llmodel_prompt,
-            args=(
-                self.model,
-                prompt_ptr,
-                PromptCallback(self._prompt_callback),
-                ResponseCallback(_generator_response_callback),
-                RecalculateCallback(self._recalculate_callback),
-                self.context,
-            ),
+            args=(prompt, _generator_callback_wrapper(callback)),
+            kwargs=kwargs,
         )
         thread.start()
 
@@ -364,18 +439,53 @@ class LLModel:
                 break
             yield response
 
+    def _callback_decoder(self, callback: ResponseCallbackType) -> RawResponseCallbackType:
+        def _raw_callback(token_id: int, response: bytes) -> bool:
+            nonlocal self, callback
+
+            decoded = []
+
+            for byte in response:
+                
+                bits = "{:08b}".format(byte)
+                (high_ones, _, _) = bits.partition('0')
+
+                if len(high_ones) == 1: 
+                    # continuation byte
+                    self.buffer.append(byte)
+                    self.buff_expecting_cont_bytes -= 1
+
+                else: 
+                    # beginning of a byte sequence
+                    if len(self.buffer) > 0:
+                        decoded.append(self.buffer.decode('utf-8', 'replace'))
+
+                        self.buffer.clear()
+
+                    self.buffer.append(byte)
+                    self.buff_expecting_cont_bytes = max(0, len(high_ones) - 1)
+
+                if self.buff_expecting_cont_bytes <= 0: 
+                    # received the whole sequence or an out of place continuation byte
+                    decoded.append(self.buffer.decode('utf-8', 'replace'))
+
+                    self.buffer.clear()
+                    self.buff_expecting_cont_bytes = 0
+                    
+            if len(decoded) == 0 and self.buff_expecting_cont_bytes > 0:
+                # wait for more continuation bytes
+                return True
+            
+            return callback(token_id, ''.join(decoded))     
+
+        return _raw_callback
+
     # Empty prompt callback
     @staticmethod
-    def _prompt_callback(token_id):
-        return True
-
-    # Empty response callback method that just prints response to be collected
-    @staticmethod
-    def _response_callback(token_id, response):
-        sys.stdout.write(response.decode('utf-8', 'replace'))
+    def _prompt_callback(token_id: int) -> bool:
         return True
 
     # Empty recalculate callback
     @staticmethod
-    def _recalculate_callback(is_recalculating):
+    def _recalculate_callback(is_recalculating: bool) -> bool:
         return is_recalculating

gpt4all-bindings/python/gpt4all/tests/test_embed_timings.py

@@ -0,0 +1,20 @@
+import sys
+import time
+from io import StringIO
+
+from gpt4all import Embed4All, GPT4All
+
+
+def time_embedding(i, embedder):
+    text = 'foo bar ' * i
+    start_time = time.time()
+    output = embedder.embed(text)
+    end_time = time.time()
+    elapsed_time = end_time - start_time
+    print(f"Time report: {2 * i / elapsed_time} tokens/second with {2 * i} tokens taking {elapsed_time} seconds")
+
+
+if __name__ == "__main__":
+    embedder = Embed4All(n_threads=8)
+    for i in [2**n for n in range(6, 14)]:
+        time_embedding(i, embedder)

gpt4all-bindings/python/gpt4all/tests/test_gpt4all.py

@@ -1,7 +1,10 @@
 import sys
 from io import StringIO
+from pathlib import Path
 
-from gpt4all import GPT4All
+from gpt4all import GPT4All, Embed4All
+import time
+import pytest
 
 
 def test_inference():
@@ -99,3 +102,31 @@ def test_inference_mpt():
     output = model.generate(prompt)
     assert isinstance(output, str)
     assert len(output) > 0
+
+
+def test_embedding():
+    text = 'The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox'
+    embedder = Embed4All()
+    output = embedder.embed(text)
+    #for i, value in enumerate(output):
+        #print(f'Value at index {i}: {value}')
+    assert len(output) == 384
+
+
+def test_empty_embedding():
+    text = ''
+    embedder = Embed4All()
+    with pytest.raises(ValueError):
+        output = embedder.embed(text)
+
+def test_download_model(tmp_path: Path):
+    import gpt4all.gpt4all
+    old_default_dir = gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY
+    gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = tmp_path  # temporary pytest directory to ensure a download happens
+    try:
+        model = GPT4All(model_name='ggml-all-MiniLM-L6-v2-f16.bin')
+        model_path = tmp_path / model.config['filename']
+        assert model_path.absolute() == Path(model.config['path']).absolute()
+        assert model_path.stat().st_size == int(model.config['filesize'])
+    finally:
+        gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = old_default_dir

gpt4all-bindings/python/mkdocs.yml

@@ -9,9 +9,12 @@ use_directory_urls: false
 
 nav:
     - 'index.md'
+    - 'GPT4All Chat Client': 'gpt4all_chat.md'
     - 'Bindings':
-      - 'GPT4All in Python': 'gpt4all_python.md'
-      - 'GPT4All Chat Client': 'gpt4all_chat.md'
+      - 'GPT4All in Python':
+        - 'Generation': 'gpt4all_python.md'
+        - 'Embedding': 'gpt4all_python_embedding.md'
+      - 'GPT4ALL in NodeJs': 'gpt4all_typescript.md'
       - 'gpt4all_cli.md'
 #    - 'Tutorials':
 #      - 'gpt4all_modal.md'
@@ -68,4 +71,4 @@ plugins:
 
   #- mkdocs-jupyter:
   #    ignore_h1_titles: True
-  #    show_input: True
+  #    show_input: True

gpt4all-bindings/python/setup.py

@@ -61,10 +61,10 @@ copy_prebuilt_C_lib(SRC_CLIB_DIRECtORY,
 
 setup(
     name=package_name,
-    version="1.0.3",
+    version="1.0.11",
     description="Python bindings for GPT4All",
-    author="Richard Guo",
-    author_email="richard@nomic.ai",
+    author="Nomic and the Open Source Community",
+    author_email="support@nomic.ai",
     url="https://pypi.org/project/gpt4all/",
     classifiers = [
         "Programming Language :: Python :: 3",

gpt4all-bindings/typescript/.gitignore

@@ -1,3 +1,10 @@
 node_modules/
 build/
 prebuilds/
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+runtimes/

gpt4all-bindings/typescript/README.md

@@ -1,98 +1,731 @@
-### Javascript Bindings
+# GPT4All Node.js API
+
+```sh
+yarn add gpt4all@alpha
+
+npm install gpt4all@alpha
+
+pnpm install gpt4all@alpha
+```
+
 The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
 
-- created by [jacoobes](https://github.com/jacoobes) and [nomic ai](https://home.nomic.ai) :D, for all to use.
+*   New bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
+*   The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
+*   Everything should work out the box.
+*   See [API Reference](#api-reference)
+
+### Chat Completion
 
-### Code (alpha)
 ```js
-import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY } from '../src/gpt4all.js'
+import { createCompletion, loadModel } from '../src/gpt4all.js'
 
-const ll = new LLModel({
-    model_name: 'ggml-vicuna-7b-1.1-q4_2.bin',
-    model_path: './', 
-    library_path: DEFAULT_LIBRARIES_DIRECTORY
-});
+const model = await loadModel('ggml-vicuna-7b-1.1-q4_2', { verbose: true });
 
-const response = await createCompletion(ll, [
+const response = await createCompletion(model, [
     { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
     { role : 'user', content: 'What is 1 + 1?'  } 
 ]);
 
 ```
-### API 
-- The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
-- [docs](./docs/api.md)
+
+### Embedding
+
+```js
+import { createEmbedding, loadModel } from '../src/gpt4all.js'
+
+const model = await loadModel('ggml-all-MiniLM-L6-v2-f16', { verbose: true });
+
+const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional");
+```
+
 ### Build Instructions
 
-- As of 05/21/2023, Tested on windows (MSVC). (somehow got it to work on MSVC 🤯)
-    - binding.gyp is compile config
-- Tested on Ubuntu. Everything seems to work fine
-- MingW works as well to build the gpt4all-backend. HOWEVER, this package works only with MSVC built dlls.
+*   binding.gyp is compile config
+*   Tested on Ubuntu. Everything seems to work fine
+*   Tested on Windows. Everything works fine.
+*   Sparse testing on mac os.
+*   MingW works as well to build the gpt4all-backend. **HOWEVER**, this package works only with MSVC built dlls.
 
 ### Requirements
-- git
-- [node.js >= 18.0.0](https://nodejs.org/en)
-- [yarn](https://yarnpkg.com/)
-- [node-gyp](https://github.com/nodejs/node-gyp)
-    - all of its requirements.
 
-### Build
+*   git
+*   [node.js >= 18.0.0](https://nodejs.org/en)
+*   [yarn](https://yarnpkg.com/)
+*   [node-gyp](https://github.com/nodejs/node-gyp)
+    *   all of its requirements.
+*   (unix) gcc version 12
+*   (win) msvc version 143
+    *   Can be obtained with visual studio 2022 build tools
+*   python 3
+
+### Build (from source)
+
 ```sh
 git clone https://github.com/nomic-ai/gpt4all.git
 cd gpt4all-bindings/typescript
 ```
-- The below shell commands assume the current working directory is `typescript`.
 
-- To Build and Rebuild: 
- ```sh
- yarn
- ```
- - llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
- ```sh
+*   The below shell commands assume the current working directory is `typescript`.
+
+*   To Build and Rebuild:
+
+```sh
+yarn
+```
+
+*   llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
+
+```sh
 git submodule update --init --depth 1 --recursive
- ```
+```
+
 **AS OF NEW BACKEND** to build the backend,
+
 ```sh
 yarn build:backend
 ```
+
 This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native The only current way to use them is to put them in the current working directory of your application. That is, **WHEREVER YOU RUN YOUR NODE APPLICATION**
-- llama-xxxx.dll is required.
-- According to whatever model you are using, you'll need to select the proper model loader.
-    - For example, if you running an Mosaic MPT model, you will need to select the mpt-(buildvariant).(dynamiclibrary)
+
+*   llama-xxxx.dll is required.
+*   According to whatever model you are using, you'll need to select the proper model loader.
+    *   For example, if you running an Mosaic MPT model, you will need to select the mpt-(buildvariant).(dynamiclibrary)
 
 ### Test
+
 ```sh
 yarn test
 ```
+
 ### Source Overview
 
 #### src/
-- Extra functions to help aid devex
-- Typings for the native node addon
-- the javascript interface
+
+*   Extra functions to help aid devex
+*   Typings for the native node addon
+*   the javascript interface
 
 #### test/
-- simple unit testings for some functions exported.
-- more advanced ai testing is not handled 
+
+*   simple unit testings for some functions exported.
+*   more advanced ai testing is not handled
 
 #### spec/
-- Average look and feel of the api
-- Should work assuming a model and libraries are installed locally in working directory
+
+*   Average look and feel of the api
+*   Should work assuming a model and libraries are installed locally in working directory
 
 #### index.cc
-- The bridge between nodejs and c. Where the bindings are.
-#### prompt.cc 
-- Handling prompting and inference of models in a threadsafe, asynchronous way.
-#### docs/
-- Autogenerated documentation using the script `yarn docs:build`
+
+*   The bridge between nodejs and c. Where the bindings are.
+
+#### prompt.cc
+
+*   Handling prompting and inference of models in a threadsafe, asynchronous way.
+
+### Known Issues
+
+*   why your model may be spewing bull 💩
+    *   The downloaded model is broken (just reinstall or download from official site)
+    *   That's it so far
 
 ### Roadmap
+
 This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
 
-- [x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
-- [ ] createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)
-- [ ] proper unit testing (integrate with circle ci)
-- [ ] publish to npm under alpha tag `gpt4all@alpha`
-- [ ] have more people test on other platforms (mac tester needed)
-- [x] switch to new pluggable backend
+*   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
+*   \[ ] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
+*   \[x] proper unit testing (integrate with circle ci)
+*   \[x] publish to npm under alpha tag `gpt4all@alpha`
+*   \[x] have more people test on other platforms (mac tester needed)
+*   \[x] switch to new pluggable backend
+*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
+    *   Should include prebuilds to avoid painful node-gyp errors
+*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
 
+### API Reference
+
+<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
+
+##### Table of Contents
+
+*   [ModelType](#modeltype)
+*   [ModelFile](#modelfile)
+    *   [gptj](#gptj)
+    *   [llama](#llama)
+    *   [mpt](#mpt)
+    *   [replit](#replit)
+*   [type](#type)
+*   [LLModel](#llmodel)
+    *   [constructor](#constructor)
+        *   [Parameters](#parameters)
+    *   [type](#type-1)
+    *   [name](#name)
+    *   [stateSize](#statesize)
+    *   [threadCount](#threadcount)
+    *   [setThreadCount](#setthreadcount)
+        *   [Parameters](#parameters-1)
+    *   [raw\_prompt](#raw_prompt)
+        *   [Parameters](#parameters-2)
+    *   [embed](#embed)
+        *   [Parameters](#parameters-3)
+    *   [isModelLoaded](#ismodelloaded)
+    *   [setLibraryPath](#setlibrarypath)
+        *   [Parameters](#parameters-4)
+    *   [getLibraryPath](#getlibrarypath)
+*   [loadModel](#loadmodel)
+    *   [Parameters](#parameters-5)
+*   [createCompletion](#createcompletion)
+    *   [Parameters](#parameters-6)
+*   [createEmbedding](#createembedding)
+    *   [Parameters](#parameters-7)
+*   [CompletionOptions](#completionoptions)
+    *   [verbose](#verbose)
+    *   [systemPromptTemplate](#systemprompttemplate)
+    *   [promptTemplate](#prompttemplate)
+    *   [promptHeader](#promptheader)
+    *   [promptFooter](#promptfooter)
+*   [PromptMessage](#promptmessage)
+    *   [role](#role)
+    *   [content](#content)
+*   [prompt\_tokens](#prompt_tokens)
+*   [completion\_tokens](#completion_tokens)
+*   [total\_tokens](#total_tokens)
+*   [CompletionReturn](#completionreturn)
+    *   [model](#model)
+    *   [usage](#usage)
+    *   [choices](#choices)
+*   [CompletionChoice](#completionchoice)
+    *   [message](#message)
+*   [LLModelPromptContext](#llmodelpromptcontext)
+    *   [logitsSize](#logitssize)
+    *   [tokensSize](#tokenssize)
+    *   [nPast](#npast)
+    *   [nCtx](#nctx)
+    *   [nPredict](#npredict)
+    *   [topK](#topk)
+    *   [topP](#topp)
+    *   [temp](#temp)
+    *   [nBatch](#nbatch)
+    *   [repeatPenalty](#repeatpenalty)
+    *   [repeatLastN](#repeatlastn)
+    *   [contextErase](#contexterase)
+*   [createTokenStream](#createtokenstream)
+    *   [Parameters](#parameters-8)
+*   [DEFAULT\_DIRECTORY](#default_directory)
+*   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
+*   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
+*   [DEFAULT\_PROMT\_CONTEXT](#default_promt_context)
+*   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
+*   [downloadModel](#downloadmodel)
+    *   [Parameters](#parameters-9)
+    *   [Examples](#examples)
+*   [DownloadModelOptions](#downloadmodeloptions)
+    *   [modelPath](#modelpath)
+    *   [verbose](#verbose-1)
+    *   [url](#url)
+    *   [md5sum](#md5sum)
+*   [DownloadController](#downloadcontroller)
+    *   [cancel](#cancel)
+    *   [promise](#promise)
+
+#### ModelType
+
+Type of the model
+
+Type: (`"gptj"` | `"llama"` | `"mpt"` | `"replit"`)
+
+#### ModelFile
+
+Full list of models available
+@deprecated These model names are outdated and this type will not be maintained, please use a string literal instead
+
+##### gptj
+
+List of GPT-J Models
+
+Type: (`"ggml-gpt4all-j-v1.3-groovy.bin"` | `"ggml-gpt4all-j-v1.2-jazzy.bin"` | `"ggml-gpt4all-j-v1.1-breezy.bin"` | `"ggml-gpt4all-j.bin"`)
+
+##### llama
+
+List Llama Models
+
+Type: (`"ggml-gpt4all-l13b-snoozy.bin"` | `"ggml-vicuna-7b-1.1-q4_2.bin"` | `"ggml-vicuna-13b-1.1-q4_2.bin"` | `"ggml-wizardLM-7B.q4_2.bin"` | `"ggml-stable-vicuna-13B.q4_2.bin"` | `"ggml-nous-gpt4-vicuna-13b.bin"` | `"ggml-v3-13b-hermes-q5_1.bin"`)
+
+##### mpt
+
+List of MPT Models
+
+Type: (`"ggml-mpt-7b-base.bin"` | `"ggml-mpt-7b-chat.bin"` | `"ggml-mpt-7b-instruct.bin"`)
+
+##### replit
+
+List of Replit Models
+
+Type: `"ggml-replit-code-v1-3b.bin"`
+
+#### type
+
+Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
+
+Type: [ModelType](#modeltype)
+
+#### LLModel
+
+LLModel class representing a language model.
+This is a base class that provides common functionality for different types of language models.
+
+##### constructor
+
+Initialize a new LLModel.
+
+###### Parameters
+
+*   `path` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Absolute path to the model file.
+
+<!---->
+
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model file does not exist.
+
+##### type
+
+either 'gpt', mpt', or 'llama' or undefined
+
+Returns **([ModelType](#modeltype) | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
+
+##### name
+
+The name of the model.
+
+Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+
+##### stateSize
+
+Get the size of the internal state of the model.
+NOTE: This state data is specific to the type of model you have created.
+
+Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the size in bytes of the internal state of the model
+
+##### threadCount
+
+Get the number of threads used for model inference.
+The default is the number of physical cores your computer has.
+
+Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The number of threads used for model inference.
+
+##### setThreadCount
+
+Set the number of threads used for model inference.
+
+###### Parameters
+
+*   `newNumber` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The new number of threads.
+
+Returns **void**&#x20;
+
+##### raw\_prompt
+
+Prompt the model with a given input and optional parameters.
+This is the raw output from model.
+Use the prompt function exported for a value
+
+###### Parameters
+
+*   `q` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
+*   `params` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
+*   `callback` **function (res: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): void**&#x20;
+
+Returns **void** The result of the model prompt.
+
+##### embed
+
+Embed text with the model. Keep in mind that
+not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
+Use the prompt function exported for a value
+
+###### Parameters
+
+*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+*   `q`  The prompt input.
+*   `params`  Optional parameters for the prompt context.
+
+Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The result of the model prompt.
+
+##### isModelLoaded
+
+Whether the model is loaded or not.
+
+Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)**&#x20;
+
+##### setLibraryPath
+
+Where to search for the pluggable backend libraries
+
+###### Parameters
+
+*   `s` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+
+Returns **void**&#x20;
+
+##### getLibraryPath
+
+Where to get the pluggable backend libraries
+
+Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+
+#### loadModel
+
+Loads a machine learning model with the specified name. The defacto way to create a model.
+By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
+
+##### Parameters
+
+*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The name of the model to load.
+*   `options` **(LoadModelOptions | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))?** (Optional) Additional options for loading the model.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<(InferenceModel | EmbeddingModel)>** A promise that resolves to an instance of the loaded LLModel.
+
+#### createCompletion
+
+The nodejs equivalent to python binding's chat\_completion
+
+##### Parameters
+
+*   `model` **InferenceModel** The language model object.
+*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
+*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
+
+Returns **[CompletionReturn](#completionreturn)** The completion result.
+
+#### createEmbedding
+
+The nodejs moral equivalent to python binding's Embed4All().embed()
+meow
+
+##### Parameters
+
+*   `model` **EmbeddingModel** The language model object.
+*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** text to embed
+
+Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The completion result.
+
+#### CompletionOptions
+
+**Extends Partial\<LLModelPromptContext>**
+
+The options for creating the completion.
+
+##### verbose
+
+Indicates if verbose logging is enabled.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### systemPromptTemplate
+
+Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
+Note that if this is not defined, system messages will not be included in the prompt.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### promptTemplate
+
+Template for user messages, with %1 being replaced by the message.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### promptHeader
+
+The initial instruction for the model, on top of the prompt
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### promptFooter
+
+The last instruction for the model, appended to the end of the prompt.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### PromptMessage
+
+A message in the conversation, identical to OpenAI's chat message.
+
+##### role
+
+The role of the message.
+
+Type: (`"system"` | `"assistant"` | `"user"`)
+
+##### content
+
+The message content.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### prompt\_tokens
+
+The number of tokens used in the prompt.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### completion\_tokens
+
+The number of tokens used in the completion.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### total\_tokens
+
+The total number of tokens used.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### CompletionReturn
+
+The result of the completion, similar to OpenAI's format.
+
+##### model
+
+The model used for the completion.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### usage
+
+Token usage report.
+
+Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
+
+##### choices
+
+The generated completions.
+
+Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[CompletionChoice](#completionchoice)>
+
+#### CompletionChoice
+
+A completion choice, similar to OpenAI's format.
+
+##### message
+
+Response message
+
+Type: [PromptMessage](#promptmessage)
+
+#### LLModelPromptContext
+
+Model inference arguments for generating completions.
+
+##### logitsSize
+
+The size of the raw logits vector.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### tokensSize
+
+The size of the raw tokens vector.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nPast
+
+The number of tokens in the past conversation.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nCtx
+
+The number of tokens possible in the context window.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nPredict
+
+The number of tokens to predict.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### topK
+
+The top-k logits to sample from.
+Top-K sampling selects the next token only from the top K most likely tokens predicted by the model.
+It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit
+the diversity of the output. A higher value for top-K (eg., 100) will consider more tokens and lead
+to more diverse text, while a lower value (eg., 10) will focus on the most probable tokens and generate
+more conservative text. 30 - 60 is a good range for most tasks.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### topP
+
+The nucleus sampling probability threshold.
+Top-P limits the selection of the next token to a subset of tokens with a cumulative probability
+above a threshold P. This method, also known as nucleus sampling, finds a balance between diversity
+and quality by considering both token probabilities and the number of tokens available for sampling.
+When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
+On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
+The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
+for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### temp
+
+The temperature to adjust the model's output distribution.
+Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
+(eg., 1.2) increase randomness, resulting in more imaginative and diverse text. Lower temperatures (eg., 0.5)
+make the output more focused, predictable, and conservative. When the temperature is set to 0, the output
+becomes completely deterministic, always selecting the most probable next token and producing identical results
+each time. A safe range would be around 0.6 - 0.85, but you are free to search what value fits best for you.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nBatch
+
+The number of predictions to generate in parallel.
+By splitting the prompt every N tokens, prompt-batch-size reduces RAM usage during processing. However,
+this can increase the processing time as a trade-off. If the N value is set too low (e.g., 10), long prompts
+with 500+ tokens will be most affected, requiring numerous processing runs to complete the prompt processing.
+To ensure optimal performance, setting the prompt-batch-size to 2048 allows processing of all tokens in a single run.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### repeatPenalty
+
+The penalty factor for repeated tokens.
+Repeat-penalty can help penalize tokens based on how frequently they occur in the text, including the input prompt.
+A token that has already appeared five times is penalized more heavily than a token that has appeared only one time.
+A value of 1 means that there is no penalty and values larger than 1 discourage repeated tokens.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### repeatLastN
+
+The number of last tokens to penalize.
+The repeat-penalty-tokens N option controls the number of tokens in the history to consider for penalizing repetition.
+A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only
+consider recent tokens.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### contextErase
+
+The percentage of context to erase if the context window is exceeded.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### createTokenStream
+
+TODO: Help wanted to implement this
+
+##### Parameters
+
+*   `llmodel` **[LLModel](#llmodel)**&#x20;
+*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>**&#x20;
+*   `options` **[CompletionOptions](#completionoptions)**&#x20;
+
+Returns **function (ll: [LLModel](#llmodel)): AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>**&#x20;
+
+#### DEFAULT\_DIRECTORY
+
+From python api:
+models will be stored in (homedir)/.cache/gpt4all/\`
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### DEFAULT\_LIBRARIES\_DIRECTORY
+
+From python api:
+The default path for dynamic libraries to be stored.
+You may separate paths by a semicolon to search in multiple areas.
+This searches DEFAULT\_DIRECTORY/libraries, cwd/libraries, and finally cwd.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### DEFAULT\_MODEL\_CONFIG
+
+Default model configuration.
+
+Type: ModelConfig
+
+#### DEFAULT\_PROMT\_CONTEXT
+
+Default prompt context.
+
+Type: [LLModelPromptContext](#llmodelpromptcontext)
+
+#### DEFAULT\_MODEL\_LIST\_URL
+
+Default model list url.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### downloadModel
+
+Initiates the download of a model file.
+By default this downloads without waiting. use the controller returned to alter this behavior.
+
+##### Parameters
+
+*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The model to be downloaded.
+*   `options` **DownloadOptions** to pass into the downloader. Default is { location: (cwd), verbose: false }.
+
+##### Examples
+
+```javascript
+const download = downloadModel('ggml-gpt4all-j-v1.3-groovy.bin')
+download.promise.then(() => console.log('Downloaded!'))
+```
+
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model already exists in the specified location.
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model cannot be found at the specified url.
+
+Returns **[DownloadController](#downloadcontroller)** object that allows controlling the download process.
+
+#### DownloadModelOptions
+
+Options for the model download process.
+
+##### modelPath
+
+location to download the model.
+Default is process.cwd(), or the current working directory
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### verbose
+
+Debug mode -- check how long it took to download in seconds
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### url
+
+Remote download url. Defaults to `https://gpt4all.io/models/<modelName>`
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### md5sum
+
+MD5 sum of the model file. If this is provided, the downloaded file will be checked against this sum.
+If the sums do not match, an error will be thrown and the file will be deleted.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### DownloadController
+
+Model download controller.
+
+##### cancel
+
+Cancel the request to download if this is called.
+
+Type: function (): void
+
+##### promise
+
+A promise resolving to the downloaded models config once the download is done
+
+Type: [Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<ModelConfig>

gpt4all-bindings/typescript/binding.ci.gyp

@@ -0,0 +1,62 @@
+{
+  "targets": [
+    {
+      "target_name": "gpt4all", # gpt4all-ts will cause compile error
+      "include_dirs": [
+        "<!@(node -p \"require('node-addon-api').include\")",
+        "gpt4all-backend",
+      ],
+      "sources": [ 
+        # PREVIOUS VERSION: had to required the sources, but with newest changes do not need to
+        #"../../gpt4all-backend/llama.cpp/examples/common.cpp",
+        #"../../gpt4all-backend/llama.cpp/ggml.c",
+        #"../../gpt4all-backend/llama.cpp/llama.cpp",
+        # "../../gpt4all-backend/utils.cpp", 
+        "gpt4all-backend/llmodel_c.cpp",
+        "gpt4all-backend/llmodel.cpp",
+        "prompt.cc",
+        "index.cc",
+       ],
+      "conditions": [
+        ['OS=="mac"', {
+            'xcode_settings': {
+                'GCC_ENABLE_CPP_EXCEPTIONS': 'YES'
+            },
+            'defines': [
+                'LIB_FILE_EXT=".dylib"',
+                'NAPI_CPP_EXCEPTIONS',
+            ],
+            'cflags_cc': [
+                "-fexceptions"
+            ]
+        }],
+        ['OS=="win"', {
+            'defines': [
+                'LIB_FILE_EXT=".dll"',
+                'NAPI_CPP_EXCEPTIONS',
+            ],
+            "msvs_settings": {
+                "VCCLCompilerTool": {
+                    "AdditionalOptions": [
+                        "/std:c++20",
+                        "/EHsc",
+                  ], 
+                },
+            },
+        }],
+        ['OS=="linux"', {
+            'defines': [
+                'LIB_FILE_EXT=".so"',
+                'NAPI_CPP_EXCEPTIONS',
+            ],
+            'cflags_cc!': [
+                '-fno-rtti',
+            ],
+            'cflags_cc': [
+                '-std=c++2a',
+                '-fexceptions'
+            ]
+        }]
+      ]
+    }]
+}

gpt4all-bindings/typescript/binding.gyp

@@ -2,7 +2,6 @@
   "targets": [
     {
       "target_name": "gpt4all", # gpt4all-ts will cause compile error
-      "cflags_cc!": [ "-fno-exceptions"],
       "include_dirs": [
         "<!@(node -p \"require('node-addon-api').include\")",
         "../../gpt4all-backend",
@@ -20,9 +19,15 @@
        ],
       "conditions": [
         ['OS=="mac"', {
+            'xcode_settings': {
+                'GCC_ENABLE_CPP_EXCEPTIONS': 'YES'
+            },
             'defines': [
                 'LIB_FILE_EXT=".dylib"',
                 'NAPI_CPP_EXCEPTIONS',
+            ],
+            'cflags_cc': [
+                "-fexceptions"
             ]
         }],
         ['OS=="win"', {
@@ -48,7 +53,8 @@
                 '-fno-rtti',
             ],
             'cflags_cc': [
-                '-std=c++20'
+                '-std=c++2a',
+                '-fexceptions'
             ]
         }]
       ]

gpt4all-bindings/typescript/docs/api.md

@@ -1,623 +0,0 @@
-<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
-
-### Table of Contents
-
-*   [download][1]
-    *   [Parameters][2]
-    *   [Examples][3]
-*   [DownloadOptions][4]
-    *   [location][5]
-    *   [debug][6]
-    *   [url][7]
-*   [DownloadController][8]
-    *   [cancel][9]
-    *   [promise][10]
-*   [ModelType][11]
-*   [ModelFile][12]
-    *   [gptj][13]
-    *   [llama][14]
-    *   [mpt][15]
-*   [type][16]
-*   [LLModel][17]
-    *   [constructor][18]
-        *   [Parameters][19]
-    *   [type][20]
-    *   [name][21]
-    *   [stateSize][22]
-    *   [threadCount][23]
-    *   [setThreadCount][24]
-        *   [Parameters][25]
-    *   [raw\_prompt][26]
-        *   [Parameters][27]
-    *   [isModelLoaded][28]
-    *   [setLibraryPath][29]
-        *   [Parameters][30]
-    *   [getLibraryPath][31]
-*   [createCompletion][32]
-    *   [Parameters][33]
-    *   [Examples][34]
-*   [CompletionOptions][35]
-    *   [verbose][36]
-    *   [hasDefaultHeader][37]
-    *   [hasDefaultFooter][38]
-*   [PromptMessage][39]
-    *   [role][40]
-    *   [content][41]
-*   [prompt\_tokens][42]
-*   [completion\_tokens][43]
-*   [total\_tokens][44]
-*   [CompletionReturn][45]
-    *   [model][46]
-    *   [usage][47]
-    *   [choices][48]
-*   [CompletionChoice][49]
-    *   [message][50]
-*   [LLModelPromptContext][51]
-    *   [logits\_size][52]
-    *   [tokens\_size][53]
-    *   [n\_past][54]
-    *   [n\_ctx][55]
-    *   [n\_predict][56]
-    *   [top\_k][57]
-    *   [top\_p][58]
-    *   [temp][59]
-    *   [n\_batch][60]
-    *   [repeat\_penalty][61]
-    *   [repeat\_last\_n][62]
-    *   [context\_erase][63]
-*   [createTokenStream][64]
-    *   [Parameters][65]
-*   [DEFAULT\_DIRECTORY][66]
-*   [DEFAULT\_LIBRARIES\_DIRECTORY][67]
-
-## download
-
-Initiates the download of a model file of a specific model type.
-By default this downloads without waiting. use the controller returned to alter this behavior.
-
-### Parameters
-
-*   `model` **[ModelFile][12]** The model file to be downloaded.
-*   `options` **[DownloadOptions][4]** to pass into the downloader. Default is { location: (cwd), debug: false }.
-
-### Examples
-
-```javascript
-const controller = download('ggml-gpt4all-j-v1.3-groovy.bin')
-controller.promise().then(() => console.log('Downloaded!'))
-```
-
-*   Throws **[Error][68]** If the model already exists in the specified location.
-*   Throws **[Error][68]** If the model cannot be found at the specified url.
-
-Returns **[DownloadController][8]** object that allows controlling the download process.
-
-## DownloadOptions
-
-Options for the model download process.
-
-### location
-
-location to download the model.
-Default is process.cwd(), or the current working directory
-
-Type: [string][69]
-
-### debug
-
-Debug mode -- check how long it took to download in seconds
-
-Type: [boolean][70]
-
-### url
-
-Remote download url. Defaults to `https://gpt4all.io/models`
-
-Type: [string][69]
-
-## DownloadController
-
-Model download controller.
-
-### cancel
-
-Cancel the request to download from gpt4all website if this is called.
-
-Type: function (): void
-
-### promise
-
-Convert the downloader into a promise, allowing people to await and manage its lifetime
-
-Type: function (): [Promise][71]\<void>
-
-## ModelType
-
-Type of the model
-
-Type: (`"gptj"` | `"llama"` | `"mpt"`)
-
-## ModelFile
-
-Full list of models available
-
-### gptj
-
-List of GPT-J Models
-
-Type: (`"ggml-gpt4all-j-v1.3-groovy.bin"` | `"ggml-gpt4all-j-v1.2-jazzy.bin"` | `"ggml-gpt4all-j-v1.1-breezy.bin"` | `"ggml-gpt4all-j.bin"`)
-
-### llama
-
-List Llama Models
-
-Type: (`"ggml-gpt4all-l13b-snoozy.bin"` | `"ggml-vicuna-7b-1.1-q4_2.bin"` | `"ggml-vicuna-13b-1.1-q4_2.bin"` | `"ggml-wizardLM-7B.q4_2.bin"` | `"ggml-stable-vicuna-13B.q4_2.bin"` | `"ggml-nous-gpt4-vicuna-13b.bin"`)
-
-### mpt
-
-List of MPT Models
-
-Type: (`"ggml-mpt-7b-base.bin"` | `"ggml-mpt-7b-chat.bin"` | `"ggml-mpt-7b-instruct.bin"`)
-
-## type
-
-Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
-
-Type: [ModelType][11]
-
-## LLModel
-
-LLModel class representing a language model.
-This is a base class that provides common functionality for different types of language models.
-
-### constructor
-
-Initialize a new LLModel.
-
-#### Parameters
-
-*   `path` **[string][69]** Absolute path to the model file.
-
-<!---->
-
-*   Throws **[Error][68]** If the model file does not exist.
-
-### type
-
-either 'gpt', mpt', or 'llama' or undefined
-
-Returns **([ModelType][11] | [undefined][72])**&#x20;
-
-### name
-
-The name of the model.
-
-Returns **[ModelFile][12]**&#x20;
-
-### stateSize
-
-Get the size of the internal state of the model.
-NOTE: This state data is specific to the type of model you have created.
-
-Returns **[number][73]** the size in bytes of the internal state of the model
-
-### threadCount
-
-Get the number of threads used for model inference.
-The default is the number of physical cores your computer has.
-
-Returns **[number][73]** The number of threads used for model inference.
-
-### setThreadCount
-
-Set the number of threads used for model inference.
-
-#### Parameters
-
-*   `newNumber` **[number][73]** The new number of threads.
-
-Returns **void**&#x20;
-
-### raw\_prompt
-
-Prompt the model with a given input and optional parameters.
-This is the raw output from std out.
-Use the prompt function exported for a value
-
-#### Parameters
-
-*   `q` **[string][69]** The prompt input.
-*   `params` **Partial<[LLModelPromptContext][51]>?** Optional parameters for the prompt context.
-
-Returns **any** The result of the model prompt.
-
-### isModelLoaded
-
-Whether the model is loaded or not.
-
-Returns **[boolean][70]**&#x20;
-
-### setLibraryPath
-
-Where to search for the pluggable backend libraries
-
-#### Parameters
-
-*   `s` **[string][69]**&#x20;
-
-Returns **void**&#x20;
-
-### getLibraryPath
-
-Where to get the pluggable backend libraries
-
-Returns **[string][69]**&#x20;
-
-## createCompletion
-
-The nodejs equivalent to python binding's chat\_completion
-
-### Parameters
-
-*   `llmodel` **[LLModel][17]** The language model object.
-*   `messages` **[Array][74]<[PromptMessage][39]>** The array of messages for the conversation.
-*   `options` **[CompletionOptions][35]** The options for creating the completion.
-
-### Examples
-
-```javascript
-const llmodel = new LLModel(model)
-const messages = [ 
-{ role: 'system', message: 'You are a weather forecaster.' },
-{ role: 'user', message: 'should i go out today?' } ]
-const completion = await createCompletion(llmodel, messages, {
- verbose: true,
- temp: 0.9,
-})
-console.log(completion.choices[0].message.content)
-// No, it's going to be cold and rainy.
-```
-
-Returns **[CompletionReturn][45]** The completion result.
-
-## CompletionOptions
-
-**Extends Partial\<LLModelPromptContext>**
-
-The options for creating the completion.
-
-### verbose
-
-Indicates if verbose logging is enabled.
-
-Type: [boolean][70]
-
-### hasDefaultHeader
-
-Indicates if the default header is included in the prompt.
-
-Type: [boolean][70]
-
-### hasDefaultFooter
-
-Indicates if the default footer is included in the prompt.
-
-Type: [boolean][70]
-
-## PromptMessage
-
-A message in the conversation, identical to OpenAI's chat message.
-
-### role
-
-The role of the message.
-
-Type: (`"system"` | `"assistant"` | `"user"`)
-
-### content
-
-The message content.
-
-Type: [string][69]
-
-## prompt\_tokens
-
-The number of tokens used in the prompt.
-
-Type: [number][73]
-
-## completion\_tokens
-
-The number of tokens used in the completion.
-
-Type: [number][73]
-
-## total\_tokens
-
-The total number of tokens used.
-
-Type: [number][73]
-
-## CompletionReturn
-
-The result of the completion, similar to OpenAI's format.
-
-### model
-
-The model name.
-
-Type: [ModelFile][12]
-
-### usage
-
-Token usage report.
-
-Type: {prompt\_tokens: [number][73], completion\_tokens: [number][73], total\_tokens: [number][73]}
-
-### choices
-
-The generated completions.
-
-Type: [Array][74]<[CompletionChoice][49]>
-
-## CompletionChoice
-
-A completion choice, similar to OpenAI's format.
-
-### message
-
-Response message
-
-Type: [PromptMessage][39]
-
-## LLModelPromptContext
-
-Model inference arguments for generating completions.
-
-### logits\_size
-
-The size of the raw logits vector.
-
-Type: [number][73]
-
-### tokens\_size
-
-The size of the raw tokens vector.
-
-Type: [number][73]
-
-### n\_past
-
-The number of tokens in the past conversation.
-
-Type: [number][73]
-
-### n\_ctx
-
-The number of tokens possible in the context window.
-
-Type: [number][73]
-
-### n\_predict
-
-The number of tokens to predict.
-
-Type: [number][73]
-
-### top\_k
-
-The top-k logits to sample from.
-
-Type: [number][73]
-
-### top\_p
-
-The nucleus sampling probability threshold.
-
-Type: [number][73]
-
-### temp
-
-The temperature to adjust the model's output distribution.
-
-Type: [number][73]
-
-### n\_batch
-
-The number of predictions to generate in parallel.
-
-Type: [number][73]
-
-### repeat\_penalty
-
-The penalty factor for repeated tokens.
-
-Type: [number][73]
-
-### repeat\_last\_n
-
-The number of last tokens to penalize.
-
-Type: [number][73]
-
-### context\_erase
-
-The percentage of context to erase if the context window is exceeded.
-
-Type: [number][73]
-
-## createTokenStream
-
-TODO: Help wanted to implement this
-
-### Parameters
-
-*   `llmodel` **[LLModel][17]**&#x20;
-*   `messages` **[Array][74]<[PromptMessage][39]>**&#x20;
-*   `options` **[CompletionOptions][35]**&#x20;
-
-Returns **function (ll: [LLModel][17]): AsyncGenerator<[string][69]>**&#x20;
-
-## DEFAULT\_DIRECTORY
-
-From python api:
-models will be stored in (homedir)/.cache/gpt4all/\`
-
-Type: [string][69]
-
-## DEFAULT\_LIBRARIES\_DIRECTORY
-
-From python api:
-The default path for dynamic libraries to be stored.
-You may separate paths by a semicolon to search in multiple areas.
-This searches DEFAULT\_DIRECTORY/libraries, cwd/libraries, and finally cwd.
-
-Type: [string][69]
-
-[1]: #download
-
-[2]: #parameters
-
-[3]: #examples
-
-[4]: #downloadoptions
-
-[5]: #location
-
-[6]: #debug
-
-[7]: #url
-
-[8]: #downloadcontroller
-
-[9]: #cancel
-
-[10]: #promise
-
-[11]: #modeltype
-
-[12]: #modelfile
-
-[13]: #gptj
-
-[14]: #llama
-
-[15]: #mpt
-
-[16]: #type
-
-[17]: #llmodel
-
-[18]: #constructor
-
-[19]: #parameters-1
-
-[20]: #type-1
-
-[21]: #name
-
-[22]: #statesize
-
-[23]: #threadcount
-
-[24]: #setthreadcount
-
-[25]: #parameters-2
-
-[26]: #raw_prompt
-
-[27]: #parameters-3
-
-[28]: #ismodelloaded
-
-[29]: #setlibrarypath
-
-[30]: #parameters-4
-
-[31]: #getlibrarypath
-
-[32]: #createcompletion
-
-[33]: #parameters-5
-
-[34]: #examples-1
-
-[35]: #completionoptions
-
-[36]: #verbose
-
-[37]: #hasdefaultheader
-
-[38]: #hasdefaultfooter
-
-[39]: #promptmessage
-
-[40]: #role
-
-[41]: #content
-
-[42]: #prompt_tokens
-
-[43]: #completion_tokens
-
-[44]: #total_tokens
-
-[45]: #completionreturn
-
-[46]: #model
-
-[47]: #usage
-
-[48]: #choices
-
-[49]: #completionchoice
-
-[50]: #message
-
-[51]: #llmodelpromptcontext
-
-[52]: #logits_size
-
-[53]: #tokens_size
-
-[54]: #n_past
-
-[55]: #n_ctx
-
-[56]: #n_predict
-
-[57]: #top_k
-
-[58]: #top_p
-
-[59]: #temp
-
-[60]: #n_batch
-
-[61]: #repeat_penalty
-
-[62]: #repeat_last_n
-
-[63]: #context_erase
-
-[64]: #createtokenstream
-
-[65]: #parameters-6
-
-[66]: #default_directory
-
-[67]: #default_libraries_directory
-
-[68]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error
-
-[69]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String
-
-[70]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean
-
-[71]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise
-
-[72]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined
-
-[73]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number
-
-[74]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array

gpt4all-bindings/typescript/index.cc

@@ -10,6 +10,7 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
        InstanceMethod("stateSize", &NodeModelWrapper::StateSize),
        InstanceMethod("raw_prompt", &NodeModelWrapper::Prompt),
        InstanceMethod("setThreadCount", &NodeModelWrapper::SetThreadCount),
+       InstanceMethod("embed", &NodeModelWrapper::GenerateEmbedding),
        InstanceMethod("threadCount", &NodeModelWrapper::ThreadCount),
        InstanceMethod("getLibraryPath", &NodeModelWrapper::GetLibraryPath),
     });
@@ -57,15 +58,19 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
         }
     }
     llmodel_set_implementation_search_path(library_path.c_str());
-    llmodel_error* e = nullptr;
-    inference_ = std::make_shared<llmodel_model>(llmodel_model_create2(full_weight_path.c_str(), "auto", e));
-    if(e != nullptr) {
-       Napi::Error::New(env, e->message).ThrowAsJavaScriptException(); 
+    llmodel_error e = {
+        .message="looks good to me",
+        .code=0,
+    };
+    inference_ = std::make_shared<llmodel_model>(llmodel_model_create2(full_weight_path.c_str(), "auto", &e));
+    if(e.code != 0) {
+       Napi::Error::New(env, e.message).ThrowAsJavaScriptException(); 
        return;
     }
     if(GetInference() == nullptr) {
        std::cerr << "Tried searching libraries in \"" << library_path << "\"" <<  std::endl;
        std::cerr << "Tried searching for model weight in \"" << full_weight_path << "\"" << std::endl;
+       std::cerr << "Do you have runtime libraries installed?" << std::endl;
        Napi::Error::New(env, "Had an issue creating llmodel object, inference is null").ThrowAsJavaScriptException(); 
        return;
     }
@@ -90,6 +95,33 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
     return Napi::Number::New(info.Env(), static_cast<int64_t>(llmodel_get_state_size(GetInference())));
   }
   
+  Napi::Value NodeModelWrapper::GenerateEmbedding(const Napi::CallbackInfo& info) {
+    auto env = info.Env();
+    std::string text = info[0].As<Napi::String>().Utf8Value();
+    size_t embedding_size = 0;
+    float* arr = llmodel_embedding(GetInference(), text.c_str(), &embedding_size);
+    if(arr == nullptr) {
+        Napi::Error::New(
+            env, 
+            "Cannot embed. native embedder returned 'nullptr'"
+        ).ThrowAsJavaScriptException(); 
+        return env.Undefined();
+    }
+
+    if(embedding_size == 0 && text.size() != 0 ) {
+        std::cout << "Warning: embedding length 0 but input text length > 0" << std::endl;
+    }
+    Napi::Float32Array js_array = Napi::Float32Array::New(env, embedding_size);
+    
+    for (size_t i = 0; i < embedding_size; ++i) {
+        float element = *(arr + i);
+        js_array[i] = element;
+    }
+
+    llmodel_free_embedding(arr);
+
+    return js_array;
+  }
 
 /**
  * Generate a response using the model.
@@ -156,12 +188,12 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
             promptContext.context_erase = inputObject.Get("context_erase").As<Napi::Number>().FloatValue();
     }
     //copy to protect llmodel resources when splitting to new thread
-
     llmodel_prompt_context copiedPrompt = promptContext;
+
     std::string copiedQuestion = question;
     PromptWorkContext pc = {
         copiedQuestion,
-        inference_.load(),
+        std::ref(inference_),
         copiedPrompt,
     };
     auto threadSafeContext = new TsfnContext(env, pc);
@@ -201,7 +233,7 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
   }
 
   llmodel_model NodeModelWrapper::GetInference() {
-    return *inference_.load();
+    return *inference_;
   }
 
 //Exports Bindings

gpt4all-bindings/typescript/index.h

@@ -23,6 +23,7 @@ public:
   void SetThreadCount(const Napi::CallbackInfo& info);
   Napi::Value getName(const Napi::CallbackInfo& info);
   Napi::Value ThreadCount(const Napi::CallbackInfo& info);
+  Napi::Value GenerateEmbedding(const Napi::CallbackInfo& info);
   /*
    * The path that is used to search for the dynamic libraries
    */
@@ -36,7 +37,7 @@ private:
   /**
    * The underlying inference that interfaces with the C interface
    */
-  std::atomic<std::shared_ptr<llmodel_model>> inference_;
+  std::shared_ptr<llmodel_model> inference_;
 
   std::string type;
   // corresponds to LLModel::name() in typescript

gpt4all-bindings/typescript/package.json

@@ -1,17 +1,29 @@
 {
   "name": "gpt4all",
-  "version": "2.0.0",
-  "packageManager": "yarn@3.5.1",
+  "version": "2.2.0",
+  "packageManager": "yarn@3.6.1",
   "main": "src/gpt4all.js",
   "repository": "nomic-ai/gpt4all",
   "scripts": {
-    "test": "node ./test/index.mjs",
-    "build:backend": "node scripts/build.js",
     "install": "node-gyp-build",
-    "prebuild": "node scripts/prebuild.js",
-    "docs:build": "documentation build ./src/gpt4all.d.ts --parse-extension d.ts --format md --output docs/api.md"
+    "test": "jest",
+    "build:backend": "node scripts/build.js",
+    "build": "node-gyp-build",
+    "predocs:build": "node scripts/docs.js",
+    "docs:build": "documentation readme ./src/gpt4all.d.ts --parse-extension js d.ts --format md --section \"API Reference\" --readme-file ../python/docs/gpt4all_typescript.md",
+    "postdocs:build": "documentation readme ./src/gpt4all.d.ts --parse-extension js d.ts --format md --section \"API Reference\" --readme-file README.md"
   },
+  "files": [
+    "src/**/*",
+    "runtimes/**/*",
+    "binding.gyp",
+    "prebuilds/**/*",
+    "*.h",
+    "*.cc",
+    "gpt4all-backend/**/*"
+  ],
   "dependencies": {
+    "md5-file": "^5.0.0",
     "mkdirp": "^3.0.1",
     "node-addon-api": "^6.1.0",
     "node-gyp-build": "^4.6.0"
@@ -19,14 +31,21 @@
   "devDependencies": {
     "@types/node": "^20.1.5",
     "documentation": "^14.0.2",
+    "jest": "^29.5.0",
     "prebuildify": "^5.0.1",
     "prettier": "^2.8.8"
   },
+  "optionalDependencies": {
+    "node-gyp": "9.x.x"
+  },
   "engines": {
     "node": ">= 18.x.x"
   },
   "prettier": {
     "endOfLine": "lf",
     "tabWidth": 4
+  },
+  "jest": {
+    "verbose": true
   }
 }

gpt4all-bindings/typescript/prompt.cc

@@ -4,11 +4,12 @@
 TsfnContext::TsfnContext(Napi::Env env, const PromptWorkContext& pc) 
     : deferred_(Napi::Promise::Deferred::New(env)), pc(pc) {
 }
+namespace {
+    static std::string *res;
+}
 
-std::mutex mtx;
-static thread_local std::string res;
 bool response_callback(int32_t token_id, const char *response) {
-   res+=response;
+   *res += response;
    return token_id != -1;
 }
 bool recalculate_callback (bool isrecalculating) {
@@ -21,10 +22,12 @@ bool prompt_callback (int32_t tid) {
 // The thread entry point. This takes as its arguments the specific
 // threadsafe-function context created inside the main thread.
 void threadEntry(TsfnContext* context) {
+  static std::mutex mtx;
   std::lock_guard<std::mutex> lock(mtx);
+  res = &context->pc.res;
   // Perform a call into JavaScript.
   napi_status status =
-    context->tsfn.NonBlockingCall(&context->pc,
+    context->tsfn.BlockingCall(&context->pc,
     [](Napi::Env env, Napi::Function jsCallback, PromptWorkContext* pc) {
         llmodel_prompt(
             *pc->inference_,
@@ -34,8 +37,6 @@ void threadEntry(TsfnContext* context) {
             &recalculate_callback,
             &pc->prompt_params
         );
-        jsCallback.Call({ Napi::String::New(env, res)} );
-        res.clear();
   });
 
   if (status != napi_ok) {
@@ -43,7 +44,6 @@ void threadEntry(TsfnContext* context) {
         "ThreadEntry",
         "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
   }
-
   // Release the thread-safe function. This decrements the internal thread
   // count, and will perform finalization since the count will reach 0.
   context->tsfn.Release();
@@ -52,11 +52,9 @@ void threadEntry(TsfnContext* context) {
 void FinalizerCallback(Napi::Env env,
                        void* finalizeData,
                        TsfnContext* context) {
-  // Join the thread
-  context->nativeThread.join();
-  // Resolve the Promise previously returned to JS via the CreateTSFN method.
-  context->deferred_.Resolve(Napi::Boolean::New(env, true));
-  delete context;
+  // Resolve the Promise previously returned to JS 
+    context->deferred_.Resolve(Napi::String::New(env, context->pc.res));
+    // Wait for the thread to finish executing before proceeding.
+    context->nativeThread.join();
+    delete context;
 }
-
-

gpt4all-bindings/typescript/prompt.h

@@ -10,8 +10,10 @@
 #include <memory>
 struct PromptWorkContext {
     std::string question;
-    std::shared_ptr<llmodel_model> inference_;
+    std::shared_ptr<llmodel_model>& inference_;
     llmodel_prompt_context prompt_params;
+    std::string res;
+
 };
 
 struct TsfnContext {
@@ -29,12 +31,12 @@ public:
 
 // The thread entry point. This takes as its arguments the specific
 // threadsafe-function context created inside the main thread.
-void threadEntry(TsfnContext* context);
+void threadEntry(TsfnContext*);
 
 // The thread-safe function finalizer callback. This callback executes
 // at destruction of thread-safe function, taking as arguments the finalizer
 // data and threadsafe-function context.
-void FinalizerCallback(Napi::Env env, void* finalizeData, TsfnContext* context);
+void FinalizerCallback(Napi::Env, void* finalizeData, TsfnContext*);
 
 bool response_callback(int32_t token_id, const char *response);
 bool recalculate_callback (bool isrecalculating);

gpt4all-bindings/typescript/scripts/build.js

@@ -2,7 +2,6 @@ const { spawn } = require("node:child_process");
 const { resolve } = require("path");
 const args = process.argv.slice(2);
 const platform = process.platform;
-
 //windows 64bit or 32
 if (platform === "win32") {
     const path = "scripts/build_msvc.bat";
@@ -10,8 +9,9 @@ if (platform === "win32") {
     process.on("data", (s) => console.log(s.toString()));
 } else if (platform === "linux" || platform === "darwin") {
     const path = "scripts/build_unix.sh";
-    const bash = spawn(`sh`, [path, ...args]);
-    bash.stdout.on("data", (s) => console.log(s.toString()), {
+    spawn(`sh `, [path, args], {
+        shell: true,
         stdio: "inherit",
     });
+    process.on("data", (s) => console.log(s.toString()));
 }

gpt4all-bindings/typescript/scripts/build_msvc.bat

@@ -0,0 +1,33 @@
+@echo off
+
+set "BUILD_TYPE=Release"
+set "BUILD_DIR=.\build\win-x64-msvc"
+set "LIBS_DIR=.\runtimes\win32-x64"
+
+REM Cleanup env
+rmdir /s /q %BUILD_DIR%
+
+REM Create directories
+mkdir %BUILD_DIR%
+mkdir %LIBS_DIR%
+
+REM Build
+cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=%BUILD_TYPE% -S ..\..\gpt4all-backend -B %BUILD_DIR% -A x64
+
+:BUILD
+REM Build the project
+cmake --build "%BUILD_DIR%" --parallel --config %BUILD_TYPE%
+
+REM Check the exit code of the build command
+if %errorlevel% neq 0 (
+    echo Build failed. Retrying...
+    goto BUILD
+)
+
+mkdir runtimes\win32-x64
+
+REM Copy the DLLs to the desired location
+del /F /A /Q %LIBS_DIR%
+xcopy /Y "%BUILD_DIR%\bin\%BUILD_TYPE%\*.dll" runtimes\win32-x64\native\
+
+echo Batch script execution completed.

gpt4all-bindings/typescript/scripts/build_unix.sh

@@ -24,7 +24,9 @@ mkdir -p "$NATIVE_DIR" "$BUILD_DIR"
 
 cmake -S ../../gpt4all-backend -B "$BUILD_DIR" &&
 cmake --build "$BUILD_DIR" -j --config Release && {
-  cp "$BUILD_DIR"/libllmodel.$LIB_EXT "$NATIVE_DIR"/
+  cp "$BUILD_DIR"/libbert*.$LIB_EXT   "$NATIVE_DIR"/
+  cp "$BUILD_DIR"/libfalcon*.$LIB_EXT "$NATIVE_DIR"/
+  cp "$BUILD_DIR"/libreplit*.$LIB_EXT "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libgptj*.$LIB_EXT   "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libllama*.$LIB_EXT  "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libmpt*.$LIB_EXT    "$NATIVE_DIR"/

gpt4all-bindings/typescript/scripts/docs.js

@@ -0,0 +1,8 @@
+//Maybe some command line piping would work better, but can't think of platform independent command line tool
+
+const fs = require('fs');
+
+const newPath = '../python/docs/gpt4all_typescript.md';
+const filepath = 'README.md';
+const data = fs.readFileSync(filepath);
+fs.writeFileSync(newPath, data);

gpt4all-bindings/typescript/scripts/prebuild.js

@@ -6,6 +6,7 @@ async function createPrebuilds(combinations) {
             platform,
             arch,
             napi: true,
+            targets: ["18.16.0"]
         };
         try {
             await createPrebuild(opts);
@@ -33,17 +34,24 @@ function createPrebuild(opts) {
     });
 }
 
-const prebuildConfigs = [
-    { platform: "win32", arch: "x64" },
-    { platform: "win32", arch: "arm64" },
-    // { platform: 'win32', arch: 'armv7' },
-    { platform: "darwin", arch: "x64" },
-    { platform: "darwin", arch: "arm64" },
-    // { platform: 'darwin', arch: 'armv7' },
+let prebuildConfigs;
+if(process.platform === 'win32') {
+   prebuildConfigs = [
+    { platform: "win32", arch: "x64" }
+   ];
+} else if(process.platform === 'linux') {
+   //Unsure if darwin works, need mac tester!
+   prebuildConfigs = [
     { platform: "linux", arch: "x64" },
-    { platform: "linux", arch: "arm64" },
-    { platform: "linux", arch: "armv7" },
-];
+    //{ platform: "linux", arch: "arm64" },
+    //{ platform: "linux", arch: "armv7" },
+   ]
+} else if(process.platform === 'darwin') {
+    prebuildConfigs = [
+       { platform: "darwin", arch: "x64" },
+       { platform: "darwin", arch: "arm64" },
+    ]
+}
 
 createPrebuilds(prebuildConfigs)
     .then(() => console.log("All builds succeeded"))

gpt4all-bindings/typescript/spec/chat.mjs

@@ -0,0 +1,66 @@
+import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY, loadModel } from '../src/gpt4all.js'
+
+const model = await loadModel(
+    'orca-mini-3b.ggmlv3.q4_0.bin',
+    { verbose: true }
+);
+const ll = model.llm;
+
+try {
+   class Extended extends LLModel {
+   }
+
+} catch(e) {
+    console.log("Extending from native class gone wrong " + e)
+}
+
+console.log("state size " + ll.stateSize())
+
+console.log("thread count " + ll.threadCount());
+ll.setThreadCount(5);
+
+console.log("thread count " + ll.threadCount());
+ll.setThreadCount(4);
+console.log("thread count " + ll.threadCount());
+console.log("name " + ll.name());
+console.log("type: " + ll.type());
+console.log("Default directory for models", DEFAULT_DIRECTORY);
+console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY);
+
+const completion1 = await createCompletion(model, [ 
+    { role : 'system', content: 'You are an advanced mathematician.'  },
+    { role : 'user', content: 'What is 1 + 1?'  }, 
+], { verbose: true })
+console.log(completion1.choices[0].message)
+
+const completion2 = await createCompletion(model, [
+    { role : 'system', content: 'You are an advanced mathematician.'  },
+    { role : 'user', content: 'What is two plus two?'  }, 
+], {  verbose: true })
+
+console.log(completion2.choices[0].message)
+
+// At the moment, from testing this code, concurrent model prompting is not possible. 
+// Behavior: The last prompt gets answered, but the rest are cancelled
+// my experience with threading is not the best, so if anyone who is good is willing to give this a shot,
+// maybe this is possible
+// INFO: threading with llama.cpp is not the best maybe not even possible, so this will be left here as reference
+
+//const responses = await Promise.all([
+//    createCompletion(ll, [ 
+//    { role : 'system', content: 'You are an advanced mathematician.'  },
+//    { role : 'user', content: 'What is 1 + 1?'  }, 
+//    ], { verbose: true }),
+//    createCompletion(ll, [ 
+//    { role : 'system', content: 'You are an advanced mathematician.'  },
+//    { role : 'user', content: 'What is 1 + 1?'  }, 
+//    ], { verbose: true }),
+//
+//createCompletion(ll, [ 
+//    { role : 'system', content: 'You are an advanced mathematician.'  },
+//    { role : 'user', content: 'What is 1 + 1?'  }, 
+//], { verbose: true })
+//
+//])
+//console.log(responses.map(s => s.choices[0].message))
+

gpt4all-bindings/typescript/spec/embed.mjs

@@ -0,0 +1,8 @@
+import {  loadModel, createEmbedding } from '../src/gpt4all.js'
+
+const embedder = await loadModel("ggml-all-MiniLM-L6-v2-f16.bin", { verbose: true })
+
+console.log(
+    createEmbedding(embedder, "Accept your current situation")
+)
+

gpt4all-bindings/typescript/spec/index.mjs

@@ -1,46 +0,0 @@
-import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY } from '../src/gpt4all.js'
-
-const ll = new LLModel({
-    model_name: 'ggml-vicuna-7b-1.1-q4_2.bin',
-    model_path: './', 
-    library_path: DEFAULT_LIBRARIES_DIRECTORY
-});
-
-try {
-   class Extended extends LLModel {
-   }
-
-} catch(e) {
-    console.log("Extending from native class gone wrong " + e)
-}
-
-console.log("state size " + ll.stateSize())
-
-console.log("thread count " + ll.threadCount());
-ll.setThreadCount(5);
-console.log("thread count " + ll.threadCount());
-ll.setThreadCount(4);
-console.log("thread count " + ll.threadCount());
-console.log("name " + ll.name());
-console.log("type: " + ll.type());
-console.log("Default directory for models", DEFAULT_DIRECTORY);
-console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY);
-
-console.log(await createCompletion(
-    ll,
-    [
-        { role : 'system', content: 'You are a girl who likes playing league of legends.'  },
-        { role : 'user', content: 'What is the best top laner to play right now?'  }, 
-    ],
-    { verbose: false}
-));
-
-
-console.log(await createCompletion(
-    ll,
-    [
-        { role : 'user', content: 'What is the best bottom laner to play right now?'  }, 
-    ],
-))
-
-

gpt4all-bindings/typescript/src/config.js

@@ -16,7 +16,26 @@ const librarySearchPaths = [
 
 const DEFAULT_LIBRARIES_DIRECTORY = librarySearchPaths.join(";");
 
+const DEFAULT_MODEL_CONFIG = {
+    systemPrompt: "",
+    promptTemplate: "### Human: \n%1\n### Assistant:\n",
+}
+
+const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models.json";
+
+const DEFAULT_PROMPT_CONTEXT = {
+    temp: 0.7,
+    topK: 40,
+    topP: 0.4,
+    repeatPenalty: 1.18,
+    repeatLastN: 64,
+    nBatch: 8,
+}
+
 module.exports = {
     DEFAULT_DIRECTORY,
     DEFAULT_LIBRARIES_DIRECTORY,
+    DEFAULT_MODEL_CONFIG,
+    DEFAULT_MODEL_LIST_URL,
+    DEFAULT_PROMPT_CONTEXT,
 };

gpt4all-bindings/typescript/src/gpt4all.d.ts

@@ -1,13 +1,13 @@
 /// <reference types="node" />
 declare module "gpt4all";
 
-export * from "./util.d.ts";
-
 /** Type of the model */
 type ModelType = "gptj" | "llama" | "mpt" | "replit";
 
+// NOTE: "deprecated" tag in below comment breaks the doc generator https://github.com/documentationjs/documentation/issues/1596
 /**
  * Full list of models available
+ * @deprecated These model names are outdated and this type will not be maintained, please use a string literal instead
  */
 interface ModelFile {
     /** List of GPT-J Models */
@@ -40,10 +40,37 @@ interface LLModelOptions {
      * Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
      */
     type?: ModelType;
-    model_name: ModelFile[ModelType];
+    model_name: string;
     model_path: string;
     library_path?: string;
 }
+
+interface ModelConfig {
+    systemPrompt: string;
+    promptTemplate: string;
+    path: string;
+    url?: string;
+}
+
+declare class InferenceModel {
+    constructor(llm: LLModel, config: ModelConfig);
+    llm: LLModel;
+    config: ModelConfig;
+
+    generate(
+        prompt: string,
+        options?: Partial<LLModelPromptContext>
+    ): Promise<string>;
+}
+
+declare class EmbeddingModel {
+    constructor(llm: LLModel, config: ModelConfig);
+    llm: LLModel;
+    config: ModelConfig;
+
+    embed(text: string): Float32Array;
+}
+
 /**
  * LLModel class representing a language model.
  * This is a base class that provides common functionality for different types of language models.
@@ -61,7 +88,7 @@ declare class LLModel {
     type(): ModelType | undefined;
 
     /** The name of the model. */
-    name(): ModelFile;
+    name(): string;
 
     /**
      * Get the size of the internal state of the model.
@@ -85,14 +112,27 @@ declare class LLModel {
 
     /**
      * Prompt the model with a given input and optional parameters.
-     * This is the raw output from std out.
+     * This is the raw output from model.
      * Use the prompt function exported for a value
      * @param q The prompt input.
      * @param params Optional parameters for the prompt context.
      * @returns The result of the model prompt.
      */
-    raw_prompt(q: string, params: Partial<LLModelPromptContext>, callback: (res: string) => void): void; // TODO work on return type
+    raw_prompt(
+        q: string,
+        params: Partial<LLModelPromptContext>,
+        callback: (res: string) => void
+    ): void; // TODO work on return type
 
+    /**
+     * Embed text with the model. Keep in mind that
+     * not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
+     * Use the prompt function exported for a value
+     * @param q The prompt input.
+     * @param params Optional parameters for the prompt context.
+     * @returns The result of the model prompt.
+     */
+    embed(text: string): Float32Array;
     /**
      * Whether the model is loaded or not.
      */
@@ -111,39 +151,67 @@ declare class LLModel {
 interface LoadModelOptions {
     modelPath?: string;
     librariesPath?: string;
+    modelConfigFile?: string;
     allowDownload?: boolean;
     verbose?: boolean;
 }
 
+interface InferenceModelOptions extends LoadModelOptions {
+    type?: "inference";
+}
+
+interface EmbeddingModelOptions extends LoadModelOptions {
+    type: "embedding";
+}
+
+/**
+ * Loads a machine learning model with the specified name. The defacto way to create a model.
+ * By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
+ *
+ * @param {string} modelName - The name of the model to load.
+ * @param {LoadModelOptions|undefined} [options] - (Optional) Additional options for loading the model.
+ * @returns {Promise<InferenceModel | EmbeddingModel>} A promise that resolves to an instance of the loaded LLModel.
+ */
 declare function loadModel(
     modelName: string,
-    options?: LoadModelOptions
-): Promise<LLModel>;
+    options?: InferenceModelOptions
+): Promise<InferenceModel>;
+
+declare function loadModel(
+    modelName: string,
+    options?: EmbeddingModelOptions
+): Promise<EmbeddingModel>;
+
+declare function loadModel(
+    modelName: string,
+    options?: EmbeddingOptions | InferenceOptions
+): Promise<InferenceModel | EmbeddingModel>;
 
 /**
  * The nodejs equivalent to python binding's chat_completion
- * @param {LLModel} llmodel - The language model object.
+ * @param {InferenceModel} model - The language model object.
  * @param {PromptMessage[]} messages - The array of messages for the conversation.
  * @param {CompletionOptions} options - The options for creating the completion.
  * @returns {CompletionReturn} The completion result.
- * @example
- * const llmodel = new LLModel(model)
- * const messages = [
- * { role: 'system', message: 'You are a weather forecaster.' },
- * { role: 'user', message: 'should i go out today?' } ]
- * const completion = await createCompletion(llmodel, messages, {
- *  verbose: true,
- *  temp: 0.9,
- * })
- * console.log(completion.choices[0].message.content)
- * // No, it's going to be cold and rainy.
  */
 declare function createCompletion(
-    llmodel: LLModel,
+    model: InferenceModel,
     messages: PromptMessage[],
     options?: CompletionOptions
 ): Promise<CompletionReturn>;
 
+/**
+ * The nodejs moral equivalent to python binding's Embed4All().embed()
+ * meow
+ * @param {EmbeddingModel} model - The language model object.
+ * @param {string} text - text to embed
+ * @returns {Float32Array} The completion result.
+ */
+declare function createEmbedding(
+    model: EmbeddingModel,
+    text: string
+): Float32Array;
+
 /**
  * The options for creating the completion.
  */
@@ -155,16 +223,25 @@ interface CompletionOptions extends Partial<LLModelPromptContext> {
     verbose?: boolean;
 
     /**
-     * Indicates if the default header is included in the prompt.
-     * @default true
+     * Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
+     * Note that if this is not defined, system messages will not be included in the prompt.
      */
-    hasDefaultHeader?: boolean;
+    systemPromptTemplate?: string;
 
     /**
-     * Indicates if the default footer is included in the prompt.
-     * @default true
+     * Template for user messages, with %1 being replaced by the message.
      */
-    hasDefaultFooter?: boolean;
+    promptTemplate?: boolean;
+
+    /**
+     * The initial instruction for the model, on top of the prompt
+     */
+    promptHeader?: string;
+
+    /**
+     * The last instruction for the model, appended to the end of the prompt.
+     */
+    promptFooter?: string;
 }
 
 /**
@@ -182,10 +259,8 @@ interface PromptMessage {
  * The result of the completion, similar to OpenAI's format.
  */
 interface CompletionReturn {
-    /** The model name.
-     * @type {ModelFile}
-     */
-    model: ModelFile[ModelType];
+    /** The model used for the completion. */
+    model: string;
 
     /** Token usage report. */
     usage: {
@@ -216,58 +291,85 @@ interface CompletionChoice {
  */
 interface LLModelPromptContext {
     /** The size of the raw logits vector. */
-    logits_size: number;
+    logitsSize: number;
 
     /** The size of the raw tokens vector. */
-    tokens_size: number;
+    tokensSize: number;
 
     /** The number of tokens in the past conversation. */
-    n_past: number;
+    nPast: number;
 
     /** The number of tokens possible in the context window.
      * @default 1024
      */
-    n_ctx: number;
+    nCtx: number;
 
     /** The number of tokens to predict.
      * @default 128
      * */
-    n_predict: number;
+    nPredict: number;
 
     /** The top-k logits to sample from.
+     * Top-K sampling selects the next token only from the top K most likely tokens predicted by the model.
+     * It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit
+     * the diversity of the output. A higher value for top-K (eg., 100) will consider more tokens and lead
+     * to more diverse text, while a lower value (eg., 10) will focus on the most probable tokens and generate
+     * more conservative text. 30 - 60 is a good range for most tasks.
      * @default 40
      * */
-    top_k: number;
+    topK: number;
 
     /** The nucleus sampling probability threshold.
-     * @default 0.9
+     * Top-P limits the selection of the next token to a subset of tokens with a cumulative probability 
+     * above a threshold P. This method, also known as nucleus sampling, finds a balance between diversity
+     * and quality by considering both token probabilities and the number of tokens available for sampling.
+     * When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
+     * On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
+     * The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
+     * for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
+     * @default 0.4
      * */
-    top_p: number;
+    topP: number;
 
     /** The temperature to adjust the model's output distribution.
-     * @default 0.72
+     * Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
+     * (eg., 1.2) increase randomness, resulting in more imaginative and diverse text. Lower temperatures (eg., 0.5)
+     * make the output more focused, predictable, and conservative. When the temperature is set to 0, the output
+     * becomes completely deterministic, always selecting the most probable next token and producing identical results
+     * each time. A safe range would be around 0.6 - 0.85, but you are free to search what value fits best for you.
+     * @default 0.7
      * */
     temp: number;
 
     /** The number of predictions to generate in parallel.
+     * By splitting the prompt every N tokens, prompt-batch-size reduces RAM usage during processing. However,
+     * this can increase the processing time as a trade-off. If the N value is set too low (e.g., 10), long prompts
+     * with 500+ tokens will be most affected, requiring numerous processing runs to complete the prompt processing.
+     * To ensure optimal performance, setting the prompt-batch-size to 2048 allows processing of all tokens in a single run.
      * @default 8
      * */
-    n_batch: number;
+    nBatch: number;
 
     /** The penalty factor for repeated tokens.
-     * @default 1
+     * Repeat-penalty can help penalize tokens based on how frequently they occur in the text, including the input prompt.
+     * A token that has already appeared five times is penalized more heavily than a token that has appeared only one time.
+     * A value of 1 means that there is no penalty and values larger than 1 discourage repeated tokens.
+     * @default 1.18
      * */
-    repeat_penalty: number;
+    repeatPenalty: number;
 
     /** The number of last tokens to penalize.
-     * @default 10
+     * The repeat-penalty-tokens N option controls the number of tokens in the history to consider for penalizing repetition.
+     * A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only
+     * consider recent tokens.
+     * @default 64
      * */
-    repeat_last_n: number;
+    repeatLastN: number;
 
     /** The percentage of context to erase if the context window is exceeded.
      * @default 0.5
      * */
-    context_erase: number;
+    contextErase: number;
 }
 
 /**
@@ -290,13 +392,104 @@ declare const DEFAULT_DIRECTORY: string;
  * This searches DEFAULT_DIRECTORY/libraries, cwd/libraries, and finally cwd.
  */
 declare const DEFAULT_LIBRARIES_DIRECTORY: string;
-interface PromptMessage {
-    role: "system" | "assistant" | "user";
-    content: string;
+
+/**
+ * Default model configuration.
+ */
+declare const DEFAULT_MODEL_CONFIG: ModelConfig;
+
+/**
+ * Default prompt context.
+ */
+declare const DEFAULT_PROMT_CONTEXT: LLModelPromptContext;
+
+/**
+ * Default model list url.
+ */
+declare const DEFAULT_MODEL_LIST_URL: string;
+
+/**
+ * Initiates the download of a model file.
+ * By default this downloads without waiting. use the controller returned to alter this behavior.
+ * @param {string} modelName - The model to be downloaded.
+ * @param {DownloadOptions} options - to pass into the downloader. Default is { location: (cwd), verbose: false }.
+ * @returns {DownloadController} object that allows controlling the download process.
+ *
+ * @throws {Error} If the model already exists in the specified location.
+ * @throws {Error} If the model cannot be found at the specified url.
+ *
+ * @example
+ * const download = downloadModel('ggml-gpt4all-j-v1.3-groovy.bin')
+ * download.promise.then(() => console.log('Downloaded!'))
+ */
+declare function downloadModel(
+    modelName: string,
+    options?: DownloadModelOptions
+): DownloadController;
+
+/**
+ * Options for the model download process.
+ */
+interface DownloadModelOptions {
+    /**
+     * location to download the model.
+     * Default is process.cwd(), or the current working directory
+     */
+    modelPath?: string;
+
+    /**
+     * Debug mode -- check how long it took to download in seconds
+     * @default false
+     */
+    verbose?: boolean;
+
+    /**
+     * Remote download url. Defaults to `https://gpt4all.io/models/<modelName>`
+     * @default https://gpt4all.io/models/<modelName>
+     */
+    url?: string;
+    /**
+     * MD5 sum of the model file. If this is provided, the downloaded file will be checked against this sum.
+     * If the sums do not match, an error will be thrown and the file will be deleted.
+     */
+    md5sum?: string;
 }
+
+interface ListModelsOptions {
+    url?: string;
+    file?: string;
+}
+
+declare function listModels(options?: ListModelsOptions): Promise<ModelConfig[]>;
+
+interface RetrieveModelOptions {
+    allowDownload?: boolean;
+    verbose?: boolean;
+    modelPath?: string;
+    modelConfigFile?: string;
+}
+
+declare function retrieveModel(
+    modelName: string,
+    options?: RetrieveModelOptions
+): Promise<ModelConfig>;
+
+/**
+ * Model download controller.
+ */
+interface DownloadController {
+    /** Cancel the request to download if this is called. */
+    cancel: () => void;
+    /** A promise resolving to the downloaded models config once the download is done */
+    promise: Promise<ModelConfig>;
+}
+
 export {
     ModelType,
     ModelFile,
+    ModelConfig,
+    InferenceModel,
+    EmbeddingModel,
     LLModel,
     LLModelPromptContext,
     PromptMessage,
@@ -304,7 +497,17 @@ export {
     LoadModelOptions,
     loadModel,
     createCompletion,
+    createEmbedding,
     createTokenStream,
     DEFAULT_DIRECTORY,
     DEFAULT_LIBRARIES_DIRECTORY,
+    DEFAULT_MODEL_CONFIG,
+    DEFAULT_PROMT_CONTEXT,
+    DEFAULT_MODEL_LIST_URL,
+    downloadModel,
+    retrieveModel,
+    listModels,
+    DownloadController,
+    RetrieveModelOptions,
+    DownloadModelOptions,
 };

gpt4all-bindings/typescript/src/gpt4all.js

@@ -10,19 +10,36 @@ const {
     downloadModel,
     appendBinSuffixIfMissing,
 } = require("./util.js");
-const config = require("./config.js");
+const {
+    DEFAULT_DIRECTORY,
+    DEFAULT_LIBRARIES_DIRECTORY,
+    DEFAULT_PROMPT_CONTEXT,
+    DEFAULT_MODEL_CONFIG,
+    DEFAULT_MODEL_LIST_URL,
+} = require("./config.js");
+const { InferenceModel, EmbeddingModel } = require("./models.js");
 
+/**
+ * Loads a machine learning model with the specified name. The defacto way to create a model.
+ * By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
+ *
+ * @param {string} modelName - The name of the model to load.
+ * @param {LoadModelOptions|undefined} [options] - (Optional) Additional options for loading the model.
+ * @returns {Promise<InferenceModel | EmbeddingModel>} A promise that resolves to an instance of the loaded LLModel.
+ */
 async function loadModel(modelName, options = {}) {
     const loadOptions = {
-        modelPath: config.DEFAULT_DIRECTORY,
-        librariesPath: config.DEFAULT_LIBRARIES_DIRECTORY,
+        modelPath: DEFAULT_DIRECTORY,
+        librariesPath: DEFAULT_LIBRARIES_DIRECTORY,
+        type: "inference",
         allowDownload: true,
         verbose: true,
         ...options,
     };
 
-    await retrieveModel(modelName, {
+    const modelConfig = await retrieveModel(modelName, {
         modelPath: loadOptions.modelPath,
+        modelConfigFile: loadOptions.modelConfigFile,
         allowDownload: loadOptions.allowDownload,
         verbose: loadOptions.verbose,
     });
@@ -37,7 +54,9 @@ async function loadModel(modelName, options = {}) {
             break;
         }
     }
-
+    if (!libPath) {
+        throw Error("Could not find a valid path from " + libSearchPaths);
+    }
     const llmOptions = {
         model_name: appendBinSuffixIfMissing(modelName),
         model_path: loadOptions.modelPath,
@@ -45,94 +64,183 @@ async function loadModel(modelName, options = {}) {
     };
 
     if (loadOptions.verbose) {
-        console.log("Creating LLModel with options:", llmOptions);
+        console.debug("Creating LLModel with options:", llmOptions);
     }
     const llmodel = new LLModel(llmOptions);
 
-    return llmodel;
+    if (loadOptions.type === "embedding") {
+        return new EmbeddingModel(llmodel, modelConfig);
+    } else if (loadOptions.type === "inference") {
+        return new InferenceModel(llmodel, modelConfig);
+    } else {
+        throw Error("Invalid model type: " + loadOptions.type);
+    }
 }
 
-function createPrompt(messages, hasDefaultHeader, hasDefaultFooter) {
+/**
+ * Formats a list of messages into a single prompt string.
+ */
+function formatChatPrompt(
+    messages,
+    {
+        systemPromptTemplate,
+        defaultSystemPrompt,
+        promptTemplate,
+        promptFooter,
+        promptHeader,
+    }
+) {
+    const systemMessages = messages
+        .filter((message) => message.role === "system")
+        .map((message) => message.content);
+
     let fullPrompt = "";
 
-    for (const message of messages) {
-        if (message.role === "system") {
-            const systemMessage = message.content + "\n";
-            fullPrompt += systemMessage;
+    if (promptHeader) {
+        fullPrompt += promptHeader + "\n\n";
+    }
+
+    if (systemPromptTemplate) {
+        // if user specified a template for the system prompt, put all system messages in the template
+        let systemPrompt = "";
+
+        if (systemMessages.length > 0) {
+            systemPrompt += systemMessages.join("\n");
         }
+
+        if (systemPrompt) {
+            fullPrompt +=
+                systemPromptTemplate.replace("%1", systemPrompt) + "\n";
+        }
+    } else if (defaultSystemPrompt) {
+        // otherwise, use the system prompt from the model config and ignore system messages
+        fullPrompt += defaultSystemPrompt + "\n\n";
     }
-    if (hasDefaultHeader) {
-        fullPrompt += `### Instruction: 
-        The prompt below is a question to answer, a task to complete, or a conversation 
-        to respond to; decide which and write an appropriate response.
-        \n### Prompt: 
-        `;
+
+    if (systemMessages.length > 0 && !systemPromptTemplate) {
+        console.warn(
+            "System messages were provided, but no systemPromptTemplate was specified. System messages will be ignored."
+        );
     }
+
     for (const message of messages) {
         if (message.role === "user") {
-            const user_message = "\n" + message["content"];
-            fullPrompt += user_message;
+            const userMessage = promptTemplate.replace(
+                "%1",
+                message["content"]
+            );
+            fullPrompt += userMessage;
         }
         if (message["role"] == "assistant") {
-            const assistant_message = "\nResponse: " + message["content"];
-            fullPrompt += assistant_message;
+            const assistantMessage = message["content"] + "\n";
+            fullPrompt += assistantMessage;
         }
     }
-    if (hasDefaultFooter) {
-        fullPrompt += "\n### Response:";
+
+    if (promptFooter) {
+        fullPrompt += "\n\n" + promptFooter;
     }
 
     return fullPrompt;
 }
 
+function createEmbedding(model, text) {
+    return model.embed(text);
+}
+
+const defaultCompletionOptions = {
+    verbose: false,
+    ...DEFAULT_PROMPT_CONTEXT,
+};
+
 async function createCompletion(
-    llmodel,
+    model,
     messages,
-    options = {
-        hasDefaultHeader: true,
-        hasDefaultFooter: false,
-        verbose: true,
-    }
+    options = defaultCompletionOptions
 ) {
-    //creating the keys to insert into promptMaker.
-    const fullPrompt = createPrompt(
-        messages,
-        options.hasDefaultHeader ?? true,
-        options.hasDefaultFooter
-    );
-    if (options.verbose) {
-        console.log("Sent: " + fullPrompt);
+    if (options.hasDefaultHeader !== undefined) {
+        console.warn(
+            "hasDefaultHeader (bool) is deprecated and has no effect, use promptHeader (string) instead"
+        );
     }
-    const promisifiedRawPrompt = new Promise((resolve, rej) => {
-        llmodel.raw_prompt(fullPrompt, options, (s) => {
-            resolve(s);
-        });
+
+    if (options.hasDefaultFooter !== undefined) {
+        console.warn(
+            "hasDefaultFooter (bool) is deprecated and has no effect, use promptFooter (string) instead"
+        );
+    }
+
+    const optionsWithDefaults = {
+        ...defaultCompletionOptions,
+        ...options,
+    };
+
+    const {
+        verbose,
+        systemPromptTemplate,
+        promptTemplate,
+        promptHeader,
+        promptFooter,
+        ...promptContext
+    } = optionsWithDefaults;
+
+    const prompt = formatChatPrompt(messages, {
+        systemPromptTemplate,
+        defaultSystemPrompt: model.config.systemPrompt,
+        promptTemplate: promptTemplate || model.config.promptTemplate || "%1",
+        promptHeader: promptHeader || "",
+        promptFooter: promptFooter || "",
+        // These were the default header/footer prompts used for non-chat single turn completions.
+        // both seem to be working well still with some models, so keeping them here for reference.
+        // promptHeader: '### Instruction: The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.',
+        // promptFooter: '### Response:',
     });
-    return promisifiedRawPrompt.then((response) => {
-        return {
-            llmodel: llmodel.name(),
-            usage: {
-                prompt_tokens: fullPrompt.length,
-                completion_tokens: response.length, //TODO
-                total_tokens: fullPrompt.length + response.length, //TODO
-            },
-            choices: [
-                {
-                    message: {
-                        role: "assistant",
-                        content: response,
-                    },
+
+    if (verbose) {
+        console.debug("Sending Prompt:\n" + prompt);
+    }
+
+    const response = await model.generate(prompt, promptContext);
+
+    if (verbose) {
+        console.debug("Received Response:\n" + response);
+    }
+
+    return {
+        llmodel: model.llm.name(),
+        usage: {
+            prompt_tokens: prompt.length,
+            completion_tokens: response.length, //TODO
+            total_tokens: prompt.length + response.length, //TODO
+        },
+        choices: [
+            {
+                message: {
+                    role: "assistant",
+                    content: response,
                 },
-            ],
-        };
-    });
+            },
+        ],
+    };
+}
+
+function createTokenStream() {
+    throw Error("This API has not been completed yet!");
 }
 
 module.exports = {
-    ...config,
+    DEFAULT_LIBRARIES_DIRECTORY,
+    DEFAULT_DIRECTORY,
+    DEFAULT_PROMPT_CONTEXT,
+    DEFAULT_MODEL_CONFIG,
+    DEFAULT_MODEL_LIST_URL,
     LLModel,
+    InferenceModel,
+    EmbeddingModel,
     createCompletion,
+    createEmbedding,
     downloadModel,
     retrieveModel,
     loadModel,
+    createTokenStream,
 };

gpt4all-bindings/typescript/src/models.js

@@ -0,0 +1,38 @@
+const { normalizePromptContext, warnOnSnakeCaseKeys } = require('./util');
+
+class InferenceModel {
+    llm;
+    config;
+
+    constructor(llmodel, config) {
+        this.llm = llmodel;
+        this.config = config;
+    }
+
+    async generate(prompt, promptContext) {
+        warnOnSnakeCaseKeys(promptContext);
+        const normalizedPromptContext = normalizePromptContext(promptContext);
+        const result = this.llm.raw_prompt(prompt, normalizedPromptContext, () => {});
+        return result;
+    }
+}
+
+class EmbeddingModel {
+    llm;
+    config;
+
+    constructor(llmodel, config) {
+        this.llm = llmodel;
+        this.config = config;
+    }
+
+    embed(text) {
+        return this.llm.embed(text)
+    }
+}
+
+
+module.exports = {
+    InferenceModel,
+    EmbeddingModel,
+};

gpt4all-bindings/typescript/src/util.d.ts

@@ -1,69 +0,0 @@
-/// <reference types="node" />
-declare module "gpt4all";
-
-/**
- * Initiates the download of a model file of a specific model type.
- * By default this downloads without waiting. use the controller returned to alter this behavior.
- * @param {ModelFile} model - The model file to be downloaded.
- * @param {DownloadOptions} options - to pass into the downloader. Default is { location: (cwd), debug: false }.
- * @returns {DownloadController} object that allows controlling the download process.
- *
- * @throws {Error} If the model already exists in the specified location.
- * @throws {Error} If the model cannot be found at the specified url.
- *
- * @example
- * const controller = download('ggml-gpt4all-j-v1.3-groovy.bin')
- * controller.promise().then(() => console.log('Downloaded!'))
- */
-declare function downloadModel(
-    modelName: string,
-    options?: DownloadModelOptions
-): DownloadController;
-
-/**
- * Options for the model download process.
- */
-export interface DownloadModelOptions {
-    /**
-     * location to download the model.
-     * Default is process.cwd(), or the current working directory
-     */
-    modelPath?: string;
-
-    /**
-     * Debug mode -- check how long it took to download in seconds
-     * @default false
-     */
-    debug?: boolean;
-
-    /**
-     * Remote download url. Defaults to `https://gpt4all.io/models`
-     * @default https://gpt4all.io/models
-     */
-    url?: string;
-}
-
-declare function listModels(): Promise<Record<string, string>[]>;
-
-interface RetrieveModelOptions {
-    allowDownload?: boolean;
-    verbose?: boolean;
-    modelPath?: string;
-}
-
-declare async function retrieveModel(
-    model: string,
-    options?: RetrieveModelOptions
-): Promise<string>;
-
-/**
- * Model download controller.
- */
-interface DownloadController {
-    /** Cancel the request to download from gpt4all website if this is called. */
-    cancel: () => void;
-    /** Convert the downloader into a promise, allowing people to await and manage its lifetime */
-    promise: () => Promise<void>;
-}
-
-export { downloadModel, DownloadModelOptions, DownloadController, listModels, retrieveModel, RetrieveModelOptions };

gpt4all-bindings/typescript/src/util.js

@@ -1,13 +1,45 @@
-const { createWriteStream, existsSync } = require("fs");
+const { createWriteStream, existsSync, statSync } = require("node:fs");
+const fsp = require("node:fs/promises");
 const { performance } = require("node:perf_hooks");
 const path = require("node:path");
-const {mkdirp} = require("mkdirp");
-const { DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY } = require("./config.js");
+const { mkdirp } = require("mkdirp");
+const md5File = require("md5-file");
+const {
+    DEFAULT_DIRECTORY,
+    DEFAULT_MODEL_CONFIG,
+    DEFAULT_MODEL_LIST_URL,
+} = require("./config.js");
 
-async function listModels() {
-    const res = await fetch("https://gpt4all.io/models/models.json");
-    const modelList = await res.json();
-    return modelList;
+async function listModels(
+    options = {
+        url: DEFAULT_MODEL_LIST_URL,
+    }
+) {
+    if (!options || (!options.url && !options.file)) {
+        throw new Error(
+            `No model list source specified. Please specify either a url or a file.`
+        );
+    }
+
+    if (options.file) {
+        if (!existsSync(options.file)) {
+            throw new Error(`Model list file ${options.file} does not exist.`);
+        }
+
+        const fileContents = await fsp.readFile(options.file, "utf-8");
+        const modelList = JSON.parse(fileContents);
+        return modelList;
+    } else if (options.url) {
+        const res = await fetch(options.url);
+
+        if (!res.ok) {
+            throw Error(
+                `Failed to retrieve model list from ${url} - ${res.status} ${res.statusText}`
+            );
+        }
+        const modelList = await res.json();
+        return modelList;
+    }
 }
 
 function appendBinSuffixIfMissing(name) {
@@ -31,76 +63,160 @@ function readChunks(reader) {
     };
 }
 
-function downloadModel(
-    modelName,
-    options = {}
-) {
+/**
+ * Prints a warning if any keys in the prompt context are snake_case.
+ */
+function warnOnSnakeCaseKeys(promptContext) {
+    const snakeCaseKeys = Object.keys(promptContext).filter((key) =>
+        key.includes("_")
+    );
+
+    if (snakeCaseKeys.length > 0) {
+        console.warn(
+            "Prompt context keys should be camelCase. Support for snake_case might be removed in the future. Found keys: " +
+                snakeCaseKeys.join(", ")
+        );
+    }
+}
+
+/**
+ * Converts all keys in the prompt context to snake_case
+ * For duplicate definitions, the value of the last occurrence will be used.
+ */
+function normalizePromptContext(promptContext) {
+    const normalizedPromptContext = {};
+
+    for (const key in promptContext) {
+        if (promptContext.hasOwnProperty(key)) {
+            const snakeKey = key.replace(
+                /[A-Z]/g,
+                (match) => `_${match.toLowerCase()}`
+            );
+            normalizedPromptContext[snakeKey] = promptContext[key];
+        }
+    }
+
+    return normalizedPromptContext;
+}
+
+function downloadModel(modelName, options = {}) {
     const downloadOptions = {
         modelPath: DEFAULT_DIRECTORY,
-        debug: false,
-        url: "https://gpt4all.io/models",
+        verbose: false,
         ...options,
     };
-
+    
     const modelFileName = appendBinSuffixIfMissing(modelName);
-    const fullModelPath = path.join(downloadOptions.modelPath, modelFileName);
-    const modelUrl = `${downloadOptions.url}/${modelFileName}`
+    const partialModelPath = path.join(
+        downloadOptions.modelPath,
+        modelName + ".part"
+    );
+    const finalModelPath = path.join(downloadOptions.modelPath, modelFileName);
+    const modelUrl =
+        downloadOptions.url ?? `https://gpt4all.io/models/${modelFileName}`;
 
-    if (existsSync(fullModelPath)) {
-        throw Error(`Model already exists at ${fullModelPath}`);
+    mkdirp.sync(downloadOptions.modelPath)
+
+    if (existsSync(finalModelPath)) {
+        throw Error(`Model already exists at ${finalModelPath}`);
+    }
+    
+    if (downloadOptions.verbose) {
+        console.log(`Downloading ${modelName} from ${modelUrl}`);
+    }
+
+    const headers = {
+        "Accept-Ranges": "arraybuffer",
+        "Response-Type": "arraybuffer",
+    };
+
+    const writeStreamOpts = {};
+
+    if (existsSync(partialModelPath)) {
+        console.log("Partial model exists, resuming download...");
+        const startRange = statSync(partialModelPath).size;
+        headers["Range"] = `bytes=${startRange}-`;
+        writeStreamOpts.flags = "a";
     }
 
     const abortController = new AbortController();
     const signal = abortController.signal;
 
-    //wrapper function to get the readable stream from request
-    // const baseUrl = options.url ?? "https://gpt4all.io/models";
-    const fetchModel = () =>
-        fetch(modelUrl, {
-            signal,
-        }).then((res) => {
-            if (!res.ok) {
-                throw Error(`Failed to download model from ${modelUrl} - ${res.statusText}`);
+    const finalizeDownload = async () => {
+        if (options.md5sum) {
+            const fileHash = await md5File(partialModelPath);
+            if (fileHash !== options.md5sum) {
+                await fsp.unlink(partialModelPath);
+                const message = `Model "${modelName}" failed verification: Hashes mismatch. Expected ${options.md5sum}, got ${fileHash}`;
+                throw Error(message);
             }
-            return res.body.getReader();
+            if (options.verbose) {
+                console.log(`MD5 hash verified: ${fileHash}`);
+            }
+        }
+
+        await fsp.rename(partialModelPath, finalModelPath);
+    };
+
+    // a promise that executes and writes to a stream. Resolves to the path the model was downloaded to when done writing.
+    const downloadPromise = new Promise((resolve, reject) => {
+        let timestampStart;
+
+        if (options.verbose) {
+            console.log(`Downloading @ ${partialModelPath} ...`);
+            timestampStart = performance.now();
+        }
+
+        const writeStream = createWriteStream(
+            partialModelPath,
+            writeStreamOpts
+        );
+
+        writeStream.on("error", (e) => {
+            writeStream.close();
+            reject(e);
         });
 
-    //a promise that executes and writes to a stream. Resolves when done writing.
-    const res = new Promise((resolve, reject) => {
-        fetchModel()
-            //Resolves an array of a reader and writestream.
-            .then((reader) => [reader, createWriteStream(fullModelPath)])
-            .then(async ([readable, wstream]) => {
-                console.log("Downloading @ ", fullModelPath);
-                let perf;
-                if (options.debug) {
-                    perf = performance.now();
+        writeStream.on("finish", () => {
+            if (options.verbose) {
+                const elapsed = performance.now() - timestampStart;
+                console.log(`Finished. Download took ${elapsed.toFixed(2)} ms`);
+            }
+
+            finalizeDownload()
+                .then(() => {
+                    resolve(finalModelPath);
+                })
+                .catch(reject);
+        });
+
+        fetch(modelUrl, {
+            signal,
+            headers,
+        })
+            .then((res) => {
+                if (!res.ok) {
+                    const message = `Failed to download model from ${modelUrl} - ${res.status} ${res.statusText}`;
+                    reject(Error(message));
                 }
-                for await (const chunk of readChunks(readable)) {
-                    wstream.write(chunk);
+                return res.body.getReader();
+            })
+            .then(async (reader) => {
+                for await (const chunk of readChunks(reader)) {
+                    writeStream.write(chunk);
                 }
-                if (options.debug) {
-                    console.log(
-                        "Time taken: ",
-                        (performance.now() - perf).toFixed(2),
-                        " ms"
-                    );
-                }
-                resolve(fullModelPath);
+                writeStream.end();
             })
             .catch(reject);
     });
 
     return {
         cancel: () => abortController.abort(),
-        promise: () => res,
+        promise: downloadPromise,
     };
-};
+}
 
-async function retrieveModel (
-    modelName,
-    options = {}
-) {
+async function retrieveModel(modelName, options = {}) {
     const retrieveOptions = {
         modelPath: DEFAULT_DIRECTORY,
         allowDownload: true,
@@ -114,43 +230,68 @@ async function retrieveModel (
     const fullModelPath = path.join(retrieveOptions.modelPath, modelFileName);
     const modelExists = existsSync(fullModelPath);
 
-    if (modelExists) {
-        return fullModelPath;
-    }
+    let config = { ...DEFAULT_MODEL_CONFIG };
 
-    if (!retrieveOptions.allowDownload) {
-        throw Error(`Model does not exist at ${fullModelPath}`);
-    }
-
-    const availableModels = await listModels();
-    const foundModel = availableModels.find((model) => model.filename === modelFileName);
-
-    if (!foundModel) {
-        throw Error(`Model "${modelName}" is not available.`);
-    }
-
-    if (retrieveOptions.verbose) {
-        console.log(`Downloading ${modelName}...`);
-    }
-
-    const downloadController = downloadModel(modelName, {
-        modelPath: retrieveOptions.modelPath,
-        debug: retrieveOptions.verbose,
+    const availableModels = await listModels({
+        file: retrieveOptions.modelConfigFile,
+        url:
+            retrieveOptions.allowDownload &&
+            "https://gpt4all.io/models/models.json",
     });
 
-    const downloadPath = await downloadController.promise();
+    const loadedModelConfig = availableModels.find(
+        (model) => model.filename === modelFileName
+    );
 
-    if (retrieveOptions.verbose) {
-        console.log(`Model downloaded to ${downloadPath}`);
+    if (loadedModelConfig) {
+        config = {
+            ...config,
+            ...loadedModelConfig,
+        };
+    } else {
+        // if there's no local modelConfigFile specified, and allowDownload is false, the default model config will be used.
+        // warning the user here because the model may not work as expected.
+        console.warn(
+            `Failed to load model config for ${modelName}. Using defaults.`
+        );
     }
 
-    return downloadPath
+    config.systemPrompt = config.systemPrompt.trim();
 
+    if (modelExists) {
+        config.path = fullModelPath;
+
+        if (retrieveOptions.verbose) {
+            console.log(`Found ${modelName} at ${fullModelPath}`);
+        }
+    } else if (retrieveOptions.allowDownload) {
+
+        const downloadController = downloadModel(modelName, {
+            modelPath: retrieveOptions.modelPath,
+            verbose: retrieveOptions.verbose,
+            filesize: config.filesize,
+            url: config.url,
+            md5sum: config.md5sum,
+        });
+
+        const downloadPath = await downloadController.promise;
+        config.path = downloadPath;
+
+        if (retrieveOptions.verbose) {
+            console.log(`Model downloaded to ${downloadPath}`);
+        }
+    } else {
+        throw Error("Failed to retrieve model.");
+    }
+
+    return config;
 }
 
-
 module.exports = {
     appendBinSuffixIfMissing,
     downloadModel,
     retrieveModel,
-};
+    listModels,
+    normalizePromptContext,
+    warnOnSnakeCaseKeys,
+};

gpt4all-bindings/typescript/test/gpt4all.test.js

@@ -0,0 +1,246 @@
+const path = require("node:path");
+const os = require("node:os");
+const fsp = require("node:fs/promises");
+const { existsSync } = require('node:fs');
+const { LLModel } = require("node-gyp-build")(path.resolve(__dirname, ".."));
+const {
+    listModels,
+    downloadModel,
+    appendBinSuffixIfMissing,
+    normalizePromptContext,
+} = require("../src/util.js");
+const {
+    DEFAULT_DIRECTORY,
+    DEFAULT_LIBRARIES_DIRECTORY,
+    DEFAULT_MODEL_LIST_URL,
+} = require("../src/config.js");
+const {
+    loadModel,
+    createPrompt,
+    createCompletion,
+} = require("../src/gpt4all.js");
+const { mock } = require("node:test");
+const { mkdirp } = require("mkdirp");
+
+describe("config", () => {
+    test("default paths constants are available and correct", () => {
+        expect(DEFAULT_DIRECTORY).toBe(
+            path.resolve(os.homedir(), ".cache/gpt4all")
+        );
+        const paths = [
+            path.join(DEFAULT_DIRECTORY, "libraries"),
+            path.resolve("./libraries"),
+            path.resolve(
+                __dirname,
+                "..",
+                `runtimes/${process.platform}-${process.arch}/native`
+            ),
+            process.cwd(),
+        ];
+        expect(typeof DEFAULT_LIBRARIES_DIRECTORY).toBe("string");
+        expect(DEFAULT_LIBRARIES_DIRECTORY).toBe(paths.join(";"));
+    });
+});
+
+describe("listModels", () => {
+    const fakeModels = require("./models.json");
+    const fakeModel = fakeModels[0];
+    const mockResponse = JSON.stringify([fakeModel]);
+
+    let mockFetch, originalFetch;
+
+    beforeAll(() => {
+        // Mock the fetch function for all tests
+        mockFetch = jest.fn().mockResolvedValue({
+            ok: true,
+            json: () => JSON.parse(mockResponse),
+        });
+        originalFetch = global.fetch;
+        global.fetch = mockFetch;
+    });
+
+    afterEach(() => {
+        // Reset the fetch counter after each test
+        mockFetch.mockClear();
+    });
+    afterAll(() => {
+        // Restore fetch
+        global.fetch = originalFetch;
+    });
+
+    it("should load the model list from remote when called without args", async () => {
+        const models = await listModels();
+        expect(fetch).toHaveBeenCalledTimes(1);
+        expect(fetch).toHaveBeenCalledWith(DEFAULT_MODEL_LIST_URL);
+        expect(models[0]).toEqual(fakeModel);
+    });
+
+    it("should load the model list from a local file, if specified", async () => {
+        const models = await listModels({
+            file: path.resolve(__dirname, "models.json"),
+        });
+        expect(fetch).toHaveBeenCalledTimes(0);
+        expect(models[0]).toEqual(fakeModel);
+    });
+    
+    it("should throw an error if neither url nor file is specified", async () => {
+        await expect(listModels(null)).rejects.toThrow(
+            "No model list source specified. Please specify either a url or a file."
+        );
+    });
+});
+
+describe("appendBinSuffixIfMissing", () => {
+    it("should make sure the suffix is there", () => {
+        expect(appendBinSuffixIfMissing("filename")).toBe("filename.bin");
+        expect(appendBinSuffixIfMissing("filename.bin")).toBe("filename.bin");
+    });
+});
+
+describe("downloadModel", () => {
+    let mockAbortController, mockFetch;
+    const fakeModelName = "fake-model";
+
+    const createMockFetch = () => {
+        const mockData = new Uint8Array([1, 2, 3, 4]);
+        const mockResponse = new ReadableStream({
+            start(controller) {
+                controller.enqueue(mockData);
+                controller.close();
+            },
+        });
+        const mockFetchImplementation = jest.fn(() =>
+            Promise.resolve({
+                ok: true,
+                body: mockResponse,
+            })
+        );
+        return mockFetchImplementation;
+    };
+
+    beforeEach(async () => {
+        // Mocking the AbortController constructor
+        mockAbortController = jest.fn();
+        global.AbortController = mockAbortController;
+        mockAbortController.mockReturnValue({
+            signal: "signal",
+            abort: jest.fn(),
+        });
+        mockFetch = createMockFetch();
+        jest.spyOn(global, "fetch").mockImplementation(mockFetch);
+
+    });
+
+    afterEach(async () => {
+        // Clean up mocks
+        mockAbortController.mockReset();
+        mockFetch.mockClear();
+        global.fetch.mockRestore();
+        
+        const rootDefaultPath = path.resolve(DEFAULT_DIRECTORY),
+              partialPath = path.resolve(rootDefaultPath, fakeModelName+'.part'),
+              fullPath = path.resolve(rootDefaultPath, fakeModelName+'.bin')  
+
+        //if tests fail, remove the created files
+        // acts as cleanup if tests fail
+        //
+        if(existsSync(fullPath)) {
+            await fsp.rm(fullPath)
+        }
+        if(existsSync(partialPath)) {
+            await fsp.rm(partialPath)
+        }
+
+    });
+
+    test("should successfully download a model file", async () => {
+        const downloadController = downloadModel(fakeModelName);
+        const modelFilePath = await downloadController.promise;
+        expect(modelFilePath).toBe(path.resolve(DEFAULT_DIRECTORY, `${fakeModelName}.bin`));
+
+        expect(global.fetch).toHaveBeenCalledTimes(1);
+        expect(global.fetch).toHaveBeenCalledWith(
+            "https://gpt4all.io/models/fake-model.bin",
+            {
+                signal: "signal",
+                headers: {
+                    "Accept-Ranges": "arraybuffer",
+                    "Response-Type": "arraybuffer",
+                },
+            }
+        );
+
+        // final model file should be present
+        await expect(fsp.access(modelFilePath)).resolves.not.toThrow();
+
+        // remove the testing model file
+        await fsp.unlink(modelFilePath);
+    });
+
+    test("should error and cleanup if md5sum is not matching", async () => {
+        const downloadController = downloadModel(fakeModelName, {
+            md5sum: "wrong-md5sum",
+        });
+        // the promise should reject with a mismatch
+        await expect(downloadController.promise).rejects.toThrow(
+            `Model "fake-model" failed verification: Hashes mismatch. Expected wrong-md5sum, got 08d6c05a21512a79a1dfeb9d2a8f262f`
+        );
+        // fetch should have been called
+        expect(global.fetch).toHaveBeenCalledTimes(1);
+        // the file should be missing
+        await expect(
+            fsp.access(path.resolve(DEFAULT_DIRECTORY, `${fakeModelName}.bin`))
+        ).rejects.toThrow();
+        // partial file should also be missing
+        await expect(
+            fsp.access(path.resolve(DEFAULT_DIRECTORY, `${fakeModelName}.part`))
+        ).rejects.toThrow();
+    });
+
+    // TODO
+    // test("should be able to cancel and resume a download", async () => {
+    // });
+});
+
+describe("normalizePromptContext", () => {
+    it("should convert a dict with camelCased keys to snake_case", () => {
+        const camelCased = {
+            topK: 20,
+            repeatLastN: 10,
+        };
+
+        const expectedSnakeCased = {
+            top_k: 20,
+            repeat_last_n: 10,
+        };
+
+        const result = normalizePromptContext(camelCased);
+        expect(result).toEqual(expectedSnakeCased);
+    });
+
+    it("should convert a mixed case dict to snake_case, last value taking precedence", () => {
+        const mixedCased = {
+            topK: 20,
+            top_k: 10,
+            repeatLastN: 10,
+        };
+
+        const expectedSnakeCased = {
+            top_k: 10,
+            repeat_last_n: 10,
+        };
+
+        const result = normalizePromptContext(mixedCased);
+        expect(result).toEqual(expectedSnakeCased);
+    });
+
+    it("should not modify already snake cased dict", () => {
+        const snakeCased = {
+            top_k: 10,
+            repeast_last_n: 10,
+        };
+
+        const result = normalizePromptContext(snakeCased);
+        expect(result).toEqual(snakeCased);
+    });
+});

gpt4all-bindings/typescript/test/index.mjs

@@ -1,8 +0,0 @@
-import * as assert from 'node:assert'
-import { download } from '../src/gpt4all.js'
-
-
-assert.rejects(async () => download('poo.bin').promise());
-
-
-console.log('OK')

gpt4all-bindings/typescript/test/models.json

@@ -0,0 +1,10 @@
+[
+  {
+    "order": "a",
+    "md5sum": "08d6c05a21512a79a1dfeb9d2a8f262f",
+    "name": "Not a real model",
+    "filename": "fake-model.bin",
+    "filesize": "4",
+    "systemPrompt": " "
+  }
+]

gpt4all-chat/CMakeLists.txt

@@ -18,7 +18,7 @@ endif()
 
 set(APP_VERSION_MAJOR 2)
 set(APP_VERSION_MINOR 4)
-set(APP_VERSION_PATCH 14)
+set(APP_VERSION_PATCH 15)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file
@@ -180,18 +180,10 @@ install(TARGETS llmodel DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 
 # We should probably iterate through the list of the cmake for backend, but these need to be installed
 # to the this component's dir for the finicky qt installer to work
-install(TARGETS gptj-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS gptj-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llama-230511-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llama-230511-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llama-230519-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llama-230519-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+#install(TARGETS gptj-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+#install(TARGETS gptj-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llama-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llama-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llamamodel-230511-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llamamodel-230511-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llamamodel-230519-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llamamodel-230519-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llamamodel-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llamamodel-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 if(APPLE)
@@ -199,13 +191,17 @@ install(TARGETS llamamodel-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_
 endif()
 install(TARGETS falcon-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS falcon-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS mpt-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS mpt-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+#install(TARGETS mpt-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+#install(TARGETS mpt-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS replit-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS replit-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 if(APPLE)
 install(TARGETS replit-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 endif()
+install(TARGETS bert-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS bert-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS starcoder-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS starcoder-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 
 set(CPACK_GENERATOR "IFW")
 set(CPACK_VERBATIM_VARIABLES YES)

gpt4all-chat/README.md

@@ -40,6 +40,7 @@ One click installers for macOS, Linux, and Windows at https://gpt4all.io
 * Syntax highlighting support for programming languages, etc.
 * REST API with a built-in webserver in the chat gui itself with a headless operation mode as well
 * Advanced settings for changing temperature, topk, etc. (DONE)
+* * Improve the accessibility of the installer for screen reader users
 * YOUR IDEA HERE
 
 ## Building and running

gpt4all-chat/chatgpt.h

@@ -46,6 +46,8 @@ public:
     ChatGPT();
     virtual ~ChatGPT();
 
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath) override;

gpt4all-chat/chatllm.cpp

@@ -14,6 +14,8 @@
 #define REPLIT_INTERNAL_STATE_VERSION 0
 #define LLAMA_INTERNAL_STATE_VERSION 0
 #define FALCON_INTERNAL_STATE_VERSION 0
+#define BERT_INTERNAL_STATE_VERSION 0
+#define STARCODER_INTERNAL_STATE_VERSION 0
 
 class LLModelStore {
 public:
@@ -240,11 +242,11 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 
 #if defined(Q_OS_MAC) && defined(__arm__)
             if (m_forceMetal)
-                m_llModelInfo.model = LLModel::construct(filePath.toStdString(), "metal");
+                m_llModelInfo.model = LLMImplementation::construct(filePath.toStdString(), "metal");
             else
-                m_llModelInfo.model = LLModel::construct(filePath.toStdString(), "auto");
+                m_llModelInfo.model = LLMImplementation::construct(filePath.toStdString(), "auto");
 #else
-            m_llModelInfo.model = LLModel::construct(filePath.toStdString(), "auto");
+            m_llModelInfo.model = LLModel::Implementation::construct(filePath.toStdString(), "auto");
 #endif
 
             if (m_llModelInfo.model) {
@@ -258,12 +260,14 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     m_llModelInfo = LLModelInfo();
                     emit modelLoadingError(QString("Could not load model due to invalid model file for %1").arg(modelInfo.filename()));
                 } else {
-                    switch (m_llModelInfo.model->implementation().modelType[0]) {
+                    switch (m_llModelInfo.model->implementation().modelType()[0]) {
                     case 'L': m_llModelType = LLModelType::LLAMA_; break;
                     case 'G': m_llModelType = LLModelType::GPTJ_; break;
                     case 'M': m_llModelType = LLModelType::MPT_; break;
                     case 'R': m_llModelType = LLModelType::REPLIT_; break;
                     case 'F': m_llModelType = LLModelType::FALCON_; break;
+                    case 'B': m_llModelType = LLModelType::BERT_; break;
+                    case 'S': m_llModelType = LLModelType::STARCODER_; break;
                     default:
                         {
                             delete std::exchange(m_llModelInfo.model, nullptr);
@@ -397,7 +401,7 @@ bool ChatLLM::handlePrompt(int32_t token)
 #endif
     ++m_promptTokens;
     ++m_promptResponseTokens;
-    m_timer->inc();
+    m_timer->start();
     return !m_stopGenerating;
 }
 
@@ -628,8 +632,8 @@ bool ChatLLM::handleNameRecalculate(bool isRecalc)
     qDebug() << "name recalc" << m_llmThread.objectName() << isRecalc;
 #endif
     Q_UNUSED(isRecalc);
-    Q_UNREACHABLE();
-    return false;
+    qt_noop();
+    return true;
 }
 
 bool ChatLLM::handleSystemPrompt(int32_t token)
@@ -669,7 +673,9 @@ bool ChatLLM::serialize(QDataStream &stream, int version)
         case MPT_: stream << MPT_INTERNAL_STATE_VERSION; break;
         case GPTJ_: stream << GPTJ_INTERNAL_STATE_VERSION; break;
         case LLAMA_: stream << LLAMA_INTERNAL_STATE_VERSION; break;
-        case FALCON_: stream << LLAMA_INTERNAL_STATE_VERSION; break;
+        case FALCON_: stream << FALCON_INTERNAL_STATE_VERSION; break;
+        case BERT_: stream << BERT_INTERNAL_STATE_VERSION; break;
+        case STARCODER_: stream << STARCODER_INTERNAL_STATE_VERSION; break;
         default: Q_UNREACHABLE();
         }
     }

gpt4all-chat/chatllm.h

@@ -16,6 +16,8 @@ enum LLModelType {
     CHATGPT_,
     REPLIT_,
     FALCON_,
+    BERT_,
+    STARCODER_
 };
 
 struct LLModelInfo {

gpt4all-chat/cmake/deploy-qt-mac.cmake.in

@@ -7,15 +7,23 @@ file(GLOB MYMPTLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NA
 file(GLOB MYLLAMALIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllama*)
 file(GLOB MYREPLITLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libreplit*)
 file(GLOB MYFALCONLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libfalcon*)
+file(GLOB MYBERTLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libbert*)
+file(GLOB MYSTARCODERLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libstarcoder*)
 file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllmodel.*)
 file(COPY ${MYGPTJLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYMPTLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
+file(COPY ${MYLLAMALIBS}
+     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYREPLITLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYFALCONLLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
+file(COPY ${MYBERTLLIBS}
+     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
+file(COPY ${MYSTARCODERLLIBS}
+     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLAMALIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLMODELLIBS}

gpt4all-chat/llm.cpp

@@ -34,7 +34,7 @@ LLM::LLM()
     if (directoryExists(frameworksDir))
         llmodelSearchPaths += ";" + frameworksDir;
 #endif
-    LLModel::setImplementationsSearchPath(llmodelSearchPaths.toStdString());
+    LLModel::Implementation::setImplementationsSearchPath(llmodelSearchPaths.toStdString());
 
 #if defined(__x86_64__)
     #ifndef _MSC_VER

gpt4all-chat/main.qml

@@ -313,6 +313,7 @@ Window {
                 Label {
                     anchors.verticalCenter: parent.verticalCenter
                     text: qsTr("Loading model...")
+                    font.pixelSize: theme.fontSizeLarge
                     color: theme.textAccent
                 }
             }
@@ -471,18 +472,21 @@ Window {
         id: copyMessage
         anchors.centerIn: parent
         text: qsTr("Conversation copied to clipboard.")
+        font.pixelSize: theme.fontSizeLarge
     }
 
     PopupDialog {
         id: copyCodeMessage
         anchors.centerIn: parent
         text: qsTr("Code copied to clipboard.")
+        font.pixelSize: theme.fontSizeLarge
     }
 
     PopupDialog {
         id: healthCheckFailed
         anchors.centerIn: parent
         text: qsTr("Connection to datalake failed.")
+        font.pixelSize: theme.fontSizeLarge
     }
 
     PopupDialog {
@@ -491,6 +495,7 @@ Window {
         shouldTimeOut: false
         shouldShowBusy: true
         text: qsTr("Recalculating context.")
+        font.pixelSize: theme.fontSizeLarge
 
         Connections {
             target: currentChat
@@ -509,6 +514,7 @@ Window {
         shouldTimeOut: false
         shouldShowBusy: true
         text: qsTr("Saving chats.")
+        font.pixelSize: theme.fontSizeLarge
     }
 
     MyToolButton {
@@ -614,6 +620,7 @@ Window {
                    If you can't start it manually, then I'm afraid you'll have to<br>
                    reinstall.")
             color: theme.textColor
+            font.pixelSize: theme.fontSizeLarge
             Accessible.role: Accessible.Dialog
             Accessible.name: text
             Accessible.description: qsTr("Dialog indicating an error")
@@ -675,7 +682,7 @@ Window {
             anchors.top: parent.top
             anchors.bottom: !currentChat.isServer ? textInputView.top : parent.bottom
             anchors.bottomMargin: !currentChat.isServer ? 30 : 0
-            ScrollBar.vertical.policy: ScrollBar.AlwaysOn
+            ScrollBar.vertical.policy: ScrollBar.AlwaysOff
 
             Rectangle {
                 anchors.fill: parent
@@ -685,6 +692,7 @@ Window {
                     id: warningLabel
                     text: qsTr("You must install a model to continue. Models are available via the download dialog or you can install them manually by downloading from <a href=\"https://gpt4all.io\">the GPT4All website</a> (look for the Models Explorer) and placing them in the model folder. The model folder can be found in the settings dialog under the application tab.")
                     color: theme.textColor
+                    font.pixelSize: theme.fontSizeLarge
                     width: 600
                     linkColor: theme.linkColor
                     wrapMode: Text.WordWrap
@@ -729,6 +737,7 @@ Window {
                     visible: ModelList.installedModels.count !== 0
                     anchors.fill: parent
                     model: chatModel
+                    ScrollBar.vertical: ScrollBar { policy: ScrollBar.AlwaysOn }
 
                     Accessible.role: Accessible.List
                     Accessible.name: qsTr("List of prompt/response pairs")
@@ -1004,6 +1013,7 @@ Window {
             anchors.rightMargin: 30
             color: theme.mutedTextColor
             text: currentChat.tokenSpeed
+            font.pixelSize: theme.fontSizeLarge
         }
 
         RectangularGlow {

gpt4all-chat/metadata/models.json

@@ -44,19 +44,6 @@
     "url": "https://huggingface.co/TheBloke/Nous-Hermes-13B-GGML/resolve/main/nous-hermes-13b.ggmlv3.q4_0.bin",
     "promptTemplate": "### Instruction:\n%1\n### Response:\n"
   },
-  {
-    "order": "e",
-    "md5sum": "81a09a0ddf89690372fc296ff7f625af",
-    "name": "Groovy",
-    "filename": "ggml-gpt4all-j-v1.3-groovy.bin",
-    "filesize": "3785248281",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "GPT-J",
-    "systemPrompt": " ",
-    "description": "<strong>Creative model can be used for commercial purposes</strong><br><ul><li>Fast responses<li>Creative responses</li><li>Instruction based</li><li>Trained by Nomic AI<li>Licensed for commercial use</ul>"
-  },
   {
     "order": "f",
     "md5sum": "11d9f060ca24575a2c303bdc39952486",
@@ -72,21 +59,6 @@
     "description": "<strong>Very good overall model</strong><br><ul><li>Instruction based<li>Based on the same dataset as Groovy<li>Slower than Groovy, with higher quality responses<li>Trained by Nomic AI<li>Cannot be used commercially</ul>",
     "url": "https://huggingface.co/TheBloke/GPT4All-13B-snoozy-GGML/resolve/main/GPT4All-13B-snoozy.ggmlv3.q4_0.bin"
   },
-  {
-    "order": "g",
-    "md5sum": "756249d3d6abe23bde3b1ae272628640",
-    "name": "MPT Chat",
-    "filename": "ggml-mpt-7b-chat.bin",
-    "filesize": "4854401050",
-    "requires": "2.4.1",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "MPT",
-    "description": "<strong>Best overall smaller model</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
-    "promptTemplate": "<|im_start|>user\n%1<|im_end|><|im_start|>assistant\n",
-    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
-  },
   {
     "order": "h",
     "md5sum": "e64e74375ce9d36a3d0af3db1523fd0a",
@@ -135,99 +107,6 @@
     "promptTemplate": "### User:\n%1\n### Response:\n",
     "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
   },
-  {
-    "order": "k",
-    "md5sum": "29119f8fa11712704c6b22ac5ab792ea",
-    "name": "Vicuna",
-    "filename": "ggml-vicuna-7b-1.1-q4_2.bin",
-    "filesize": "4212859520",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_2",
-    "type": "LLaMA",
-    "systemPrompt": " ",
-    "description": "<strong>Good small model - trained by teams from UC Berkeley, CMU, Stanford, MBZUAI, and UC San Diego</strong><br><ul><li>Instruction based<li>Cannot be used commercially</ul>"
-  },
-  {
-    "order": "l",
-    "md5sum": "95999b7b0699e2070af63bf5d34101a8",
-    "name": "Vicuna (large)",
-    "filename": "ggml-vicuna-13b-1.1-q4_2.bin",
-    "filesize": "8136770688",
-    "ramrequired": "16",
-    "parameters": "13 billion",
-    "quant": "q4_2",
-    "type": "LLaMA",
-    "systemPrompt": " ",
-    "description": "<strong>Good larger model - trained by teams from UC Berkeley, CMU, Stanford, MBZUAI, and UC San Diego</strong><br><ul><li>Instruction based<li>Cannot be used commercially</ul>"
-  },
-  {
-    "order": "m",
-    "md5sum": "99e6d129745a3f1fb1121abed747b05a",
-    "name": "Wizard",
-    "filename": "ggml-wizardLM-7B.q4_2.bin",
-    "filesize": "4212864640",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_2",
-    "type": "LLaMA",
-    "systemPrompt": " ",
-    "description": "<strong>Good small model - trained by by Microsoft and Peking University</strong><br><ul><li>Instruction based<li>Cannot be used commercially</ul>"
-  },
-  {
-    "order": "n",
-    "md5sum": "6cb4ee297537c9133bddab9692879de0",
-    "name": "Stable Vicuna",
-    "filename": "ggml-stable-vicuna-13B.q4_2.bin",
-    "filesize": "8136777088",
-    "ramrequired": "16",
-    "parameters": "13 billion",
-    "quant": "q4_2",
-    "type": "LLaMA",
-    "description": "<strong>Trained with RLHF by Stability AI</strong><br><ul><li>Instruction based<li>Cannot be used commercially</ul>",
-    "systemPrompt": "## Assistant: I am StableVicuna, a large language model created by CarperAI. I am here to chat!\n\n"
-  },
-  {
-    "order": "o",
-    "md5sum": "1cfa4958f489f0a0d1ffdf6b37322809",
-    "name": "MPT Instruct",
-    "filename": "ggml-mpt-7b-instruct.bin",
-    "filesize": "4854401028",
-    "requires": "2.4.1",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "MPT",
-    "systemPrompt": " ",
-    "description": "<strong>Mosaic's instruction model</strong><br><ul><li>Instruction based<li>Trained by Mosaic ML<li>Licensed for commercial use</ul>"
-  },
-  {
-    "order": "p",
-    "md5sum": "120c32a51d020066288df045ef5d52b9",
-    "name": "MPT Base",
-    "filename": "ggml-mpt-7b-base.bin",
-    "filesize": "4854401028",
-    "requires": "2.4.1",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "MPT",
-    "systemPrompt": " ",
-    "description": "<strong>Trained for text completion with no assistant finetuning</strong><br><ul><li>Completion based<li>Trained by Mosaic ML<li>Licensed for commercial use</ul>"
-  },
-  {
-    "order": "q",
-    "md5sum": "d5eafd5b0bd0d615cfd5fd763f642dfe",
-    "name": "Nous Vicuna",
-    "filename": "ggml-nous-gpt4-vicuna-13b.bin",
-    "filesize": "8136777088",
-    "ramrequired": "16",
-    "parameters": "13 billion",
-    "quant": "q4_0",
-    "type": "LLaMA",
-    "systemPrompt": " ",
-    "description": "<strong>Trained on ~180,000 instructions</strong><br><ul><li>Instruction based<li>Trained by Nous Research<li>Cannot be used commercially</ul>"
-  },
   {
     "order": "r",
     "md5sum": "489d21fd48840dcb31e5f92f453f3a20",
@@ -256,7 +135,71 @@
     "quant": "f16",
     "type": "Replit",
     "systemPrompt": " ",
+    "promptTemplate": "%1",
     "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>Licensed for commercial use</ul>",
     "url": "https://huggingface.co/nomic-ai/ggml-replit-code-v1-3b/resolve/main/ggml-replit-code-v1-3b.bin"
+  },
+  {
+    "order": "t",
+    "md5sum": "031bb5d5722c08d13e3e8eaf55c37391",
+    "disableGUI": "true",
+    "name": "Bert",
+    "filename": "ggml-all-MiniLM-L6-v2-f16.bin",
+    "filesize": "45521167",
+    "requires": "2.4.14",
+    "ramrequired": "1",
+    "parameters": "1 million",
+    "quant": "f16",
+    "type": "Bert",
+    "systemPrompt": " ",
+    "description": "<strong>Sbert</strong><br><ul><li>For embeddings"
+  },
+  {
+    "order": "u",
+    "md5sum": "379ee1bab9a7a9c27c2314daa097528e",
+    "disableGUI": "true",
+    "name": "Starcoder (Small)",
+    "filename": "starcoderbase-3b-ggml.bin",
+    "filesize": "7503121552",
+    "requires": "2.4.14",
+    "ramrequired": "8",
+    "parameters": "3 billion",
+    "quant": "f16",
+    "type": "Starcoder",
+    "systemPrompt": " ",
+    "promptTemplate": "%1",
+    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based</ul>"
+  },
+  {
+    "order": "w",
+    "md5sum": "f981ab8fbd1ebbe4932ddd667c108ba7",
+    "disableGUI": "true",
+    "name": "Starcoder",
+    "filename": "starcoderbase-7b-ggml.bin",
+    "filesize": "17860448016",
+    "requires": "2.4.14",
+    "ramrequired": "16",
+    "parameters": "7 billion",
+    "quant": "f16",
+    "type": "Starcoder",
+    "systemPrompt": " ",
+    "promptTemplate": "%1",
+    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based</ul>"
+  },
+  {
+    "order": "w",
+    "md5sum": "c7ebc61eec1779bddae1f2bcbf2007cc",
+    "name": "Llama-2-7B Chat",
+    "filename": "llama-2-7b-chat.ggmlv3.q4_0.bin",
+    "filesize": "3791725184",
+    "requires": "2.4.14",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "LLaMA2",
+    "description": "<strong>New LLaMA2 model from Meta AI.</strong><br><ul><li>Fine-tuned for dialogue.<li>static model trained on an offline dataset<li>RLHF dataset<li>Licensed for commercial use</ul>",
+    "url": "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/resolve/main/llama-2-7b-chat.ggmlv3.q4_0.bin",
+    "promptTemplate": "[INST] %1 [/INST] ",
+    "systemPrompt": "[INST]<<SYS>>You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.<</SYS>>[/INST] "
   }
 ]

gpt4all-chat/metadata/release.json

@@ -450,6 +450,20 @@
 * Aaron Miller (Nomic AI)
 * Adam Treat (Nomic AI)
 * Community (beta testers, bug reporters)
+"
+  },
+  {
+    "version": "2.4.14",
+    "notes":
+"
+* Add starcoder model support
+* Add ability to switch between light mode/dark mode
+* Increase the size of fonts in monospace code blocks a bit
+",
+    "contributors":
+"
+* Lakshay Kansal (Nomic AI)
+* Adam Treat (Nomic AI)
 "
   }
 ]

gpt4all-chat/modellist.h

@@ -108,7 +108,7 @@ private:
     QString m_name;
     QString m_filename;
     double  m_temperature         = 0.7;
-    double  m_topP                = 0.1;
+    double  m_topP                = 0.4;
     int     m_topK                = 40;
     int     m_maxLength           = 4096;
     int     m_promptBatchSize     = 128;