Revert an incorrect renaming that slipped in.

Signed-off-by: Adam Treat <treat.adam@gmail.com>

.circleci/config.yml

@@ -15,6 +15,5 @@ workflows:
             gpt4all-backend/.* run-all-workflows true
             gpt4all-bindings/python/.* run-python-workflow true
             gpt4all-bindings/typescript/.* run-ts-workflow true
-            gpt4all-bindings/csharp/.* run-csharp-workflow true
             gpt4all-chat/.* run-chat-workflow true
             .* run-default-workflow true

.circleci/continue_config.yml

@@ -20,9 +20,6 @@ parameters:
   run-ts-workflow:
     type: boolean
     default: false
-  run-csharp-workflow:
-    type: boolean
-    default: false
 
 jobs:
   default-job:
@@ -43,6 +40,9 @@ jobs:
       - restore_cache:  # this is the new step to restore cache
           keys:
             - macos-qt-cache-v3
+      - run:
+          name: Install Rosetta
+          command: softwareupdate --install-rosetta --agree-to-license  # needed for QtIFW
       - run:
           name: Installing Qt
           command: |
@@ -77,8 +77,90 @@ jobs:
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package
             mkdir upload
             cp gpt4all-installer-* upload
+      # persist the unsigned installer
       - store_artifacts:
           path: build/upload
+      # add workspace so signing jobs can connect & obtain dmg
+      - persist_to_workspace:
+          root: build
+          # specify path to only include components we want to persist
+          # accross builds
+          paths:
+            - upload
+
+  sign-offline-chat-installer-macos:
+    macos:
+      xcode: 14.0.0
+    steps:
+      - checkout
+      # attach to a workspace containing unsigned dmg
+      - attach_workspace:
+          at: build
+      - run:
+          name: "Setup Keychain"
+          command: |
+            echo $MAC_SIGNING_CERT | base64 --decode > cert.p12
+            # cat \<<< "$MAC_SIGNING_CERT" > certs1.pem
+            # file certs1.pem
+            # iconv -c -f UTF8 -t ASCII certs1.pem > certs.pem
+            # openssl pkcs12 -legacy -export -out cert.p12 -in certs.pem -inkey certs.pem -passin pass:"$MAC_SIGNING_CERT_PWD" -passout pass:"$MAC_SIGNING_CERT_PWD"
+            security create-keychain -p "$MAC_KEYCHAIN_KEY" sign.keychain
+            security default-keychain -s sign.keychain
+            security unlock-keychain -p "$MAC_KEYCHAIN_KEY" sign.keychain
+            security import cert.p12 -k sign.keychain -P "$MAC_SIGNING_CERT_PWD" -T /usr/bin/codesign
+            security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$MAC_KEYCHAIN_KEY" sign.keychain
+            rm cert.p12
+      - run:
+          name: "Sign App Bundle"
+          command: |
+            python3 -m pip install click
+            python3 gpt4all-chat/cmake/sign_dmg.py --input-dmg build/upload/gpt4all-installer-darwin.dmg --output-dmg build/upload/gpt4all-installer-darwin-signed.dmg --signing-identity "$MAC_SIGNING_CERT_NAME"
+      - run:
+          name: "Sign DMG"
+          command: |
+            codesign --options runtime --timestamp -s "$MAC_SIGNING_CERT_NAME" build/upload/gpt4all-installer-darwin-signed.dmg
+      # add workspace so signing jobs can connect & obtain dmg
+      - persist_to_workspace:
+          root: build
+          # specify path to only include components we want to persist
+          # accross builds
+          paths:
+            - upload
+
+  notarize-offline-chat-installer-macos:
+    macos:
+      xcode: 14.0.0
+    steps:
+      - checkout
+      - attach_workspace:
+          at: build
+      # - run:
+      #     name: "Setup Notarize Keychain"
+      #     command: |
+      #       security create-keychain
+      #       sudo xcrun notarytool store-credentials "notarytool-profile" --apple-id "$MAC_NOTARIZATION_ID" --team-id "$MAC_NOTARIZATION_TID" --password "$MAC_NOTARIZATION_KEY" --keychain /Library/Keychains/System.keychain
+      - run:
+          name: "Notarize"
+          command: |
+            xcrun notarytool submit build/upload/gpt4all-installer-darwin-signed.dmg --apple-id "$MAC_NOTARIZATION_ID" --team-id "$MAC_NOTARIZATION_TID" --password "$MAC_NOTARIZATION_KEY" --wait  | tee notarize_log.txt
+      - run:
+          name: "Report Notarization Failure"
+          command: |
+            NID=`python3 .circleci/grab_notary_id.py notarize_log.txt` && export NID
+            xcrun notarytool log $NID --keychain-profile "notary-profile"
+            exit 1
+          when: on_fail
+      # - run:
+      #     name: "Rename and move"
+      #     command: |
+      #       mv build/upload/gpt4all-installer-darwin-signed.dmg build/upload-signed/gpt4all-installer-darwin-signed.dmg
+      - run:
+          name: "Staple"
+          command: |
+            xcrun stapler staple build/upload/gpt4all-installer-darwin-signed.dmg
+      - store_artifacts:
+          path: build/upload
+
   build-offline-chat-installer-linux:
     machine:
       image: ubuntu-2204:2023.04.2
@@ -97,7 +179,9 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Installing Qt
           command: |
@@ -121,6 +205,7 @@ jobs:
             set -eo pipefail
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
             export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
+            export PATH=$PATH:/usr/local/cuda/bin
             mkdir build
             cd build
             mkdir upload
@@ -162,6 +247,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Build
           command: |
@@ -218,7 +308,9 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Installing Qt
           command: |
@@ -235,6 +327,7 @@ jobs:
           name: Build
           command: |
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
+            export PATH=$PATH:/usr/local/cuda/bin
             ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S gpt4all-chat -B build
             ~/Qt/Tools/CMake/bin/cmake --build build --target all
 
@@ -269,6 +362,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Build
           command: |
@@ -312,6 +410,9 @@ jobs:
       - restore_cache:  # this is the new step to restore cache
           keys:
             - macos-qt-cache-v3
+      - run:
+          name: Install Rosetta
+          command: softwareupdate --install-rosetta --agree-to-license  # needed for QtIFW
       - run:
           name: Installing Qt
           command: |
@@ -394,15 +495,18 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk
+            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
             pip install setuptools wheel cmake
       - run:
           name: Build C library
           command: |
+            export PATH=$PATH:/usr/local/cuda/bin
             git submodule update --init --recursive
             cd gpt4all-backend
-            cmake -B build
+            cmake -B build -DCMAKE_BUILD_TYPE=Release
             cmake --build build --parallel
       - run:
           name: Build wheel
@@ -432,7 +536,7 @@ jobs:
           command: |
             git submodule update --init  # don't use --recursive because macOS doesn't use Kompute
             cd gpt4all-backend
-            cmake -B build -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
             cmake --build build --parallel
       - run:
           name: Build wheel
@@ -447,46 +551,64 @@ jobs:
             - "*.whl"
 
   build-py-windows:
-    executor:
-      name: win/default
+    machine:
+      image: 'windows-server-2019-vs2019:2022.08.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
     steps:
       - checkout
       - run:
-          name: Install MinGW64
-          command: choco install -y mingw --force --no-progress
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
       - run:
           name: Install VulkanSDK
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Install dependencies
           command:
-            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+            choco install -y cmake ninja --installargs 'ADD_CMAKE_TO_PATH=System'
       - run:
           name: Install Python dependencies
           command: pip install setuptools wheel cmake
       - run:
           name: Build C library
           command: |
-            git submodule update --init --recursive
-            cd gpt4all-backend
-            $Env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
-            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            # Visual Studio setup
+            # I would use Enter-VsDevShell but it causes cudafe++ to segfault
+            $Env:PATH += ";C:\Program Files (x86)\Windows Kits\10\bin\x64"
+            $Env:PATH += ";C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
+            $Env:PATH += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
+            $Env:LIB  = "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
+            $Env:LIB += ";C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
+            $Env:LIB += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
+            $Env:LIB += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64"
+            $Env:INCLUDE  = "C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
+
+            $Env:PATH += ";C:\VulkanSDK\1.3.261.1\bin"
             $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
-            cmake -G "MinGW Makefiles" -B build -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
+            cd gpt4all-backend
+            cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=Release -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON
             cmake --build build --parallel
       - run:
           name: Build wheel
-          # TODO: As part of this task, we need to move mingw64 binaries into package.
-          # This is terrible and needs a more robust solution eventually.
           command: |
             cd gpt4all-bindings/python
-            cd gpt4all
-            mkdir llmodel_DO_NOT_MODIFY
-            mkdir llmodel_DO_NOT_MODIFY/build/
-            cp 'C:\ProgramData\mingw64\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
-            cd ..
             python setup.py bdist_wheel --plat-name=win_amd64
       - store_artifacts:
           path: gpt4all-bindings/python/dist
@@ -530,11 +652,14 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list            
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk
+            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Build Libraries
           command: |
+            export PATH=$PATH:/usr/local/cuda/bin
             cd gpt4all-backend
             mkdir -p runtimes/build
             cd runtimes/build
@@ -580,52 +705,6 @@ jobs:
             - runtimes/osx-x64/*.metal
 
   build-bindings-backend-windows:
-    executor:
-      name: win/default
-      size: large
-      shell: powershell.exe -ExecutionPolicy Bypass
-    steps:
-      - checkout
-      - run:
-          name: Update Submodules
-          command: |
-            git submodule sync
-            git submodule update --init --recursive
-      - run:
-          name: Install MinGW64
-          command: choco install -y mingw --force --no-progress
-      - run:
-          name: Install VulkanSDK
-          command: |
-            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
-            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
-      - run:
-          name: Install dependencies
-          command: |
-            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
-      - run:
-          name: Build Libraries
-          command: |
-            $MinGWBin = "C:\ProgramData\mingw64\mingw64\bin"
-            $Env:Path += ";$MinGwBin"
-            $Env:Path += ";C:\Program Files\CMake\bin"
-            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
-            $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
-            cd gpt4all-backend
-            mkdir runtimes/win-x64
-            cd runtimes/win-x64
-            cmake -G "MinGW Makefiles" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON ../..
-            cmake --build . --parallel --config Release
-            cp "$MinGWBin\libgcc*.dll" .
-            cp "$MinGWBin\libstdc++*.dll" .
-            cp "$MinGWBin\libwinpthread*.dll" .
-            cp bin/*.dll .
-      - persist_to_workspace:
-          root: gpt4all-backend
-          paths:
-            - runtimes/win-x64/*.dll
-
-  build-bindings-backend-windows-msvc:
     machine:
       image: 'windows-server-2022-gui:2023.03.1'
       resource_class: windows.large
@@ -642,6 +721,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Install dependencies
           command: |
@@ -663,182 +747,6 @@ jobs:
           paths:
             - runtimes/win-x64_msvc/*.dll
 
-  build-csharp-linux:
-    docker:
-      - image: mcr.microsoft.com/dotnet/sdk:8.0
-    steps:
-      - checkout
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: "Prepare Native Libs"
-          command: |
-            cd gpt4all-bindings/csharp
-            mkdir -p runtimes/linux-x64/native
-            cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
-            ls -R runtimes
-      - restore_cache:
-          keys:
-            - gpt4all-csharp-nuget-packages-nix
-      - run:
-          name: "Install project dependencies"
-          command: |
-            cd gpt4all-bindings/csharp
-            dotnet restore Gpt4All
-      - save_cache:
-          paths:
-            - ~/.nuget/packages
-          key: gpt4all-csharp-nuget-packages-nix
-      - run:
-          name: Build C# Project
-          command: |
-            cd gpt4all-bindings/csharp
-            dotnet build Gpt4All --configuration Release --nologo
-      - run:
-          name: "Run C# Tests"
-          command: |
-            cd gpt4all-bindings/csharp
-            dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
-      - run:
-          name: Test results
-          command: |
-              cd gpt4all-bindings/csharp/Gpt4All.Tests
-              dotnet tool install -g trx2junit
-              export PATH="$PATH:$HOME/.dotnet/tools"
-              trx2junit TestResults/*.trx
-      - store_test_results:
-          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
-
-  build-csharp-windows:
-    executor:
-      name: win/default
-      size: large
-      shell: powershell.exe -ExecutionPolicy Bypass
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - gpt4all-csharp-nuget-packages-win
-      - attach_workspace:
-          at: C:\Users\circleci\workspace
-      - run:
-          name: "Install .NET"
-          command: |
-            choco install -y dotnet-8.0-sdk
-      - run:
-          name: "Prepare Native Libs"
-          command: |
-            cd gpt4all-bindings/csharp
-            mkdir -p runtimes\win-x64\native
-            cp C:\Users\circleci\workspace\runtimes\win-x64\*.dll runtimes\win-x64\native\
-            ls -R runtimes
-      - run:
-          name: "Install project dependencies"
-          command: |
-            cd gpt4all-bindings/csharp
-            dotnet.exe restore Gpt4All
-      - save_cache:
-          paths:
-            - C:\Users\circleci\.nuget\packages
-          key: gpt4all-csharp-nuget-packages-win
-      - run:
-          name: Build C# Project
-          command: |
-            cd gpt4all-bindings/csharp
-            dotnet.exe build Gpt4All --configuration Release --nologo
-      - run:
-          name: "Run C# Tests"
-          command: |
-            cd gpt4all-bindings/csharp
-            dotnet.exe test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
-      - run:
-          name: Test results
-          command: |
-              cd gpt4all-bindings/csharp/Gpt4All.Tests
-              dotnet tool install -g trx2junit
-              $Env:Path += ";$Env:USERPROFILE\.dotnet\tools"
-              trx2junit TestResults/*.trx
-      - store_test_results:
-          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
-
-  build-csharp-macos:
-    macos:
-      xcode: "14.0.0"
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - gpt4all-csharp-nuget-packages-nix
-      - run:
-          name: Install dependencies
-          command: |
-            brew tap isen-ng/dotnet-sdk-versions
-            brew install --cask dotnet-sdk8-0-100
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: "Prepare Native Libs"
-          command: |
-            cd gpt4all-bindings/csharp
-            mkdir -p runtimes/osx/native
-            cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
-            cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
-            ls -R runtimes
-      - run:
-          name: "Install project dependencies"
-          command: |
-            cd gpt4all-bindings/csharp
-            dotnet restore Gpt4All
-      - save_cache:
-          paths:
-            - ~/.nuget/packages
-          key: gpt4all-csharp-nuget-packages-nix
-      - run:
-          name: Build C# Project
-          command: |
-            cd gpt4all-bindings/csharp
-            dotnet build Gpt4All --configuration Release --nologo
-      - run:
-          name: "Run C# Tests"
-          command: |
-            cd gpt4all-bindings/csharp
-            dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
-      - run:
-          name: Test results
-          command: |
-              cd gpt4all-bindings/csharp/Gpt4All.Tests
-              dotnet tool install -g trx2junit
-              export PATH="$PATH:$HOME/.dotnet/tools"
-              trx2junit TestResults/*.trx
-      - store_test_results:
-          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
-
-  store-and-upload-nupkgs:
-    docker:
-      - image: mcr.microsoft.com/dotnet/sdk:8.0
-    steps:
-      - attach_workspace:
-          at: /tmp/workspace
-      - checkout
-      - restore_cache:
-           keys:
-             - gpt4all-csharp-nuget-packages-nix
-      - run:
-          name: NuGet Pack
-          command: |
-            cd gpt4all-bindings/csharp
-            mkdir -p runtimes/linux-x64/native
-            cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
-            mkdir -p runtimes/win-x64/native
-            cp /tmp/workspace/runtimes/win-x64/*.dll runtimes/win-x64/native/
-            #mkdir -p runtimes/osx/native
-            #cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
-            #cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
-            dotnet pack ./Gpt4All/Gpt4All.csproj -p:IncludeSymbols=true -p:SymbolPackageFormat=snupkg -c Release
-            dotnet nuget push ./Gpt4All/bin/Release/Gpt4All.*.nupkg -s $NUGET_URL -k $NUGET_TOKEN --skip-duplicate
-      - store_artifacts:
-          path: gpt4all-bindings/csharp/Gpt4All/bin/Release
-
   build-nodejs-linux: 
     docker: 
       - image: cimg/base:stable
@@ -1022,6 +930,12 @@ workflows:
       - build-offline-chat-installer-macos:
           requires:
             - hold
+      - sign-offline-chat-installer-macos:
+          requires:
+            - build-offline-chat-installer-macos
+      - notarize-offline-chat-installer-macos:
+          requires:
+            - sign-offline-chat-installer-macos
       - build-offline-chat-installer-windows:
           requires:
             - hold
@@ -1103,13 +1017,10 @@ workflows:
       or:
        - << pipeline.parameters.run-all-workflows >>
        - << pipeline.parameters.run-python-workflow >>
-       - << pipeline.parameters.run-csharp-workflow >>
        - << pipeline.parameters.run-ts-workflow >>
     jobs:
       - hold:
           type: approval
-      - csharp-hold:
-          type: approval
       - nuget-hold:
           type: approval
       - nodejs-hold:
@@ -1134,12 +1045,6 @@ workflows:
               only:
           requires:
             - hold
-      - build-bindings-backend-windows-msvc:
-          filters:
-            branches:
-              only:
-          requires:
-            - hold
 
       # NodeJs Jobs 
       - prepare-npm-pkg: 
@@ -1164,7 +1069,7 @@ workflows:
               only:
           requires:
             - nodejs-hold
-            - build-bindings-backend-windows-msvc
+            - build-bindings-backend-windows
       - build-nodejs-macos:
           filters:
             branches:
@@ -1172,36 +1077,3 @@ workflows:
           requires:
             - nodejs-hold
             - build-bindings-backend-macos
-
-
-      # CSharp Jobs
-      - build-csharp-linux:
-          filters:
-            branches:
-              only:
-          requires:
-            - csharp-hold
-            - build-bindings-backend-linux
-      - build-csharp-windows:
-          filters:
-            branches:
-              only:
-          requires:
-            - csharp-hold
-            - build-bindings-backend-windows
-      - build-csharp-macos:
-          filters:
-            branches:
-              only:
-          requires:
-            - csharp-hold
-            - build-bindings-backend-macos
-      - store-and-upload-nupkgs:
-          filters:
-            branches:
-              only:
-          requires:
-            - nuget-hold
-            - build-csharp-windows
-            - build-csharp-linux
-            #- build-csharp-macos

.circleci/grab_notary_id.py

@@ -0,0 +1,17 @@
+import re
+import sys
+
+ID_REG = r"id: (.*)"
+
+def main() -> None:
+    notary_log = sys.argv[1]
+    with open(notary_log, "r") as f:
+        notary_output = f.read()
+        id_m = re.search(ID_REG, notary_output)
+        if id_m:
+            print(id_m.group(1))
+        else:
+            raise RuntimeError("Unable to parse ID from notarization logs")
+
+if __name__ == "__main__":
+    main()

.github/workflows/codespell.yml

@@ -14,6 +14,6 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v2

.gitmodules

@@ -2,3 +2,6 @@
 	path = gpt4all-backend/llama.cpp-mainline
 	url = https://github.com/nomic-ai/llama.cpp.git
 	branch = master
+[submodule "gpt4all-chat/usearch"]
+	path = gpt4all-chat/usearch
+	url = https://github.com/nomic-ai/usearch.git

LICENSE_SOM.txt

@@ -1,30 +0,0 @@
-Software for Open Models License (SOM)
-Version 1.0 dated August 30th, 2023
-
-This license governs use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
-
-This license is intended to encourage open release of models created, modified, processed, or otherwise used via the Software under open licensing terms, and should be interpreted in light of that intent.
-
-1. Definitions
-The “Licensor” is the person or entity who is making the Software available under this license. “Software” is the software made available by Licensor under this license.
-A “Model” is the output of a machine learning algorithm, and excludes the Software.
-“Model Source Materials” must include the Model and model weights, and may include any input data, input data descriptions, documentation or training descriptions for the Model.
-“Open Licensing Terms” means: (a) any open source license approved by the Open Source Initiative, or (b) any other terms that make the Model Source Materials publicly available free of charge, and allow recipients to use, modify and distribute the Model Source Materials. Terms described in (b) may include reasonable restrictions such as non-commercial or non-production limitations, or require use in compliance with law.
-
-2. Grant of Rights. Subject to the conditions and limitations in section 3:
-(A) Copyright Grant. Licensor grants you a non-exclusive, worldwide, royalty-free copyright license to copy, modify, and distribute the Software and any modifications of the Software you create under this license. The foregoing license includes without limitation the right to create, modify, and use Models using this Software.
-
-(B) Patent Grant. Licensor grants you a non-exclusive, worldwide, royalty-free license, under any patents owned or controlled by Licensor, to make, have made, use, sell, offer for sale, import, or otherwise exploit the Software.  No license is granted to patent rights that are not embodied in the operation of the Software in the form provided by Licensor.
-
-3. Conditions and Limitations
-(A) Model Licensing and Access. If you use the Software to create, modify, process, or otherwise use any Model, including usage to create inferences with a Model, whether or not you make the Model available to others, you must make that Model Source Materials publicly available under Open Licensing Terms. 
-
-(B) No Re-Licensing. If you redistribute the Software, or modifications to the Software made under the license granted above, you must make it available only under the terms of this license. You may offer additional terms such as warranties, maintenance and support, but You, and not Licensor, are responsible for performing such terms.
-
-(C) No Trademark License. This license does not grant you rights to use the Licensor’s name, logo, or trademarks.
-
-(D) If you assert in writing a claim against any person or entity alleging that the use of the Software infringes any patent, all of your licenses to the Software under Section 2 end automatically as of the date you asserted the claim.
-
-(E) If you distribute any portion of the Software, you must retain all copyright, patent, trademark, and attribution notices that are present in the Software, and you must include a copy of this license.
-
-(F) The Software is licensed “as-is.” You bear the entire risk of using it. Licensor gives You no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws that this license cannot change. To the extent permitted under your local laws, the Licensor disclaims and excludes the implied warranties of merchantability, fitness for a particular purpose and non-infringement. To the extent this disclaimer is unlawful, you, and not Licensor, are responsible for any liability.

README.md

@@ -1,82 +1,74 @@
 <h1 align="center">GPT4All</h1>
-
-<p align="center">Open-source large language models that run locally on your CPU and nearly any GPU</p>
-
-
+<p align="center">Privacy-oriented software for chatting with large language models that run on your own computer.</p>
 <p align="center">
-Join the <a href="https://discord.gg/tyc74KNVK3?event=1227642051294658621">GPT4All 2024 Roadmap Townhall</a> on April 18, 2024 at 12pm EST
+  <a href="https://gpt4all.io">Official Website</a> &bull; <a href="https://docs.gpt4all.io">Documentation</a> &bull; <a href="https://discord.gg/mGZE39AS3e">Discord</a>
 </p>
-
 <p align="center">
-<a href="https://gpt4all.io">GPT4All Website and Models</a> • <a href="https://docs.gpt4all.io">GPT4All Documentation</a> • <a href="https://discord.gg/mGZE39AS3e">Discord</a>
+  Official Download Links: <a href="https://gpt4all.io/installers/gpt4all-installer-win64.exe">Windows</a> &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-darwin.dmg">macOS</a> &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-linux.run">Ubuntu</a>
 </p>
-
-
 <p align="center">
-<a href="https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html">🦜️🔗 Official Langchain Backend</a> 
+  <b>NEW:</b> <a href="https://forms.nomic.ai/gpt4all-release-notes-signup">Subscribe to our mailing list</a> for updates and news!
 </p>
-
 <p align="center">
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
 </p>
-
 <p align="center">
- <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg?&logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNSIgaGVpZ2h0PSI0IiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxwYXRoIGQ9Ik00LjQzIDEuODgyYTEuNDQgMS40NCAwIDAgMS0uMDk4LjQyNmMtLjA1LjEyMy0uMTE1LjIzLS4xOTIuMzIyLS4wNzUuMDktLjE2LjE2NS0uMjU1LjIyNmExLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxMmMtLjA5OS4wMTItLjE5Mi4wMTQtLjI3OS4wMDZsLTEuNTkzLS4xNHYtLjQwNmgxLjY1OGMuMDkuMDAxLjE3LS4xNjkuMjQ2LS4xOTFhLjYwMy42MDMgMCAwIDAgLjItLjEwNi41MjkuNTI5IDAgMCAwIC4xMzgtLjE3LjY1NC42NTQgMCAwIDAgLjA2NS0uMjRsLjAyOC0uMzJhLjkzLjkzIDAgMCAwLS4wMzYtLjI0OS41NjcuNTY3IDAgMCAwLS4xMDMtLjIuNTAyLjUwMiAwIDAgMC0uMTY4LS4xMzguNjA4LjYwOCAwIDAgMC0uMjQtLjA2N0wyLjQzNy43MjkgMS42MjUuNjcxYS4zMjIuMzIyIDAgMCAwLS4yMzIuMDU4LjM3NS4zNzUgMCAwIDAtLjExNi4yMzJsLS4xMTYgMS40NS0uMDU4LjY5Ny0uMDU4Ljc1NEwuNzA1IDRsLS4zNTctLjA3OUwuNjAyLjkwNkMuNjE3LjcyNi42NjMuNTc0LjczOS40NTRhLjk1OC45NTggMCAwIDEgLjI3NC0uMjg1Ljk3MS45NzEgMCAwIDEgLjMzNy0uMTRjLjExOS0uMDI2LjIyNy0uMDM0LjMyNS0uMDI2TDMuMjMyLjE2Yy4xNTkuMDE0LjMzNi4wMy40NTkuMDgyYTEuMTczIDEuMTczIDAgMCAxIC41NDUuNDQ3Yy4wNi4wOTQuMTA5LjE5Mi4xNDQuMjkzYTEuMzkyIDEuMzkyIDAgMCAxIC4wNzguNThsLS4wMjkuMzJaIiBmaWxsPSIjRjI3NzdBIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+Cjwvc3ZnPgo=" alt="phorm.ai"></a>
+ <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg" alt="phorm.ai"></a>
 </p>
 
 <p align="center">
-  <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
+  <img width="auto" height="400" src="https://github.com/nomic-ai/gpt4all/assets/14168726/495fce3e-769b-4e5a-a394-99f072ac4d29">
 </p>
 <p align="center">
-Run on an M1 macOS Device (not sped up!)
+Run on an M2 MacBook Pro (not sped up!)
 </p>
 
-## GPT4All: An ecosystem of open-source on-edge large language models.
 
-GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and any GPU. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
+## About GPT4All
+
+GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and NVIDIA and AMD GPUs. Note that your CPU needs to support [AVX instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
 
 Learn more in the [documentation](https://docs.gpt4all.io).
 
-A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models.
+A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily deploy their own on-edge large language models.
 
-### What's New ([Issue Tracker](https://github.com/orgs/nomic-ai/projects/2))
+
+### Installation
+
+The recommended way to install GPT4All is to use one of the online installers linked above in this README, which are also available at the [GPT4All website](https://gpt4all.io/). These require an internet connection at install time, are slightly easier to use on macOS due to code signing, and provide a version of GPT4All that can check for updates.
+
+An alternative way to install GPT4All is to use one of the offline installers available on the [Releases page](https://github.com/nomic-ai/gpt4all/releases). These do not require an internet connection at install time, and can be used to install an older version of GPT4All if so desired. But using these requires acknowledging a security warning on macOS, and they provide a version of GPT4All that is unable to notify you of updates, so you should enable notifications for Releases on this repository (Watch > Custom > Releases) or sign up for announcements in our [Discord server](https://discord.gg/mGZE39AS3e).
+
+
+### What's New
 - **October 19th, 2023**: GGUF Support Launches with Support for:
     - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
     - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4\_0 and Q4\_1 quantizations in GGUF.
     - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
-- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on AMD, Intel, Samsung, Qualcomm and NVIDIA GPUs.
-- **August 15th, 2023**: GPT4All API launches allowing inference of local LLMs from docker containers.
-- **July 2023**: Stable support for LocalDocs, a GPT4All Plugin that allows you to privately and locally chat with your data.
+- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on NVIDIA and AMD GPUs.
+- **July 2023**: Stable support for LocalDocs, a feature that allows you to privately and locally chat with your data.
+- **June 28th, 2023**: [Docker-based API server] launches allowing inference of local LLMs from an OpenAI-compatible HTTP endpoint.
+
+[Docker-based API server]: https://github.com/nomic-ai/gpt4all/tree/cef74c2be20f5b697055d5b8b506861c7b997fab/gpt4all-api
 
 
-### Chat Client
-Run any GPT4All model natively on your home desktop with the auto-updating desktop chat client. See <a href="https://gpt4all.io">GPT4All Website</a> for a full list of open-source models you can run with this powerful desktop application.
+### Building From Source
 
-Direct Installer Links:
+* Follow the instructions [here](gpt4all-chat/build_and_run.md) to build the GPT4All Chat UI from source.
 
-* [macOS](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
-
-* [Windows](https://gpt4all.io/installers/gpt4all-installer-win64.exe)
-
-* [Ubuntu](https://gpt4all.io/installers/gpt4all-installer-linux.run)
-
-Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io/)
-
-### Chat Client building and running
-
-* Follow the visual instructions on the chat client [build_and_run](gpt4all-chat/build_and_run.md) page
 
 ### Bindings
 
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/java">:computer: Official Java Bindings</a>
+* :snake: <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python">Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
+* :computer: <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">Typescript Bindings</a>
+
 
 ### Integrations
 
-* 🗃️ [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
+* :parrot::link: [Langchain](https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html)
+* :card_file_box: [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
+* :telescope: [OpenLIT (OTel-native Monitoring)](https://github.com/openlit/openlit) - [Docs](https://docs.openlit.io/latest/integrations/gpt4all)
+
 
 ## Contributing
 GPT4All welcomes contributions, involvement, and discussion from the open source community!
@@ -86,6 +78,59 @@ Check project discord, with project owners, or through existing issues/PRs to av
 Please make sure to tag all of the above with relevant project identifiers or your contribution could potentially get lost.
 Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 
+
+## GPT4All 2024 Roadmap
+To contribute to the development of any of the below roadmap items, make or find the corresponding issue and cross-reference the [in-progress task](https://github.com/orgs/nomic-ai/projects/2/views/1).
+
+Each item should have an issue link below.
+
+- Chat UI Language Localization (localize UI into the native languages of users)
+    - [ ] Chinese
+    - [ ] German
+    - [ ] French
+    - [ ] Portuguese
+    - [ ] Your native language here. 
+- UI Redesign: an internal effort at Nomic to improve the UI/UX of gpt4all for all users.
+    - [ ] Design new user interface and gather community feedback
+    - [ ] Implement the new user interface and experience.
+- Installer and Update Improvements
+    - [ ] Seamless native installation and update process on OSX
+    - [ ] Seamless native installation and update process on Windows
+    - [ ] Seamless native installation and update process on Linux
+- Model discoverability improvements:
+    - [x] Support huggingface model discoverability
+    - [ ] Support Nomic hosted model discoverability
+- LocalDocs (towards a local perplexity)
+    - Multilingual LocalDocs Support
+        - [ ] Create a multilingual experience
+        - [ ] Incorporate a multilingual embedding model
+        - [ ] Specify a preferred multilingual LLM for localdocs
+    - Improved RAG techniques
+        - [ ] Query augmentation and re-writing
+        - [ ] Improved chunking and text extraction from arbitrary modalities
+            - [ ] Custom PDF extractor past the QT default (charts, tables, text)
+        - [ ] Faster indexing and local exact search with v1.5 hamming embeddings and reranking (skip ANN index construction!)
+    - Support queries like 'summarize X document'
+    - Multimodal LocalDocs support with Nomic Embed
+    - Nomic Dataset Integration with real-time LocalDocs
+        - [ ] Include an option to allow the export of private LocalDocs collections to Nomic Atlas for debugging data/chat quality
+        - [ ] Allow optional sharing of LocalDocs collections between users.
+        - [ ] Allow the import of a LocalDocs collection from an Atlas Datasets
+            - Chat with live version of Wikipedia, Chat with Pubmed, chat with the latest snapshot of world news.
+- First class Multilingual LLM Support
+    - [ ] Recommend and set a default LLM for German
+    - [ ] Recommend and set a default LLM for English
+    - [ ] Recommend and set a default LLM for Chinese
+    - [ ] Recommend and set a default LLM for Spanish
+
+- Server Mode improvements
+    - Improved UI and new requested features:
+        - [ ] Fix outstanding bugs and feature requests around networking configurations.
+        - [ ] Support Nomic Embed inferencing
+        - [ ] First class documentation
+        - [ ] Improving developer use and quality of server mode (e.g. support larger batches)
+
+
 ## Technical Reports
 
 <p align="center">
@@ -100,6 +145,7 @@ Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 <a href="https://s3.amazonaws.com/static.nomic.ai/gpt4all/2023_GPT4All_Technical_Report.pdf">:green_book: Technical Report 1: GPT4All</a>
 </p>
 
+
 ## Citation
 
 If you utilize this repository, models or data in a downstream project, please consider citing it with:

gpt4all-api/.gitignore

@@ -1,112 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-app/__pycache__/
-gpt4all_api/__pycache__/
-gpt4all_api/app/api_v1/__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# VS Code
-.vscode/
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-
-*.lock
-*.cache

gpt4all-api/.isort.cfg

@@ -1,7 +0,0 @@
-[settings]
-known_third_party=geopy,nltk,np,numpy,pandas,pysbd,fire,torch
-
-line_length=120
-include_trailing_comma=True
-multi_line_output=3
-use_parentheses=True

gpt4all-api/LICENSE

@@ -1,13 +0,0 @@
-Copyright 2023 Nomic, Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

gpt4all-api/README.md

@@ -1,90 +0,0 @@
-# GPT4All REST API
-
-NOTICE: We are considering to deprecate this API as it has become challenging to maintain and test. If you have any interest in maintaining this or would like to takeover and adopt or discuss the future of this API please speak up in the discord channel.
-
-This directory contains the source code to run and build docker images that run a FastAPI app
-for serving inference from GPT4All models. The API matches the OpenAI API spec.
-
-## Tutorial
-
-The following tutorial assumes that you have checked out this repo and cd'd into it.
-
-### Starting the app
-
-First change your working directory to `gpt4all/gpt4all-api`.
-
-Now you can build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
-```bash
-DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
-```
-
-Then, start the backend with:
-
-```bash
-docker compose up --build
-```
-
-This will run both the API and locally hosted GPU inference server. If you want to run the API without the GPU inference server, you can run:
-
-```bash
-docker compose up --build gpt4all_api
-```
-
-To run the API with the GPU inference server, you will need to include environment variables (like the `MODEL_ID`). Edit the `.env` file and run
-```bash
-docker compose --env-file .env up --build
-```
-
-
-#### Spinning up your app
-Run `docker compose up` to spin up the backend. Monitor the logs for errors in-case you forgot to set an environment variable above.
-
-
-#### Development
-Run
-
-```bash
-docker compose up --build
-```
-and edit files in the `app` directory. The api will hot-reload on changes.
-
-You can run the unit tests with
-
-```bash
-make test
-```
-
-#### Viewing API documentation
-
-Once the FastAPI ap is started you can access its documentation and test the search endpoint by going to:
-```
-localhost:80/docs
-```
-
-This documentation should match the OpenAI OpenAPI spec located at https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-#### Running inference
-```python
-import openai
-openai.api_base = "http://localhost:4891/v1"
-
-openai.api_key = "not needed for a local LLM"
-
-
-def test_completion():
-    model = "gpt4all-j-v1.3-groovy"
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model,
-        prompt=prompt,
-        max_tokens=50,
-        temperature=0.28,
-        top_p=0.95,
-        n=1,
-        echo=True,
-        stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-    print(response)
-```

gpt4all-api/docker-compose.gpu.yaml

@@ -1,24 +0,0 @@
-version: "3.8"
-
-services:
-  gpt4all_gpu:
-    image: ghcr.io/huggingface/text-generation-inference:0.9.3
-    container_name: gpt4all_gpu
-    restart: always #restart on error (usually code compilation from save during bad state)
-    environment:
-      - HUGGING_FACE_HUB_TOKEN=token
-      - USE_FLASH_ATTENTION=false
-      - MODEL_ID=''
-      - NUM_SHARD=1
-    command: --model-id $MODEL_ID --num-shard $NUM_SHARD
-    volumes:
-      - ./:/data
-    ports:
-      - "8080:80"
-    shm_size: 1g
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              capabilities: [gpu]

gpt4all-api/docker-compose.yaml

@@ -1,22 +0,0 @@
-version: "3.8"
-
-services:
-  gpt4all_api:
-    image: gpt4all_api
-    container_name: gpt4all_api
-    restart: always #restart on error (usually code compilation from save during bad state)
-    ports:
-      - "4891:4891"
-    env_file:
-      - .env
-    environment:
-      - APP_ENVIRONMENT=dev
-      - WEB_CONCURRENCY=2
-      - LOGLEVEL=debug
-      - PORT=4891
-      - model=${MODEL_BIN} # using variable from .env file
-      - inference_mode=cpu
-    volumes:
-      - './gpt4all_api/app:/app'
-      - './gpt4all_api/models:/models' # models are mounted in the container
-    command: ["/start-reload.sh"]

gpt4all-api/gpt4all_api/Dockerfile.buildkit

@@ -1,17 +0,0 @@
-# syntax=docker/dockerfile:1.0.0-experimental
-FROM tiangolo/uvicorn-gunicorn:python3.11
-
-# Put first so anytime this file changes other cached layers are invalidated.
-COPY gpt4all_api/requirements.txt /requirements.txt
-
-RUN pip install --upgrade pip
-
-# Run various pip install commands with ssh keys from host machine.
-RUN --mount=type=ssh pip install -r /requirements.txt && \
-  rm -Rf /root/.cache && rm -Rf /tmp/pip-install*
-
-# Finally, copy app and client.
-COPY gpt4all_api/app /app
-
-RUN mkdir -p /models
-

gpt4all-api/gpt4all_api/README.md

@@ -1 +0,0 @@
-# FastAPI app for serving GPT4All models

gpt4all-api/gpt4all_api/app/api_v1/api.py

@@ -1,9 +0,0 @@
-from api_v1.routes import chat, completions, engines, health
-from fastapi import APIRouter
-
-router = APIRouter()
-
-router.include_router(chat.router)
-router.include_router(completions.router)
-router.include_router(engines.router)
-router.include_router(health.router)

gpt4all-api/gpt4all_api/app/api_v1/events.py

@@ -1,29 +0,0 @@
-import logging
-
-from api_v1.settings import settings
-from fastapi import HTTPException
-from fastapi.responses import JSONResponse
-from starlette.requests import Request
-
-log = logging.getLogger(__name__)
-
-
-startup_msg_fmt = """
- Starting up GPT4All API
-"""
-
-
-async def on_http_error(request: Request, exc: HTTPException):
-    return JSONResponse({'detail': exc.detail}, status_code=exc.status_code)
-
-
-async def on_startup(app):
-    startup_msg = startup_msg_fmt.format(settings=settings)
-    log.info(startup_msg)
-
-
-def startup_event_handler(app):
-    async def start_app() -> None:
-        await on_startup(app)
-
-    return start_app

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -1,103 +0,0 @@
-import logging
-import time
-from typing import List
-from uuid import uuid4
-from fastapi import APIRouter, HTTPException
-from gpt4all import GPT4All
-from pydantic import BaseModel, Field
-from api_v1.settings import settings
-from fastapi.responses import StreamingResponse
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-class ChatCompletionMessage(BaseModel):
-    role: str
-    content: str
-
-class ChatCompletionRequest(BaseModel):
-    model: str = Field(settings.model, description='The model to generate a completion from.')
-    messages: List[ChatCompletionMessage] = Field(..., description='Messages for the chat completion.')
-    temperature: float = Field(settings.temp, description='Model temperature')
-
-class ChatCompletionChoice(BaseModel):
-    message: ChatCompletionMessage
-    index: int
-    logprobs: float
-    finish_reason: str
-
-class ChatCompletionUsage(BaseModel):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-class ChatCompletionResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[ChatCompletionChoice]
-    usage: ChatCompletionUsage
-
-router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
-
-@router.post("/completions", response_model=ChatCompletionResponse)
-async def chat_completion(request: ChatCompletionRequest):
-    '''
-    Completes a GPT4All model response based on the last message in the chat.
-    '''
-    # GPU is not implemented yet
-    if settings.inference_mode == "gpu":
-        raise HTTPException(status_code=400,
-              detail=f"Not implemented yet: Can only infer in CPU mode.")
-
-    # we only support the configured model
-    if request.model != settings.model:
-        raise HTTPException(status_code=400,
-              detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
-
-    # run only of we have a message
-    if request.messages:
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        # format system message and conversation history correctly
-        formatted_messages = ""
-        for message in request.messages:
-            formatted_messages += f"<|im_start|>{message.role}\n{message.content}<|im_end|>\n"
-
-        # the LLM will complete the response of the assistant
-        formatted_messages += "<|im_start|>assistant\n"
-        response = model.generate(
-            prompt=formatted_messages,
-            temp=request.temperature
-            )
-
-        # the LLM may continue to hallucinate the conversation, but we want only the first response
-        # so, cut off everything after first <|im_end|>
-        index = response.find("<|im_end|>")
-        response_content = response[:index].strip()
-    else:
-        response_content = "No messages received."
-
-    # Create a chat message for the response
-    response_message = ChatCompletionMessage(role="assistant", content=response_content)
-
-    # Create a choice object with the response message
-    response_choice = ChatCompletionChoice(
-        message=response_message,
-        index=0,
-        logprobs=-1.0,  # Placeholder value
-        finish_reason="length"  # Placeholder value
-    )
-
-    # Create the response object
-    chat_response = ChatCompletionResponse(
-        id=str(uuid4()),
-        created=int(time.time()),
-        model=request.model,
-        choices=[response_choice],
-        usage=ChatCompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0),  # Placeholder values
-    )
-
-    return chat_response

gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py

@@ -1,215 +0,0 @@
-import json
-from typing import List, Dict, Iterable, AsyncIterable
-import logging
-import time
-from typing import Dict, List, Union, Optional
-from uuid import uuid4
-import aiohttp
-import asyncio
-from api_v1.settings import settings
-from fastapi import APIRouter, Depends, Response, Security, status, HTTPException
-from fastapi.responses import StreamingResponse
-from gpt4all import GPT4All
-from pydantic import BaseModel, Field
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class CompletionRequest(BaseModel):
-    model: str = Field(settings.model, description='The model to generate a completion from.')
-    prompt: Union[List[str], str] = Field(..., description='The prompt to begin completing from.')
-    max_tokens: int = Field(None, description='Max tokens to generate')
-    temperature: float = Field(settings.temp, description='Model temperature')
-    top_p: Optional[float] = Field(settings.top_p, description='top_p')
-    top_k: Optional[int] = Field(settings.top_k, description='top_k')
-    n: int = Field(1, description='How many completions to generate for each prompt')
-    stream: bool = Field(False, description='Stream responses')
-    repeat_penalty: float = Field(settings.repeat_penalty, description='Repeat penalty')
-
-
-class CompletionChoice(BaseModel):
-    text: str
-    index: int
-    logprobs: float
-    finish_reason: str
-
-
-class CompletionUsage(BaseModel):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-
-class CompletionResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[CompletionChoice]
-    usage: CompletionUsage
-
-
-class CompletionStreamResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[CompletionChoice]
-
-
-router = APIRouter(prefix="/completions", tags=["Completion Endpoints"])
-
-def stream_completion(output: Iterable, base_response: CompletionStreamResponse):
-    """
-    Streams a GPT4All output to the client.
-
-    Args:
-        output: The output of GPT4All.generate(), which is an iterable of tokens.
-        base_response: The base response object, which is cloned and modified for each token.
-
-    Returns:
-        A Generator of CompletionStreamResponse objects, which are serialized to JSON Event Stream format.
-    """
-    for token in output:
-        chunk = base_response.copy()
-        chunk.choices = [dict(CompletionChoice(
-            text=token,
-            index=0,
-            logprobs=-1,
-            finish_reason=''
-        ))]
-        yield f"data: {json.dumps(dict(chunk))}\n\n"
-
-async def gpu_infer(payload, header):
-    async with aiohttp.ClientSession() as session:
-        try:
-            async with session.post(
-                settings.hf_inference_server_host, headers=header, data=json.dumps(payload)
-            ) as response:
-                resp = await response.json()
-            return resp
-
-        except aiohttp.ClientError as e:
-            # Handle client-side errors (e.g., connection error, invalid URL)
-            logger.error(f"Client error: {e}")
-        except aiohttp.ServerError as e:
-            # Handle server-side errors (e.g., internal server error)
-            logger.error(f"Server error: {e}")
-        except json.JSONDecodeError as e:
-            # Handle JSON decoding errors
-            logger.error(f"JSON decoding error: {e}")
-        except Exception as e:
-            # Handle other unexpected exceptions
-            logger.error(f"Unexpected error: {e}")
-
-@router.post("/", response_model=CompletionResponse)
-async def completions(request: CompletionRequest):
-    '''
-    Completes a GPT4All model response.
-    '''
-    if settings.inference_mode == "gpu":
-        params = request.dict(exclude={'model', 'prompt', 'max_tokens', 'n'})
-        params["max_new_tokens"] = request.max_tokens
-        params["num_return_sequences"] = request.n
-
-        header = {"Content-Type": "application/json"}
-        if isinstance(request.prompt, list):
-            tasks = []
-            for prompt in request.prompt:
-                payload = {"parameters": params}
-                payload["inputs"] = prompt
-                task = gpu_infer(payload, header)
-                tasks.append(task)
-            results = await asyncio.gather(*tasks)
-
-            choices = []
-            for response in results:
-                scores = response["scores"] if "scores" in response else -1.0
-                choices.append(
-                    dict(
-                        CompletionChoice(
-                            text=response["generated_text"], index=0, logprobs=scores, finish_reason='stop'
-                        )
-                    )
-                )
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=choices,
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
-
-        else:
-            payload = {"parameters": params}
-            # If streaming, we need to return a StreamingResponse
-            payload["inputs"] = request.prompt
-
-            resp = await gpu_infer(payload, header)
-
-            output = resp["generated_text"]
-            # this returns all logprobs
-            scores = resp["scores"] if "scores" in resp else -1.0
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(text=output, index=0, logprobs=scores, finish_reason='stop'))],
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
-
-    else:
-
-        if request.model != settings.model:
-            raise HTTPException(status_code=400,
-                                detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
-
-        if isinstance(request.prompt, list):
-            if len(request.prompt) > 1:
-                raise HTTPException(status_code=400, detail="Can only infer one inference per request in CPU mode.")
-            else:
-                request.prompt = request.prompt[0]
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        output = model.generate(prompt=request.prompt,
-                                max_tokens=request.max_tokens,
-                                streaming=request.stream,
-                                top_k=request.top_k,
-                                top_p=request.top_p,
-                                temp=request.temperature,
-                                )
-
-        # If streaming, we need to return a StreamingResponse
-        if request.stream:
-            base_chunk = CompletionStreamResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[]
-            )
-            return StreamingResponse((response for response in stream_completion(output, base_chunk)),
-                                     media_type="text/event-stream")
-        else:
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(
-                    text=output,
-                    index=0,
-                    logprobs=-1,
-                    finish_reason='stop'
-                ))],
-                usage={
-                    'prompt_tokens': 0,  # TODO how to compute this?
-                    'completion_tokens': 0,
-                    'total_tokens': 0
-                }
-            )

gpt4all-api/gpt4all_api/app/api_v1/routes/embeddings.py

@@ -1,65 +0,0 @@
-from typing import List, Union
-from fastapi import APIRouter
-from api_v1.settings import settings
-from gpt4all import Embed4All
-from pydantic import BaseModel, Field
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class EmbeddingRequest(BaseModel):
-    model: str = Field(
-        settings.model, description="The model to generate an embedding from."
-    )
-    input: Union[str, List[str], List[int], List[List[int]]] = Field(
-        ..., description="Input text to embed, encoded as a string or array of tokens."
-    )
-
-
-class EmbeddingUsage(BaseModel):
-    prompt_tokens: int = 0
-    total_tokens: int = 0
-
-
-class Embedding(BaseModel):
-    index: int = 0
-    object: str = "embedding"
-    embedding: List[float]
-
-
-class EmbeddingResponse(BaseModel):
-    object: str = "list"
-    model: str
-    data: List[Embedding]
-    usage: EmbeddingUsage
-
-
-router = APIRouter(prefix="/embeddings", tags=["Embedding Endpoints"])
-
-embedder = Embed4All()
-
-
-def get_embedding(data: EmbeddingRequest) -> EmbeddingResponse:
-    """
-    Calculates the embedding for the given input using a specified model.
-
-    Args:
-        data (EmbeddingRequest): An EmbeddingRequest object containing the input data
-        and model name.
-
-    Returns:
-        EmbeddingResponse: An EmbeddingResponse object encapsulating the calculated embedding,
-        usage info, and the model name.
-    """
-    embedding = embedder.embed(data.input)
-    return EmbeddingResponse(
-        data=[Embedding(embedding=embedding)], usage=EmbeddingUsage(), model=data.model
-    )
-
-
-@router.post("/", response_model=EmbeddingResponse)
-def embeddings(data: EmbeddingRequest):
-    """
-    Creates a GPT4All embedding
-    """
-    return get_embedding(data)

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -1,39 +0,0 @@
-import requests
-from fastapi import APIRouter, HTTPException
-from pydantic import BaseModel, Field
-from typing import List, Dict
-
-# Define the router for the engines module
-router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
-
-# Define the models for the engines module
-class ListEnginesResponse(BaseModel):
-    data: List[Dict] = Field(..., description="All available models.")
-
-class EngineResponse(BaseModel):
-    data: List[Dict] = Field(..., description="All available models.")
-
-
-# Define the routes for the engines module
-@router.get("/", response_model=ListEnginesResponse)
-async def list_engines():
-    try:
-        response = requests.get('https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json')
-        response.raise_for_status()  # This will raise an HTTPError if the HTTP request returned an unsuccessful status code
-        engines = response.json()
-        return ListEnginesResponse(data=engines)
-    except requests.RequestException as e:
-        logger.error(f"Error fetching engine list: {e}")
-        raise HTTPException(status_code=500, detail="Error fetching engine list")
-
-# Define the routes for the engines module
-@router.get("/{engine_id}", response_model=EngineResponse)
-async def retrieve_engine(engine_id: str):
-    try:
-        # Implement logic to fetch a specific engine's details
-        # This is a placeholder, replace with your actual data retrieval logic
-        engine_details = {"id": engine_id, "name": "Engine Name", "description": "Engine Description"}
-        return EngineResponse(data=[engine_details])
-    except Exception as e:
-        logger.error(f"Error fetching engine details: {e}")
-        raise HTTPException(status_code=500, detail=f"Error fetching details for engine {engine_id}")

gpt4all-api/gpt4all_api/app/api_v1/routes/health.py

@@ -1,13 +0,0 @@
-import logging
-from fastapi import APIRouter
-from fastapi.responses import JSONResponse
-
-log = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/health", tags=["Health"])
-
-
-@router.get('/', response_class=JSONResponse)
-async def health_check():
-    """Runs a health check on this instance of the API."""
-    return JSONResponse({'status': 'ok'}, headers={'Access-Control-Allow-Origin': '*'})

gpt4all-api/gpt4all_api/app/api_v1/settings.py

@@ -1,19 +0,0 @@
-from pydantic import BaseSettings
-
-
-class Settings(BaseSettings):
-    app_environment = 'dev'
-    model: str = 'ggml-mpt-7b-chat.bin'
-    gpt4all_path: str = '/models'
-    inference_mode: str = "cpu"
-    hf_inference_server_host: str = "http://gpt4all_gpu:80/generate"
-    sentry_dns: str = None
-
-    temp: float = 0.18
-    top_p: float = 1.0
-    top_k: int = 50
-    repeat_penalty: float = 1.18
-
-
-
-settings = Settings()

gpt4all-api/gpt4all_api/app/docs.py

@@ -1,3 +0,0 @@
-desc = 'GPT4All API'
-
-endpoint_paths = {'health': '/health'}

gpt4all-api/gpt4all_api/app/main.py

@@ -1,84 +0,0 @@
-import logging
-import os
-
-import docs
-from api_v1 import events
-from api_v1.api import router as v1_router
-from api_v1.settings import settings
-from fastapi import FastAPI, HTTPException, Request
-from fastapi.logger import logger as fastapi_logger
-from starlette.middleware.cors import CORSMiddleware
-
-logger = logging.getLogger(__name__)
-
-app = FastAPI(title='GPT4All API', description=docs.desc)
-
-# CORS Configuration (in-case you want to deploy)
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["GET", "POST", "OPTIONS"],
-    allow_headers=["*"],
-)
-
-logger.info('Adding v1 endpoints..')
-
-# add v1
-app.include_router(v1_router, prefix='/v1')
-app.add_event_handler('startup', events.startup_event_handler(app))
-app.add_exception_handler(HTTPException, events.on_http_error)
-
-
-@app.on_event("startup")
-async def startup():
-    global model
-    if settings.inference_mode == "cpu":
-        logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
-        from gpt4all import GPT4All
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
-    else:
-        # is it possible to do this once the server is up?
-        ## TODO block until HF inference server is up.
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
-
-
-@app.on_event("shutdown")
-async def shutdown():
-    logger.info("Shutting down API")
-
-
-if settings.sentry_dns is not None:
-    import sentry_sdk
-
-    def traces_sampler(sampling_context):
-        if 'health' in sampling_context['transaction_context']['name']:
-            return False
-
-    sentry_sdk.init(
-        dsn=settings.sentry_dns, traces_sample_rate=0.1, traces_sampler=traces_sampler, send_default_pii=False
-    )
-
-# This is needed to get logs to show up in the app
-if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
-    gunicorn_error_logger = logging.getLogger("gunicorn.error")
-    gunicorn_logger = logging.getLogger("gunicorn")
-
-    root_logger = logging.getLogger()
-    fastapi_logger.setLevel(gunicorn_logger.level)
-    fastapi_logger.handlers = gunicorn_error_logger.handlers
-    root_logger.setLevel(gunicorn_logger.level)
-
-    uvicorn_logger = logging.getLogger("uvicorn.access")
-    uvicorn_logger.handlers = gunicorn_error_logger.handlers
-else:
-    # https://github.com/tiangolo/fastapi/issues/2019
-    LOG_FORMAT2 = (
-        "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
-    )
-    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -1,93 +0,0 @@
-"""
-Use the OpenAI python API to test gpt4all models.
-"""
-from typing import List, get_args
-import os
-from dotenv import load_dotenv
-
-import openai
-
-openai.api_base = "http://localhost:4891/v1"
-openai.api_key = "not needed for a local LLM"
-
-# Load the .env file
-env_path = 'gpt4all-api/gpt4all_api/.env'
-load_dotenv(dotenv_path=env_path)
-
-# Fetch MODEL_ID from .env file
-model_id = os.getenv('MODEL_BIN', 'default_model_id')
-embedding = os.getenv('EMBEDDING', 'default_embedding_model_id')
-print (model_id)
-print (embedding)
-
-def test_completion():
-    model = model_id
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-
-def test_streaming_completion():
-    model = model_id
-    prompt = "Who is Michael Jordan?"
-    tokens = []
-    for resp in openai.Completion.create(
-            model=model,
-            prompt=prompt,
-            max_tokens=50,
-            temperature=0.28,
-            top_p=0.95,
-            n=1,
-            echo=True,
-            stream=True):
-        tokens.append(resp.choices[0].text)
-
-    assert (len(tokens) > 0)
-    assert (len("".join(tokens)) > len(prompt))
-
-# Modified test batch, problems with keyerror in response
-def test_batched_completion():
-    model = model_id  # replace with your specific model ID
-    prompt = "Who is Michael Jordan?"
-    responses = []
-
-    # Loop to create completions one at a time
-    for _ in range(3):
-        response = openai.Completion.create(
-            model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-        )
-        responses.append(response)
-
-    # Assertions to check the responses
-    for response in responses:
-        assert len(response['choices'][0]['text']) > len(prompt)
-
-    assert len(responses) == 3
-
-def test_embedding():
-    model = embedding
-    prompt = "Who is Michael Jordan?"
-    response = openai.Embedding.create(model=model, input=prompt)
-    output = response["data"][0]["embedding"]
-    args = get_args(List[float])
-
-    assert response["model"] == model
-    assert isinstance(output, list)
-    assert all(isinstance(x, args) for x in output)
-
-def test_chat_completion():
-    model = model_id
-
-    response = openai.ChatCompletion.create(
-        model=model,
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": "Knock knock."},
-            {"role": "assistant", "content": "Who's there?"},
-            {"role": "user", "content": "Orange."},
-            ]
-    )
-
-    assert response.choices[0].message.role == "assistant"
-    assert len(response.choices[0].message.content) > 0

gpt4all-api/gpt4all_api/env

@@ -1,3 +0,0 @@
-# Add your GGUF compatible model LLM here. ie: MODEL_BIN="mistral-7b-instruct-v0.1.Q4_0", rename file ".env"
-# Make sure this LLM matches the model you placed inside the models folder
-MODEL_BIN=""

gpt4all-api/gpt4all_api/models/README.md

@@ -1 +0,0 @@
-### Drop GGUF compatible models here, make sure it matches MODEL_BIN on your .env file

gpt4all-api/gpt4all_api/requirements.txt

@@ -1,13 +0,0 @@
-aiohttp>=3.6.2
-aiofiles
-pydantic>=1.4.0,<2.0.0
-requests>=2.24.0
-ujson>=2.0.2
-fastapi>=0.95.0
-Jinja2>=3.0
-gpt4all>=1.0.0
-pytest
-openai==0.28.0
-black
-isort
-python-dotenv

gpt4all-api/makefile

@@ -1,46 +0,0 @@
-ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-APP_NAME:=gpt4all_api
-PYTHON:=python3.8
-SHELL := /bin/bash
-
-all: dependencies
-
-fresh: clean dependencies
-
-testenv: clean_testenv test_build
-	docker compose -f docker-compose.yaml up --build
-
-testenv_gpu: clean_testenv test_build
-	docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up --build
-
-testenv_d: clean_testenv test_build
-	docker compose env up --build -d
-
-test:
-	docker compose exec $(APP_NAME) pytest -svv --disable-warnings -p no:cacheprovider /app/tests
-
-test_build:
-    DOCKER_BUILDKIT=1 docker build -t $(APP_NAME) --progress plain -f $(APP_NAME)/Dockerfile.buildkit .
-
-clean_testenv:
-	docker compose down -v
-
-fresh_testenv: clean_testenv testenv
-
-venv:
-	if [ ! -d $(ROOT_DIR)/venv ]; then $(PYTHON) -m venv $(ROOT_DIR)/venv; fi
-
-dependencies: venv
-	source $(ROOT_DIR)/venv/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
-
-clean: clean_testenv
-	# Remove existing environment
-	rm -rf $(ROOT_DIR)/venv;
-	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
-
-
-black:
-	source $(ROOT_DIR)/venv/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
-
-isort:
-	source $(ROOT_DIR)/venv/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)

gpt4all-backend/CMakeLists.txt

@@ -1,16 +1,24 @@
-cmake_minimum_required(VERSION 3.16)
+cmake_minimum_required(VERSION 3.21)  # for PROJECT_IS_TOP_LEVEL
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-if(APPLE)
-  option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
-  if(BUILD_UNIVERSAL)
+if (APPLE)
+    option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
+else()
+    option(LLMODEL_KOMPUTE "llmodel: use Kompute"              ON)
+    option(LLMODEL_VULKAN  "llmodel: use Vulkan"               OFF)
+    option(LLMODEL_CUDA    "llmodel: use CUDA"                 ON)
+    option(LLMODEL_ROCM    "llmodel: use ROCm"                 OFF)
+endif()
+
+if (APPLE)
+  if (BUILD_UNIVERSAL)
     # Build a Universal binary on macOS
     # This requires that the found Qt library is compiled as Universal binaries.
     set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
   else()
     # Build for the host architecture on macOS
-    if(NOT CMAKE_OSX_ARCHITECTURES)
+    if (NOT CMAKE_OSX_ARCHITECTURES)
       set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
     endif()
   endif()
@@ -39,11 +47,39 @@ else()
     message(STATUS "Interprocedural optimization support detected")
 endif()
 
+set(DIRECTORY llama.cpp-mainline)
 include(llama.cpp.cmake)
 
-set(BUILD_VARIANTS default avxonly)
-if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-    set(BUILD_VARIANTS ${BUILD_VARIANTS} metal)
+set(BUILD_VARIANTS)
+set(GPTJ_BUILD_VARIANT cpu)
+if (APPLE)
+    list(APPEND BUILD_VARIANTS metal)
+endif()
+if (LLMODEL_KOMPUTE)
+    list(APPEND BUILD_VARIANTS kompute kompute-avxonly)
+    set(GPTJ_BUILD_VARIANT kompute)
+else()
+    list(PREPEND BUILD_VARIANTS cpu cpu-avxonly)
+endif()
+if (LLMODEL_VULKAN)
+    list(APPEND BUILD_VARIANTS vulkan vulkan-avxonly)
+endif()
+if (LLMODEL_CUDA)
+    if (DEFINED CMAKE_CUDA_ARCHITECTURES)
+        set(GGML_CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}")
+    endif()
+
+    include(CheckLanguage)
+    check_language(CUDA)
+    if (NOT CMAKE_CUDA_COMPILER)
+        message(WARNING "CUDA Toolkit not found. To build without CUDA, use -DLLMODEL_CUDA=OFF.")
+    endif()
+    enable_language(CUDA)
+    list(APPEND BUILD_VARIANTS cuda cuda-avxonly)
+endif()
+if (LLMODEL_ROCM)
+    enable_language(HIP)
+    list(APPEND BUILD_VARIANTS rocm rocm-avxonly)
 endif()
 
 set(CMAKE_VERBOSE_MAKEFILE ON)
@@ -51,24 +87,34 @@ set(CMAKE_VERBOSE_MAKEFILE ON)
 # Go through each build variant
 foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     # Determine flags
-    if (BUILD_VARIANT STREQUAL avxonly)
-        set(GPT4ALL_ALLOW_NON_AVX NO)
+    if (BUILD_VARIANT MATCHES avxonly)
+        set(GPT4ALL_ALLOW_NON_AVX OFF)
     else()
-        set(GPT4ALL_ALLOW_NON_AVX YES)
+        set(GPT4ALL_ALLOW_NON_AVX ON)
     endif()
     set(LLAMA_AVX2 ${GPT4ALL_ALLOW_NON_AVX})
     set(LLAMA_F16C ${GPT4ALL_ALLOW_NON_AVX})
     set(LLAMA_FMA  ${GPT4ALL_ALLOW_NON_AVX})
 
-    if (BUILD_VARIANT STREQUAL metal)
-        set(LLAMA_METAL YES)
-    else()
-        set(LLAMA_METAL NO)
+    set(LLAMA_METAL   OFF)
+    set(LLAMA_KOMPUTE OFF)
+    set(LLAMA_VULKAN  OFF)
+    set(LLAMA_CUDA    OFF)
+    set(LLAMA_ROCM    OFF)
+    if (BUILD_VARIANT MATCHES metal)
+        set(LLAMA_METAL   ON)
+    elseif (BUILD_VARIANT MATCHES kompute)
+        set(LLAMA_KOMPUTE ON)
+    elseif (BUILD_VARIANT MATCHES vulkan)
+        set(LLAMA_VULKAN  ON)
+    elseif (BUILD_VARIANT MATCHES cuda)
+        set(LLAMA_CUDA    ON)
+    elseif (BUILD_VARIANT MATCHES rocm)
+        set(LLAMA_HIPBLAS ON)
     endif()
 
     # Include GGML
-    set(LLAMA_K_QUANTS YES)
-    include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
+    include_ggml(-mainline-${BUILD_VARIANT})
 
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
@@ -93,17 +139,21 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    if (NOT LLAMA_METAL)
+    if (BUILD_VARIANT MATCHES ${GPTJ_BUILD_VARIANT})
         add_library(gptj-${BUILD_VARIANT} SHARED
             gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
         prepare_target(gptj llama-mainline)
     endif()
+
+    if (NOT PROJECT_IS_TOP_LEVEL AND BUILD_VARIANT STREQUAL cuda)
+        set(CUDAToolkit_BIN_DIR ${CUDAToolkit_BIN_DIR} PARENT_SCOPE)
+    endif()
 endforeach()
 
 add_library(llmodel
     llmodel.h llmodel.cpp llmodel_shared.cpp
     llmodel_c.h llmodel_c.cpp
-    dlhandle.h
+    dlhandle.cpp
 )
 target_compile_definitions(llmodel PRIVATE LIB_FILE_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}")
 

gpt4all-backend/dlhandle.cpp

@@ -0,0 +1,73 @@
+#include "dlhandle.h"
+
+#include <string>
+
+#ifndef _WIN32
+#   include <dlfcn.h>
+#else
+#   include <cassert>
+#   include <sstream>
+#   define WIN32_LEAN_AND_MEAN
+#   ifndef NOMINMAX
+#       define NOMINMAX
+#   endif
+#   include <windows.h>
+#endif
+
+using namespace std::string_literals;
+namespace fs = std::filesystem;
+
+
+#ifndef _WIN32
+
+Dlhandle::Dlhandle(const fs::path &fpath)
+{
+    chandle = dlopen(fpath.c_str(), RTLD_LAZY | RTLD_LOCAL);
+    if (!chandle) {
+        throw Exception("dlopen: "s + dlerror());
+    }
+}
+
+Dlhandle::~Dlhandle()
+{
+    if (chandle) dlclose(chandle);
+}
+
+void *Dlhandle::get_internal(const char *symbol) const
+{
+    return dlsym(chandle, symbol);
+}
+
+#else // defined(_WIN32)
+
+Dlhandle::Dlhandle(const fs::path &fpath)
+{
+    fs::path afpath = fs::absolute(fpath);
+
+    // Suppress the "Entry Point Not Found" dialog, caused by outdated nvcuda.dll from the GPU driver
+    UINT lastErrorMode = GetErrorMode();
+    SetErrorMode(lastErrorMode | SEM_FAILCRITICALERRORS);
+
+    chandle = LoadLibraryExW(afpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
+
+    SetErrorMode(lastErrorMode);
+
+    if (!chandle) {
+        DWORD err = GetLastError();
+        std::ostringstream ss;
+        ss << "LoadLibraryExW failed with error 0x" << std::hex << err;
+        throw Exception(ss.str());
+    }
+}
+
+Dlhandle::~Dlhandle()
+{
+    if (chandle) FreeLibrary(HMODULE(chandle));
+}
+
+void *Dlhandle::get_internal(const char *symbol) const
+{
+    return GetProcAddress(HMODULE(chandle), symbol);
+}
+
+#endif // defined(_WIN32)

gpt4all-backend/dlhandle.h

@@ -1,73 +1,15 @@
-#ifndef DLHANDLE_H
-#define DLHANDLE_H
-#ifndef _WIN32
-#include <string>
-#include <stdexcept>
-#include <utility>
-#include <dlfcn.h>
+#pragma once
 
-
-
-class Dlhandle {
-    void *chandle;
-
-public:
-    class Exception : public std::runtime_error {
-    public:
-        using std::runtime_error::runtime_error;
-    };
-
-    Dlhandle() : chandle(nullptr) {}
-    Dlhandle(const std::string& fpath, int flags = RTLD_LAZY | RTLD_LOCAL) {
-        chandle = dlopen(fpath.c_str(), flags);
-        if (!chandle) {
-            throw Exception("dlopen(\""+fpath+"\"): "+dlerror());
-        }
-    }
-    Dlhandle(const Dlhandle& o) = delete;
-    Dlhandle(Dlhandle&& o) : chandle(o.chandle) {
-        o.chandle = nullptr;
-    }
-    ~Dlhandle() {
-        if (chandle) dlclose(chandle);
-    }
-
-    auto operator =(Dlhandle&& o) {
-        chandle = std::exchange(o.chandle, nullptr);
-    }
-
-    bool is_valid() const {
-        return chandle != nullptr;
-    }
-    operator bool() const {
-        return is_valid();
-    }
-
-    template<typename T>
-    T* get(const std::string& fname) const {
-        auto fres = reinterpret_cast<T*>(dlsym(chandle, fname.c_str()));
-        return (dlerror()==NULL)?fres:nullptr;
-    }
-    auto get_fnc(const std::string& fname) const {
-        return get<void*(...)>(fname);
-    }
-};
-#else
-#include <algorithm>
 #include <filesystem>
-#include <string>
-#include <exception>
 #include <stdexcept>
-#ifndef NOMINMAX
-    #define NOMINMAX
-#endif
-#include <windows.h>
-#include <libloaderapi.h>
+#include <string>
+#include <utility>
 
+namespace fs = std::filesystem;
 
 
 class Dlhandle {
-    HMODULE chandle;
+    void *chandle = nullptr;
 
 public:
     class Exception : public std::runtime_error {
@@ -75,34 +17,31 @@ public:
         using std::runtime_error::runtime_error;
     };
 
-    Dlhandle() : chandle(nullptr) {}
-    Dlhandle(const std::string& fpath) {
-        std::string afpath = std::filesystem::absolute(fpath).string();
-        std::replace(afpath.begin(), afpath.end(), '/', '\\');
-        chandle = LoadLibraryExA(afpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
-        if (!chandle) {
-            throw Exception("dlopen(\""+fpath+"\"): Error");
-        }
-    }
-    Dlhandle(const Dlhandle& o) = delete;
-    Dlhandle(Dlhandle&& o) : chandle(o.chandle) {
+    Dlhandle() = default;
+    Dlhandle(const fs::path &fpath);
+    Dlhandle(const Dlhandle &o) = delete;
+    Dlhandle(Dlhandle &&o)
+        : chandle(o.chandle)
+    {
         o.chandle = nullptr;
     }
-    ~Dlhandle() {
-        if (chandle) FreeLibrary(chandle);
+
+    ~Dlhandle();
+
+    Dlhandle &operator=(Dlhandle &&o) {
+        chandle = std::exchange(o.chandle, nullptr);
+        return *this;
     }
 
-    bool is_valid() const {
-        return chandle != nullptr;
+    template <typename T>
+    T *get(const std::string &symbol) const {
+        return reinterpret_cast<T *>(get_internal(symbol.c_str()));
     }
 
-    template<typename T>
-    T* get(const std::string& fname) const {
-        return reinterpret_cast<T*>(GetProcAddress(chandle, fname.c_str()));
-    }
-    auto get_fnc(const std::string& fname) const {
-        return get<void*(...)>(fname);
+    auto get_fnc(const std::string &symbol) const {
+        return get<void*(...)>(symbol);
     }
+
+private:
+    void *get_internal(const char *symbol) const;
 };
-#endif
-#endif // DLHANDLE_H

gpt4all-backend/gptj.cpp

@@ -1,33 +1,28 @@
 #define GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
 #include "gptj_impl.h"
 
-#include "utils.h"
+#include "llmodel.h"
 #include "llmodel_shared.h"
+#include "utils.h"
 
+#include <ggml.h>
+
+#include <algorithm>
 #include <cassert>
 #include <cinttypes>
 #include <cmath>
 #include <cstdio>
 #include <cstring>
-#include <map>
-#include <string>
-#include <vector>
+#include <ctime>
 #include <iostream>
-#if defined(_WIN32) && defined(_MSC_VER)
-    #define WIN32_LEAN_AND_MEAN
-    #ifndef NOMINMAX
-        #define NOMINMAX
-    #endif
-    #include <windows.h>
-    #include <io.h>
-    #include <stdio.h>
-#else
-    #include <unistd.h>
-#endif
+#include <map>
+#include <memory>
+#include <random>
 #include <sstream>
-#include <unordered_set>
-#include <ggml.h>
-
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
 
 namespace {
 const char *modelType_ = "GPT-J";
@@ -128,7 +123,8 @@ static bool kv_cache_init(
 }
 
 // load the model's weights from a file path
-bool gptj_model_load(const std::string &fname, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
+bool gptj_model_load(const std::string &fname, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr)
+{
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
     if(mem_req != nullptr) {
         *mem_req = 0;
@@ -672,7 +668,8 @@ GPTJ::GPTJ()
     d_ptr->modelLoaded = false;
 }
 
-size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx, int ngl) {
+size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
+{
     (void)n_ctx;
     (void)ngl;
     gptj_model dummy_model;
@@ -682,7 +679,8 @@ size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx, int ngl) {
     return mem_req;
 }
 
-bool GPTJ::loadModel(const std::string &modelPath, int n_ctx, int ngl) {
+bool GPTJ::loadModel(const std::string &modelPath, int n_ctx, int ngl)
+{
     (void)n_ctx;
     (void)ngl;
     d_ptr->modelLoaded = false;
@@ -703,7 +701,8 @@ bool GPTJ::loadModel(const std::string &modelPath, int n_ctx, int ngl) {
     return true;
 }
 
-void GPTJ::setThreadCount(int32_t n_threads) {
+void GPTJ::setThreadCount(int32_t n_threads)
+{
     d_ptr->n_threads = n_threads;
 }
 
@@ -785,13 +784,16 @@ const std::vector<LLModel::Token> &GPTJ::endTokens() const
     return fres;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
+const char *get_arch_name(gguf_context *ctx_gguf)
+{
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    if (kid == -1)
+        throw std::runtime_error("key not found in model: general.architecture");
+
     enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
+    if (ktype != GGUF_TYPE_STRING)
+        throw std::runtime_error("key general.architecture has wrong type");
+
     return gguf_get_val_str(ctx_gguf, kid);
 }
 
@@ -802,36 +804,50 @@ std::string get_arch_name(gguf_context *ctx_gguf) {
 #endif
 
 extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
+DLL_EXPORT bool is_g4a_backend_model_implementation()
+{
     return true;
 }
 
-DLL_EXPORT const char *get_model_type() {
+DLL_EXPORT const char *get_model_type()
+{
     return modelType_;
 }
 
-DLL_EXPORT const char *get_build_variant() {
+DLL_EXPORT const char *get_build_variant()
+{
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
+DLL_EXPORT char *get_file_arch(const char *fname)
+{
     struct ggml_context * ctx_meta = NULL;
     struct gguf_init_params params = {
         /*.no_alloc = */ true,
         /*.ctx      = */ &ctx_meta,
     };
     gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
 
-    bool isValid = gguf_get_version(ctx_gguf) <= 3;
-    isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
+    char *arch = nullptr;
+    if (ctx_gguf && gguf_get_version(ctx_gguf) <= 3) {
+        try {
+            arch = strdup(get_arch_name(ctx_gguf));
+        } catch (const std::runtime_error &) {
+            // cannot read key -> return null
+        }
+    }
 
     gguf_free(ctx_gguf);
-    return isValid;
+    return arch;
 }
 
-DLL_EXPORT LLModel *construct() {
+DLL_EXPORT bool is_arch_supported(const char *arch)
+{
+    return !strcmp(arch, "gptj");
+}
+
+DLL_EXPORT LLModel *construct()
+{
     return new GPTJ;
 }
 }

gpt4all-backend/gptj_impl.h

@@ -4,11 +4,12 @@
 #ifndef GPTJ_H
 #define GPTJ_H
 
-#include <string>
-#include <functional>
-#include <vector>
 #include "llmodel.h"
 
+#include <functional>
+#include <string>
+#include <vector>
+
 struct GPTJPrivate;
 class GPTJ : public LLModel {
 public:

gpt4all-backend/llamamodel.cpp

@@ -1,56 +1,102 @@
 #define LLAMAMODEL_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
 #include "llamamodel_impl.h"
 
+#include "llmodel.h"
+
+#include <ggml.h>
+#include <llama.h>
+
+#include <algorithm>
 #include <cassert>
 #include <cmath>
+#include <cstdint>
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
 #include <fstream>
+#include <functional>
 #include <initializer_list>
 #include <iomanip>
 #include <iostream>
-#include <map>
+#include <iterator>
+#include <memory>
 #include <numeric>
-#include <random>
+#include <optional>
 #include <sstream>
 #include <stdexcept>
 #include <string>
 #include <thread>
-#include <unordered_set>
 #include <vector>
 
-#include <llama.h>
-#include <ggml.h>
 #ifdef GGML_USE_KOMPUTE
-#include <ggml-kompute.h>
+#   include <ggml-kompute.h>
+#elif GGML_USE_VULKAN
+#   include <ggml-vulkan.h>
+#elif GGML_USE_CUDA
+#   include <ggml-cuda.h>
 #endif
 
 using namespace std::string_literals;
 
+
 // Maximum supported GGUF version
 static constexpr int GGUF_VER_MAX = 3;
 
 static const char * const modelType_ = "LLaMA";
 
+// note: same order as LLM_ARCH_NAMES in llama.cpp
 static const std::vector<const char *> KNOWN_ARCHES {
-    "baichuan", "bert", "bloom", "codeshell", "falcon", "gemma", "gpt2", "llama", "mpt", "nomic-bert", "orion",
-    "persimmon", "phi2", "plamo", "qwen", "qwen2", "refact", "stablelm", "starcoder"
+    "llama",
+    "falcon",
+    // "grok", -- 314B parameters
+    "gpt2",
+    // "gptj", -- no inference code
+    // "gptneox", -- no inference code
+    "mpt",
+    "baichuan",
+    "starcoder",
+    // "persimmon", -- CUDA generates garbage
+    "refact",
+    "bert",
+    "nomic-bert",
+    "bloom",
+    "stablelm",
+    "qwen",
+    "qwen2",
+    "qwen2moe",
+    "phi2",
+    "phi3",
+    // "plamo", -- https://github.com/ggerganov/llama.cpp/issues/5669
+    "codeshell",
+    "orion",
+    "internlm2",
+    // "minicpm", -- CUDA generates garbage
+    "gemma",
+    "starcoder2",
+    // "mamba", -- CUDA missing SSM_CONV
+    "xverse",
+    "command-r",
+    // "dbrx", -- 16x12B parameters
+    "olmo",
 };
 
 static const std::vector<const char *> EMBEDDING_ARCHES {
-    "bert", "nomic-bert"
+    "bert", "nomic-bert",
 };
 
-static bool is_embedding_arch(const std::string &arch) {
+static bool is_embedding_arch(const std::string &arch)
+{
     return std::find(EMBEDDING_ARCHES.begin(), EMBEDDING_ARCHES.end(), arch) < EMBEDDING_ARCHES.end();
 }
 
-static bool llama_verbose() {
+static bool llama_verbose()
+{
     const char* var = getenv("GPT4ALL_VERBOSE_LLAMACPP");
     return var && *var;
 }
 
-static void llama_log_callback(enum ggml_log_level level, const char *text, void *userdata) {
+static void llama_log_callback(enum ggml_log_level level, const char *text, void *userdata)
+{
     (void)userdata;
     if (llama_verbose() || level <= GGML_LOG_LEVEL_ERROR) {
         fputs(text, stderr);
@@ -104,17 +150,21 @@ static int llama_sample_top_p_top_k(
     return llama_sample_token(ctx, &candidates_p);
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
+const char *get_arch_name(gguf_context *ctx_gguf)
+{
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    if (kid == -1)
+        throw std::runtime_error("key not found in model: general.architecture");
+
     enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != (GGUF_TYPE_STRING)) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
+    if (ktype != GGUF_TYPE_STRING)
+        throw std::runtime_error("key general.architecture has wrong type");
+
     return gguf_get_val_str(ctx_gguf, kid);
 }
 
-static gguf_context *load_gguf(const char *fname) {
+static gguf_context *load_gguf(const char *fname)
+{
     struct gguf_init_params params = {
         /*.no_alloc = */ true,
         /*.ctx      = */ nullptr,
@@ -135,14 +185,22 @@ static gguf_context *load_gguf(const char *fname) {
     return ctx;
 }
 
-static int32_t get_arch_key_u32(std::string const &modelPath, std::string const &archKey) {
+static int32_t get_arch_key_u32(std::string const &modelPath, std::string const &archKey)
+{
+    int32_t value = -1;
+    std::string arch;
+
     auto * ctx = load_gguf(modelPath.c_str());
     if (!ctx)
-        return -1;
-    std::string arch = get_arch_name(ctx);
+        goto cleanup;
 
-    int32_t value = -1;
-    if (ctx) {
+    try {
+        arch = get_arch_name(ctx);
+    } catch (const std::runtime_error &) {
+        goto cleanup; // cannot read key
+    }
+
+    {
         auto key = arch + "." + archKey;
         int keyidx = gguf_find_key(ctx, key.c_str());
         if (keyidx != -1) {
@@ -152,6 +210,7 @@ static int32_t get_arch_key_u32(std::string const &modelPath, std::string const
         }
     }
 
+cleanup:
     gguf_free(ctx);
     return value;
 }
@@ -160,6 +219,7 @@ struct LLamaPrivate {
     const std::string modelPath;
     bool modelLoaded = false;
     int device = -1;
+    std::string deviceName;
     llama_model *model = nullptr;
     llama_context *ctx = nullptr;
     llama_model_params model_params;
@@ -183,7 +243,8 @@ struct llama_file_hparams {
     enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16;
 };
 
-size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx, int ngl) {
+size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
+{
     // TODO(cebtenzzre): update to GGUF
     (void)ngl; // FIXME(cetenzzre): use this value
     auto fin = std::ifstream(modelPath, std::ios::binary);
@@ -207,7 +268,8 @@ size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
     return filesize + est_kvcache_size;
 }
 
-bool LLamaModel::isModelBlacklisted(const std::string &modelPath) const {
+bool LLamaModel::isModelBlacklisted(const std::string &modelPath) const
+{
     auto * ctx = load_gguf(modelPath.c_str());
     if (!ctx) {
         std::cerr << __func__ << ": failed to load " << modelPath << "\n";
@@ -243,16 +305,28 @@ bool LLamaModel::isModelBlacklisted(const std::string &modelPath) const {
     return res;
 }
 
-bool LLamaModel::isEmbeddingModel(const std::string &modelPath) const {
+bool LLamaModel::isEmbeddingModel(const std::string &modelPath) const
+{
+    bool result = false;
+    std::string arch;
+
     auto *ctx_gguf = load_gguf(modelPath.c_str());
     if (!ctx_gguf) {
         std::cerr << __func__ << ": failed to load GGUF from " <<  modelPath << "\n";
-        return false;
+        goto cleanup;
     }
 
-    std::string arch = get_arch_name(ctx_gguf);
+    try {
+        arch = get_arch_name(ctx_gguf);
+    } catch (const std::runtime_error &) {
+        goto cleanup; // cannot read key
+    }
+
+    result = is_embedding_arch(arch);
+
+cleanup:
     gguf_free(ctx_gguf);
-    return is_embedding_arch(arch);
+    return result;
 }
 
 bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
@@ -292,10 +366,16 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
 
     d_ptr->backend_name = "cpu"; // default
 
-#ifdef GGML_USE_KOMPUTE
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
     if (d_ptr->device != -1) {
         d_ptr->model_params.main_gpu = d_ptr->device;
         d_ptr->model_params.n_gpu_layers = ngl;
+        d_ptr->model_params.split_mode = LLAMA_SPLIT_MODE_NONE;
+    } else {
+#ifdef GGML_USE_CUDA
+        std::cerr << "Llama ERROR: CUDA loadModel was called without a device\n";
+        return false;
+#endif // GGML_USE_CUDA
     }
 #elif defined(GGML_USE_METAL)
     (void)ngl;
@@ -308,14 +388,17 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     // always fully offload on Metal
     // TODO(cebtenzzre): use this parameter to allow using more than 53% of system RAM to load a model
     d_ptr->model_params.n_gpu_layers = 100;
-#else
+#else // !KOMPUTE && !VULKAN && !CUDA && !METAL
     (void)ngl;
 #endif
 
-    d_ptr->model = llama_load_model_from_file_gpt4all(modelPath.c_str(), &d_ptr->model_params);
+    d_ptr->model = llama_load_model_from_file(modelPath.c_str(), d_ptr->model_params);
     if (!d_ptr->model) {
         fflush(stdout);
+#ifndef GGML_USE_CUDA
         d_ptr->device = -1;
+        d_ptr->deviceName.clear();
+#endif
         std::cerr << "LLAMA ERROR: failed to load model from " << modelPath << std::endl;
         return false;
     }
@@ -327,7 +410,8 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     bool isEmbedding = is_embedding_arch(llama_model_arch(d_ptr->model));
     const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
     if (isEmbedding) {
-        d_ptr->ctx_params.n_batch = n_ctx;
+        d_ptr->ctx_params.n_batch  = n_ctx;
+        d_ptr->ctx_params.n_ubatch = n_ctx;
     } else {
         if (n_ctx > n_ctx_train) {
             std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
@@ -357,18 +441,27 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
         std::cerr << "LLAMA ERROR: failed to init context for model " <<  modelPath << std::endl;
         llama_free_model(d_ptr->model);
         d_ptr->model = nullptr;
+#ifndef GGML_USE_CUDA
         d_ptr->device = -1;
+        d_ptr->deviceName.clear();
+#endif
         return false;
     }
 
     d_ptr->end_tokens = {llama_token_eos(d_ptr->model)};
 
+    if (usingGPUDevice()) {
 #ifdef GGML_USE_KOMPUTE
-    if (usingGPUDevice() && ggml_vk_has_device()) {
-        std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
+        if (llama_verbose()) {
+            std::cerr << "llama.cpp: using Vulkan on " << d_ptr->deviceName << std::endl;
+        }
         d_ptr->backend_name = "kompute";
-    }
+#elif defined(GGML_USE_VULKAN)
+        d_ptr->backend_name = "vulkan";
+#elif defined(GGML_USE_CUDA)
+        d_ptr->backend_name = "cuda";
 #endif
+    }
 
     m_supportsEmbedding = isEmbedding;
     m_supportsCompletion = !isEmbedding;
@@ -378,12 +471,14 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     return true;
 }
 
-void LLamaModel::setThreadCount(int32_t n_threads) {
+void LLamaModel::setThreadCount(int32_t n_threads)
+{
     d_ptr->n_threads = n_threads;
     llama_set_n_threads(d_ptr->ctx, n_threads, n_threads);
 }
 
-int32_t LLamaModel::threadCount() const {
+int32_t LLamaModel::threadCount() const
+{
     return d_ptr->n_threads;
 }
 
@@ -429,7 +524,18 @@ std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::
 
 std::string LLamaModel::tokenToString(Token id) const
 {
-    return llama_token_to_piece(d_ptr->ctx, id);
+    std::vector<char> result(8, 0);
+    const int n_tokens = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), false);
+    if (n_tokens < 0) {
+        result.resize(-n_tokens);
+        int check = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), false);
+        GGML_ASSERT(check == -n_tokens);
+    }
+    else {
+        result.resize(n_tokens);
+    }
+
+    return std::string(result.data(), result.size());
 }
 
 LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
@@ -494,34 +600,78 @@ int32_t LLamaModel::layerCount(std::string const &modelPath) const
     return get_arch_key_u32(modelPath, "block_count");
 }
 
+#ifdef GGML_USE_VULKAN
+static const char *getVulkanVendorName(uint32_t vendorID)
+{
+    switch (vendorID) {
+        case 0x10DE: return "nvidia";
+        case 0x1002: return "amd";
+        case 0x8086: return "intel";
+        default:     return "unknown";
+    }
+}
+#endif
+
 std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryRequired) const
 {
-#ifdef GGML_USE_KOMPUTE
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
     size_t count = 0;
-    auto * vkDevices = ggml_vk_available_devices(memoryRequired, &count);
 
-    if (vkDevices) {
+#ifdef GGML_USE_KOMPUTE
+    auto *lcppDevices = ggml_vk_available_devices(memoryRequired, &count);
+#elif defined(GGML_USE_VULKAN)
+    (void)memoryRequired; // hasn't been used since GGUF was added
+    auto *lcppDevices = ggml_vk_available_devices(&count);
+#else // defined(GGML_USE_CUDA)
+    (void)memoryRequired;
+    auto *lcppDevices = ggml_cuda_available_devices(&count);
+#endif
+
+    if (lcppDevices) {
         std::vector<LLModel::GPUDevice> devices;
         devices.reserve(count);
 
         for (size_t i = 0; i < count; ++i) {
-            auto & dev = vkDevices[i];
+            auto & dev = lcppDevices[i];
+
             devices.emplace_back(
+#ifdef GGML_USE_KOMPUTE
+                /* backend  = */ "kompute",
                 /* index    = */ dev.index,
                 /* type     = */ dev.type,
                 /* heapSize = */ dev.heapSize,
                 /* name     = */ dev.name,
                 /* vendor   = */ dev.vendor
+#elif defined(GGML_USE_VULKAN)
+                /* backend  = */ "vulkan",
+                /* index    = */ dev.index,
+                /* type     = */ dev.type,
+                /* heapSize = */ dev.heapSize,
+                /* name     = */ dev.name,
+                /* vendor   = */ getVulkanVendorName(dev.vendorID)
+#else // defined(GGML_USE_CUDA)
+                /* backend  = */ "cuda",
+                /* index    = */ dev.index,
+                /* type     = */ 2, // vk::PhysicalDeviceType::eDiscreteGpu
+                /* heapSize = */ dev.heapSize,
+                /* name     = */ dev.name,
+                /* vendor   = */ "nvidia"
+#endif
             );
+
+#ifndef GGML_USE_CUDA
             ggml_vk_device_destroy(&dev);
+#else
+            ggml_cuda_device_destroy(&dev);
+#endif
         }
 
-        free(vkDevices);
+        free(lcppDevices);
         return devices;
     }
 #else
     (void)memoryRequired;
-    std::cerr << __func__ << ": built without Kompute\n";
+    std::cerr << __func__ << ": built without a GPU backend\n";
 #endif
 
     return {};
@@ -529,11 +679,32 @@ std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryReq
 
 bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string &name) const
 {
-#if defined(GGML_USE_KOMPUTE)
+#if defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
+    auto devices = availableGPUDevices(memoryRequired);
+
+    auto dev_it = devices.begin();
+#ifndef GGML_USE_CUDA
+    if (name == "amd" || name == "nvidia" || name == "intel") {
+        dev_it = std::find_if(dev_it, devices.end(), [&name](auto &dev) { return dev.vendor == name; });
+    } else
+#endif
+    if (name != "gpu") {
+        dev_it = std::find_if(dev_it, devices.end(), [&name](auto &dev) { return dev.name == name; });
+    }
+
+    if (dev_it < devices.end()) {
+        d_ptr->device     = dev_it->index;
+        d_ptr->deviceName = dev_it->name;
+        return true;
+    }
+    return false;
+#elif defined(GGML_USE_KOMPUTE)
     ggml_vk_device device;
     bool ok = ggml_vk_get_device(&device, memoryRequired, name.c_str());
     if (ok) {
         d_ptr->device = device.index;
+        d_ptr->deviceName = device.name;
+        ggml_vk_device_destroy(&device);
         return true;
     }
 #else
@@ -545,37 +716,49 @@ bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string &n
 
 bool LLamaModel::initializeGPUDevice(int device, std::string *unavail_reason) const
 {
-#if defined(GGML_USE_KOMPUTE)
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
     (void)unavail_reason;
+    auto devices = availableGPUDevices();
+    auto it = std::find_if(devices.begin(), devices.end(), [device](auto &dev) { return dev.index == device; });
     d_ptr->device = device;
+    d_ptr->deviceName = it < devices.end() ? it->name : "(unknown)";
     return true;
 #else
     (void)device;
     if (unavail_reason) {
-        *unavail_reason = "built without Kompute";
+        *unavail_reason = "built without a GPU backend";
     }
     return false;
 #endif
 }
 
-bool LLamaModel::hasGPUDevice()
+bool LLamaModel::usingGPUDevice() const
 {
-#if defined(GGML_USE_KOMPUTE)
-    return d_ptr->device != -1;
-#else
-    return false;
+    if (!d_ptr->model)
+        return false;
+
+    bool usingGPU = llama_model_using_gpu(d_ptr->model);
+#ifdef GGML_USE_KOMPUTE
+    assert(!usingGPU || ggml_vk_has_device());
 #endif
+    return usingGPU;
 }
 
-bool LLamaModel::usingGPUDevice()
+const char *LLamaModel::backendName() const
 {
-#if defined(GGML_USE_KOMPUTE)
-    return hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
+    return d_ptr->backend_name;
+}
+
+const char *LLamaModel::gpuDeviceName() const
+{
+    if (usingGPUDevice()) {
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
+        return d_ptr->deviceName.c_str();
 #elif defined(GGML_USE_METAL)
-    return true;
-#else
-    return false;
+        return "Metal";
 #endif
+    }
+    return nullptr;
 }
 
 void llama_batch_add(
@@ -595,13 +778,15 @@ void llama_batch_add(
     batch.n_tokens++;
 }
 
-static void batch_add_seq(llama_batch &batch, const std::vector<LLModel::Token> &tokens, int seq_id) {
+static void batch_add_seq(llama_batch &batch, const std::vector<LLModel::Token> &tokens, int seq_id)
+{
     for (unsigned i = 0; i < tokens.size(); i++) {
         llama_batch_add(batch, tokens[i], i, { seq_id }, i == tokens.size() - 1);
     }
 }
 
-size_t LLamaModel::embeddingSize() const {
+size_t LLamaModel::embeddingSize() const
+{
     return llama_n_embd(d_ptr->model);
 }
 
@@ -721,12 +906,14 @@ void LLamaModel::embed(
 // MD5 hash of "nomic empty"
 static const char EMPTY_PLACEHOLDER[] = "24df574ea1c998de59d5be15e769658e";
 
-auto product(double a) -> std::function<double(double)> {
+auto product(double a) -> std::function<double(double)>
+{
     return [a](double b) { return a * b; };
 }
 
 template <typename T>
-double getL2NormScale(T *start, T *end) {
+double getL2NormScale(T *start, T *end)
+{
     double magnitude = std::sqrt(std::inner_product(start, end, start, 0.0));
     return 1.0 / std::max(magnitude, 1e-12);
 }
@@ -755,11 +942,12 @@ void LLamaModel::embedInternal(
         tokens.resize(text.length()+4);
         int32_t n_tokens = llama_tokenize(d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), wantBOS, false);
         if (n_tokens) {
-            assert(useEOS == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
-            tokens.resize(n_tokens - useEOS); // erase EOS/SEP
-        } else {
-            tokens.clear();
+            (void)eos_token;
+            assert((useEOS && wantBOS) == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
+            if (useEOS && wantBOS)
+                n_tokens--; // erase EOS/SEP
         }
+        tokens.resize(n_tokens);
     };
 
     // tokenize the texts
@@ -811,14 +999,14 @@ void LLamaModel::embedInternal(
     size_t totalTokens = 0;
     for (unsigned i = 0; i < inputs.size(); i++) {
         auto &input = inputs[i];
-        for (auto it = input.begin(); it < input.end(); it += max_len) {
-            if (it > input.begin()) { it -= chunkOverlap; }
-            auto end = std::min(it + max_len, input.end());
+        for (unsigned j = 0; j < input.size(); j += max_len) {
+            if (j) { j -= chunkOverlap; }
+            unsigned end = std::min(j + max_len, unsigned(input.size()));
             batches.push_back({ i, {} });
             auto &batch = batches.back().batch;
             batch = prefixTokens;
-            batch.insert(batch.end(), it, end);
-            totalTokens += end - it;
+            batch.insert(batch.end(), input.begin() + j, input.begin() + end);
+            totalTokens += end - j;
             batch.push_back(eos_token);
             if (!doMean) { break; /* limit text to one chunk */ }
         }
@@ -922,6 +1110,8 @@ void LLamaModel::embedInternal(
     }
 
     if (tokenCount) { *tokenCount = totalTokens; }
+
+    llama_batch_free(batch);
 }
 
 #if defined(_WIN32)
@@ -931,40 +1121,54 @@ void LLamaModel::embedInternal(
 #endif
 
 extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
+DLL_EXPORT bool is_g4a_backend_model_implementation()
+{
     return true;
 }
 
-DLL_EXPORT const char *get_model_type() {
+DLL_EXPORT const char *get_model_type()
+{
     return modelType_;
 }
 
-DLL_EXPORT const char *get_build_variant() {
+DLL_EXPORT const char *get_build_variant()
+{
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char *fname) {
-    auto * ctx = load_gguf(fname);
-    std::string arch = get_arch_name(ctx);
+DLL_EXPORT char *get_file_arch(const char *fname)
+{
+    char *arch = nullptr;
+    std::string archStr;
 
-    bool valid = true;
+    auto *ctx = load_gguf(fname);
+    if (!ctx)
+        goto cleanup;
 
-    if (std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), arch) == KNOWN_ARCHES.end()) {
-        // not supported by this version of llama.cpp
-        if (arch != "gptj") { // we support this via another module
-            std::cerr << __func__ << ": unsupported model architecture: " << arch << "\n";
-        }
-        valid = false;
+    try {
+        archStr = get_arch_name(ctx);
+    } catch (const std::runtime_error &) {
+        goto cleanup; // cannot read key
     }
 
-    if (valid && is_embedding_arch(arch) && gguf_find_key(ctx, (arch + ".pooling_type").c_str()) < 0)
-        valid = false; // old pre-llama.cpp embedding model, e.g. all-MiniLM-L6-v2-f16.gguf
+    if (is_embedding_arch(archStr) && gguf_find_key(ctx, (archStr + ".pooling_type").c_str()) < 0) {
+        // old bert.cpp embedding model
+    } else {
+        arch = strdup(archStr.c_str());
+    }
 
+cleanup:
     gguf_free(ctx);
-    return valid;
+    return arch;
 }
 
-DLL_EXPORT LLModel *construct() {
+DLL_EXPORT bool is_arch_supported(const char *arch)
+{
+    return std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), std::string(arch)) < KNOWN_ARCHES.end();
+}
+
+DLL_EXPORT LLModel *construct()
+{
     llama_log_set(llama_log_callback, nullptr);
     return new LLamaModel;
 }

gpt4all-backend/llamamodel_impl.h

@@ -4,11 +4,12 @@
 #ifndef LLAMAMODEL_H
 #define LLAMAMODEL_H
 
+#include "llmodel.h"
+
 #include <functional>
 #include <memory>
 #include <string>
 #include <vector>
-#include "llmodel.h"
 
 struct LLamaPrivate;
 struct EmbModelSpec;
@@ -30,11 +31,12 @@ public:
     size_t restoreState(const uint8_t *src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
-    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const override;
+    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0) const override;
     bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const override;
     bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const override;
-    bool hasGPUDevice() override;
-    bool usingGPUDevice() override;
+    bool usingGPUDevice() const override;
+    const char *backendName() const override;
+    const char *gpuDeviceName() const override;
 
     size_t embeddingSize() const override;
     // user-specified prefix

gpt4all-backend/llmodel.cpp

@@ -1,20 +1,45 @@
 #include "llmodel.h"
+
 #include "dlhandle.h"
-#include "sysinfo.h"
 
 #include <cassert>
 #include <cstdlib>
 #include <filesystem>
 #include <fstream>
 #include <iostream>
+#include <iterator>
 #include <memory>
+#include <optional>
 #include <regex>
 #include <sstream>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
+#ifdef _WIN32
+#   define WIN32_LEAN_AND_MEAN
+#   ifndef NOMINMAX
+#       define NOMINMAX
+#   endif
+#   include <windows.h>
+#endif
+
 #ifdef _MSC_VER
-#include <intrin.h>
+#   include <intrin.h>
+#endif
+
+#if defined(__APPLE__) && defined(__aarch64__)
+#   include "sysinfo.h" // for getSystemTotalRAMInBytes
+#endif
+
+namespace fs = std::filesystem;
+
+#ifndef __APPLE__
+static const std::string DEFAULT_BACKENDS[] = {"kompute", "cpu"};
+#elif defined(__aarch64__)
+static const std::string DEFAULT_BACKENDS[] = {"metal", "cpu"};
+#else
+static const std::string DEFAULT_BACKENDS[] = {"cpu"};
 #endif
 
 std::string s_implementations_search_path = ".";
@@ -32,13 +57,13 @@ std::string s_implementations_search_path = ".";
     }
 
     // AVX via EAX=1: Processor Info and Feature Bits, bit 28 of ECX
-    #define cpu_supports_avx()  (get_cpu_info(1, 2) & (1 << 28))
+    #define cpu_supports_avx()  !!(get_cpu_info(1, 2) & (1 << 28))
     // AVX2 via EAX=7, ECX=0: Extended Features, bit 5 of EBX
-    #define cpu_supports_avx2() (get_cpu_info(7, 1) & (1 <<  5))
+    #define cpu_supports_avx2() !!(get_cpu_info(7, 1) & (1 <<  5))
 #else
     // gcc/clang
-    #define cpu_supports_avx()  __builtin_cpu_supports("avx")
-    #define cpu_supports_avx2() __builtin_cpu_supports("avx2")
+    #define cpu_supports_avx()  !!__builtin_cpu_supports("avx")
+    #define cpu_supports_avx2() !!__builtin_cpu_supports("avx2")
 #endif
 
 LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
@@ -49,14 +74,17 @@ LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
     m_buildVariant = get_build_variant();
-    m_magicMatch = m_dlhandle->get<bool(const char*)>("magic_match");
-    assert(m_magicMatch);
+    m_getFileArch = m_dlhandle->get<char *(const char *)>("get_file_arch");
+    assert(m_getFileArch);
+    m_isArchSupported = m_dlhandle->get<bool(const char *)>("is_arch_supported");
+    assert(m_isArchSupported);
     m_construct = m_dlhandle->get<LLModel *()>("construct");
     assert(m_construct);
 }
 
 LLModel::Implementation::Implementation(Implementation &&o)
-    : m_magicMatch(o.m_magicMatch)
+    : m_getFileArch(o.m_getFileArch)
+    , m_isArchSupported(o.m_isArchSupported)
     , m_construct(o.m_construct)
     , m_modelType(o.m_modelType)
     , m_buildVariant(o.m_buildVariant)
@@ -64,15 +92,33 @@ LLModel::Implementation::Implementation(Implementation &&o)
     o.m_dlhandle = nullptr;
 }
 
-LLModel::Implementation::~Implementation() {
+LLModel::Implementation::~Implementation()
+{
     delete m_dlhandle;
 }
 
-static bool isImplementation(const Dlhandle &dl) {
+static bool isImplementation(const Dlhandle &dl)
+{
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
-const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
+// Add the CUDA Toolkit to the DLL search path on Windows.
+// This is necessary for chat.exe to find CUDA when started from Qt Creator.
+static void addCudaSearchPath()
+{
+#ifdef _WIN32
+    if (const auto *cudaPath = _wgetenv(L"CUDA_PATH")) {
+        auto libDir = std::wstring(cudaPath) + L"\\bin";
+        if (!AddDllDirectory(libDir.c_str())) {
+            auto err = GetLastError();
+            std::wcerr << L"AddDllDirectory(\"" << libDir << L"\") failed with error 0x" << std::hex << err << L"\n";
+        }
+    }
+#endif
+}
+
+const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList()
+{
     if (cpu_supports_avx() == 0) {
         throw std::runtime_error("CPU does not support AVX");
     }
@@ -82,11 +128,11 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     static auto* libs = new std::vector<Implementation>([] () {
         std::vector<Implementation> fres;
 
-        std::string impl_name_re = "(gptj|llamamodel-mainline)";
+        addCudaSearchPath();
+
+        std::string impl_name_re = "(gptj|llamamodel-mainline)-(cpu|metal|kompute|vulkan|cuda)";
         if (cpu_supports_avx2() == 0) {
             impl_name_re += "-avxonly";
-        } else {
-            impl_name_re += "-(default|metal)";
         }
         std::regex re(impl_name_re);
         auto search_in_directory = [&](const std::string& paths) {
@@ -94,21 +140,27 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
             std::string path;
             // Split the paths string by the delimiter and process each path.
             while (std::getline(ss, path, ';')) {
-                std::filesystem::path fs_path(path);
+                std::u8string u8_path(path.begin(), path.end());
                 // Iterate over all libraries
-                for (const auto& f : std::filesystem::directory_iterator(fs_path)) {
-                    const std::filesystem::path& p = f.path();
+                for (const auto &f : fs::directory_iterator(u8_path)) {
+                    const fs::path &p = f.path();
 
                     if (p.extension() != LIB_FILE_EXT) continue;
                     if (!std::regex_search(p.stem().string(), re)) continue;
 
                     // Add to list if model implementation
+                    Dlhandle dl;
                     try {
-                        Dlhandle dl(p.string());
-                        if (!isImplementation(dl))
-                            continue;
-                        fres.emplace_back(Implementation(std::move(dl)));
-                    } catch (...) {}
+                        dl = Dlhandle(p);
+                    } catch (const Dlhandle::Exception &e) {
+                        std::cerr << "Failed to load " << p.filename().string() << ": " << e.what() << "\n";
+                        continue;
+                    }
+                    if (!isImplementation(dl)) {
+                        std::cerr << "Not an implementation: " << p.filename().string() << "\n";
+                        continue;
+                    }
+                    fres.emplace_back(Implementation(std::move(dl)));
                 }
             }
         };
@@ -121,127 +173,175 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     return *libs;
 }
 
-const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
+static std::string applyCPUVariant(const std::string &buildVariant)
+{
+    if (buildVariant != "metal" && cpu_supports_avx2() == 0) {
+        return buildVariant + "-avxonly";
+    }
+    return buildVariant;
+}
+
+const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant)
+{
     bool buildVariantMatched = false;
+    std::optional<std::string> archName;
     for (const auto& i : implementationList()) {
         if (buildVariant != i.m_buildVariant) continue;
         buildVariantMatched = true;
 
-        if (!i.m_magicMatch(fname)) continue;
-        return &i;
+        char *arch = i.m_getFileArch(fname);
+        if (!arch) continue;
+        archName = arch;
+
+        bool archSupported = i.m_isArchSupported(arch);
+        free(arch);
+        if (archSupported) return &i;
     }
 
     if (!buildVariantMatched)
-        throw std::runtime_error("Could not find any implementations for build variant: " + buildVariant);
+        return nullptr;
+    if (!archName)
+        throw UnsupportedModelError("Unsupported file format");
 
-    return nullptr; // unsupported model format
+    throw BadArchError(std::move(*archName));
 }
 
-LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant, int n_ctx) {
-    // Get correct implementation
-    const Implementation* impl = nullptr;
-
-    #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
-        if (buildVariant == "auto") {
-            size_t total_mem = getSystemTotalRAMInBytes();
-            impl = implementation(modelPath.c_str(), "metal");
-            if(impl) {
-                LLModel* metalimpl = impl->m_construct();
-                metalimpl->m_implementation = impl;
-                /* TODO(cebtenzzre): after we fix requiredMem, we should change this to happen at
-                 * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
-                 * most (all?) places where this is called, causing underestimation of required
-                 * memory. */
-                size_t req_mem = metalimpl->requiredMem(modelPath, n_ctx, 100);
-                float req_to_total = (float) req_mem / (float) total_mem;
-                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
-                if (req_to_total >= 0.53) {
-                    delete metalimpl;
-                    impl = nullptr;
-                } else {
-                    return metalimpl;
-                }
-            }
-        }
-    #else
-        (void)n_ctx;
-    #endif
-
-    if (!impl) {
-        //TODO: Auto-detect CUDA/OpenCL
-        if (buildVariant == "auto") {
-            if (cpu_supports_avx2() == 0) {
-                buildVariant = "avxonly";
-            } else {
-                buildVariant = "default";
-            }
-        }
-        impl = implementation(modelPath.c_str(), buildVariant);
-        if (!impl) return nullptr;
+LLModel *LLModel::Implementation::construct(const std::string &modelPath, const std::string &backend, int n_ctx)
+{
+    std::vector<std::string> desiredBackends;
+    if (backend != "auto") {
+        desiredBackends.push_back(backend);
+    } else {
+        desiredBackends.insert(desiredBackends.end(), DEFAULT_BACKENDS, std::end(DEFAULT_BACKENDS));
     }
 
-    // Construct and return llmodel implementation
-    auto fres = impl->m_construct();
-    fres->m_implementation = impl;
-    return fres;
+    for (const auto &desiredBackend: desiredBackends) {
+        const auto *impl = implementation(modelPath.c_str(), applyCPUVariant(desiredBackend));
+
+        if (impl) {
+            // Construct llmodel implementation
+            auto *fres = impl->m_construct();
+            fres->m_implementation = impl;
+
+#if defined(__APPLE__) && defined(__aarch64__) // FIXME: See if metal works for intel macs
+            /* TODO(cebtenzzre): after we fix requiredMem, we should change this to happen at
+             * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
+             * most (all?) places where this is called, causing underestimation of required
+             * memory. */
+            if (backend == "auto" && desiredBackend == "metal") {
+                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
+                size_t req_mem = fres->requiredMem(modelPath, n_ctx, 100);
+                if (req_mem >= size_t(0.53f * getSystemTotalRAMInBytes())) {
+                    delete fres;
+                    continue;
+                }
+            }
+#else
+            (void)n_ctx;
+#endif
+
+            return fres;
+        }
+    }
+
+    throw MissingImplementationError("Could not find any implementations for backend: " + backend);
 }
 
-LLModel *LLModel::Implementation::constructDefaultLlama() {
-    static std::unique_ptr<LLModel> llama([]() -> LLModel * {
-        const std::vector<LLModel::Implementation> *impls;
-        try {
-            impls = &implementationList();
-        } catch (const std::runtime_error &e) {
-            std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
-            return nullptr;
-        }
+LLModel *LLModel::Implementation::constructGlobalLlama(const std::optional<std::string> &backend)
+{
+    static std::unordered_map<std::string, std::unique_ptr<LLModel>> implCache;
+
+    const std::vector<Implementation> *impls;
+    try {
+        impls = &implementationList();
+    } catch (const std::runtime_error &e) {
+        std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
+        return nullptr;
+    }
+
+    std::vector<std::string> desiredBackends;
+    if (backend) {
+        desiredBackends.push_back(backend.value());
+    } else {
+        desiredBackends.insert(desiredBackends.end(), DEFAULT_BACKENDS, std::end(DEFAULT_BACKENDS));
+    }
+
+    const Implementation *impl = nullptr;
+
+    for (const auto &desiredBackend: desiredBackends) {
+        auto cacheIt = implCache.find(desiredBackend);
+        if (cacheIt != implCache.end())
+            return cacheIt->second.get(); // cached
 
-        const LLModel::Implementation *impl = nullptr;
         for (const auto &i: *impls) {
-            if (i.m_buildVariant == "metal" || i.m_modelType != "LLaMA") continue;
-            impl = &i;
-        }
-        if (!impl) {
-            std::cerr << __func__ << ": could not find llama.cpp implementation\n";
-            return nullptr;
+            if (i.m_modelType == "LLaMA" && i.m_buildVariant == applyCPUVariant(desiredBackend)) {
+                impl = &i;
+                break;
+            }
         }
 
-        auto fres = impl->m_construct();
-        fres->m_implementation = impl;
-        return fres;
-    }());
-    return llama.get();
+        if (impl) {
+            auto *fres = impl->m_construct();
+            fres->m_implementation = impl;
+            implCache[desiredBackend] = std::unique_ptr<LLModel>(fres);
+            return fres;
+        }
+    }
+
+    std::cerr << __func__ << ": could not find Llama implementation for backend: " << backend.value_or("default") << "\n";
+    return nullptr;
 }
 
-std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices(size_t memoryRequired) {
-    auto *llama = constructDefaultLlama();
-    if (llama) { return llama->availableGPUDevices(memoryRequired); }
-    return {};
+std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices(size_t memoryRequired)
+{
+    std::vector<LLModel::GPUDevice> devices;
+#ifndef __APPLE__
+    static const std::string backends[] = {"kompute", "cuda"};
+    for (const auto &backend: backends) {
+        auto *llama = constructGlobalLlama(backend);
+        if (llama) {
+            auto backendDevs = llama->availableGPUDevices(memoryRequired);
+            devices.insert(devices.end(), backendDevs.begin(), backendDevs.end());
+        }
+    }
+#endif
+    return devices;
 }
 
-int32_t LLModel::Implementation::maxContextLength(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
+int32_t LLModel::Implementation::maxContextLength(const std::string &modelPath)
+{
+    auto *llama = constructGlobalLlama();
     return llama ? llama->maxContextLength(modelPath) : -1;
 }
 
-int32_t LLModel::Implementation::layerCount(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
+int32_t LLModel::Implementation::layerCount(const std::string &modelPath)
+{
+    auto *llama = constructGlobalLlama();
     return llama ? llama->layerCount(modelPath) : -1;
 }
 
-bool LLModel::Implementation::isEmbeddingModel(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
+bool LLModel::Implementation::isEmbeddingModel(const std::string &modelPath)
+{
+    auto *llama = constructGlobalLlama();
     return llama && llama->isEmbeddingModel(modelPath);
 }
 
-void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
+void LLModel::Implementation::setImplementationsSearchPath(const std::string& path)
+{
     s_implementations_search_path = path;
 }
 
-const std::string& LLModel::Implementation::implementationsSearchPath() {
+const std::string& LLModel::Implementation::implementationsSearchPath()
+{
     return s_implementations_search_path;
 }
 
-bool LLModel::Implementation::hasSupportedCPU() {
+bool LLModel::Implementation::hasSupportedCPU()
+{
     return cpu_supports_avx() != 0;
 }
+
+int LLModel::Implementation::cpuSupportsAVX2()
+{
+    return cpu_supports_avx2();
+}

gpt4all-backend/llmodel.h

@@ -1,15 +1,21 @@
 #ifndef LLMODEL_H
 #define LLMODEL_H
 
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
 #include <cstdint>
-#include <fstream>
 #include <functional>
-#include <limits>
 #include <optional>
+#include <stdexcept>
 #include <string>
 #include <string_view>
+#include <unordered_map>
+#include <utility>
 #include <vector>
 
+using namespace std::string_literals;
+
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
 class Dlhandle;
@@ -17,15 +23,66 @@ class LLModel {
 public:
     using Token = int32_t;
 
+    class BadArchError: public std::runtime_error {
+    public:
+        BadArchError(std::string arch)
+            : runtime_error("Unsupported model architecture: " + arch)
+            , m_arch(std::move(arch))
+            {}
+
+        const std::string &arch() const noexcept { return m_arch; }
+
+    private:
+        std::string m_arch;
+    };
+
+    class MissingImplementationError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
+    class UnsupportedModelError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
     struct GPUDevice {
+        const char *backend;
         int index;
         int type;
         size_t heapSize;
         std::string name;
         std::string vendor;
 
-        GPUDevice(int index, int type, size_t heapSize, std::string name, std::string vendor):
-            index(index), type(type), heapSize(heapSize), name(std::move(name)), vendor(std::move(vendor)) {}
+        GPUDevice(const char *backend, int index, int type, size_t heapSize, std::string name, std::string vendor):
+            backend(backend), index(index), type(type), heapSize(heapSize), name(std::move(name)),
+            vendor(std::move(vendor)) {}
+
+        std::string selectionName() const
+        {
+            assert(backend == "cuda"s || backend == "kompute"s);
+            return backendName() + ": " + name;
+        }
+
+        std::string backendName() const { return backendIdToName(backend); }
+
+        static std::string backendIdToName(const std::string &backend) { return s_backendNames.at(backend); }
+
+        static std::string updateSelectionName(const std::string &name) {
+            if (name == "Auto" || name == "CPU" || name == "Metal")
+                return name;
+            auto it = std::find_if(s_backendNames.begin(), s_backendNames.end(), [&name](const auto &entry) {
+                return name.starts_with(entry.second + ": ");
+            });
+            if (it != s_backendNames.end())
+                return name;
+            return "Vulkan: " + name; // previously, there were only Vulkan devices
+        }
+
+    private:
+        static inline const std::unordered_map<std::string, std::string> s_backendNames {
+            {"cpu", "CPU"}, {"metal", "Metal"}, {"cuda", "CUDA"}, {"kompute", "Vulkan"},
+        };
     };
 
     class Implementation {
@@ -37,7 +94,7 @@ public:
         std::string_view modelType() const { return m_modelType; }
         std::string_view buildVariant() const { return m_buildVariant; }
 
-        static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto", int n_ctx = 2048);
+        static LLModel *construct(const std::string &modelPath, const std::string &backend = "auto", int n_ctx = 2048);
         static std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0);
         static int32_t maxContextLength(const std::string &modelPath);
         static int32_t layerCount(const std::string &modelPath);
@@ -45,15 +102,18 @@ public:
         static void setImplementationsSearchPath(const std::string &path);
         static const std::string &implementationsSearchPath();
         static bool hasSupportedCPU();
+        // 0 for no, 1 for yes, -1 for non-x86_64
+        static int cpuSupportsAVX2();
 
     private:
         Implementation(Dlhandle &&);
 
         static const std::vector<Implementation> &implementationList();
         static const Implementation *implementation(const char *fname, const std::string &buildVariant);
-        static LLModel *constructDefaultLlama();
+        static LLModel *constructGlobalLlama(const std::optional<std::string> &backend = std::nullopt);
 
-        bool (*m_magicMatch)(const char *fname);
+        char *(*m_getFileArch)(const char *fname);
+        bool (*m_isArchSupported)(const char *arch);
         LLModel *(*m_construct)();
 
         std::string_view m_modelType;
@@ -144,8 +204,9 @@ public:
         return false;
     }
 
-    virtual bool hasGPUDevice() { return false; }
-    virtual bool usingGPUDevice() { return false; }
+    virtual bool usingGPUDevice() const { return false; }
+    virtual const char *backendName() const { return "cpu"; }
+    virtual const char *gpuDeviceName() const { return nullptr; }
 
     void setProgressCallback(ProgressCallback callback) { m_progressCallback = callback; }
 

gpt4all-backend/llmodel_c.cpp

@@ -1,12 +1,18 @@
 #include "llmodel_c.h"
+
 #include "llmodel.h"
 
-#include <cerrno>
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
 #include <cstring>
+#include <exception>
+#include <functional>
 #include <iostream>
 #include <memory>
 #include <optional>
-#include <utility>
+#include <string>
+#include <vector>
 
 struct LLModelWrapper {
     LLModel *llModel = nullptr;
@@ -14,7 +20,8 @@ struct LLModelWrapper {
     ~LLModelWrapper() { delete llModel; }
 };
 
-llmodel_model llmodel_model_create(const char *model_path) {
+llmodel_model llmodel_model_create(const char *model_path)
+{
     const char *error;
     auto fres = llmodel_model_create2(model_path, "auto", &error);
     if (!fres) {
@@ -23,7 +30,8 @@ llmodel_model llmodel_model_create(const char *model_path) {
     return fres;
 }
 
-static void llmodel_set_error(const char **errptr, const char *message) {
+static void llmodel_set_error(const char **errptr, const char *message)
+{
     thread_local static std::string last_error_message;
     if (errptr) {
         last_error_message = message;
@@ -31,26 +39,23 @@ static void llmodel_set_error(const char **errptr, const char *message) {
     }
 }
 
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error) {
+llmodel_model llmodel_model_create2(const char *model_path, const char *backend, const char **error)
+{
     LLModel *llModel;
     try {
-        llModel = LLModel::Implementation::construct(model_path, build_variant);
+        llModel = LLModel::Implementation::construct(model_path, backend);
     } catch (const std::exception& e) {
         llmodel_set_error(error, e.what());
         return nullptr;
     }
 
-    if (!llModel) {
-        llmodel_set_error(error, "Model format not supported (no matching implementation found)");
-        return nullptr;
-    }
-
     auto wrapper = new LLModelWrapper;
     wrapper->llModel = llModel;
     return wrapper;
 }
 
-void llmodel_model_destroy(llmodel_model model) {
+void llmodel_model_destroy(llmodel_model model)
+{
     delete static_cast<LLModelWrapper *>(model);
 }
 
@@ -253,6 +258,7 @@ struct llmodel_gpu_device *llmodel_available_gpu_devices(size_t memoryRequired,
     for (unsigned i = 0; i < devices.size(); i++) {
         const auto &dev  =   devices[i];
               auto &cdev = c_devices[i];
+        cdev.backend  = dev.backend;
         cdev.index    = dev.index;
         cdev.type     = dev.type;
         cdev.heapSize = dev.heapSize;
@@ -281,8 +287,14 @@ bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
     return wrapper->llModel->initializeGPUDevice(device);
 }
 
-bool llmodel_has_gpu_device(llmodel_model model)
+const char *llmodel_model_backend_name(llmodel_model model)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
-    return wrapper->llModel->hasGPUDevice();
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->backendName();
+}
+
+const char *llmodel_model_gpu_device_name(llmodel_model model)
+{
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->gpuDeviceName();
 }

gpt4all-backend/llmodel_c.h

@@ -1,9 +1,9 @@
 #ifndef LLMODEL_C_H
 #define LLMODEL_C_H
 
-#include <stdint.h>
-#include <stddef.h>
 #include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
 
 #ifdef __GNUC__
 #define DEPRECATED __attribute__ ((deprecated))
@@ -48,6 +48,7 @@ struct llmodel_prompt_context {
 };
 
 struct llmodel_gpu_device {
+    const char * backend;
     int index;
     int type; // same as VkPhysicalDeviceType
     size_t heapSize;
@@ -86,7 +87,7 @@ typedef bool (*llmodel_recalculate_callback)(bool is_recalculating);
  * Embedding cancellation callback for use with llmodel_embed.
  * @param batch_sizes The number of tokens in each batch that will be embedded.
  * @param n_batch The number of batches that will be embedded.
- * @param backend The backend that will be used for embedding. One of "cpu", "kompute", or "metal".
+ * @param backend The backend that will be used for embedding. One of "cpu", "kompute", "cuda", or "metal".
  * @return True to cancel llmodel_embed, false to continue.
  */
 typedef bool (*llmodel_emb_cancel_callback)(unsigned *batch_sizes, unsigned n_batch, const char *backend);
@@ -103,11 +104,11 @@ DEPRECATED llmodel_model llmodel_model_create(const char *model_path);
  * Create a llmodel instance.
  * Recognises correct model type from file at model_path
  * @param model_path A string representing the path to the model file; will only be used to detect model type.
- * @param build_variant A string representing the implementation to use (auto, default, avxonly, ...),
+ * @param backend A string representing the implementation to use. One of 'auto', 'cpu', 'metal', 'kompute', or 'cuda'.
  * @param error A pointer to a string; will only be set on error.
  * @return A pointer to the llmodel_model instance; NULL on error.
  */
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error);
+llmodel_model llmodel_model_create2(const char *model_path, const char *backend, const char **error);
 
 /**
  * Destroy a llmodel instance.
@@ -291,9 +292,14 @@ bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gp
 bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
 
 /**
- * @return True if a GPU device is successfully initialized, false otherwise.
+ * @return The name of the llama.cpp backend currently in use. One of "cpu", "kompute", or "metal".
  */
-bool llmodel_has_gpu_device(llmodel_model model);
+const char *llmodel_model_backend_name(llmodel_model model);
+
+/**
+ * @return The name of the GPU device currently in use, or NULL for backends other than Kompute.
+ */
+const char *llmodel_model_gpu_device_name(llmodel_model model);
 
 #ifdef __cplusplus
 }

gpt4all-backend/llmodel_shared.cpp

@@ -1,13 +1,21 @@
 #include "llmodel.h"
 
+#include <algorithm>
 #include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
 #include <iostream>
+#include <optional>
 #include <regex>
+#include <stdexcept>
 #include <string>
 #include <unordered_set>
+#include <vector>
 
 // TODO(cebtenzzre): replace this with llama_kv_cache_seq_shift for llamamodel (GPT-J needs this as-is)
-void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
+void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate)
+{
     int n_keep = shouldAddBOS();
     const int32_t n_discard = (promptCtx.n_ctx - n_keep) * promptCtx.contextErase;
 
@@ -36,7 +44,8 @@ stop_generating:
     recalculate(false);
 }
 
-static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err) {
+static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err)
+{
     static const std::regex placeholderRegex(R"(%[1-2](?![0-9]))");
 
     auto it = std::sregex_iterator(tmpl.begin(), tmpl.end(), placeholderRegex);

gpt4all-backend/llmodel_shared.h

@@ -1,9 +1,11 @@
 #pragma once
-#include <cstdint>
-#include <cstddef>
-#include <vector>
+
 #include <ggml.h>
 
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
 struct llm_buffer {
     uint8_t * addr = NULL;
     size_t size = 0;
@@ -36,7 +38,8 @@ struct llm_kv_cache {
     }
 };
 
-inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads) {
+inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads)
+{
     struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
     if (plan.work_size > 0) {
         buf.resize(plan.work_size);

gpt4all-backend/sysinfo.h

@@ -2,17 +2,21 @@
 #define SYSINFO_H
 
 #include <fstream>
-#include <string>
-#include <sstream>
 #include <iomanip>
+#include <sstream>
+#include <string>
 
 #if defined(__linux__)
-#include <unistd.h>
+#   include <unistd.h>
 #elif defined(__APPLE__)
-#include <sys/types.h>
-#include <sys/sysctl.h>
+#   include <sys/types.h>
+#   include <sys/sysctl.h>
 #elif defined(_WIN32)
-#include <windows.h>
+#   define WIN32_LEAN_AND_MEAN
+#   ifndef NOMINMAX
+#       define NOMINMAX
+#   endif
+#   include <windows.h>
 #endif
 
 static long long getSystemTotalRAMInBytes()

gpt4all-backend/utils.cpp

@@ -1,9 +1,15 @@
 #include "utils.h"
 
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
 #include <fstream>
+#include <iterator>
 #include <regex>
+#include <utility>
 
-void replace(std::string & str, const std::string & needle, const std::string & replacement) {
+void replace(std::string & str, const std::string & needle, const std::string & replacement)
+{
     size_t pos = 0;
     while ((pos = str.find(needle, pos)) != std::string::npos) {
         str.replace(pos, needle.length(), replacement);
@@ -11,7 +17,8 @@ void replace(std::string & str, const std::string & needle, const std::string &
     }
 }
 
-std::map<std::string, int32_t> json_parse(const std::string & fname) {
+std::map<std::string, int32_t> json_parse(const std::string & fname)
+{
     std::map<std::string, int32_t> result;
 
     // read file into string
@@ -102,7 +109,8 @@ std::map<std::string, int32_t> json_parse(const std::string & fname) {
     return result;
 }
 
-std::vector<gpt_vocab::id> gpt_tokenize_inner(const gpt_vocab & vocab, const std::string & text) {
+std::vector<gpt_vocab::id> gpt_tokenize_inner(const gpt_vocab & vocab, const std::string & text)
+{
     std::vector<std::string> words;
 
     // first split the text into words
@@ -157,12 +165,14 @@ std::vector<gpt_vocab::id> gpt_tokenize_inner(const gpt_vocab & vocab, const std
     return tokens;
 }
 
-std::string regex_escape(const std::string &s) {
+std::string regex_escape(const std::string &s)
+{
   static const std::regex metacharacters(R"([\.\^\$\-\+\(\)\[\]\{\}\|\?\*])");
   return std::regex_replace(s, metacharacters, "\\$&");
 }
 
-std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {
+std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text)
+{
     // Generate the subpattern from the special_tokens vector if it's not empty
     if (!vocab.special_tokens.empty()) {
         std::vector<gpt_vocab::id> out;
@@ -198,7 +208,8 @@ std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::stri
 }
 
 
-bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {
+bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab)
+{
     printf("%s: loading vocab from '%s'\n", __func__, fname.c_str());
 
     vocab.token_to_id = ::json_parse(fname);
@@ -325,4 +336,4 @@ gpt_vocab::id gpt_sample_top_k_top_p(
     int idx = dist(rng);
 
     return logits_id[idx].second;
-}
+}

gpt4all-backend/utils.h

@@ -2,16 +2,20 @@
 
 #pragma once
 
-#include <string>
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
 #include <map>
-#include <vector>
 #include <random>
+#include <string>
 #include <thread>
+#include <vector>
 
 //
 // General purpose inline functions
 //
-constexpr inline unsigned long long operator ""_MiB(unsigned long long bytes) {
+constexpr inline unsigned long long operator ""_MiB(unsigned long long bytes)
+{
     return bytes*1024*1024;
 }
 

gpt4all-bindings/README.md

@@ -1,3 +1,21 @@
-# GPT4All Bindings
-This directory will contain language specific bindings on top of the C/C++ model backends.
-We will have one directory per language binding (e.g. Python, Typescript, Golang, etc.).
+# GPT4All Language Bindings
+These are the language bindings for the GPT4All backend. They provide functionality to load GPT4All models (and other llama.cpp models), generate text, and (in the case of the Python bindings) embed text as a vector representation.
+
+See their respective folders for language-specific documentation.
+
+### Languages
+- [Python](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python) (Nomic official, maintained by [@cebtenzzre](https://github.com/cebtenzzre))
+- [Node.js/Typescript](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript) (community, maintained by [@jacoobes](https://github.com/jacoobes) and [@iimez](https://github.com/iimez))
+
+<br/>
+<br/>
+
+<details><summary><b>Archived Bindings</b></summary>
+<br/>
+
+The following bindings have been removed from this repository due to lack of maintenance. If adopted, they can be brought back&mdash;feel free to message a developer on Dicsord if you are interested in maintaining one of them. Below are links to their last available version (not necessarily the last working version).
+- C#: [41c9013f](https://github.com/nomic-ai/gpt4all/tree/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-bindings/csharp)
+- Java: [41c9013f](https://github.com/nomic-ai/gpt4all/tree/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-bindings/java)
+- Go: [41c9013f](https://github.com/nomic-ai/gpt4all/tree/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-bindings/golang)
+
+</details>

gpt4all-bindings/csharp/.editorconfig

@@ -1,348 +0,0 @@
-# EditorConfig is awesome: https://EditorConfig.org
-
-# top-most EditorConfig file
-root = true
-
-# Don't use tabs for indentation.
-[*]
-indent_style = space
-# (Please don't specify an indent_size here; that has too many unintended consequences.)
-
-# Code files
-[*.{cs,csx,vb,vbx}]
-indent_size = 4
-insert_final_newline = true
-charset = utf-8-bom
-
-# XML project files
-[*.{csproj,vbproj,vcxproj,vcxproj.filters,proj,projitems,shproj}]
-indent_size = 4
-
-# XML config files
-[*.{props,targets,ruleset,config,nuspec,resx,vsixmanifest,vsct}]
-indent_size = 2
-
-# JSON files
-[*.json]
-indent_size = 2
-
-# Powershell files
-[*.ps1]
-indent_size = 2
-
-# Shell script files
-[*.sh]
-end_of_line = lf
-indent_size = 2
-insert_final_newline = true
-
-# Dotnet code style settings:
-[*.{cs,vb}]
-
-# IDE0055: Fix formatting
-dotnet_diagnostic.IDE0055.severity = error
-dotnet_diagnostic.CS1573.severity = suggestion
-dotnet_diagnostic.CS1591.severity = suggestion
-
-# Sort using and Import directives with System.* appearing first
-dotnet_sort_system_directives_first = true
-dotnet_separate_import_directive_groups = false
-
-# Avoid "this." and "Me." if not necessary
-dotnet_style_qualification_for_field = false:suggestion
-dotnet_style_qualification_for_property = false:suggestion
-dotnet_style_qualification_for_method = false:suggestion
-dotnet_style_qualification_for_event = false:suggestion
-
-# Use language keywords instead of framework type names for type references
-dotnet_style_predefined_type_for_locals_parameters_members = true:warning
-dotnet_style_predefined_type_for_member_access = true:warning
-
-# Suggest more modern language features when available
-dotnet_style_object_initializer = true:suggestion
-dotnet_style_collection_initializer = true:suggestion
-dotnet_style_coalesce_expression = true:suggestion
-dotnet_style_null_propagation = true:suggestion
-dotnet_style_explicit_tuple_names = true:suggestion
-
-# Whitespace options
-dotnet_style_allow_multiple_blank_lines_experimental = false
-
-# Private fields are camelCase with '_' prefix
-dotnet_naming_rule.private_members_with_underscore.symbols  = private_fields
-dotnet_naming_rule.private_members_with_underscore.style    = prefix_underscore
-dotnet_naming_rule.private_members_with_underscore.severity = error
-dotnet_naming_symbols.private_fields.applicable_kinds           = field
-dotnet_naming_symbols.private_fields.applicable_accessibilities = private
-dotnet_naming_style.prefix_underscore.capitalization = camel_case
-dotnet_naming_style.prefix_underscore.required_prefix = _
-
-# Non-private static fields are PascalCase
-dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.severity = suggestion
-dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.symbols = non_private_static_fields
-dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.style = non_private_static_field_style
-
-dotnet_naming_symbols.non_private_static_fields.applicable_kinds = field
-dotnet_naming_symbols.non_private_static_fields.applicable_accessibilities = public, protected, internal, protected_internal, private_protected
-dotnet_naming_symbols.non_private_static_fields.required_modifiers = static
-
-dotnet_naming_style.non_private_static_field_style.capitalization = pascal_case
-
-# Non-private readonly fields are PascalCase
-dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.severity = suggestion
-dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.symbols = non_private_readonly_fields
-dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.style = non_private_static_field_style
-
-dotnet_naming_symbols.non_private_readonly_fields.applicable_kinds = field
-dotnet_naming_symbols.non_private_readonly_fields.applicable_accessibilities = public, protected, internal, protected_internal, private_protected
-dotnet_naming_symbols.non_private_readonly_fields.required_modifiers = readonly
-
-dotnet_naming_style.non_private_readonly_field_style.capitalization = pascal_case
-
-# Constants are PascalCase
-dotnet_naming_rule.constants_should_be_pascal_case.severity = suggestion
-dotnet_naming_rule.constants_should_be_pascal_case.symbols = constants
-dotnet_naming_rule.constants_should_be_pascal_case.style = non_private_static_field_style
-
-dotnet_naming_symbols.constants.applicable_kinds = field, local
-dotnet_naming_symbols.constants.required_modifiers = const
-
-dotnet_naming_style.constant_style.capitalization = pascal_case
-
-# Static fields are camelCase and start with s_
-dotnet_naming_rule.static_fields_should_be_camel_case.severity = none
-dotnet_naming_rule.static_fields_should_be_camel_case.symbols = static_fields
-dotnet_naming_rule.static_fields_should_be_camel_case.style = static_field_style
-
-dotnet_naming_symbols.static_fields.applicable_kinds = field
-dotnet_naming_symbols.static_fields.required_modifiers = static
-
-dotnet_naming_style.static_field_style.capitalization = camel_case
-dotnet_naming_style.static_field_style.required_prefix = s_
-
-# Instance fields are camelCase and start with _
-dotnet_naming_rule.instance_fields_should_be_camel_case.severity = none
-dotnet_naming_rule.instance_fields_should_be_camel_case.symbols = instance_fields
-dotnet_naming_rule.instance_fields_should_be_camel_case.style = instance_field_style
-
-dotnet_naming_symbols.instance_fields.applicable_kinds = field
-
-dotnet_naming_style.instance_field_style.capitalization = camel_case
-dotnet_naming_style.instance_field_style.required_prefix = _
-
-# Locals and parameters are camelCase
-dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion
-dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters
-dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style
-
-dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local
-
-dotnet_naming_style.camel_case_style.capitalization = camel_case
-
-# Local functions are PascalCase
-dotnet_naming_rule.local_functions_should_be_pascal_case.severity = suggestion
-dotnet_naming_rule.local_functions_should_be_pascal_case.symbols = local_functions
-dotnet_naming_rule.local_functions_should_be_pascal_case.style = non_private_static_field_style
-
-dotnet_naming_symbols.local_functions.applicable_kinds = local_function
-
-dotnet_naming_style.local_function_style.capitalization = pascal_case
-
-# By default, name items with PascalCase
-dotnet_naming_rule.members_should_be_pascal_case.severity = suggestion
-dotnet_naming_rule.members_should_be_pascal_case.symbols = all_members
-dotnet_naming_rule.members_should_be_pascal_case.style = non_private_static_field_style
-
-dotnet_naming_symbols.all_members.applicable_kinds = *
-
-dotnet_naming_style.pascal_case_style.capitalization = pascal_case
-
-# error RS2008: Enable analyzer release tracking for the analyzer project containing rule '{0}'
-dotnet_diagnostic.RS2008.severity = none
-
-# IDE0073: File header
-dotnet_diagnostic.IDE0073.severity = none
-#file_header_template = Licensed to the .NET Foundation under one or more agreements.\nThe .NET Foundation licenses this file to you under the MIT license.\nSee the LICENSE file in the project root for more information.
-
-# IDE0035: Remove unreachable code
-dotnet_diagnostic.IDE0035.severity = warning
-
-# IDE0036: Order modifiers
-dotnet_diagnostic.IDE0036.severity = warning
-
-# IDE0043: Format string contains invalid placeholder
-dotnet_diagnostic.IDE0043.severity = warning
-
-# IDE0044: Make field readonly
-dotnet_diagnostic.IDE0044.severity = warning
-
-# IDE1006: Naming rule violation
-#dotnet_diagnostic.IDE1006.severity = none
-
-# RS0016: Only enable if API files are present
-dotnet_public_api_analyzer.require_api_files = true
-dotnet_style_operator_placement_when_wrapping = beginning_of_line
-tab_width = 4
-end_of_line = crlf
-dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion
-dotnet_style_prefer_auto_properties = true:silent
-dotnet_style_prefer_simplified_boolean_expressions = true:suggestion
-dotnet_style_prefer_conditional_expression_over_assignment = true:silent
-dotnet_style_prefer_conditional_expression_over_return = true:silent
-dotnet_style_prefer_inferred_tuple_names = true:suggestion
-dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion
-dotnet_style_prefer_compound_assignment = true:suggestion
-dotnet_style_prefer_simplified_interpolation = true:suggestion
-dotnet_style_namespace_match_folder = true:suggestion
-
-# CSharp code style settings:
-[*.cs]
-# Newline settings
-csharp_new_line_before_open_brace = all
-csharp_new_line_before_else = true
-csharp_new_line_before_catch = true
-csharp_new_line_before_finally = true
-csharp_new_line_before_members_in_object_initializers = true
-csharp_new_line_before_members_in_anonymous_types = true
-csharp_new_line_between_query_expression_clauses = true
-
-# Indentation preferences
-csharp_indent_block_contents = true
-csharp_indent_braces = false
-csharp_indent_case_contents = true
-csharp_indent_case_contents_when_block = true
-csharp_indent_switch_labels = true
-csharp_indent_labels = flush_left
-
-# Whitespace options
-csharp_style_allow_embedded_statements_on_same_line_experimental = false
-csharp_style_allow_blank_lines_between_consecutive_braces_experimental = false
-csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = false
-
-# Prefer "var" everywhere
-csharp_style_var_for_built_in_types = true:suggestion
-csharp_style_var_when_type_is_apparent = true:suggestion
-csharp_style_var_elsewhere = true:suggestion
-
-# Prefer method-like constructs to have a block body
-csharp_style_expression_bodied_methods = false:none
-csharp_style_expression_bodied_constructors = false:none
-csharp_style_expression_bodied_operators = false:none
-
-# Prefer property-like constructs to have an expression-body
-csharp_style_expression_bodied_properties = true:none
-csharp_style_expression_bodied_indexers = true:none
-csharp_style_expression_bodied_accessors = true:none
-
-# Suggest more modern language features when available
-csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion
-csharp_style_pattern_matching_over_as_with_null_check = true:suggestion
-csharp_style_inlined_variable_declaration = true:suggestion
-csharp_style_throw_expression = true:suggestion
-csharp_style_conditional_delegate_call = true:suggestion
-
-# Space preferences
-csharp_space_after_cast = false
-csharp_space_after_colon_in_inheritance_clause = true
-csharp_space_after_comma = true
-csharp_space_after_dot = false
-csharp_space_after_keywords_in_control_flow_statements = true
-csharp_space_after_semicolon_in_for_statement = true
-csharp_space_around_binary_operators = before_and_after
-csharp_space_around_declaration_statements = do_not_ignore
-csharp_space_before_colon_in_inheritance_clause = true
-csharp_space_before_comma = false
-csharp_space_before_dot = false
-csharp_space_before_open_square_brackets = false
-csharp_space_before_semicolon_in_for_statement = false
-csharp_space_between_empty_square_brackets = false
-csharp_space_between_method_call_empty_parameter_list_parentheses = false
-csharp_space_between_method_call_name_and_opening_parenthesis = false
-csharp_space_between_method_call_parameter_list_parentheses = false
-csharp_space_between_method_declaration_empty_parameter_list_parentheses = false
-csharp_space_between_method_declaration_name_and_open_parenthesis = false
-csharp_space_between_method_declaration_parameter_list_parentheses = false
-csharp_space_between_parentheses = false
-csharp_space_between_square_brackets = false
-
-# Blocks are allowed
-csharp_prefer_braces = true:silent
-csharp_preserve_single_line_blocks = true
-csharp_preserve_single_line_statements = true
-
-# Target-type new expressio
-csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion
-
-# Currently only enabled for C# due to crash in VB analyzer.  VB can be enabled once
-# https://github.com/dotnet/roslyn/pull/54259 has been published.
-dotnet_style_allow_statement_immediately_after_block_experimental = false
-dotnet_diagnostic.RCS0003.severity=warning
-dotnet_diagnostic.RCS1036.severity=error
-dotnet_diagnostic.IDE0005.severity=warning
-dotnet_diagnostic.IDE0007.severity=error
-csharp_using_directive_placement = outside_namespace:silent
-csharp_prefer_simple_using_statement = true:suggestion
-csharp_style_namespace_declarations = block_scoped:silent
-csharp_style_expression_bodied_lambdas = true:silent
-csharp_style_expression_bodied_local_functions = false:silent
-csharp_style_prefer_null_check_over_type_check = true:suggestion
-dotnet_diagnostic.RCS1075.severity = suggestion
-
-[src/CodeStyle/**.{cs,vb}]
-# warning RS0005: Do not use generic CodeAction.Create to create CodeAction
-dotnet_diagnostic.RS0005.severity = none
-
-[src/{Analyzers,CodeStyle,Features,Workspaces,EditorFeatures,VisualStudio}/**/*.{cs,vb}]
-
-# IDE0011: Add braces
-csharp_prefer_braces = when_multiline:warning
-# NOTE: We need the below severity entry for Add Braces due to https://github.com/dotnet/roslyn/issues/44201
-dotnet_diagnostic.IDE0011.severity = warning
-
-# IDE0040: Add accessibility modifiers
-dotnet_diagnostic.IDE0040.severity = warning
-
-# CONSIDER: Are IDE0051 and IDE0052 too noisy to be warnings for IDE editing scenarios? Should they be made build-only warnings?
-# IDE0051: Remove unused private member
-dotnet_diagnostic.IDE0051.severity = warning
-
-# IDE0052: Remove unread private member
-dotnet_diagnostic.IDE0052.severity = warning
-
-# IDE0059: Unnecessary assignment to a value
-dotnet_diagnostic.IDE0059.severity = warning
-
-# IDE0060: Remove unused parameter
-dotnet_diagnostic.IDE0060.severity = warning
-
-# CA1012: Abstract types should not have public constructors
-dotnet_diagnostic.CA1012.severity = warning
-
-# CA1822: Make member static
-dotnet_diagnostic.CA1822.severity = warning
-
-# Prefer "var" everywhere
-dotnet_diagnostic.IDE0007.severity = warning
-csharp_style_var_for_built_in_types = true:warning
-csharp_style_var_when_type_is_apparent = true:warning
-csharp_style_var_elsewhere = true:warning
-
-# dotnet_style_allow_multiple_blank_lines_experimental
-dotnet_diagnostic.IDE2000.severity = warning
-
-# csharp_style_allow_embedded_statements_on_same_line_experimental
-dotnet_diagnostic.IDE2001.severity = warning
-
-# csharp_style_allow_blank_lines_between_consecutive_braces_experimental
-dotnet_diagnostic.IDE2002.severity = warning
-
-# dotnet_style_allow_statement_immediately_after_block_experimental
-dotnet_diagnostic.IDE2003.severity = warning
-
-# csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental
-dotnet_diagnostic.IDE2004.severity = warning
-
-[src/{VisualStudio}/**/*.{cs,vb}]
-# CA1822: Make member static
-# There is a risk of accidentally breaking an internal API that partners rely on though IVT.
-dotnet_code_quality.CA1822.api_surface = private

gpt4all-bindings/csharp/.gitignore

@@ -1,379 +0,0 @@
-## Ignore Visual Studio temporary files, build results, and
-## files generated by popular Visual Studio add-ons.
-##
-## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
-
-runtimes
-**/*nuget
-
-*.zip
-include/
-*.exp
-*.lib
-*.dll
-
-# User-specific files
-*.rsuser
-*.suo
-*.user
-*.userosscache
-*.sln.docstates
-
-# User-specific files (MonoDevelop/Xamarin Studio)
-*.userprefs
-
-# Mono auto generated files
-mono_crash.*
-Tests/**/launchSettings.json
-
-# Build results
-[Dd]ebug/
-[Dd]ebugPublic/
-[Rr]elease/
-[Rr]eleases/
-x64/
-x86/
-[Ww][Ii][Nn]32/
-[Aa][Rr][Mm]/
-[Aa][Rr][Mm]64/
-bld/
-[Bb]in/
-[Oo]bj/
-[Oo]ut/
-[Ll]og/
-[Ll]ogs/
-
-# Visual Studio 2015/2017 cache/options directory
-.vs/
-# Uncomment if you have tasks that create the project's static files in wwwroot
-#wwwroot/
-
-# Visual Studio 2017 auto generated files
-Generated\ Files/
-
-# MSTest test Results
-[Tt]est[Rr]esult*/
-[Bb]uild[Ll]og.*
-
-# NUnit
-*.VisualState.xml
-TestResult.xml
-nunit-*.xml
-
-# Build Results of an ATL Project
-[Dd]ebugPS/
-[Rr]eleasePS/
-dlldata.c
-
-# Benchmark Results
-BenchmarkDotNet.Artifacts/
-
-# .NET Core
-project.lock.json
-project.fragment.lock.json
-artifacts/
-
-# ASP.NET Scaffolding
-ScaffoldingReadMe.txt
-
-# StyleCop
-StyleCopReport.xml
-
-# Files built by Visual Studio
-*_i.c
-*_p.c
-*_h.h
-*.ilk
-*.meta
-*.obj
-*.iobj
-*.pch
-*.pdb
-*.ipdb
-*.pgc
-*.pgd
-*.rsp
-*.sbr
-*.tlb
-*.tli
-*.tlh
-*.tmp
-*.tmp_proj
-*_wpftmp.csproj
-*.log
-*.vspscc
-*.vssscc
-.builds
-*.pidb
-*.svclog
-*.scc
-
-# Chutzpah Test files
-_Chutzpah*
-
-# Visual C++ cache files
-ipch/
-*.aps
-*.ncb
-*.opendb
-*.opensdf
-*.sdf
-*.cachefile
-*.VC.db
-*.VC.VC.opendb
-
-# Visual Studio profiler
-*.psess
-*.vsp
-*.vspx
-*.sap
-
-# Visual Studio Trace Files
-*.e2e
-
-# TFS 2012 Local Workspace
-$tf/
-
-# Guidance Automation Toolkit
-*.gpState
-
-# ReSharper is a .NET coding add-in
-_ReSharper*/
-*.[Rr]e[Ss]harper
-*.DotSettings.user
-
-# TeamCity is a build add-in
-_TeamCity*
-
-# DotCover is a Code Coverage Tool
-*.dotCover
-
-# AxoCover is a Code Coverage Tool
-.axoCover/*
-!.axoCover/settings.json
-
-# Coverlet is a free, cross platform Code Coverage Tool
-coverage*.json
-coverage*.xml
-coverage*.info
-
-# Visual Studio code coverage results
-*.coverage
-*.coveragexml
-
-# NCrunch
-_NCrunch_*
-.*crunch*.local.xml
-nCrunchTemp_*
-
-# MightyMoose
-*.mm.*
-AutoTest.Net/
-
-# Web workbench (sass)
-.sass-cache/
-
-# Installshield output folder
-[Ee]xpress/
-
-# DocProject is a documentation generator add-in
-DocProject/buildhelp/
-DocProject/Help/*.HxT
-DocProject/Help/*.HxC
-DocProject/Help/*.hhc
-DocProject/Help/*.hhk
-DocProject/Help/*.hhp
-DocProject/Help/Html2
-DocProject/Help/html
-
-# Click-Once directory
-publish/
-
-# Publish Web Output
-*.[Pp]ublish.xml
-*.azurePubxml
-# Note: Comment the next line if you want to checkin your web deploy settings,
-# but database connection strings (with potential passwords) will be unencrypted
-*.pubxml
-*.publishproj
-
-# Microsoft Azure Web App publish settings. Comment the next line if you want to
-# checkin your Azure Web App publish settings, but sensitive information contained
-# in these scripts will be unencrypted
-PublishScripts/
-
-# NuGet Packages
-*.nupkg
-# NuGet Symbol Packages
-*.snupkg
-# The packages folder can be ignored because of Package Restore
-**/[Pp]ackages/*
-# except build/, which is used as an MSBuild target.
-!**/[Pp]ackages/build/
-# Uncomment if necessary however generally it will be regenerated when needed
-#!**/[Pp]ackages/repositories.config
-# NuGet v3's project.json files produces more ignorable files
-*.nuget.props
-*.nuget.targets
-
-# Microsoft Azure Build Output
-csx/
-*.build.csdef
-
-# Microsoft Azure Emulator
-ecf/
-rcf/
-
-# Windows Store app package directories and files
-AppPackages/
-BundleArtifacts/
-Package.StoreAssociation.xml
-_pkginfo.txt
-*.appx
-*.appxbundle
-*.appxupload
-
-# Visual Studio cache files
-# files ending in .cache can be ignored
-*.[Cc]ache
-# but keep track of directories ending in .cache
-!?*.[Cc]ache/
-
-# Others
-ClientBin/
-~$*
-*~
-*.dbmdl
-*.dbproj.schemaview
-*.jfm
-*.pfx
-*.publishsettings
-orleans.codegen.cs
-
-# Including strong name files can present a security risk
-# (https://github.com/github/gitignore/pull/2483#issue-259490424)
-#*.snk
-
-# Since there are multiple workflows, uncomment next line to ignore bower_components
-# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
-#bower_components/
-
-# RIA/Silverlight projects
-Generated_Code/
-
-# Backup & report files from converting an old project file
-# to a newer Visual Studio version. Backup files are not needed,
-# because we have git ;-)
-_UpgradeReport_Files/
-Backup*/
-UpgradeLog*.XML
-UpgradeLog*.htm
-ServiceFabricBackup/
-*.rptproj.bak
-
-# SQL Server files
-*.mdf
-*.ldf
-*.ndf
-
-# Business Intelligence projects
-*.rdl.data
-*.bim.layout
-*.bim_*.settings
-*.rptproj.rsuser
-*- [Bb]ackup.rdl
-*- [Bb]ackup ([0-9]).rdl
-*- [Bb]ackup ([0-9][0-9]).rdl
-
-# Microsoft Fakes
-FakesAssemblies/
-
-# GhostDoc plugin setting file
-*.GhostDoc.xml
-
-# Node.js Tools for Visual Studio
-.ntvs_analysis.dat
-node_modules/
-
-# Visual Studio 6 build log
-*.plg
-
-# Visual Studio 6 workspace options file
-*.opt
-
-# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
-*.vbw
-
-# Visual Studio LightSwitch build output
-**/*.HTMLClient/GeneratedArtifacts
-**/*.DesktopClient/GeneratedArtifacts
-**/*.DesktopClient/ModelManifest.xml
-**/*.Server/GeneratedArtifacts
-**/*.Server/ModelManifest.xml
-_Pvt_Extensions
-
-# Paket dependency manager
-.paket/paket.exe
-paket-files/
-
-# FAKE - F# Make
-.fake/
-
-# CodeRush personal settings
-.cr/personal
-
-# Python Tools for Visual Studio (PTVS)
-__pycache__/
-*.pyc
-
-# Cake - Uncomment if you are using it
-# tools/**
-# !tools/packages.config
-
-# Tabs Studio
-*.tss
-
-# Telerik's JustMock configuration file
-*.jmconfig
-
-# BizTalk build output
-*.btp.cs
-*.btm.cs
-*.odx.cs
-*.xsd.cs
-
-# OpenCover UI analysis results
-OpenCover/
-
-# Azure Stream Analytics local run output
-ASALocalRun/
-
-# MSBuild Binary and Structured Log
-*.binlog
-
-# NVidia Nsight GPU debugger configuration file
-*.nvuser
-
-# MFractors (Xamarin productivity tool) working folder
-.mfractor/
-
-# Local History for Visual Studio
-.localhistory/
-
-# BeatPulse healthcheck temp database
-healthchecksdb
-
-# Backup folder for Package Reference Convert tool in Visual Studio 2017
-MigrationBackup/
-
-# Ionide (cross platform F# VS Code tools) working folder
-.ionide/
-
-# Fody - auto-generated XML schema
-FodyWeavers.xsd
-
-# JetBrains Rider
-.idea
-
-# Visual Studio Code
-.vscode

gpt4all-bindings/csharp/Directory.Build.props

@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project>
-
-    <PropertyGroup>
-        <Company></Company>
-        <Copyright></Copyright>
-        <NeutralLanguage>en-US</NeutralLanguage>
-        <Version>0.6.4-alpha</Version>
-        <VersionSuffix>$(VersionSuffix)</VersionSuffix>
-        <Version Condition=" '$(VersionSuffix)' != '' ">$(Version)$(VersionSuffix)</Version>
-        <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
-        <RepositoryUrl></RepositoryUrl>
-        <RepositoryType>git</RepositoryType>
-        <IncludeSymbols>true</IncludeSymbols>
-        <IncludeSource>true</IncludeSource>
-        <AnalysisLevel>latest-minimum</AnalysisLevel>
-		<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
-    </PropertyGroup>
-
-    <ItemGroup>
-        <Using Include="System"/>
-    </ItemGroup>
-
-    <PropertyGroup>
-        <LangVersion>preview</LangVersion>
-        <Features>strict</Features>
-    </PropertyGroup>
-	
-	<ItemGroup>
-        <PackageReference Include="Roslynator.Analyzers" Version="4.2.0">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-        </PackageReference>
-        <PackageReference Include="Roslynator.CodeAnalysis.Analyzers" Version="4.2.0">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-        </PackageReference>
-        <PackageReference Include="Roslynator.Formatting.Analyzers" Version="4.2.0">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-        </PackageReference>
-    </ItemGroup>
-
-</Project>

gpt4all-bindings/csharp/Gpt4All.Samples/Gpt4All.Samples.csproj

@@ -1,33 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-        <GenerateDocumentationFile>true</GenerateDocumentationFile>
-    </PropertyGroup>
-
-    <ItemGroup>
-        <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
-    </ItemGroup>
-
-    <ItemGroup>
-        <!-- Windows -->
-        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
-        <!-- Linux -->
-        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
-        <!-- MacOS -->
-        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
-    </ItemGroup>
-
-    <ItemGroup>
-        <!-- Windows -->
-        <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-        <!-- Linux -->
-        <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-         <!-- MacOS -->
-        <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-        <Content Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.metal" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-    </ItemGroup>
-</Project>

gpt4all-bindings/csharp/Gpt4All.Samples/Program.cs

@@ -1,22 +0,0 @@
-using Gpt4All;
-
-var modelFactory = new Gpt4AllModelFactory();
-if (args.Length < 2)
-{
-    Console.WriteLine($"Usage: Gpt4All.Samples <model-path> <prompt>");
-    return;
-}
-
-var modelPath = args[0];
-var prompt = args[1];
-
-using var model = modelFactory.LoadModel(modelPath);
-
-var result = await model.GetStreamingPredictionAsync(
-    prompt,
-    PredictRequestOptions.Defaults);
-
-await foreach (var token in result.GetPredictionStreamingAsync())
-{
-    Console.Write(token);
-}

gpt4all-bindings/csharp/Gpt4All.Tests/Constants.cs

@@ -1,9 +0,0 @@
-namespace Gpt4All.Tests;
-
-public static class Constants
-{
-    public const string MODELS_BASE_DIR = "../../../models";
-    public const string LLAMA_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-l13b-snoozy.bin";
-    public const string GPTJ_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-j-v1.3-groovy.bin";
-    public const string MPT_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-mpt-7b-chat.bin";
-}

gpt4all-bindings/csharp/Gpt4All.Tests/Gpt4All.Tests.csproj

@@ -1,60 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
-        <Nullable>enable</Nullable>
-
-        <IsPackable>false</IsPackable>
-        <GenerateDocumentationFile>true</GenerateDocumentationFile>
-    </PropertyGroup>
-
-    <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2" />
-        <PackageReference Include="xunit" Version="2.4.2" />
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-            <PrivateAssets>all</PrivateAssets>
-        </PackageReference>
-        <PackageReference Include="coverlet.collector" Version="6.0.0">
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-            <PrivateAssets>all</PrivateAssets>
-        </PackageReference>
-    </ItemGroup>
-
-    <ItemGroup>
-        <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
-    </ItemGroup>
-
-    <ItemGroup>
-        <!-- Windows -->
-        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
-        <!-- Linux -->
-        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
-        <!-- MacOS -->
-        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
-    </ItemGroup>
-
-    <ItemGroup>
-        <!-- Windows -->
-        <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-        <!-- Linux -->
-        <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-         <!-- MacOS -->
-        <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-    </ItemGroup>
-
-    <ItemGroup>
-      <PackageReference Update="Roslynator.Analyzers" Version="4.3.0">
-        <PrivateAssets>all</PrivateAssets>
-        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-      </PackageReference>
-      <PackageReference Update="Roslynator.CodeAnalysis.Analyzers" Version="4.3.0">
-        <PrivateAssets>all</PrivateAssets>
-        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-      </PackageReference>
-      <PackageReference Update="Roslynator.Formatting.Analyzers" Version="4.3.0">
-        <PrivateAssets>all</PrivateAssets>
-        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-      </PackageReference>
-    </ItemGroup>
-</Project>

gpt4all-bindings/csharp/Gpt4All.Tests/ModelFactoryTests.cs

@@ -1,34 +0,0 @@
-using Xunit;
-
-namespace Gpt4All.Tests;
-
-public class ModelFactoryTests
-{
-    private readonly Gpt4AllModelFactory _modelFactory;
-
-    public ModelFactoryTests()
-    {
-        _modelFactory = new Gpt4AllModelFactory();
-    }
-
-    [Fact]
-    [Trait(Traits.SkipOnCI, "True")]
-    public void CanLoadLlamaModel()
-    {
-        using var model = _modelFactory.LoadModel(Constants.LLAMA_MODEL_PATH);
-    }
-
-    [Fact]
-    [Trait(Traits.SkipOnCI, "True")]
-    public void CanLoadGptjModel()
-    {
-        using var model = _modelFactory.LoadModel(Constants.GPTJ_MODEL_PATH);
-    }
-
-    [Fact]
-    [Trait(Traits.SkipOnCI, "True")]
-    public void CanLoadMptModel()
-    {
-        using var model = _modelFactory.LoadModel(Constants.MPT_MODEL_PATH);
-    }
-}

gpt4all-bindings/csharp/Gpt4All.Tests/NativeLibraryLoaderTests.cs

@@ -1,56 +0,0 @@
-using System.IO;
-using Gpt4All.LibraryLoader;
-using Xunit;
-
-namespace Gpt4All.Tests;
-
-public class NativeLibraryLoaderTests
-{
-    [Fact]
-    public void NativeLibraryShouldLoad()
-    {
-        var result = NativeLibraryLoader.LoadNativeLibrary(bypassLoading: false);
-        Assert.True(result.IsSuccess);
-    }
-
-    private const string LLModelLib = "libllmodel.{0}";
-
-    [PlatformSpecificFact(Platforms.Windows)]
-    public void NativeLibraryShouldLoad_Windows()
-    {
-        var libraryLoader = new WindowsLibraryLoader();
-
-        var libraryPath = Path.Combine(
-            Environment.CurrentDirectory,
-            string.Format(LLModelLib, "dll"));
-
-        var result = libraryLoader.OpenLibrary(libraryPath);
-        Assert.True(result.IsSuccess);
-    }
-
-    [PlatformSpecificFact(Platforms.Linux)]
-    public void NativeLibraryShouldLoad_Linux()
-    {
-        var libraryLoader = new LinuxLibraryLoader();
-
-        var libraryPath = Path.Combine(
-            Environment.CurrentDirectory,
-            string.Format(LLModelLib, "so"));
-
-        var result = libraryLoader.OpenLibrary(libraryPath);
-        Assert.True(result.IsSuccess);
-    }
-
-    [PlatformSpecificFact(Platforms.MacOS)]
-    public void NativeLibraryShouldLoad_MacOS()
-    {
-        var libraryLoader = new MacOsLibraryLoader();
-
-        var libraryPath = Path.Combine(
-            Environment.CurrentDirectory,
-            string.Format(LLModelLib, "dylib"));
-
-        var result = libraryLoader.OpenLibrary(libraryPath);
-        Assert.True(result.IsSuccess);
-    }
-}

gpt4all-bindings/csharp/Gpt4All.Tests/PlatformSpecificFactAttribute.cs

@@ -1,27 +0,0 @@
-using Xunit;
-
-namespace Gpt4All.Tests;
-
-public static class Platforms
-{
-    public const string Windows = "windows";
-    public const string Linux = "linux";
-    public const string MacOS = "macOS";
-}
-
-/// <summary>
-/// This attribute ensures the Fact is only run on the specified platform.
-/// </summary>
-/// <remarks>
-/// <see cref="OperatingSystem.IsOSPlatform(string)"/> for info about the platform string.
-/// </remarks>
-public class PlatformSpecificFactAttribute : FactAttribute
-{
-    public PlatformSpecificFactAttribute(string platform)
-    {
-        if (!OperatingSystem.IsOSPlatform(platform))
-        {
-            Skip = $"Test only runs on {platform}.";
-        }
-    }
-}

gpt4all-bindings/csharp/Gpt4All.Tests/Traits.cs

@@ -1,6 +0,0 @@
-namespace Gpt4All.Tests;
-
-public static class Traits
-{
-    public const string SkipOnCI = "SKIP_ON_CI";
-}

gpt4all-bindings/csharp/Gpt4All.sln

@@ -1,47 +0,0 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.5.33516.290
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Gpt4All.Samples", "Gpt4All.Samples\Gpt4All.Samples.csproj", "{59864AE8-E45D-42F7-A7C0-1308EF185F39}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{DA396C11-CEAD-4368-8234-FB12255A30D2}"
-	ProjectSection(SolutionItems) = preProject
-		.gitignore = .gitignore
-		build_linux.sh = build_linux.sh
-		build_win-mingw.ps1 = build_win-mingw.ps1
-		build_win-msvc.ps1 = build_win-msvc.ps1
-		docs\gpt4all_csharp.md = docs\gpt4all_csharp.md
-		README.md = README.md
-	EndProjectSection
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Gpt4All", "Gpt4All\Gpt4All.csproj", "{6015C62B-2008-426B-A334-740D6F1FE38B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gpt4All.Tests", "Gpt4All.Tests\Gpt4All.Tests.csproj", "{33A72341-52C1-4EAE-878B-A98BC77F686A}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Release|Any CPU = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6015C62B-2008-426B-A334-740D6F1FE38B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6015C62B-2008-426B-A334-740D6F1FE38B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6015C62B-2008-426B-A334-740D6F1FE38B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6015C62B-2008-426B-A334-740D6F1FE38B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Release|Any CPU.Build.0 = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {17632027-F4C2-4903-B88F-310CE3DE386B}
-	EndGlobalSection
-EndGlobal

gpt4all-bindings/csharp/Gpt4All/Bindings/ILLModel.cs

@@ -1,29 +0,0 @@
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Represents the interface exposed by the universal wrapper for GPT4All language models built around llmodel C-API.
-/// </summary>
-public interface ILLModel : IDisposable
-{
-    ulong GetStateSizeBytes();
-
-    int GetThreadCount();
-
-    void SetThreadCount(int threadCount);
-
-    bool IsLoaded();
-
-    bool Load(string modelPath);
-
-    void Prompt(
-        string text,
-        LLModelPromptContext context,
-        Func<ModelPromptEventArgs, bool>? promptCallback = null,
-        Func<ModelResponseEventArgs, bool>? responseCallback = null,
-        Func<ModelRecalculatingEventArgs, bool>? recalculateCallback = null,
-        CancellationToken cancellationToken = default);
-
-    unsafe ulong RestoreStateData(byte* destination);
-
-    unsafe ulong SaveStateData(byte* source);
-}

gpt4all-bindings/csharp/Gpt4All/Bindings/LLModel.cs

@@ -1,212 +0,0 @@
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
-
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Arguments for the response processing callback
-/// </summary>
-/// <param name="TokenId">The token id of the response</param>
-/// <param name="Response"> The response string. NOTE: a token_id of -1 indicates the string is an error string</param>
-/// <return>
-/// A bool indicating whether the model should keep generating
-/// </return>
-public record ModelResponseEventArgs(int TokenId, string Response)
-{
-    public bool IsError => TokenId == -1;
-}
-
-/// <summary>
-/// Arguments for the prompt processing callback
-/// </summary>
-/// <param name="TokenId">The token id of the prompt</param>
-/// <return>
-/// A bool indicating whether the model should keep processing
-/// </return>
-public record ModelPromptEventArgs(int TokenId)
-{
-}
-
-/// <summary>
-/// Arguments for the recalculating callback
-/// </summary>
-/// <param name="IsRecalculating"> whether the model is recalculating the context.</param>
-/// <return>
-/// A bool indicating whether the model should keep generating
-/// </return>
-public record ModelRecalculatingEventArgs(bool IsRecalculating);
-
-/// <summary>
-/// Base class and universal wrapper for GPT4All language models built around llmodel C-API.
-/// </summary>
-public class LLModel : ILLModel
-{
-    protected readonly IntPtr _handle;
-    private readonly ILogger _logger;
-    private bool _disposed;
-
-    internal LLModel(IntPtr handle, ILogger? logger = null)
-    {
-        _handle = handle;
-        _logger = logger ?? NullLogger.Instance;
-    }
-
-    /// <summary>
-    /// Create a new model from a pointer
-    /// </summary>
-    /// <param name="handle">Pointer to underlying model</param>
-    public static LLModel Create(IntPtr handle, ILogger? logger = null)
-    {
-        return new LLModel(handle, logger: logger);
-    }
-
-    /// <summary>
-    /// Generate a response using the model
-    /// </summary>
-    /// <param name="text">The input promp</param>
-    /// <param name="context">The context</param>
-    /// <param name="promptCallback">A callback function for handling the processing of prompt</param>
-    /// <param name="responseCallback">A callback function for handling the generated response</param>
-    /// <param name="recalculateCallback">A callback function for handling recalculation requests</param>
-    /// <param name="cancellationToken"></param>
-    public void Prompt(
-        string text,
-        LLModelPromptContext context,
-        Func<ModelPromptEventArgs, bool>? promptCallback = null,
-        Func<ModelResponseEventArgs, bool>? responseCallback = null,
-        Func<ModelRecalculatingEventArgs, bool>? recalculateCallback = null,
-        CancellationToken cancellationToken = default)
-    {
-        GC.KeepAlive(promptCallback);
-        GC.KeepAlive(responseCallback);
-        GC.KeepAlive(recalculateCallback);
-        GC.KeepAlive(cancellationToken);
-
-        _logger.LogInformation("Prompt input='{Prompt}' ctx={Context}", text, context.Dump());
-
-        NativeMethods.llmodel_prompt(
-            _handle,
-            text,
-            (tokenId) =>
-            {
-                if (cancellationToken.IsCancellationRequested) return false;
-                if (promptCallback == null) return true;
-                var args = new ModelPromptEventArgs(tokenId);
-                return promptCallback(args);
-            },
-            (tokenId, response) =>
-            {
-                if (cancellationToken.IsCancellationRequested)
-                {
-                    _logger.LogDebug("ResponseCallback evt=CancellationRequested");
-                    return false;
-                }
-
-                if (responseCallback == null) return true;
-                var args = new ModelResponseEventArgs(tokenId, response);
-                return responseCallback(args);
-            },
-            (isRecalculating) =>
-            {
-                if (cancellationToken.IsCancellationRequested) return false;
-                if (recalculateCallback == null) return true;
-                var args = new ModelRecalculatingEventArgs(isRecalculating);
-                return recalculateCallback(args);
-            },
-            ref context.UnderlyingContext
-        );
-    }
-
-    /// <summary>
-    ///  Set the number of threads to be used by the model.
-    /// </summary>
-    /// <param name="threadCount">The new thread count</param>
-    public void SetThreadCount(int threadCount)
-    {
-        NativeMethods.llmodel_setThreadCount(_handle, threadCount);
-    }
-
-    /// <summary>
-    /// Get  the number of threads used by the model.
-    /// </summary>
-    /// <returns>the number of threads used by the model</returns>
-    public int GetThreadCount()
-    {
-        return NativeMethods.llmodel_threadCount(_handle);
-    }
-
-    /// <summary>
-    /// Get the size of the internal state of the model.
-    /// </summary>
-    /// <remarks>
-    /// This state data is specific to the type of model you have created.
-    /// </remarks>
-    /// <returns>the size in bytes of the internal state of the model</returns>
-    public ulong GetStateSizeBytes()
-    {
-        return NativeMethods.llmodel_get_state_size(_handle);
-    }
-
-    /// <summary>
-    /// Saves the internal state of the model to the specified destination address.
-    /// </summary>
-    /// <param name="source">A pointer to the src</param>
-    /// <returns>The number of bytes copied</returns>
-    public unsafe ulong SaveStateData(byte* source)
-    {
-        return NativeMethods.llmodel_save_state_data(_handle, source);
-    }
-
-    /// <summary>
-    /// Restores the internal state of the model using data from the specified address.
-    /// </summary>
-    /// <param name="destination">A pointer to destination</param>
-    /// <returns>the number of bytes read</returns>
-    public unsafe ulong RestoreStateData(byte* destination)
-    {
-        return NativeMethods.llmodel_restore_state_data(_handle, destination);
-    }
-
-    /// <summary>
-    /// Check if the model is loaded.
-    /// </summary>
-    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
-    public bool IsLoaded()
-    {
-        return NativeMethods.llmodel_isModelLoaded(_handle);
-    }
-
-    /// <summary>
-    /// Load the model from a file.
-    /// </summary>
-    /// <param name="modelPath">The path to the model file.</param>
-    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
-    public bool Load(string modelPath)
-    {
-        return NativeMethods.llmodel_loadModel(_handle, modelPath, 2048, 100);
-    }
-
-    protected void Destroy()
-    {
-        NativeMethods.llmodel_model_destroy(_handle);
-    }
-    protected virtual void Dispose(bool disposing)
-    {
-        if (_disposed) return;
-
-        if (disposing)
-        {
-            // dispose managed state
-        }
-
-        Destroy();
-
-        _disposed = true;
-    }
-
-    public void Dispose()
-    {
-        Dispose(disposing: true);
-        GC.SuppressFinalize(this);
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Bindings/LLPromptContext.cs

@@ -1,147 +0,0 @@
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Wrapper around the llmodel_prompt_context structure for holding the prompt context.
-/// </summary>
-/// <remarks>
-/// The implementation takes care of all the memory handling of the raw logits pointer and the
-/// raw tokens pointer.Attempting to resize them or modify them in any way can lead to undefined behavior
-/// </remarks>
-public unsafe class LLModelPromptContext
-{
-    private llmodel_prompt_context _ctx;
-
-    internal ref llmodel_prompt_context UnderlyingContext => ref _ctx;
-
-    public LLModelPromptContext()
-    {
-        _ctx = new();
-    }
-
-    /// <summary>
-    /// logits of current context
-    /// </summary>
-    public Span<float> Logits => new(_ctx.logits, (int)_ctx.logits_size);
-
-    /// <summary>
-    /// the size of the raw logits vector
-    /// </summary>
-    public nuint LogitsSize
-    {
-        get => _ctx.logits_size;
-        set => _ctx.logits_size = value;
-    }
-
-    /// <summary>
-    /// current tokens in the context window
-    /// </summary>
-    public Span<int> Tokens => new(_ctx.tokens, (int)_ctx.tokens_size);
-
-    /// <summary>
-    /// the size of the raw tokens vector
-    /// </summary>
-    public nuint TokensSize
-    {
-        get => _ctx.tokens_size;
-        set => _ctx.tokens_size = value;
-    }
-
-    /// <summary>
-    /// top k logits to sample from
-    /// </summary>
-    public int TopK
-    {
-        get => _ctx.top_k;
-        set => _ctx.top_k = value;
-    }
-
-    /// <summary>
-    /// nucleus sampling probability threshold
-    /// </summary>
-    public float TopP
-    {
-        get => _ctx.top_p;
-        set => _ctx.top_p = value;
-    }
-
-    /// <summary>
-    /// min p sampling probability threshold
-    /// </summary>
-    public float MinP
-    {
-        get => _ctx.min_p;
-        set => _ctx.min_p = value;
-    }
-
-    /// <summary>
-    /// temperature to adjust model's output distribution
-    /// </summary>
-    public float Temperature
-    {
-        get => _ctx.temp;
-        set => _ctx.temp = value;
-    }
-
-    /// <summary>
-    /// number of tokens in past conversation
-    /// </summary>
-    public int PastNum
-    {
-        get => _ctx.n_past;
-        set => _ctx.n_past = value;
-    }
-
-    /// <summary>
-    /// number of predictions to generate in parallel
-    /// </summary>
-    public int Batches
-    {
-        get => _ctx.n_batch;
-        set => _ctx.n_batch = value;
-    }
-
-    /// <summary>
-    /// number of tokens to predict
-    /// </summary>
-    public int TokensToPredict
-    {
-        get => _ctx.n_predict;
-        set => _ctx.n_predict = value;
-    }
-
-    /// <summary>
-    /// penalty factor for repeated tokens
-    /// </summary>
-    public float RepeatPenalty
-    {
-        get => _ctx.repeat_penalty;
-        set => _ctx.repeat_penalty = value;
-    }
-
-    /// <summary>
-    /// last n tokens to penalize
-    /// </summary>
-    public int RepeatLastN
-    {
-        get => _ctx.repeat_last_n;
-        set => _ctx.repeat_last_n = value;
-    }
-
-    /// <summary>
-    /// number of tokens possible in context window
-    /// </summary>
-    public int ContextSize
-    {
-        get => _ctx.n_ctx;
-        set => _ctx.n_ctx = value;
-    }
-
-    /// <summary>
-    /// percent of context to erase if we exceed the context window
-    /// </summary>
-    public float ContextErase
-    {
-        get => _ctx.context_erase;
-        set => _ctx.context_erase = value;
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeMethods.cs

@@ -1,112 +0,0 @@
-using System.Runtime.InteropServices;
-
-namespace Gpt4All.Bindings;
-
-public unsafe partial struct llmodel_prompt_context
-{
-    public float* logits;
-
-    [NativeTypeName("size_t")]
-    public nuint logits_size;
-
-    [NativeTypeName("int32_t *")]
-    public int* tokens;
-
-    [NativeTypeName("size_t")]
-    public nuint tokens_size;
-
-    [NativeTypeName("int32_t")]
-    public int n_past;
-
-    [NativeTypeName("int32_t")]
-    public int n_ctx;
-
-    [NativeTypeName("int32_t")]
-    public int n_predict;
-
-    [NativeTypeName("int32_t")]
-    public int top_k;
-
-    public float top_p;
-
-    public float min_p;
-
-    public float temp;
-
-    [NativeTypeName("int32_t")]
-    public int n_batch;
-
-    public float repeat_penalty;
-
-    [NativeTypeName("int32_t")]
-    public int repeat_last_n;
-
-    public float context_erase;
-}
-#pragma warning disable CA2101
-internal static unsafe partial class NativeMethods
-{
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelResponseCallback(int token_id, [MarshalAs(UnmanagedType.LPUTF8Str)] string response);
-
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelPromptCallback(int token_id);
-
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelRecalculateCallback(bool isRecalculating);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_model_create2(
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string build_variant,
-        out IntPtr error);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_model_destroy([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public static extern bool llmodel_loadModel(
-        [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path,
-        [NativeTypeName("int32_t")] int n_ctx,
-        [NativeTypeName("int32_t")] int ngl);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-
-    [return: MarshalAs(UnmanagedType.I1)]
-    public static extern bool llmodel_isModelLoaded([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_get_state_size([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_save_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("uint8_t *")] byte* dest);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_restore_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("const uint8_t *")] byte* src);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    public static extern void llmodel_prompt(
-        [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string prompt,
-        LlmodelPromptCallback prompt_callback,
-        LlmodelResponseCallback response_callback,
-        LlmodelRecalculateCallback recalculate_callback,
-        ref llmodel_prompt_context ctx);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_setThreadCount([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("int32_t")] int n_threads);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("int32_t")]
-    public static extern int llmodel_threadCount([NativeTypeName("llmodel_model")] IntPtr model);
-}
-#pragma warning restore CA2101

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeTypeNameAttribute.cs

@@ -1,21 +0,0 @@
-using System.Diagnostics;
-
-namespace Gpt4All.Bindings;
-
-/// <summary>Defines the type of a member as it was used in the native signature.</summary>
-[AttributeUsage(AttributeTargets.Struct | AttributeTargets.Enum | AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.ReturnValue, AllowMultiple = false, Inherited = true)]
-[Conditional("DEBUG")]
-internal sealed partial class NativeTypeNameAttribute : Attribute
-{
-    private readonly string _name;
-
-    /// <summary>Initializes a new instance of the <see cref="NativeTypeNameAttribute" /> class.</summary>
-    /// <param name="name">The name of the type that was used in the native signature.</param>
-    public NativeTypeNameAttribute(string name)
-    {
-        _name = name;
-    }
-
-    /// <summary>Gets the name of the type that was used in the native signature.</summary>
-    public string Name => _name;
-}

gpt4all-bindings/csharp/Gpt4All/Extensions/LLPromptContextExtensions.cs

@@ -1,27 +0,0 @@
-using Gpt4All.Bindings;
-
-namespace Gpt4All;
-
-internal static class LLPromptContextExtensions
-{
-    public static string Dump(this LLModelPromptContext context)
-    {
-        var ctx = context.UnderlyingContext;
-        return @$"
-        {{
-            logits_size = {ctx.logits_size}
-            tokens_size = {ctx.tokens_size}
-            n_past = {ctx.n_past}
-            n_ctx = {ctx.n_ctx}
-            n_predict = {ctx.n_predict}
-            top_k = {ctx.top_k}
-            top_p = {ctx.top_p}
-            min_p = {ctx.min_p}
-            temp = {ctx.temp}
-            n_batch = {ctx.n_batch}
-            repeat_penalty = {ctx.repeat_penalty}
-            repeat_last_n = {ctx.repeat_last_n}
-            context_erase = {ctx.context_erase}
-        }}";
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Extensions/PredictRequestOptionsExtensions.cs

@@ -1,26 +0,0 @@
-using Gpt4All.Bindings;
-
-namespace Gpt4All;
-
-public static class PredictRequestOptionsExtensions
-{
-    public static LLModelPromptContext ToPromptContext(this PredictRequestOptions opts)
-    {
-        return new LLModelPromptContext
-        {
-            LogitsSize = opts.LogitsSize,
-            TokensSize = opts.TokensSize,
-            TopK = opts.TopK,
-            TopP = opts.TopP,
-            MinP = opts.MinP,
-            PastNum = opts.PastConversationTokensNum,
-            RepeatPenalty = opts.RepeatPenalty,
-            Temperature = opts.Temperature,
-            RepeatLastN = opts.RepeatLastN,
-            Batches = opts.Batches,
-            ContextErase = opts.ContextErase,
-            ContextSize = opts.ContextSize,
-            TokensToPredict = opts.TokensToPredict
-        };
-    }
-}

gpt4all-bindings/csharp/Gpt4All/GenLLModelBindings.rsp

@@ -1,21 +0,0 @@
---config
-exclude-funcs-with-body
---with-access-specifier
-*=Public
---include-directory
-..\..\..\gpt4all-backend\
---file
-..\..\..\gpt4all-backend\llmodel_c.h
---libraryPath
-libllmodel
---remap
-sbyte*=IntPtr
-void*=IntPtr
---namespace
-Gpt4All.Bindings
---methodClassName
-NativeMethods
---output
-.\Bindings\NativeMethods.cs
---output-mode
-CSharp

gpt4all-bindings/csharp/Gpt4All/Gpt4All.cs

@@ -1,135 +0,0 @@
-using System.Diagnostics;
-using System.Runtime.CompilerServices;
-using Gpt4All.Bindings;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
-
-[assembly: InternalsVisibleTo("Gpt4All.Tests")]
-
-namespace Gpt4All;
-
-public class Gpt4All : IGpt4AllModel
-{
-    private readonly ILLModel _model;
-    private readonly ILogger _logger;
-
-    private const string ResponseErrorMessage =
-        "The model reported an error during token generation error={ResponseError}";
-
-    /// <inheritdoc/>
-    public IPromptFormatter? PromptFormatter { get; set; }
-
-    internal Gpt4All(ILLModel model, ILogger? logger = null)
-    {
-        _model = model;
-        _logger = logger ?? NullLogger.Instance;
-        PromptFormatter = new DefaultPromptFormatter();
-    }
-
-    private string FormatPrompt(string prompt)
-    {
-        if (PromptFormatter == null) return prompt;
-
-        return PromptFormatter.FormatPrompt(prompt);
-    }
-
-    public Task<ITextPredictionResult> GetPredictionAsync(string text, PredictRequestOptions opts, CancellationToken cancellationToken = default)
-    {
-        ArgumentNullException.ThrowIfNull(text);
-
-        return Task.Run(() =>
-        {
-            _logger.LogInformation("Start prediction task");
-
-            var sw = Stopwatch.StartNew();
-            var result = new TextPredictionResult();
-            var context = opts.ToPromptContext();
-            var prompt = FormatPrompt(text);
-
-            try
-            {
-                _model.Prompt(prompt, context, responseCallback: e =>
-                {
-                    if (e.IsError)
-                    {
-                        _logger.LogWarning(ResponseErrorMessage, e.Response);
-                        result.Success = false;
-                        result.ErrorMessage = e.Response;
-                        return false;
-                    }
-                    result.Append(e.Response);
-                    return true;
-                }, cancellationToken: cancellationToken);
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, "Prompt error");
-                result.Success = false;
-            }
-
-            sw.Stop();
-            _logger.LogInformation("Prediction task completed elapsed={Elapsed}s", sw.Elapsed.TotalSeconds);
-
-            return (ITextPredictionResult)result;
-        }, CancellationToken.None);
-    }
-
-    public Task<ITextPredictionStreamingResult> GetStreamingPredictionAsync(string text, PredictRequestOptions opts, CancellationToken cancellationToken = default)
-    {
-        ArgumentNullException.ThrowIfNull(text);
-
-        var result = new TextPredictionStreamingResult();
-
-        _ = Task.Run(() =>
-        {
-            _logger.LogInformation("Start streaming prediction task");
-            var sw = Stopwatch.StartNew();
-
-            try
-            {
-                var context = opts.ToPromptContext();
-                var prompt = FormatPrompt(text);
-
-                _model.Prompt(prompt, context, responseCallback: e =>
-                {
-                    if (e.IsError)
-                    {
-                        _logger.LogWarning(ResponseErrorMessage, e.Response);
-                        result.Success = false;
-                        result.ErrorMessage = e.Response;
-                        return false;
-                    }
-                    result.Append(e.Response);
-                    return true;
-                }, cancellationToken: cancellationToken);
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, "Prompt error");
-                result.Success = false;
-            }
-            finally
-            {
-                result.Complete();
-                sw.Stop();
-                _logger.LogInformation("Prediction task completed elapsed={Elapsed}s", sw.Elapsed.TotalSeconds);
-            }
-        }, CancellationToken.None);
-
-        return Task.FromResult((ITextPredictionStreamingResult)result);
-    }
-
-    protected virtual void Dispose(bool disposing)
-    {
-        if (disposing)
-        {
-            _model.Dispose();
-        }
-    }
-
-    public void Dispose()
-    {
-        Dispose(true);
-        GC.SuppressFinalize(this);
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Gpt4All.csproj

@@ -1,23 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-    <PropertyGroup>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-        <GenerateDocumentationFile>true</GenerateDocumentationFile>
-        <TargetFramework>net8.0</TargetFramework>
-    </PropertyGroup>
-    <ItemGroup>
-        <!-- Windows -->
-        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
-        <!-- Linux -->
-        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
-        <!-- MacOS -->
-        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
-        <Content Include="..\runtimes\osx\native\*.metal" Pack="true" PackagePath="contentFiles\any\any;content">
-            <PackageCopyToOutput>true</PackageCopyToOutput>
-        </Content>
-    </ItemGroup>
-    <ItemGroup>
-        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
-    </ItemGroup>
-</Project>

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/ILibraryLoader.cs

@@ -1,6 +0,0 @@
-namespace Gpt4All.LibraryLoader;
-
-public interface ILibraryLoader
-{
-    LoadResult OpenLibrary(string? fileName);
-}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LinuxLibraryLoader.cs

@@ -1,53 +0,0 @@
-using System.Runtime.InteropServices;
-
-namespace Gpt4All.LibraryLoader;
-
-internal class LinuxLibraryLoader : ILibraryLoader
-{
-#pragma warning disable CA2101
-    [DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
-#pragma warning restore CA2101
-    public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags);
-
-#pragma warning disable CA2101
-    [DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
-#pragma warning restore CA2101
-    public static extern IntPtr NativeOpenLibraryLibdl2(string? filename, int flags);
-
-    [DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
-    public static extern IntPtr GetLoadError();
-
-    [DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
-    public static extern IntPtr GetLoadError2();
-
-    public LoadResult OpenLibrary(string? fileName)
-    {
-        IntPtr loadedLib;
-        try
-        {
-            // open with rtls lazy flag
-            loadedLib = NativeOpenLibraryLibdl2(fileName, 0x00001);
-        }
-        catch (DllNotFoundException)
-        {
-            loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001);
-        }
-
-        if (loadedLib == IntPtr.Zero)
-        {
-            string errorMessage;
-            try
-            {
-                errorMessage = Marshal.PtrToStringAnsi(GetLoadError2()) ?? "Unknown error";
-            }
-            catch (DllNotFoundException)
-            {
-                errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error";
-            }
-
-            return LoadResult.Failure(errorMessage);
-        }
-
-        return LoadResult.Success;
-    }
-}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LoadResult.cs

@@ -1,20 +0,0 @@
-namespace Gpt4All.LibraryLoader;
-
-public class LoadResult
-{
-    private LoadResult(bool isSuccess, string? errorMessage)
-    {
-        IsSuccess = isSuccess;
-        ErrorMessage = errorMessage;
-    }
-
-    public static LoadResult Success { get; } = new(true, null);
-
-    public static LoadResult Failure(string errorMessage)
-    {
-        return new(false, errorMessage);
-    }
-
-    public bool IsSuccess { get; }
-    public string? ErrorMessage { get; }
-}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/MacOsLibraryLoader.cs

@@ -1,28 +0,0 @@
-using System.Runtime.InteropServices;
-
-namespace Gpt4All.LibraryLoader;
-
-internal class MacOsLibraryLoader : ILibraryLoader
-{
-#pragma warning disable CA2101
-    [DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
-#pragma warning restore CA2101
-    public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags);
-
-    [DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
-    public static extern IntPtr GetLoadError();
-
-    public LoadResult OpenLibrary(string? fileName)
-    {
-        var loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001);
-
-        if (loadedLib == IntPtr.Zero)
-        {
-            var errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error";
-
-            return LoadResult.Failure(errorMessage);
-        }
-
-        return LoadResult.Success;
-    }
-}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/NativeLibraryLoader.cs

@@ -1,81 +0,0 @@
-#if !IOS && !MACCATALYST && !TVOS && !ANDROID
-using System.Runtime.InteropServices;
-#endif
-
-namespace Gpt4All.LibraryLoader;
-
-public static class NativeLibraryLoader
-{
-    private static ILibraryLoader? defaultLibraryLoader;
-
-    /// <summary>
-    /// Sets the library loader used to load the native libraries. Overwrite this only if you want some custom loading.
-    /// </summary>
-    /// <param name="libraryLoader">The library loader to be used.</param>
-    public static void SetLibraryLoader(ILibraryLoader libraryLoader)
-    {
-        defaultLibraryLoader = libraryLoader;
-    }
-
-    internal static LoadResult LoadNativeLibrary(string? path = default, bool bypassLoading = true)
-    {
-        // If the user has handled loading the library themselves, we don't need to do anything.
-        if (bypassLoading)
-        {
-            return LoadResult.Success;
-        }
-
-        var architecture = RuntimeInformation.OSArchitecture switch
-        {
-            Architecture.X64 => "x64",
-            Architecture.X86 => "x86",
-            Architecture.Arm => "arm",
-            Architecture.Arm64 => "arm64",
-            _ => throw new PlatformNotSupportedException(
-                $"Unsupported OS platform, architecture: {RuntimeInformation.OSArchitecture}")
-        };
-
-        var (platform, extension) = Environment.OSVersion.Platform switch
-        {
-            _ when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => ("win", "dll"),
-            _ when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => ("linux", "so"),
-            _ when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => ("osx", "dylib"),
-            _ => throw new PlatformNotSupportedException(
-                $"Unsupported OS platform, architecture: {RuntimeInformation.OSArchitecture}")
-        };
-
-        // If the user hasn't set the path, we'll try to find it ourselves.
-        if (string.IsNullOrEmpty(path))
-        {
-            var libraryName = "libllmodel";
-            var assemblySearchPath = new[]
-            {
-                AppDomain.CurrentDomain.RelativeSearchPath,
-                Path.GetDirectoryName(typeof(NativeLibraryLoader).Assembly.Location),
-                Path.GetDirectoryName(Environment.GetCommandLineArgs()[0])
-            }.FirstOrDefault(it => !string.IsNullOrEmpty(it));
-            // Search for the library dll within the assembly search path. If it doesn't exist, for whatever reason, use the default path.
-            path = Directory.EnumerateFiles(assemblySearchPath ?? string.Empty, $"{libraryName}.{extension}", SearchOption.AllDirectories).FirstOrDefault() ?? Path.Combine("runtimes", $"{platform}-{architecture}", $"{libraryName}.{extension}");
-        }
-
-        if (defaultLibraryLoader != null)
-        {
-            return defaultLibraryLoader.OpenLibrary(path);
-        }
-
-        if (!File.Exists(path))
-        {
-            throw new FileNotFoundException($"Native Library not found in path {path}. " +
-                                            $"Verify you have have included the native Gpt4All library in your application.");
-        }
-
-        ILibraryLoader libraryLoader = platform switch
-        {
-            "win" => new WindowsLibraryLoader(),
-            "osx" => new MacOsLibraryLoader(),
-            "linux" => new LinuxLibraryLoader(),
-            _ => throw new PlatformNotSupportedException($"Currently {platform} platform is not supported")
-        };
-        return libraryLoader.OpenLibrary(path);
-    }
-}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/WindowsLibraryLoader.cs

@@ -1,24 +0,0 @@
-using System.ComponentModel;
-using System.Runtime.InteropServices;
-
-namespace Gpt4All.LibraryLoader;
-
-internal class WindowsLibraryLoader : ILibraryLoader
-{
-    public LoadResult OpenLibrary(string? fileName)
-    {
-        var loadedLib = LoadLibrary(fileName);
-
-        if (loadedLib == IntPtr.Zero)
-        {
-            var errorCode = Marshal.GetLastWin32Error();
-            var errorMessage = new Win32Exception(errorCode).Message;
-            return LoadResult.Failure(errorMessage);
-        }
-
-        return LoadResult.Success;
-    }
-
-    [DllImport("kernel32", SetLastError = true, CharSet = CharSet.Auto)]
-    private static extern IntPtr LoadLibrary([MarshalAs(UnmanagedType.LPWStr)] string? lpFileName);
-}

gpt4all-bindings/csharp/Gpt4All/Model/DefaultPromptFormatter.cs

@@ -1,16 +0,0 @@
-namespace Gpt4All;
-
-public class DefaultPromptFormatter : IPromptFormatter
-{
-    public string FormatPrompt(string prompt)
-    {
-        return $"""
-        ### Instruction: 
-        The prompt below is a question to answer, a task to complete, or a conversation
-        to respond to; decide which and write an appropriate response.
-        ### Prompt:
-        {prompt}
-        ### Response:
-        """;
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Model/Gpt4AllModelFactory.cs

@@ -1,62 +0,0 @@
-using System.Diagnostics;
-using Microsoft.Extensions.Logging.Abstractions;
-using Microsoft.Extensions.Logging;
-using Gpt4All.Bindings;
-using Gpt4All.LibraryLoader;
-using System.Runtime.InteropServices;
-
-namespace Gpt4All;
-
-public class Gpt4AllModelFactory : IGpt4AllModelFactory
-{
-    private readonly ILoggerFactory _loggerFactory;
-    private readonly ILogger _logger;
-    private static bool bypassLoading;
-    private static string? libraryPath;
-
-    private static readonly Lazy<LoadResult> libraryLoaded = new(() =>
-    {
-        return NativeLibraryLoader.LoadNativeLibrary(Gpt4AllModelFactory.libraryPath, Gpt4AllModelFactory.bypassLoading);
-    }, true);
-
-    public Gpt4AllModelFactory(string? libraryPath = default, bool bypassLoading = true, ILoggerFactory? loggerFactory = null)
-    {
-        _loggerFactory = loggerFactory ?? NullLoggerFactory.Instance;
-        _logger = _loggerFactory.CreateLogger<Gpt4AllModelFactory>();
-        Gpt4AllModelFactory.libraryPath = libraryPath;
-        Gpt4AllModelFactory.bypassLoading = bypassLoading;
-
-        if (!libraryLoaded.Value.IsSuccess)
-        {
-            throw new Exception($"Failed to load native gpt4all library. Error: {libraryLoaded.Value.ErrorMessage}");
-        }
-    }
-
-    private Gpt4All CreateModel(string modelPath)
-    {
-        _logger.LogInformation("Creating model path={ModelPath}", modelPath);
-        IntPtr error;
-        var handle = NativeMethods.llmodel_model_create2(modelPath, "auto", out error);
-        if (error != IntPtr.Zero)
-        {
-            throw new Exception(Marshal.PtrToStringAnsi(error));
-        }
-        _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
-        _logger.LogInformation("Model loading started");
-        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath, 2048, 100);
-        _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
-        if (!loadedSuccessfully)
-        {
-            throw new Exception($"Failed to load model: '{modelPath}'");
-        }
-
-        var logger = _loggerFactory.CreateLogger<LLModel>();
-        var underlyingModel = LLModel.Create(handle, logger: logger);
-
-        Debug.Assert(underlyingModel.IsLoaded());
-
-        return new Gpt4All(underlyingModel, logger: logger);
-    }
-
-    public IGpt4AllModel LoadModel(string modelPath) => CreateModel(modelPath);
-}

gpt4all-bindings/csharp/Gpt4All/Model/IGpt4AllModel.cs

@@ -1,10 +0,0 @@
-namespace Gpt4All;
-
-public interface IGpt4AllModel : ITextPrediction, IDisposable
-{
-    /// <summary>
-    /// The prompt formatter used to format the prompt before
-    /// feeding it to the model, if null no transformation is applied
-    /// </summary>
-    IPromptFormatter? PromptFormatter { get; set; }
-}

gpt4all-bindings/csharp/Gpt4All/Model/IGpt4AllModelFactory.cs

@@ -1,6 +0,0 @@
-namespace Gpt4All;
-
-public interface IGpt4AllModelFactory
-{
-    IGpt4AllModel LoadModel(string modelPath);
-}

gpt4all-bindings/csharp/Gpt4All/Model/IPromptFormatter.cs

@@ -1,14 +0,0 @@
-namespace Gpt4All;
-
-/// <summary>
-/// Formats a prompt
-/// </summary>
-public interface IPromptFormatter
-{
-    /// <summary>
-    /// Format the provided prompt
-    /// </summary>
-    /// <param name="prompt">the input prompt</param>
-    /// <returns>The formatted prompt</returns>
-    string FormatPrompt(string prompt);
-}

gpt4all-bindings/csharp/Gpt4All/Model/ModelOptions.cs

@@ -1,6 +0,0 @@
-namespace Gpt4All;
-
-public record ModelOptions
-{
-    public int Threads { get; init; } = 4;
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/ITextPrediction.cs

@@ -1,31 +0,0 @@
-namespace Gpt4All;
-
-/// <summary>
-/// Interface for text prediction services
-/// </summary>
-public interface ITextPrediction
-{
-    /// <summary>
-    /// Get prediction results for the prompt and provided options.
-    /// </summary>
-    /// <param name="text">The text to complete</param>
-    /// <param name="opts">The prediction settings</param>
-    /// <param name="cancellation">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
-    /// <returns>The prediction result generated by the model</returns>
-    Task<ITextPredictionResult> GetPredictionAsync(
-        string text,
-        PredictRequestOptions opts,
-        CancellationToken cancellation = default);
-
-    /// <summary>
-    /// Get streaming prediction results for the prompt and provided options.
-    /// </summary>
-    /// <param name="text">The text to complete</param>
-    /// <param name="opts">The prediction settings</param>
-    /// <param name="cancellationToken">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
-    /// <returns>The prediction result generated by the model</returns>
-    Task<ITextPredictionStreamingResult> GetStreamingPredictionAsync(
-        string text,
-        PredictRequestOptions opts,
-        CancellationToken cancellationToken = default);
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/ITextPredictionResult.cs

@@ -1,10 +0,0 @@
-namespace Gpt4All;
-
-public interface ITextPredictionResult
-{
-    bool Success { get; }
-
-    string? ErrorMessage { get; }
-
-    Task<string> GetPredictionAsync(CancellationToken cancellationToken = default);
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/ITextPredictionStreamingResult.cs

@@ -1,6 +0,0 @@
-namespace Gpt4All;
-
-public interface ITextPredictionStreamingResult : ITextPredictionResult
-{
-    IAsyncEnumerable<string> GetPredictionStreamingAsync(CancellationToken cancellationToken = default);
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/PredictRequestOptions.cs

@@ -1,32 +0,0 @@
-namespace Gpt4All;
-
-public record PredictRequestOptions
-{
-    public nuint LogitsSize { get; init; } = 0;
-
-    public nuint TokensSize { get; init; } = 0;
-
-    public int PastConversationTokensNum { get; init; } = 0;
-
-    public int ContextSize { get; init; } = 1024;
-
-    public int TokensToPredict { get; init; } = 128;
-
-    public int TopK { get; init; } = 40;
-
-    public float TopP { get; init; } = 0.9f;
-
-    public float MinP { get; init; } = 0.0f;
-
-    public float Temperature { get; init; } = 0.1f;
-
-    public int Batches { get; init; } = 8;
-
-    public float RepeatPenalty { get; init; } = 1.2f;
-
-    public int RepeatLastN { get; init; } = 10;
-
-    public float ContextErase { get; init; } = 0.5f;
-
-    public static readonly PredictRequestOptions Defaults = new();
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/TextPredictionResult.cs

@@ -1,27 +0,0 @@
-using System.Text;
-
-namespace Gpt4All;
-
-public record TextPredictionResult : ITextPredictionResult
-{
-    private readonly StringBuilder _result;
-
-    public bool Success { get; internal set; } = true;
-
-    public string? ErrorMessage { get; internal set; }
-
-    internal TextPredictionResult()
-    {
-        _result = new StringBuilder();
-    }
-
-    internal void Append(string token)
-    {
-        _result.Append(token);
-    }
-
-    public Task<string> GetPredictionAsync(CancellationToken cancellationToken = default)
-    {
-        return Task.FromResult(_result.ToString());
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/TextPredictionStreamingResult.cs

@@ -1,49 +0,0 @@
-using System.Text;
-using System.Threading.Channels;
-
-namespace Gpt4All;
-
-public record TextPredictionStreamingResult : ITextPredictionStreamingResult
-{
-    private readonly Channel<string> _channel;
-
-    public bool Success { get; internal set; } = true;
-
-    public string? ErrorMessage { get; internal set; }
-
-    public Task Completion => _channel.Reader.Completion;
-
-    internal TextPredictionStreamingResult()
-    {
-        _channel = Channel.CreateUnbounded<string>();
-    }
-
-    internal bool Append(string token)
-    {
-        return _channel.Writer.TryWrite(token);
-    }
-
-    internal void Complete()
-    {
-        _channel.Writer.Complete();
-    }
-
-    public async Task<string> GetPredictionAsync(CancellationToken cancellationToken = default)
-    {
-        var sb = new StringBuilder();
-
-        var tokens = GetPredictionStreamingAsync(cancellationToken).ConfigureAwait(false);
-
-        await foreach (var token in tokens)
-        {
-            sb.Append(token);
-        }
-
-        return sb.ToString();
-    }
-
-    public IAsyncEnumerable<string> GetPredictionStreamingAsync(CancellationToken cancellationToken = default)
-    {
-        return _channel.Reader.ReadAllAsync(cancellationToken);
-    }
-}

gpt4all-bindings/csharp/Gpt4All/gen_bindings.ps1

@@ -1 +0,0 @@
-ClangSharpPInvokeGenerator @(Get-Content .\GenLLModelBindings.rsp)

gpt4all-bindings/csharp/README.md

@@ -1,124 +0,0 @@
-# C# GPT4All
-
-This package contains a set of C# bindings around the `llmodel` C-API.
-
-## Documentation
-TBD
-
-## Installation
-
-Windows and Linux builds are available on NuGet: https://www.nuget.org/packages/Gpt4All
-
-macOS is WIP due to code signing issues, contributions are welcome.
-
-## Project Structure
-```
-gpt4all-bindings/
-└── csharp                
-    ├── Gpt4All               // .NET Bindigs
-    ├── Gpt4All.Samples       // Sample project
-    ├── build_win-msvc.ps1    // Native build scripts
-    ├── build_win-mingw.ps1   
-    ├── build_linux.sh        
-    └── runtimes              // [POST-BUILD] Platform-specific native libraries
-          ├── win-x64
-          ├── ...
-          └── linux-x64
-```
-
-## Prerequisites
-
-On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
-
-macOS users do not need Vulkan, as GPT4All will use Metal instead.
-
-## Local Build Instructions
-> **Note** 
-> Tested On:
->  - Windows 11 22H + VS2022 (CE) x64
->  - Linux Ubuntu x64
->  - Linux Ubuntu (WSL2) x64
-
-1. Setup the repository
-2. Build the native libraries for the platform of choice (see below)
-3. Build the C# Bindings (NET6+ SDK is required)
-```
-git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
-cd gpt4all/gpt4all-bindings/csharp
-```
-### Linux
-1. Setup build environment and install NET6+ SDK with the appropriate procedure for your distribution
-```
-sudo apt-get update
-sudo apt-get install -y cmake build-essential
-chmod +x ./build_linux.sh
-```
-2. `./build_linux.sh`
-3. The native libraries should be present at `.\native\linux-x64`
-
-### Windows - MinGW64
-#### Additional requirements
-  - [MinGW64](https://www.mingw-w64.org/) 
-  - CMAKE
-1. Setup
-```
-choco install mingw
-$env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
-choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
-```
-2. Run the `./build_win-mingw.ps1` build script
-3. The native libraries should be present at `.\native\win-x64`
-
-### Windows - MSVC
-#### Additional requirements
-  - Visual Studio 2022
-1. Open a terminal using the  `x64 Native Tools Command Prompt for VS 2022` (`vcvars64.bat`)
-2. Run the `./build_win-msvc.ps1` build script
-3. `libllmodel.dll` and `libllama.dll` should be present at `.\native\win-x64`
-
-> **Warning** 
-> If the build fails with: '**error C7555: use of designated initializers requires at least '/std:c++20'**'
->
-> Modify `cd gpt4all/gpt4all-backends/CMakeLists.txt` adding `CXX_STANDARD_20` to `llmodel` properties.
-> ```cmake
-> set_target_properties(llmodel PROPERTIES
->                              VERSION ${PROJECT_VERSION}
->                              CXX_STANDARD 20 # <---- ADD THIS -----------------------
->                              SOVERSION ${PROJECT_VERSION_MAJOR})
-> ```
-## C# Bindings Build Instructions
-Build the `Gpt4All` (or `Gpt4All.Samples`) projects from within VisualStudio.
-### Try the bindings
-```csharp
-using Gpt4All;
-
-// load the model
-var modelFactory = new ModelFactory();
-
-using var model = modelFactory.LoadModel("./path/to/ggml-gpt4all-j-v1.3-groovy.bin");
-
-var input = "Name 3 Colors";
-
-// request a prediction
-var result = await model.GetStreamingPredictionAsync(
-    input, 
-    PredictRequestOptions.Defaults);
-
-// asynchronously print the tokens as soon as they are produces by the model
-await foreach(var token in result.GetPredictionStreamingAsync())
-{
-    Console.Write(token);
-}
-```
-Output:
-```
-gptj_model_load: loading model from 'ggml-gpt4all-j-v1.3-groovy.bin' - please wait ...
-gptj_model_load: n_vocab = 50400
-[...TRUNCATED...]
-gptj_model_load: ggml ctx size = 5401.45 MB
-gptj_model_load: kv self size  =  896.00 MB
-gptj_model_load: ................................... done
-gptj_model_load: model size =  3609.38 MB / num tensors = 285
-
-Black, Blue and White
-```

gpt4all-bindings/csharp/build_linux.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-mkdir -p runtimes
-rm -rf runtimes/linux-x64
-mkdir -p runtimes/linux-x64/native
-mkdir runtimes/linux-x64/build
-cmake -S ../../gpt4all-backend -B runtimes/linux-x64/build
-cmake --build runtimes/linux-x64/build --parallel --config Release
-cp runtimes/linux-x64/build/libllmodel.so  runtimes/linux-x64/native/libllmodel.so
-cp runtimes/linux-x64/build/libgptj*.so  runtimes/linux-x64/native/
-cp runtimes/linux-x64/build/libllama*.so  runtimes/linux-x64/native/

gpt4all-bindings/csharp/build_win-mingw.ps1

@@ -1,16 +0,0 @@
-$ROOT_DIR = '.\runtimes\win-x64'
-$BUILD_DIR = '.\runtimes\win-x64\build\mingw'
-$LIBS_DIR = '.\runtimes\win-x64\native'
-
-# cleanup env
-Remove-Item -Force -Recurse $ROOT_DIR -ErrorAction SilentlyContinue | Out-Null
-mkdir $BUILD_DIR | Out-Null
-mkdir $LIBS_DIR  | Out-Null
-
-# build
-cmake -G "MinGW Makefiles" -S ..\..\gpt4all-backend -B $BUILD_DIR
-cmake --build $BUILD_DIR --parallel --config Release
-
-# copy native dlls
-cp "C:\ProgramData\mingw64\mingw64\bin\*dll" $LIBS_DIR
-cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR

gpt4all-bindings/csharp/build_win-msvc.ps1

@@ -1,6 +0,0 @@
-Remove-Item -Force -Recurse .\runtimes\win-x64\msvc -ErrorAction SilentlyContinue
-mkdir .\runtimes\win-x64\msvc\build | Out-Null
-cmake -G "Visual Studio 17 2022" -A X64 -S ..\..\gpt4all-backend -B .\runtimes\win-x64\msvc\build
-cmake --build .\runtimes\win-x64\msvc\build --parallel --config Release
-cp .\runtimes\win-x64\msvc\build\bin\Release\*.dll .\runtimes\win-x64
-mv .\runtimes\win-x64\llmodel.dll .\runtimes\win-x64\libllmodel.dll

gpt4all-bindings/csharp/docs/gpt4all_csharp.md

@@ -1 +0,0 @@
-# GPT4All C# API

gpt4all-bindings/golang/Makefile

@@ -1,163 +0,0 @@
-INCLUDE_PATH := $(abspath ./)
-LIBRARY_PATH := $(abspath ./)
-CMAKEFLAGS=
-
-ifndef UNAME_S
-UNAME_S := $(shell uname -s)
-endif
-
-ifndef UNAME_P
-UNAME_P := $(shell uname -p)
-endif
-
-ifndef UNAME_M
-UNAME_M := $(shell uname -m)
-endif
-
-CCV := $(shell $(CC) --version | head -n 1)
-CXXV := $(shell $(CXX) --version | head -n 1)
-
-# Mac OS + Arm can report x86_64
-# ref: https://github.com/ggerganov/whisper.cpp/issues/66#issuecomment-1282546789
-ifeq ($(UNAME_S),Darwin)
-	ifneq ($(UNAME_P),arm)
-		SYSCTL_M := $(shell sysctl -n hw.optional.arm64 2>/dev/null)
-		ifeq ($(SYSCTL_M),1)
-			# UNAME_P := arm
-			# UNAME_M := arm64
-			warn := $(warning Your arch is announced as x86_64, but it seems to actually be ARM64. Not fixing that can lead to bad performance. For more info see: https://github.com/ggerganov/whisper.cpp/issues/66\#issuecomment-1282546789)
-		endif
-	endif
-endif
-
-#
-# Compile flags
-#
-
-# keep standard at C11 and C++11
-CFLAGS   = -I. -I../../gpt4all-backend/llama.cpp -I../../gpt4all-backend -I -O3 -DNDEBUG -std=c11 -fPIC
-CXXFLAGS = -I. -I../../gpt4all-backend/llama.cpp -I../../gpt4all-backend -O3 -DNDEBUG -std=c++17 -fPIC
-LDFLAGS  =
-
-# warnings
-CFLAGS   += -Wall -Wextra -Wpedantic -Wcast-qual -Wdouble-promotion -Wshadow -Wstrict-prototypes -Wpointer-arith -Wno-unused-function
-CXXFLAGS += -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wno-multichar
-
-# OS specific
-# TODO: support Windows
-ifeq ($(UNAME_S),Linux)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-ifeq ($(UNAME_S),Darwin)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-ifeq ($(UNAME_S),FreeBSD)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-ifeq ($(UNAME_S),NetBSD)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-ifeq ($(UNAME_S),OpenBSD)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-ifeq ($(UNAME_S),Haiku)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-
-# Architecture specific
-# TODO: probably these flags need to be tweaked on some architectures
-#       feel free to update the Makefile for your architecture and send a pull request or issue
-ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686))
-	# Use all CPU extensions that are available:
-	CFLAGS += -march=native -mtune=native
-	CXXFLAGS += -march=native -mtune=native
-endif
-ifneq ($(filter ppc64%,$(UNAME_M)),)
-	POWER9_M := $(shell grep "POWER9" /proc/cpuinfo)
-	ifneq (,$(findstring POWER9,$(POWER9_M)))
-		CFLAGS += -mcpu=power9
-		CXXFLAGS += -mcpu=power9
-	endif
-	# Require c++23's std::byteswap for big-endian support.
-	ifeq ($(UNAME_M),ppc64)
-		CXXFLAGS += -std=c++23 -DGGML_BIG_ENDIAN
-	endif
-endif
-ifndef LLAMA_NO_ACCELERATE
-	# Mac M1 - include Accelerate framework.
-	# `-framework Accelerate` works on Mac Intel as well, with negliable performance boost (as of the predict time).
-	ifeq ($(UNAME_S),Darwin)
-		CFLAGS  += -DGGML_USE_ACCELERATE
-		LDFLAGS += -framework Accelerate
-	endif
-endif
-ifdef LLAMA_OPENBLAS
-	CFLAGS  += -DGGML_USE_OPENBLAS -I/usr/local/include/openblas
-	LDFLAGS += -lopenblas
-endif
-ifdef LLAMA_GPROF
-	CFLAGS   += -pg
-	CXXFLAGS += -pg
-endif
-ifneq ($(filter aarch64%,$(UNAME_M)),)
-	CFLAGS += -mcpu=native
-	CXXFLAGS += -mcpu=native
-endif
-ifneq ($(filter armv6%,$(UNAME_M)),)
-	# Raspberry Pi 1, 2, 3
-	CFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access
-endif
-ifneq ($(filter armv7%,$(UNAME_M)),)
-	# Raspberry Pi 4
-	CFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access -funsafe-math-optimizations
-endif
-ifneq ($(filter armv8%,$(UNAME_M)),)
-	# Raspberry Pi 4
-	CFLAGS += -mfp16-format=ieee -mno-unaligned-access
-endif
-
-#
-# Print build information
-#
-
-$(info I go-gpt4all build info: )
-$(info I UNAME_S:  $(UNAME_S))
-$(info I UNAME_P:  $(UNAME_P))
-$(info I UNAME_M:  $(UNAME_M))
-$(info I CFLAGS:   $(CFLAGS))
-$(info I CXXFLAGS: $(CXXFLAGS))
-$(info I LDFLAGS:  $(LDFLAGS))
-$(info I CMAKEFLAGS:  $(CMAKEFLAGS))
-$(info I CC:       $(CCV))
-$(info I CXX:      $(CXXV))
-$(info )
-
-llmodel.o:
-	[ -e buildllm ] || mkdir buildllm
-	cd buildllm && cmake ../../../gpt4all-backend/ $(CMAKEFLAGS) && make
-	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel_c.cpp.o ../llmodel_c.o
-	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel.cpp.o ../llmodel.o
-
-clean:
-	rm -f *.o
-	rm -f *.a
-	rm -rf buildllm
-	rm -rf example/main
-
-binding.o: binding.cpp binding.h
-	$(CXX) $(CXXFLAGS) binding.cpp -o binding.o -c $(LDFLAGS)
-
-libgpt4all.a: binding.o llmodel.o
-	ar src libgpt4all.a llmodel.o binding.o
-
-test: libgpt4all.a
-	@C_INCLUDE_PATH=${INCLUDE_PATH} LIBRARY_PATH=${LIBRARY_PATH} go test -v ./...
-
-example/main: libgpt4all.a
-	C_INCLUDE_PATH=$(INCLUDE_PATH) LIBRARY_PATH=$(INCLUDE_PATH) go build -o example/main ./example/

gpt4all-bindings/golang/README.md

@@ -1,59 +0,0 @@
-# GPT4All Golang bindings
-
-The golang bindings have been tested on:
-- MacOS
-- Linux
-
-### Usage
-
-```
-import (
-	"github.com/nomic-ai/gpt4all/gpt4all-bindings/golang"
-)
-
-func main() {
-	// Load the model
-	model, err := gpt4all.New("model.bin", gpt4all.SetModelType(gpt4all.GPTJType))
-	if err != nil {
-		panic(err)
-	}
-	defer model.Free()
-
-	model.SetTokenCallback(func(s string) bool {
-		fmt.Print(s)
-		return true
-	})
-
-	_, err = model.Predict("Here are 4 steps to create a website:", gpt4all.SetTemperature(0.1))
-	if err != nil {
-		panic(err)
-	}
-}
-```
-
-## Building
-
-In order to use the bindings you will need to build `libgpt4all.a`:
-
-```
-git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
-cd gpt4all/gpt4all-bindings/golang
-make libgpt4all.a
-```
-
-To use the bindings in your own software:
-
-- Import `github.com/nomic-ai/gpt4all/gpt4all-bindings/golang`;
-- Compile `libgpt4all.a` (you can use `make libgpt4all.a` in the bindings/go directory);
-- Link your go binary by setting the environment variables `C_INCLUDE_PATH` and `LIBRARY_PATH` to point to the `binding.h` file directory and `libgpt4all.a` file directory respectively.
-- Note: you need to have *.so/*.dynlib/*.dll files of the implementation nearby the binary produced by the binding in order to make this to work
-
-## Testing
-
-To run tests, run `make test`:
-
-```
-git clone https://github.com/nomic-ai/gpt4all
-cd gpt4all/gpt4all-bindings/golang
-make test
-```

gpt4all-bindings/golang/binding.cpp

@@ -1,106 +0,0 @@
-#include "../../gpt4all-backend/llmodel_c.h"
-#include "../../gpt4all-backend/llmodel.h"
-#include "../../gpt4all-backend/llmodel_c.cpp"
-
-#include "binding.h"
-#include <cassert>
-#include <cmath>
-#include <cstddef>
-#include <cstdio>
-#include <cstring>
-#include <fstream>
-#include <map>
-#include <string>
-#include <vector>
-#include <iostream>
-#include <unistd.h>
-
-void* load_model(const char *fname, int n_threads) {
-    // load the model
-    const char *new_error;
-    auto model = llmodel_model_create2(fname, "auto", &new_error);
-    if (model == nullptr) {
-        fprintf(stderr, "%s: error '%s'\n", __func__, new_error);
-        return nullptr;
-    }
-    if (!llmodel_loadModel(model, fname, 2048, 100)) {
-        llmodel_model_destroy(model);
-        return nullptr;
-    }
-
-    llmodel_setThreadCount(model,  n_threads);
-    return model;
-}
-
-std::string res = "";
-void * mm;
-
-void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float min_p, float temp, int n_batch,float ctx_erase)
-{
-    llmodel_model* model = (llmodel_model*) m;
-
-   // std::string res = "";
- 
-    auto lambda_prompt = [](int token_id)  {
-	        return true;
-    };
-
-    mm=model;
-    res="";
-
-    auto lambda_response = [](int token_id, const char *responsechars) {
-        res.append((char*)responsechars);
-        return !!getTokenCallback(mm, (char*)responsechars);
-	};
-	
-	auto lambda_recalculate = [](bool is_recalculating) {
-	        // You can handle recalculation requests here if needed
-	    return is_recalculating;
-	};
-
-    llmodel_prompt_context* prompt_context = new llmodel_prompt_context{
-        .logits = NULL,
-        .logits_size = 0,
-        .tokens = NULL,
-        .tokens_size = 0,
-        .n_past = 0,
-        .n_ctx = 1024,
-        .n_predict = 50,
-        .top_k = 10,
-        .top_p = 0.9,
-        .min_p = 0.0,
-        .temp = 1.0,
-        .n_batch = 1,
-        .repeat_penalty = 1.2,
-        .repeat_last_n = 10,
-        .context_erase = 0.5
-    };
-
-    prompt_context->n_predict = tokens;
-    prompt_context->repeat_last_n = repeat_last_n;
-    prompt_context->repeat_penalty = repeat_penalty;
-    prompt_context->n_ctx = n_ctx;
-    prompt_context->top_k = top_k;
-    prompt_context->context_erase = ctx_erase;
-    prompt_context->top_p = top_p;
-    prompt_context->min_p = min_p;
-    prompt_context->temp = temp;
-    prompt_context->n_batch = n_batch;    
-
-    llmodel_prompt(model, prompt,
-                        lambda_prompt,
-                        lambda_response,
-                    lambda_recalculate,
-                    prompt_context );
-
-    strcpy(result, res.c_str()); 
-
-    free(prompt_context);
-}
-
-void free_model(void *state_ptr) {
-    llmodel_model* ctx = (llmodel_model*) state_ptr;
-    llmodel_model_destroy(*ctx);
-}
-

gpt4all-bindings/golang/binding.h

@@ -1,18 +0,0 @@
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdbool.h>
-
-void* load_model(const char *fname, int n_threads);
-
-void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float min_p, float temp, int n_batch,float ctx_erase);
-
-void free_model(void *state_ptr);
-
-extern unsigned char getTokenCallback(void *, char *);
-
-#ifdef __cplusplus
-}
-#endif

gpt4all-bindings/golang/example/main.go

@@ -1,82 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"flag"
-	"fmt"
-	"io"
-	"os"
-	"runtime"
-	"strings"
-
-	gpt4all "github.com/nomic-ai/gpt4all/gpt4all-bindings/golang"
-)
-
-var (
-	threads = 4
-	tokens  = 128
-)
-
-func main() {
-	var model string
-
-	flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
-	flags.StringVar(&model, "m", "./models/7B/ggml-model-q4_0.bin", "path to q4_0.bin model file to load")
-	flags.IntVar(&threads, "t", runtime.NumCPU(), "number of threads to use during computation")
-	flags.IntVar(&tokens, "n", 512, "number of tokens to predict")
-
-	err := flags.Parse(os.Args[1:])
-	if err != nil {
-		fmt.Printf("Parsing program arguments failed: %s", err)
-		os.Exit(1)
-	}
-	l, err := gpt4all.New(model, gpt4all.SetThreads(threads))
-	if err != nil {
-		fmt.Println("Loading the model failed:", err.Error())
-		os.Exit(1)
-	}
-	fmt.Printf("Model loaded successfully.\n")
-
-	l.SetTokenCallback(func(token string) bool {
-		fmt.Print(token)
-		return true
-	})
-
-	reader := bufio.NewReader(os.Stdin)
-
-	for {
-		text := readMultiLineInput(reader)
-
-		_, err := l.Predict(text, gpt4all.SetTokens(tokens), gpt4all.SetTopK(90), gpt4all.SetTopP(0.86))
-		if err != nil {
-			panic(err)
-		}
-		fmt.Printf("\n\n")
-	}
-}
-
-// readMultiLineInput reads input until an empty line is entered.
-func readMultiLineInput(reader *bufio.Reader) string {
-	var lines []string
-	fmt.Print(">>> ")
-
-	for {
-		line, err := reader.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				os.Exit(0)
-			}
-			fmt.Printf("Reading the prompt failed: %s", err)
-			os.Exit(1)
-		}
-
-		if len(strings.TrimSpace(line)) == 0 {
-			break
-		}
-
-		lines = append(lines, line)
-	}
-
-	text := strings.Join(lines, "")
-	return text
-}