cmake: fix Metal build after #2310 (#2350)

I don't understand why this is needed, but it works.

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/continue_config.yml

@@ -97,7 +97,9 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Installing Qt
           command: |
@@ -121,6 +123,7 @@ jobs:
             set -eo pipefail
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
             export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
+            export PATH=$PATH:/usr/local/cuda/bin
             mkdir build
             cd build
             mkdir upload
@@ -162,6 +165,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Build
           command: |
@@ -218,7 +226,9 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Installing Qt
           command: |
@@ -235,6 +245,7 @@ jobs:
           name: Build
           command: |
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
+            export PATH=$PATH:/usr/local/cuda/bin
             ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S gpt4all-chat -B build
             ~/Qt/Tools/CMake/bin/cmake --build build --target all
 
@@ -269,6 +280,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Build
           command: |
@@ -370,13 +386,13 @@ jobs:
       - run:
           name: Make Documentation
           command: |
-            cd gpt4all-bindings/python/
+            cd gpt4all-bindings/python
             mkdocs build
       - run:
           name: Deploy Documentation
           command: |
-            cd gpt4all-bindings/python/
-            aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat
+            cd gpt4all-bindings/python
+            aws s3 sync --delete site/ s3://docs.gpt4all.io/
       - run:
           name: Invalidate docs.gpt4all.io cloudfront
           command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"
@@ -394,12 +410,15 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk
+            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
             pip install setuptools wheel cmake
       - run:
           name: Build C library
           command: |
+            export PATH=$PATH:/usr/local/cuda/bin
             git submodule update --init --recursive
             cd gpt4all-backend
             cmake -B build
@@ -459,6 +478,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Install dependencies
           command:
@@ -530,11 +554,14 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list            
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk
+            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Build Libraries
           command: |
+            export PATH=$PATH:/usr/local/cuda/bin
             cd gpt4all-backend
             mkdir -p runtimes/build
             cd runtimes/build
@@ -599,6 +626,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Install dependencies
           command: |
@@ -642,6 +674,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Install dependencies
           command: |

LICENSE_SOM.txt

@@ -1,30 +0,0 @@
-Software for Open Models License (SOM)
-Version 1.0 dated August 30th, 2023
-
-This license governs use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
-
-This license is intended to encourage open release of models created, modified, processed, or otherwise used via the Software under open licensing terms, and should be interpreted in light of that intent.
-
-1. Definitions
-The “Licensor” is the person or entity who is making the Software available under this license. “Software” is the software made available by Licensor under this license.
-A “Model” is the output of a machine learning algorithm, and excludes the Software.
-“Model Source Materials” must include the Model and model weights, and may include any input data, input data descriptions, documentation or training descriptions for the Model.
-“Open Licensing Terms” means: (a) any open source license approved by the Open Source Initiative, or (b) any other terms that make the Model Source Materials publicly available free of charge, and allow recipients to use, modify and distribute the Model Source Materials. Terms described in (b) may include reasonable restrictions such as non-commercial or non-production limitations, or require use in compliance with law.
-
-2. Grant of Rights. Subject to the conditions and limitations in section 3:
-(A) Copyright Grant. Licensor grants you a non-exclusive, worldwide, royalty-free copyright license to copy, modify, and distribute the Software and any modifications of the Software you create under this license. The foregoing license includes without limitation the right to create, modify, and use Models using this Software.
-
-(B) Patent Grant. Licensor grants you a non-exclusive, worldwide, royalty-free license, under any patents owned or controlled by Licensor, to make, have made, use, sell, offer for sale, import, or otherwise exploit the Software.  No license is granted to patent rights that are not embodied in the operation of the Software in the form provided by Licensor.
-
-3. Conditions and Limitations
-(A) Model Licensing and Access. If you use the Software to create, modify, process, or otherwise use any Model, including usage to create inferences with a Model, whether or not you make the Model available to others, you must make that Model Source Materials publicly available under Open Licensing Terms. 
-
-(B) No Re-Licensing. If you redistribute the Software, or modifications to the Software made under the license granted above, you must make it available only under the terms of this license. You may offer additional terms such as warranties, maintenance and support, but You, and not Licensor, are responsible for performing such terms.
-
-(C) No Trademark License. This license does not grant you rights to use the Licensor’s name, logo, or trademarks.
-
-(D) If you assert in writing a claim against any person or entity alleging that the use of the Software infringes any patent, all of your licenses to the Software under Section 2 end automatically as of the date you asserted the claim.
-
-(E) If you distribute any portion of the Software, you must retain all copyright, patent, trademark, and attribution notices that are present in the Software, and you must include a copy of this license.
-
-(F) The Software is licensed “as-is.” You bear the entire risk of using it. Licensor gives You no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws that this license cannot change. To the extent permitted under your local laws, the Licensor disclaims and excludes the implied warranties of merchantability, fitness for a particular purpose and non-infringement. To the extent this disclaimer is unlawful, you, and not Licensor, are responsible for any liability.

README.md

@@ -1,87 +1,73 @@
 <h1 align="center">GPT4All</h1>
-
-<p align="center">Open-source large language models that run locally on your CPU and nearly any GPU</p>
-
+<p align="center">Privacy-oriented software for chatting with large language models that run on your own computer.</p>
 <p align="center">
-<a href="https://gpt4all.io">GPT4All Website and Models</a>
+  <a href="https://gpt4all.io">Official Website</a> &bull; <a href="https://docs.gpt4all.io">Documentation</a> &bull; <a href="https://discord.gg/mGZE39AS3e">Discord</a>
 </p>
-
 <p align="center">
-<a href="https://docs.gpt4all.io">GPT4All Documentation</a>
+  Official Download Links: <a href="https://gpt4all.io/installers/gpt4all-installer-win64.exe">Windows</a> &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-darwin.dmg">macOS</a> &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-linux.run">Ubuntu</a>
 </p>
-
 <p align="center">
-<a href="https://discord.gg/mGZE39AS3e">Discord</a>
+  <b>NEW:</b> <a href="https://forms.nomic.ai/gpt4all-release-notes-signup">Subscribe to our mailing list</a> for updates and news!
 </p>
-
-<p align="center">
-<a href="https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html">🦜️🔗 Official Langchain Backend</a> 
-</p>
-
 <p align="center">
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
 </p>
-
 <p align="center">
- <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg?&logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNSIgaGVpZ2h0PSI0IiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxwYXRoIGQ9Ik00LjQzIDEuODgyYTEuNDQgMS40NCAwIDAgMS0uMDk4LjQyNmMtLjA1LjEyMy0uMTE1LjIzLS4xOTIuMzIyLS4wNzUuMDktLjE2LjE2NS0uMjU1LjIyNmExLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxMmMtLjA5OS4wMTItLjE5Mi4wMTQtLjI3OS4wMDZsLTEuNTkzLS4xNHYtLjQwNmgxLjY1OGMuMDkuMDAxLjE3LS4xNjkuMjQ2LS4xOTFhLjYwMy42MDMgMCAwIDAgLjItLjEwNi41MjkuNTI5IDAgMCAwIC4xMzgtLjE3LjY1NC42NTQgMCAwIDAgLjA2NS0uMjRsLjAyOC0uMzJhLjkzLjkzIDAgMCAwLS4wMzYtLjI0OS41NjcuNTY3IDAgMCAwLS4xMDMtLjIuNTAyLjUwMiAwIDAgMC0uMTY4LS4xMzguNjA4LjYwOCAwIDAgMC0uMjQtLjA2N0wyLjQzNy43MjkgMS42MjUuNjcxYS4zMjIuMzIyIDAgMCAwLS4yMzIuMDU4LjM3NS4zNzUgMCAwIDAtLjExNi4yMzJsLS4xMTYgMS40NS0uMDU4LjY5Ny0uMDU4Ljc1NEwuNzA1IDRsLS4zNTctLjA3OUwuNjAyLjkwNkMuNjE3LjcyNi42NjMuNTc0LjczOS40NTRhLjk1OC45NTggMCAwIDEgLjI3NC0uMjg1Ljk3MS45NzEgMCAwIDEgLjMzNy0uMTRjLjExOS0uMDI2LjIyNy0uMDM0LjMyNS0uMDI2TDMuMjMyLjE2Yy4xNTkuMDE0LjMzNi4wMy40NTkuMDgyYTEuMTczIDEuMTczIDAgMCAxIC41NDUuNDQ3Yy4wNi4wOTQuMTA5LjE5Mi4xNDQuMjkzYTEuMzkyIDEuMzkyIDAgMCAxIC4wNzguNThsLS4wMjkuMzJaIiBmaWxsPSIjRjI3NzdBIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+Cjwvc3ZnPgo=" alt="phorm.ai"></a>
+ <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg" alt="phorm.ai"></a>
 </p>
 
 <p align="center">
-  <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
+  <img width="auto" height="400" src="https://github.com/nomic-ai/gpt4all/assets/14168726/495fce3e-769b-4e5a-a394-99f072ac4d29">
 </p>
 <p align="center">
-Run on an M1 macOS Device (not sped up!)
+Run on an M2 MacBook Pro (not sped up!)
 </p>
 
-## GPT4All: An ecosystem of open-source on-edge large language models.
 
-> [!IMPORTANT]
-> GPT4All v2.5.0 and newer only supports models in GGUF format (.gguf). Models used with a previous version of GPT4All (.bin extension) will no longer work.
+## About GPT4All
 
-GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and any GPU. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
+GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and NVIDIA and AMD GPUs. Note that your CPU needs to support [AVX instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
 
 Learn more in the [documentation](https://docs.gpt4all.io).
 
-A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models.
+A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily deploy their own on-edge large language models.
 
-### What's New ([Issue Tracker](https://github.com/orgs/nomic-ai/projects/2))
+
+### Installation
+
+The recommended way to install GPT4All is to use one of the online installers linked above in this README, which are also available at the [GPT4All website](https://gpt4all.io/). These require an internet connection at install time, are slightly easier to use on macOS due to code signing, and provide a version of GPT4All that can check for updates.
+
+An alternative way to install GPT4All is to use one of the offline installers available on the [Releases page](https://github.com/nomic-ai/gpt4all/releases). These do not require an internet connection at install time, and can be used to install an older version of GPT4All if so desired. But using these requires acknowledging a security warning on macOS, and they provide a version of GPT4All that is unable to notify you of updates, so you should enable notifications for Releases on this repository (Watch > Custom > Releases) or sign up for announcements in our [Discord server](https://discord.gg/mGZE39AS3e).
+
+
+### What's New
 - **October 19th, 2023**: GGUF Support Launches with Support for:
     - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
     - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4\_0 and Q4\_1 quantizations in GGUF.
     - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
-- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on AMD, Intel, Samsung, Qualcomm and NVIDIA GPUs.
-- **August 15th, 2023**: GPT4All API launches allowing inference of local LLMs from docker containers.
-- **July 2023**: Stable support for LocalDocs, a GPT4All Plugin that allows you to privately and locally chat with your data.
+- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on NVIDIA and AMD GPUs.
+- **July 2023**: Stable support for LocalDocs, a feature that allows you to privately and locally chat with your data.
+- **June 28th, 2023**: [Docker-based API server] launches allowing inference of local LLMs from an OpenAI-compatible HTTP endpoint.
+
+[Docker-based API server]: https://github.com/nomic-ai/gpt4all/tree/cef74c2be20f5b697055d5b8b506861c7b997fab/gpt4all-api
 
 
-### Chat Client
-Run any GPT4All model natively on your home desktop with the auto-updating desktop chat client. See <a href="https://gpt4all.io">GPT4All Website</a> for a full list of open-source models you can run with this powerful desktop application.
+### Building From Source
 
-Direct Installer Links:
+* Follow the instructions [here](gpt4all-chat/build_and_run.md) to build the GPT4All Chat UI from source.
 
-* [macOS](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
-
-* [Windows](https://gpt4all.io/installers/gpt4all-installer-win64.exe)
-
-* [Ubuntu](https://gpt4all.io/installers/gpt4all-installer-linux.run)
-
-Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io/)
-
-### Chat Client building and running
-
-* Follow the visual instructions on the chat client [build_and_run](gpt4all-chat/build_and_run.md) page
 
 ### Bindings
 
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/java">:computer: Official Java Bindings</a>
+* :snake: <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python">Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
+* :computer: <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">Typescript Bindings</a>
+
 
 ### Integrations
 
-* 🗃️ [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
+* :parrot::link: [Langchain](https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html)
+* :card_file_box: [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
+
 
 ## Contributing
 GPT4All welcomes contributions, involvement, and discussion from the open source community!
@@ -91,6 +77,59 @@ Check project discord, with project owners, or through existing issues/PRs to av
 Please make sure to tag all of the above with relevant project identifiers or your contribution could potentially get lost.
 Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 
+
+## GPT4All 2024 Roadmap
+To contribute to the development of any of the below roadmap items, make or find the corresponding issue and cross-reference the [in-progress task](https://github.com/orgs/nomic-ai/projects/2/views/1).
+
+Each item should have an issue link below.
+
+- Chat UI Language Localization (localize UI into the native languages of users)
+    - [ ] Chinese
+    - [ ] German
+    - [ ] French
+    - [ ] Portuguese
+    - [ ] Your native language here. 
+- UI Redesign: an internal effort at Nomic to improve the UI/UX of gpt4all for all users.
+    - [ ] Design new user interface and gather community feedback
+    - [ ] Implement the new user interface and experience.
+- Installer and Update Improvements
+    - [ ] Seamless native installation and update process on OSX
+    - [ ] Seamless native installation and update process on Windows
+    - [ ] Seamless native installation and update process on Linux
+- Model discoverability improvements:
+    - [x] Support huggingface model discoverability
+    - [ ] Support Nomic hosted model discoverability
+- LocalDocs (towards a local perplexity)
+    - Multilingual LocalDocs Support
+        - [ ] Create a multilingual experience
+        - [ ] Incorporate a multilingual embedding model
+        - [ ] Specify a preferred multilingual LLM for localdocs
+    - Improved RAG techniques
+        - [ ] Query augmentation and re-writing
+        - [ ] Improved chunking and text extraction from arbitrary modalities
+            - [ ] Custom PDF extractor past the QT default (charts, tables, text)
+        - [ ] Faster indexing and local exact search with v1.5 hamming embeddings and reranking (skip ANN index construction!)
+    - Support queries like 'summarize X document'
+    - Multimodal LocalDocs support with Nomic Embed
+    - Nomic Dataset Integration with real-time LocalDocs
+        - [ ] Include an option to allow the export of private LocalDocs collections to Nomic Atlas for debugging data/chat quality
+        - [ ] Allow optional sharing of LocalDocs collections between users.
+        - [ ] Allow the import of a LocalDocs collection from an Atlas Datasets
+            - Chat with live version of Wikipedia, Chat with Pubmed, chat with the latest snapshot of world news.
+- First class Multilingual LLM Support
+    - [ ] Recommend and set a default LLM for German
+    - [ ] Recommend and set a default LLM for English
+    - [ ] Recommend and set a default LLM for Chinese
+    - [ ] Recommend and set a default LLM for Spanish
+
+- Server Mode improvements
+    - Improved UI and new requested features:
+        - [ ] Fix outstanding bugs and feature requests around networking configurations.
+        - [ ] Support Nomic Embed inferencing
+        - [ ] First class documentation
+        - [ ] Improving developer use and quality of server mode (e.g. support larger batches)
+
+
 ## Technical Reports
 
 <p align="center">
@@ -105,6 +144,7 @@ Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 <a href="https://s3.amazonaws.com/static.nomic.ai/gpt4all/2023_GPT4All_Technical_Report.pdf">:green_book: Technical Report 1: GPT4All</a>
 </p>
 
+
 ## Citation
 
 If you utilize this repository, models or data in a downstream project, please consider citing it with:

gpt4all-api/.gitignore

@@ -1,112 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-app/__pycache__/
-gpt4all_api/__pycache__/
-gpt4all_api/app/api_v1/__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# VS Code
-.vscode/
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-
-*.lock
-*.cache

gpt4all-api/.isort.cfg

@@ -1,7 +0,0 @@
-[settings]
-known_third_party=geopy,nltk,np,numpy,pandas,pysbd,fire,torch
-
-line_length=120
-include_trailing_comma=True
-multi_line_output=3
-use_parentheses=True

gpt4all-api/LICENSE

@@ -1,13 +0,0 @@
-Copyright 2023 Nomic, Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

gpt4all-api/README.md

@@ -1,90 +0,0 @@
-# GPT4All REST API
-
-NOTICE: We are considering to deprecate this API as it has become challenging to maintain and test. If you have any interest in maintaining this or would like to takeover and adopt or discuss the future of this API please speak up in the discord channel.
-
-This directory contains the source code to run and build docker images that run a FastAPI app
-for serving inference from GPT4All models. The API matches the OpenAI API spec.
-
-## Tutorial
-
-The following tutorial assumes that you have checked out this repo and cd'd into it.
-
-### Starting the app
-
-First change your working directory to `gpt4all/gpt4all-api`.
-
-Now you can build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
-```bash
-DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
-```
-
-Then, start the backend with:
-
-```bash
-docker compose up --build
-```
-
-This will run both the API and locally hosted GPU inference server. If you want to run the API without the GPU inference server, you can run:
-
-```bash
-docker compose up --build gpt4all_api
-```
-
-To run the API with the GPU inference server, you will need to include environment variables (like the `MODEL_ID`). Edit the `.env` file and run
-```bash
-docker compose --env-file .env up --build
-```
-
-
-#### Spinning up your app
-Run `docker compose up` to spin up the backend. Monitor the logs for errors in-case you forgot to set an environment variable above.
-
-
-#### Development
-Run
-
-```bash
-docker compose up --build
-```
-and edit files in the `app` directory. The api will hot-reload on changes.
-
-You can run the unit tests with
-
-```bash
-make test
-```
-
-#### Viewing API documentation
-
-Once the FastAPI ap is started you can access its documentation and test the search endpoint by going to:
-```
-localhost:80/docs
-```
-
-This documentation should match the OpenAI OpenAPI spec located at https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-#### Running inference
-```python
-import openai
-openai.api_base = "http://localhost:4891/v1"
-
-openai.api_key = "not needed for a local LLM"
-
-
-def test_completion():
-    model = "gpt4all-j-v1.3-groovy"
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model,
-        prompt=prompt,
-        max_tokens=50,
-        temperature=0.28,
-        top_p=0.95,
-        n=1,
-        echo=True,
-        stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-    print(response)
-```

gpt4all-api/docker-compose.gpu.yaml

@@ -1,24 +0,0 @@
-version: "3.8"
-
-services:
-  gpt4all_gpu:
-    image: ghcr.io/huggingface/text-generation-inference:0.9.3
-    container_name: gpt4all_gpu
-    restart: always #restart on error (usually code compilation from save during bad state)
-    environment:
-      - HUGGING_FACE_HUB_TOKEN=token
-      - USE_FLASH_ATTENTION=false
-      - MODEL_ID=''
-      - NUM_SHARD=1
-    command: --model-id $MODEL_ID --num-shard $NUM_SHARD
-    volumes:
-      - ./:/data
-    ports:
-      - "8080:80"
-    shm_size: 1g
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              capabilities: [gpu]

gpt4all-api/docker-compose.yaml

@@ -1,22 +0,0 @@
-version: "3.8"
-
-services:
-  gpt4all_api:
-    image: gpt4all_api
-    container_name: gpt4all_api
-    restart: always #restart on error (usually code compilation from save during bad state)
-    ports:
-      - "4891:4891"
-    env_file:
-      - .env
-    environment:
-      - APP_ENVIRONMENT=dev
-      - WEB_CONCURRENCY=2
-      - LOGLEVEL=debug
-      - PORT=4891
-      - model=${MODEL_BIN} # using variable from .env file
-      - inference_mode=cpu
-    volumes:
-      - './gpt4all_api/app:/app'
-      - './gpt4all_api/models:/models' # models are mounted in the container
-    command: ["/start-reload.sh"]

gpt4all-api/gpt4all_api/Dockerfile.buildkit

@@ -1,17 +0,0 @@
-# syntax=docker/dockerfile:1.0.0-experimental
-FROM tiangolo/uvicorn-gunicorn:python3.11
-
-# Put first so anytime this file changes other cached layers are invalidated.
-COPY gpt4all_api/requirements.txt /requirements.txt
-
-RUN pip install --upgrade pip
-
-# Run various pip install commands with ssh keys from host machine.
-RUN --mount=type=ssh pip install -r /requirements.txt && \
-  rm -Rf /root/.cache && rm -Rf /tmp/pip-install*
-
-# Finally, copy app and client.
-COPY gpt4all_api/app /app
-
-RUN mkdir -p /models
-

gpt4all-api/gpt4all_api/README.md

@@ -1 +0,0 @@
-# FastAPI app for serving GPT4All models

gpt4all-api/gpt4all_api/app/api_v1/api.py

@@ -1,9 +0,0 @@
-from api_v1.routes import chat, completions, engines, health
-from fastapi import APIRouter
-
-router = APIRouter()
-
-router.include_router(chat.router)
-router.include_router(completions.router)
-router.include_router(engines.router)
-router.include_router(health.router)

gpt4all-api/gpt4all_api/app/api_v1/events.py

@@ -1,29 +0,0 @@
-import logging
-
-from api_v1.settings import settings
-from fastapi import HTTPException
-from fastapi.responses import JSONResponse
-from starlette.requests import Request
-
-log = logging.getLogger(__name__)
-
-
-startup_msg_fmt = """
- Starting up GPT4All API
-"""
-
-
-async def on_http_error(request: Request, exc: HTTPException):
-    return JSONResponse({'detail': exc.detail}, status_code=exc.status_code)
-
-
-async def on_startup(app):
-    startup_msg = startup_msg_fmt.format(settings=settings)
-    log.info(startup_msg)
-
-
-def startup_event_handler(app):
-    async def start_app() -> None:
-        await on_startup(app)
-
-    return start_app

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -1,103 +0,0 @@
-import logging
-import time
-from typing import List
-from uuid import uuid4
-from fastapi import APIRouter, HTTPException
-from gpt4all import GPT4All
-from pydantic import BaseModel, Field
-from api_v1.settings import settings
-from fastapi.responses import StreamingResponse
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-class ChatCompletionMessage(BaseModel):
-    role: str
-    content: str
-
-class ChatCompletionRequest(BaseModel):
-    model: str = Field(settings.model, description='The model to generate a completion from.')
-    messages: List[ChatCompletionMessage] = Field(..., description='Messages for the chat completion.')
-    temperature: float = Field(settings.temp, description='Model temperature')
-
-class ChatCompletionChoice(BaseModel):
-    message: ChatCompletionMessage
-    index: int
-    logprobs: float
-    finish_reason: str
-
-class ChatCompletionUsage(BaseModel):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-class ChatCompletionResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[ChatCompletionChoice]
-    usage: ChatCompletionUsage
-
-router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
-
-@router.post("/completions", response_model=ChatCompletionResponse)
-async def chat_completion(request: ChatCompletionRequest):
-    '''
-    Completes a GPT4All model response based on the last message in the chat.
-    '''
-    # GPU is not implemented yet
-    if settings.inference_mode == "gpu":
-        raise HTTPException(status_code=400,
-              detail=f"Not implemented yet: Can only infer in CPU mode.")
-
-    # we only support the configured model
-    if request.model != settings.model:
-        raise HTTPException(status_code=400,
-              detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
-
-    # run only of we have a message
-    if request.messages:
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        # format system message and conversation history correctly
-        formatted_messages = ""
-        for message in request.messages:
-            formatted_messages += f"<|im_start|>{message.role}\n{message.content}<|im_end|>\n"
-
-        # the LLM will complete the response of the assistant
-        formatted_messages += "<|im_start|>assistant\n"
-        response = model.generate(
-            prompt=formatted_messages,
-            temp=request.temperature
-            )
-
-        # the LLM may continue to hallucinate the conversation, but we want only the first response
-        # so, cut off everything after first <|im_end|>
-        index = response.find("<|im_end|>")
-        response_content = response[:index].strip()
-    else:
-        response_content = "No messages received."
-
-    # Create a chat message for the response
-    response_message = ChatCompletionMessage(role="assistant", content=response_content)
-
-    # Create a choice object with the response message
-    response_choice = ChatCompletionChoice(
-        message=response_message,
-        index=0,
-        logprobs=-1.0,  # Placeholder value
-        finish_reason="length"  # Placeholder value
-    )
-
-    # Create the response object
-    chat_response = ChatCompletionResponse(
-        id=str(uuid4()),
-        created=int(time.time()),
-        model=request.model,
-        choices=[response_choice],
-        usage=ChatCompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0),  # Placeholder values
-    )
-
-    return chat_response

gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py

@@ -1,215 +0,0 @@
-import json
-from typing import List, Dict, Iterable, AsyncIterable
-import logging
-import time
-from typing import Dict, List, Union, Optional
-from uuid import uuid4
-import aiohttp
-import asyncio
-from api_v1.settings import settings
-from fastapi import APIRouter, Depends, Response, Security, status, HTTPException
-from fastapi.responses import StreamingResponse
-from gpt4all import GPT4All
-from pydantic import BaseModel, Field
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class CompletionRequest(BaseModel):
-    model: str = Field(settings.model, description='The model to generate a completion from.')
-    prompt: Union[List[str], str] = Field(..., description='The prompt to begin completing from.')
-    max_tokens: int = Field(None, description='Max tokens to generate')
-    temperature: float = Field(settings.temp, description='Model temperature')
-    top_p: Optional[float] = Field(settings.top_p, description='top_p')
-    top_k: Optional[int] = Field(settings.top_k, description='top_k')
-    n: int = Field(1, description='How many completions to generate for each prompt')
-    stream: bool = Field(False, description='Stream responses')
-    repeat_penalty: float = Field(settings.repeat_penalty, description='Repeat penalty')
-
-
-class CompletionChoice(BaseModel):
-    text: str
-    index: int
-    logprobs: float
-    finish_reason: str
-
-
-class CompletionUsage(BaseModel):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-
-class CompletionResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[CompletionChoice]
-    usage: CompletionUsage
-
-
-class CompletionStreamResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[CompletionChoice]
-
-
-router = APIRouter(prefix="/completions", tags=["Completion Endpoints"])
-
-def stream_completion(output: Iterable, base_response: CompletionStreamResponse):
-    """
-    Streams a GPT4All output to the client.
-
-    Args:
-        output: The output of GPT4All.generate(), which is an iterable of tokens.
-        base_response: The base response object, which is cloned and modified for each token.
-
-    Returns:
-        A Generator of CompletionStreamResponse objects, which are serialized to JSON Event Stream format.
-    """
-    for token in output:
-        chunk = base_response.copy()
-        chunk.choices = [dict(CompletionChoice(
-            text=token,
-            index=0,
-            logprobs=-1,
-            finish_reason=''
-        ))]
-        yield f"data: {json.dumps(dict(chunk))}\n\n"
-
-async def gpu_infer(payload, header):
-    async with aiohttp.ClientSession() as session:
-        try:
-            async with session.post(
-                settings.hf_inference_server_host, headers=header, data=json.dumps(payload)
-            ) as response:
-                resp = await response.json()
-            return resp
-
-        except aiohttp.ClientError as e:
-            # Handle client-side errors (e.g., connection error, invalid URL)
-            logger.error(f"Client error: {e}")
-        except aiohttp.ServerError as e:
-            # Handle server-side errors (e.g., internal server error)
-            logger.error(f"Server error: {e}")
-        except json.JSONDecodeError as e:
-            # Handle JSON decoding errors
-            logger.error(f"JSON decoding error: {e}")
-        except Exception as e:
-            # Handle other unexpected exceptions
-            logger.error(f"Unexpected error: {e}")
-
-@router.post("/", response_model=CompletionResponse)
-async def completions(request: CompletionRequest):
-    '''
-    Completes a GPT4All model response.
-    '''
-    if settings.inference_mode == "gpu":
-        params = request.dict(exclude={'model', 'prompt', 'max_tokens', 'n'})
-        params["max_new_tokens"] = request.max_tokens
-        params["num_return_sequences"] = request.n
-
-        header = {"Content-Type": "application/json"}
-        if isinstance(request.prompt, list):
-            tasks = []
-            for prompt in request.prompt:
-                payload = {"parameters": params}
-                payload["inputs"] = prompt
-                task = gpu_infer(payload, header)
-                tasks.append(task)
-            results = await asyncio.gather(*tasks)
-
-            choices = []
-            for response in results:
-                scores = response["scores"] if "scores" in response else -1.0
-                choices.append(
-                    dict(
-                        CompletionChoice(
-                            text=response["generated_text"], index=0, logprobs=scores, finish_reason='stop'
-                        )
-                    )
-                )
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=choices,
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
-
-        else:
-            payload = {"parameters": params}
-            # If streaming, we need to return a StreamingResponse
-            payload["inputs"] = request.prompt
-
-            resp = await gpu_infer(payload, header)
-
-            output = resp["generated_text"]
-            # this returns all logprobs
-            scores = resp["scores"] if "scores" in resp else -1.0
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(text=output, index=0, logprobs=scores, finish_reason='stop'))],
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
-
-    else:
-
-        if request.model != settings.model:
-            raise HTTPException(status_code=400,
-                                detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
-
-        if isinstance(request.prompt, list):
-            if len(request.prompt) > 1:
-                raise HTTPException(status_code=400, detail="Can only infer one inference per request in CPU mode.")
-            else:
-                request.prompt = request.prompt[0]
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        output = model.generate(prompt=request.prompt,
-                                max_tokens=request.max_tokens,
-                                streaming=request.stream,
-                                top_k=request.top_k,
-                                top_p=request.top_p,
-                                temp=request.temperature,
-                                )
-
-        # If streaming, we need to return a StreamingResponse
-        if request.stream:
-            base_chunk = CompletionStreamResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[]
-            )
-            return StreamingResponse((response for response in stream_completion(output, base_chunk)),
-                                     media_type="text/event-stream")
-        else:
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(
-                    text=output,
-                    index=0,
-                    logprobs=-1,
-                    finish_reason='stop'
-                ))],
-                usage={
-                    'prompt_tokens': 0,  # TODO how to compute this?
-                    'completion_tokens': 0,
-                    'total_tokens': 0
-                }
-            )

gpt4all-api/gpt4all_api/app/api_v1/routes/embeddings.py

@@ -1,65 +0,0 @@
-from typing import List, Union
-from fastapi import APIRouter
-from api_v1.settings import settings
-from gpt4all import Embed4All
-from pydantic import BaseModel, Field
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class EmbeddingRequest(BaseModel):
-    model: str = Field(
-        settings.model, description="The model to generate an embedding from."
-    )
-    input: Union[str, List[str], List[int], List[List[int]]] = Field(
-        ..., description="Input text to embed, encoded as a string or array of tokens."
-    )
-
-
-class EmbeddingUsage(BaseModel):
-    prompt_tokens: int = 0
-    total_tokens: int = 0
-
-
-class Embedding(BaseModel):
-    index: int = 0
-    object: str = "embedding"
-    embedding: List[float]
-
-
-class EmbeddingResponse(BaseModel):
-    object: str = "list"
-    model: str
-    data: List[Embedding]
-    usage: EmbeddingUsage
-
-
-router = APIRouter(prefix="/embeddings", tags=["Embedding Endpoints"])
-
-embedder = Embed4All()
-
-
-def get_embedding(data: EmbeddingRequest) -> EmbeddingResponse:
-    """
-    Calculates the embedding for the given input using a specified model.
-
-    Args:
-        data (EmbeddingRequest): An EmbeddingRequest object containing the input data
-        and model name.
-
-    Returns:
-        EmbeddingResponse: An EmbeddingResponse object encapsulating the calculated embedding,
-        usage info, and the model name.
-    """
-    embedding = embedder.embed(data.input)
-    return EmbeddingResponse(
-        data=[Embedding(embedding=embedding)], usage=EmbeddingUsage(), model=data.model
-    )
-
-
-@router.post("/", response_model=EmbeddingResponse)
-def embeddings(data: EmbeddingRequest):
-    """
-    Creates a GPT4All embedding
-    """
-    return get_embedding(data)

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -1,39 +0,0 @@
-import requests
-from fastapi import APIRouter, HTTPException
-from pydantic import BaseModel, Field
-from typing import List, Dict
-
-# Define the router for the engines module
-router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
-
-# Define the models for the engines module
-class ListEnginesResponse(BaseModel):
-    data: List[Dict] = Field(..., description="All available models.")
-
-class EngineResponse(BaseModel):
-    data: List[Dict] = Field(..., description="All available models.")
-
-
-# Define the routes for the engines module
-@router.get("/", response_model=ListEnginesResponse)
-async def list_engines():
-    try:
-        response = requests.get('https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json')
-        response.raise_for_status()  # This will raise an HTTPError if the HTTP request returned an unsuccessful status code
-        engines = response.json()
-        return ListEnginesResponse(data=engines)
-    except requests.RequestException as e:
-        logger.error(f"Error fetching engine list: {e}")
-        raise HTTPException(status_code=500, detail="Error fetching engine list")
-
-# Define the routes for the engines module
-@router.get("/{engine_id}", response_model=EngineResponse)
-async def retrieve_engine(engine_id: str):
-    try:
-        # Implement logic to fetch a specific engine's details
-        # This is a placeholder, replace with your actual data retrieval logic
-        engine_details = {"id": engine_id, "name": "Engine Name", "description": "Engine Description"}
-        return EngineResponse(data=[engine_details])
-    except Exception as e:
-        logger.error(f"Error fetching engine details: {e}")
-        raise HTTPException(status_code=500, detail=f"Error fetching details for engine {engine_id}")

gpt4all-api/gpt4all_api/app/api_v1/routes/health.py

@@ -1,13 +0,0 @@
-import logging
-from fastapi import APIRouter
-from fastapi.responses import JSONResponse
-
-log = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/health", tags=["Health"])
-
-
-@router.get('/', response_class=JSONResponse)
-async def health_check():
-    """Runs a health check on this instance of the API."""
-    return JSONResponse({'status': 'ok'}, headers={'Access-Control-Allow-Origin': '*'})

gpt4all-api/gpt4all_api/app/api_v1/settings.py

@@ -1,19 +0,0 @@
-from pydantic import BaseSettings
-
-
-class Settings(BaseSettings):
-    app_environment = 'dev'
-    model: str = 'ggml-mpt-7b-chat.bin'
-    gpt4all_path: str = '/models'
-    inference_mode: str = "cpu"
-    hf_inference_server_host: str = "http://gpt4all_gpu:80/generate"
-    sentry_dns: str = None
-
-    temp: float = 0.18
-    top_p: float = 1.0
-    top_k: int = 50
-    repeat_penalty: float = 1.18
-
-
-
-settings = Settings()

gpt4all-api/gpt4all_api/app/docs.py

@@ -1,3 +0,0 @@
-desc = 'GPT4All API'
-
-endpoint_paths = {'health': '/health'}

gpt4all-api/gpt4all_api/app/main.py

@@ -1,84 +0,0 @@
-import logging
-import os
-
-import docs
-from api_v1 import events
-from api_v1.api import router as v1_router
-from api_v1.settings import settings
-from fastapi import FastAPI, HTTPException, Request
-from fastapi.logger import logger as fastapi_logger
-from starlette.middleware.cors import CORSMiddleware
-
-logger = logging.getLogger(__name__)
-
-app = FastAPI(title='GPT4All API', description=docs.desc)
-
-# CORS Configuration (in-case you want to deploy)
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["GET", "POST", "OPTIONS"],
-    allow_headers=["*"],
-)
-
-logger.info('Adding v1 endpoints..')
-
-# add v1
-app.include_router(v1_router, prefix='/v1')
-app.add_event_handler('startup', events.startup_event_handler(app))
-app.add_exception_handler(HTTPException, events.on_http_error)
-
-
-@app.on_event("startup")
-async def startup():
-    global model
-    if settings.inference_mode == "cpu":
-        logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
-        from gpt4all import GPT4All
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
-    else:
-        # is it possible to do this once the server is up?
-        ## TODO block until HF inference server is up.
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
-
-
-@app.on_event("shutdown")
-async def shutdown():
-    logger.info("Shutting down API")
-
-
-if settings.sentry_dns is not None:
-    import sentry_sdk
-
-    def traces_sampler(sampling_context):
-        if 'health' in sampling_context['transaction_context']['name']:
-            return False
-
-    sentry_sdk.init(
-        dsn=settings.sentry_dns, traces_sample_rate=0.1, traces_sampler=traces_sampler, send_default_pii=False
-    )
-
-# This is needed to get logs to show up in the app
-if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
-    gunicorn_error_logger = logging.getLogger("gunicorn.error")
-    gunicorn_logger = logging.getLogger("gunicorn")
-
-    root_logger = logging.getLogger()
-    fastapi_logger.setLevel(gunicorn_logger.level)
-    fastapi_logger.handlers = gunicorn_error_logger.handlers
-    root_logger.setLevel(gunicorn_logger.level)
-
-    uvicorn_logger = logging.getLogger("uvicorn.access")
-    uvicorn_logger.handlers = gunicorn_error_logger.handlers
-else:
-    # https://github.com/tiangolo/fastapi/issues/2019
-    LOG_FORMAT2 = (
-        "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
-    )
-    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -1,93 +0,0 @@
-"""
-Use the OpenAI python API to test gpt4all models.
-"""
-from typing import List, get_args
-import os
-from dotenv import load_dotenv
-
-import openai
-
-openai.api_base = "http://localhost:4891/v1"
-openai.api_key = "not needed for a local LLM"
-
-# Load the .env file
-env_path = 'gpt4all-api/gpt4all_api/.env'
-load_dotenv(dotenv_path=env_path)
-
-# Fetch MODEL_ID from .env file
-model_id = os.getenv('MODEL_BIN', 'default_model_id')
-embedding = os.getenv('EMBEDDING', 'default_embedding_model_id')
-print (model_id)
-print (embedding)
-
-def test_completion():
-    model = model_id
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-
-def test_streaming_completion():
-    model = model_id
-    prompt = "Who is Michael Jordan?"
-    tokens = []
-    for resp in openai.Completion.create(
-            model=model,
-            prompt=prompt,
-            max_tokens=50,
-            temperature=0.28,
-            top_p=0.95,
-            n=1,
-            echo=True,
-            stream=True):
-        tokens.append(resp.choices[0].text)
-
-    assert (len(tokens) > 0)
-    assert (len("".join(tokens)) > len(prompt))
-
-# Modified test batch, problems with keyerror in response
-def test_batched_completion():
-    model = model_id  # replace with your specific model ID
-    prompt = "Who is Michael Jordan?"
-    responses = []
-
-    # Loop to create completions one at a time
-    for _ in range(3):
-        response = openai.Completion.create(
-            model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-        )
-        responses.append(response)
-
-    # Assertions to check the responses
-    for response in responses:
-        assert len(response['choices'][0]['text']) > len(prompt)
-
-    assert len(responses) == 3
-
-def test_embedding():
-    model = embedding
-    prompt = "Who is Michael Jordan?"
-    response = openai.Embedding.create(model=model, input=prompt)
-    output = response["data"][0]["embedding"]
-    args = get_args(List[float])
-
-    assert response["model"] == model
-    assert isinstance(output, list)
-    assert all(isinstance(x, args) for x in output)
-
-def test_chat_completion():
-    model = model_id
-
-    response = openai.ChatCompletion.create(
-        model=model,
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": "Knock knock."},
-            {"role": "assistant", "content": "Who's there?"},
-            {"role": "user", "content": "Orange."},
-            ]
-    )
-
-    assert response.choices[0].message.role == "assistant"
-    assert len(response.choices[0].message.content) > 0

gpt4all-api/gpt4all_api/env

@@ -1,3 +0,0 @@
-# Add your GGUF compatible model LLM here. ie: MODEL_BIN="mistral-7b-instruct-v0.1.Q4_0", rename file ".env"
-# Make sure this LLM matches the model you placed inside the models folder
-MODEL_BIN=""

gpt4all-api/gpt4all_api/models/README.md

@@ -1 +0,0 @@
-### Drop GGUF compatible models here, make sure it matches MODEL_BIN on your .env file

gpt4all-api/gpt4all_api/requirements.txt

@@ -1,13 +0,0 @@
-aiohttp>=3.6.2
-aiofiles
-pydantic>=1.4.0,<2.0.0
-requests>=2.24.0
-ujson>=2.0.2
-fastapi>=0.95.0
-Jinja2>=3.0
-gpt4all>=1.0.0
-pytest
-openai==0.28.0
-black
-isort
-python-dotenv

gpt4all-api/makefile

@@ -1,46 +0,0 @@
-ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-APP_NAME:=gpt4all_api
-PYTHON:=python3.8
-SHELL := /bin/bash
-
-all: dependencies
-
-fresh: clean dependencies
-
-testenv: clean_testenv test_build
-	docker compose -f docker-compose.yaml up --build
-
-testenv_gpu: clean_testenv test_build
-	docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up --build
-
-testenv_d: clean_testenv test_build
-	docker compose env up --build -d
-
-test:
-	docker compose exec $(APP_NAME) pytest -svv --disable-warnings -p no:cacheprovider /app/tests
-
-test_build:
-    DOCKER_BUILDKIT=1 docker build -t $(APP_NAME) --progress plain -f $(APP_NAME)/Dockerfile.buildkit .
-
-clean_testenv:
-	docker compose down -v
-
-fresh_testenv: clean_testenv testenv
-
-venv:
-	if [ ! -d $(ROOT_DIR)/venv ]; then $(PYTHON) -m venv $(ROOT_DIR)/venv; fi
-
-dependencies: venv
-	source $(ROOT_DIR)/venv/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
-
-clean: clean_testenv
-	# Remove existing environment
-	rm -rf $(ROOT_DIR)/venv;
-	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
-
-
-black:
-	source $(ROOT_DIR)/venv/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
-
-isort:
-	source $(ROOT_DIR)/venv/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)

gpt4all-backend/CMakeLists.txt

@@ -2,15 +2,23 @@ cmake_minimum_required(VERSION 3.16)
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-if(APPLE)
-  option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
-  if(BUILD_UNIVERSAL)
+if (APPLE)
+    option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
+else()
+    option(LLMODEL_KOMPUTE "llmodel: use Kompute"              ON)
+    option(LLMODEL_VULKAN  "llmodel: use Vulkan"               OFF)
+    option(LLMODEL_CUDA    "llmodel: use CUDA"                 ON)
+    option(LLMODEL_ROCM    "llmodel: use ROCm"                 OFF)
+endif()
+
+if (APPLE)
+  if (BUILD_UNIVERSAL)
     # Build a Universal binary on macOS
     # This requires that the found Qt library is compiled as Universal binaries.
     set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
   else()
     # Build for the host architecture on macOS
-    if(NOT CMAKE_OSX_ARCHITECTURES)
+    if (NOT CMAKE_OSX_ARCHITECTURES)
       set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
     endif()
   endif()
@@ -39,11 +47,35 @@ else()
     message(STATUS "Interprocedural optimization support detected")
 endif()
 
+set(DIRECTORY llama.cpp-mainline)
 include(llama.cpp.cmake)
 
-set(BUILD_VARIANTS default avxonly)
-if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-    set(BUILD_VARIANTS ${BUILD_VARIANTS} metal)
+set(BUILD_VARIANTS)
+set(GPTJ_BUILD_VARIANT cpu)
+if (APPLE)
+    list(APPEND BUILD_VARIANTS metal)
+endif()
+if (LLMODEL_KOMPUTE)
+    list(APPEND BUILD_VARIANTS kompute kompute-avxonly)
+    set(GPTJ_BUILD_VARIANT kompute)
+else()
+    list(PREPEND BUILD_VARIANTS cpu cpu-avxonly)
+endif()
+if (LLMODEL_VULKAN)
+    list(APPEND BUILD_VARIANTS vulkan vulkan-avxonly)
+endif()
+if (LLMODEL_CUDA)
+    include(CheckLanguage)
+    check_language(CUDA)
+    if (NOT CMAKE_CUDA_COMPILER)
+        message(WARNING "CUDA Toolkit not found. To build without CUDA, use -DLLMODEL_CUDA=OFF.")
+    endif()
+    enable_language(CUDA)
+    list(APPEND BUILD_VARIANTS cuda cuda-avxonly)
+endif()
+if (LLMODEL_ROCM)
+    enable_language(HIP)
+    list(APPEND BUILD_VARIANTS rocm rocm-avxonly)
 endif()
 
 set(CMAKE_VERBOSE_MAKEFILE ON)
@@ -51,24 +83,34 @@ set(CMAKE_VERBOSE_MAKEFILE ON)
 # Go through each build variant
 foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     # Determine flags
-    if (BUILD_VARIANT STREQUAL avxonly)
-        set(GPT4ALL_ALLOW_NON_AVX NO)
+    if (BUILD_VARIANT MATCHES avxonly)
+        set(GPT4ALL_ALLOW_NON_AVX OFF)
     else()
-        set(GPT4ALL_ALLOW_NON_AVX YES)
+        set(GPT4ALL_ALLOW_NON_AVX ON)
     endif()
     set(LLAMA_AVX2 ${GPT4ALL_ALLOW_NON_AVX})
     set(LLAMA_F16C ${GPT4ALL_ALLOW_NON_AVX})
     set(LLAMA_FMA  ${GPT4ALL_ALLOW_NON_AVX})
 
-    if (BUILD_VARIANT STREQUAL metal)
-        set(LLAMA_METAL YES)
-    else()
-        set(LLAMA_METAL NO)
+    set(LLAMA_METAL   OFF)
+    set(LLAMA_KOMPUTE OFF)
+    set(LLAMA_VULKAN  OFF)
+    set(LLAMA_CUDA    OFF)
+    set(LLAMA_ROCM    OFF)
+    if (BUILD_VARIANT MATCHES metal)
+        set(LLAMA_METAL   ON)
+    elseif (BUILD_VARIANT MATCHES kompute)
+        set(LLAMA_KOMPUTE ON)
+    elseif (BUILD_VARIANT MATCHES vulkan)
+        set(LLAMA_VULKAN  ON)
+    elseif (BUILD_VARIANT MATCHES cuda)
+        set(LLAMA_CUDA    ON)
+    elseif (BUILD_VARIANT MATCHES rocm)
+        set(LLAMA_HIPBLAS ON)
     endif()
 
     # Include GGML
-    set(LLAMA_K_QUANTS YES)
-    include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
+    include_ggml(-mainline-${BUILD_VARIANT})
 
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
@@ -93,11 +135,15 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    if (NOT LLAMA_METAL)
+    if (BUILD_VARIANT MATCHES ${GPTJ_BUILD_VARIANT})
         add_library(gptj-${BUILD_VARIANT} SHARED
             gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
         prepare_target(gptj llama-mainline)
     endif()
+
+    if (BUILD_VARIANT STREQUAL cuda)
+        set(CUDAToolkit_BIN_DIR ${CUDAToolkit_BIN_DIR} PARENT_SCOPE)
+    endif()
 endforeach()
 
 add_library(llmodel

gpt4all-backend/gptj.cpp

@@ -785,13 +785,15 @@ const std::vector<LLModel::Token> &GPTJ::endTokens() const
     return fres;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
+const char *get_arch_name(gguf_context *ctx_gguf) {
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    if (kid == -1)
+        throw std::runtime_error("key not found in model: general.architecture");
+
     enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
+    if (ktype != GGUF_TYPE_STRING)
+        throw std::runtime_error("key general.architecture has wrong type");
+
     return gguf_get_val_str(ctx_gguf, kid);
 }
 
@@ -814,21 +816,29 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
+DLL_EXPORT char *get_file_arch(const char *fname) {
     struct ggml_context * ctx_meta = NULL;
     struct gguf_init_params params = {
         /*.no_alloc = */ true,
         /*.ctx      = */ &ctx_meta,
     };
     gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
 
-    bool isValid = gguf_get_version(ctx_gguf) <= 3;
-    isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
+    char *arch = nullptr;
+    if (ctx_gguf && gguf_get_version(ctx_gguf) <= 3) {
+        try {
+            arch = strdup(get_arch_name(ctx_gguf));
+        } catch (const std::runtime_error &) {
+            // cannot read key -> return null
+        }
+    }
 
     gguf_free(ctx_gguf);
-    return isValid;
+    return arch;
+}
+
+DLL_EXPORT bool is_arch_supported(const char *arch) {
+    return !strcmp(arch, "gptj");
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llamamodel.cpp

@@ -22,7 +22,11 @@
 #include <llama.h>
 #include <ggml.h>
 #ifdef GGML_USE_KOMPUTE
-#include <ggml-kompute.h>
+#   include <ggml-kompute.h>
+#elif GGML_USE_VULKAN
+#   include <ggml-vulkan.h>
+#elif GGML_USE_CUDA
+#   include <ggml-cuda.h>
 #endif
 
 using namespace std::string_literals;
@@ -32,13 +36,44 @@ static constexpr int GGUF_VER_MAX = 3;
 
 static const char * const modelType_ = "LLaMA";
 
+// note: same order as LLM_ARCH_NAMES in llama.cpp
 static const std::vector<const char *> KNOWN_ARCHES {
-    "baichuan", "bert", "bloom", "codeshell", "falcon", "gemma", "gpt2", "llama", "mpt", "nomic-bert", "orion",
-    "persimmon", "phi2", "plamo", "qwen", "qwen2", "refact", "stablelm", "starcoder"
+    "llama",
+    "falcon",
+    // "grok", -- 314B parameters
+    "gpt2",
+    // "gptj", -- no inference code
+    // "gptneox", -- no inference code
+    "mpt",
+    "baichuan",
+    "starcoder",
+    // "persimmon", -- CUDA generates garbage
+    "refact",
+    "bert",
+    "nomic-bert",
+    "bloom",
+    "stablelm",
+    "qwen",
+    "qwen2",
+    "qwen2moe",
+    "phi2",
+    "phi3",
+    // "plamo", -- https://github.com/ggerganov/llama.cpp/issues/5669
+    "codeshell",
+    "orion",
+    "internlm2",
+    // "minicpm", -- CUDA generates garbage
+    "gemma",
+    "starcoder2",
+    // "mamba", -- CUDA missing SSM_CONV
+    "xverse",
+    "command-r",
+    // "dbrx", -- 16x12B parameters
+    "olmo",
 };
 
 static const std::vector<const char *> EMBEDDING_ARCHES {
-    "bert", "nomic-bert"
+    "bert", "nomic-bert",
 };
 
 static bool is_embedding_arch(const std::string &arch) {
@@ -104,13 +139,15 @@ static int llama_sample_top_p_top_k(
     return llama_sample_token(ctx, &candidates_p);
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
+const char *get_arch_name(gguf_context *ctx_gguf) {
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    if (kid == -1)
+        throw std::runtime_error("key not found in model: general.architecture");
+
     enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != (GGUF_TYPE_STRING)) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
+    if (ktype != GGUF_TYPE_STRING)
+        throw std::runtime_error("key general.architecture has wrong type");
+
     return gguf_get_val_str(ctx_gguf, kid);
 }
 
@@ -136,13 +173,20 @@ static gguf_context *load_gguf(const char *fname) {
 }
 
 static int32_t get_arch_key_u32(std::string const &modelPath, std::string const &archKey) {
+    int32_t value = -1;
+    std::string arch;
+
     auto * ctx = load_gguf(modelPath.c_str());
     if (!ctx)
-        return -1;
-    std::string arch = get_arch_name(ctx);
+        goto cleanup;
 
-    int32_t value = -1;
-    if (ctx) {
+    try {
+        arch = get_arch_name(ctx);
+    } catch (const std::runtime_error &) {
+        goto cleanup; // cannot read key
+    }
+
+    {
         auto key = arch + "." + archKey;
         int keyidx = gguf_find_key(ctx, key.c_str());
         if (keyidx != -1) {
@@ -152,26 +196,27 @@ static int32_t get_arch_key_u32(std::string const &modelPath, std::string const
         }
     }
 
+cleanup:
     gguf_free(ctx);
     return value;
 }
 
 struct LLamaPrivate {
     const std::string modelPath;
-    bool modelLoaded;
+    bool modelLoaded = false;
     int device = -1;
+    std::string deviceName;
     llama_model *model = nullptr;
     llama_context *ctx = nullptr;
     llama_model_params model_params;
     llama_context_params ctx_params;
     int64_t n_threads = 0;
     std::vector<LLModel::Token> end_tokens;
+    const char *backend_name = nullptr;
 };
 
 LLamaModel::LLamaModel()
-    : d_ptr(new LLamaPrivate) {
-    d_ptr->modelLoaded = false;
-}
+    : d_ptr(new LLamaPrivate) {}
 
 // default hparams (LLaMA 7B)
 struct llama_file_hparams {
@@ -245,15 +290,26 @@ bool LLamaModel::isModelBlacklisted(const std::string &modelPath) const {
 }
 
 bool LLamaModel::isEmbeddingModel(const std::string &modelPath) const {
+    bool result = false;
+    std::string arch;
+
     auto *ctx_gguf = load_gguf(modelPath.c_str());
     if (!ctx_gguf) {
         std::cerr << __func__ << ": failed to load GGUF from " <<  modelPath << "\n";
-        return false;
+        goto cleanup;
     }
 
-    std::string arch = get_arch_name(ctx_gguf);
+    try {
+        arch = get_arch_name(ctx_gguf);
+    } catch (const std::runtime_error &) {
+        goto cleanup; // cannot read key
+    }
+
+    result = is_embedding_arch(arch);
+
+cleanup:
     gguf_free(ctx_gguf);
-    return is_embedding_arch(arch);
+    return result;
 }
 
 bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
@@ -291,10 +347,13 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     d_ptr->model_params.progress_callback = &LLModel::staticProgressCallback;
     d_ptr->model_params.progress_callback_user_data = this;
 
-#ifdef GGML_USE_KOMPUTE
+    d_ptr->backend_name = "cpu"; // default
+
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
     if (d_ptr->device != -1) {
         d_ptr->model_params.main_gpu = d_ptr->device;
         d_ptr->model_params.n_gpu_layers = ngl;
+        d_ptr->model_params.split_mode = LLAMA_SPLIT_MODE_NONE;
     }
 #elif defined(GGML_USE_METAL)
     (void)ngl;
@@ -302,6 +361,7 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     if (llama_verbose()) {
         std::cerr << "llama.cpp: using Metal" << std::endl;
     }
+    d_ptr->backend_name = "metal";
 
     // always fully offload on Metal
     // TODO(cebtenzzre): use this parameter to allow using more than 53% of system RAM to load a model
@@ -314,6 +374,7 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     if (!d_ptr->model) {
         fflush(stdout);
         d_ptr->device = -1;
+        d_ptr->deviceName.clear();
         std::cerr << "LLAMA ERROR: failed to load model from " << modelPath << std::endl;
         return false;
     }
@@ -325,7 +386,7 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     bool isEmbedding = is_embedding_arch(llama_model_arch(d_ptr->model));
     const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
     if (isEmbedding) {
-        d_ptr->ctx_params.n_batch = n_ctx_train;
+        d_ptr->ctx_params.n_batch = n_ctx;
     } else {
         if (n_ctx > n_ctx_train) {
             std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
@@ -356,16 +417,24 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
         llama_free_model(d_ptr->model);
         d_ptr->model = nullptr;
         d_ptr->device = -1;
+        d_ptr->deviceName.clear();
         return false;
     }
 
     d_ptr->end_tokens = {llama_token_eos(d_ptr->model)};
 
+    if (usingGPUDevice()) {
 #ifdef GGML_USE_KOMPUTE
-    if (usingGPUDevice() && ggml_vk_has_device()) {
-        std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
-    }
+        if (llama_verbose()) {
+            std::cerr << "llama.cpp: using Vulkan on " << d_ptr->deviceName << std::endl;
+        }
+        d_ptr->backend_name = "kompute";
+#elif defined(GGML_USE_VULKAN)
+        d_ptr->backend_name = "vulkan";
+#elif defined(GGML_USE_CUDA)
+        d_ptr->backend_name = "cuda";
 #endif
+    }
 
     m_supportsEmbedding = isEmbedding;
     m_supportsCompletion = !isEmbedding;
@@ -426,7 +495,18 @@ std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::
 
 std::string LLamaModel::tokenToString(Token id) const
 {
-    return llama_token_to_piece(d_ptr->ctx, id);
+    std::vector<char> result(8, 0);
+    const int n_tokens = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), false);
+    if (n_tokens < 0) {
+        result.resize(-n_tokens);
+        int check = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), false);
+        GGML_ASSERT(check == -n_tokens);
+    }
+    else {
+        result.resize(n_tokens);
+    }
+
+    return std::string(result.data(), result.size());
 }
 
 LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
@@ -491,34 +571,77 @@ int32_t LLamaModel::layerCount(std::string const &modelPath) const
     return get_arch_key_u32(modelPath, "block_count");
 }
 
+#ifdef GGML_USE_VULKAN
+static const char *getVulkanVendorName(uint32_t vendorID) {
+    switch (vendorID) {
+        case 0x10DE: return "nvidia";
+        case 0x1002: return "amd";
+        case 0x8086: return "intel";
+        default:     return "unknown";
+    }
+}
+#endif
+
 std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryRequired) const
 {
-#ifdef GGML_USE_KOMPUTE
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
     size_t count = 0;
-    auto * vkDevices = ggml_vk_available_devices(memoryRequired, &count);
 
-    if (vkDevices) {
+#ifdef GGML_USE_KOMPUTE
+    auto *lcppDevices = ggml_vk_available_devices(memoryRequired, &count);
+#elif defined(GGML_USE_VULKAN)
+    (void)memoryRequired; // hasn't been used since GGUF was added
+    auto *lcppDevices = ggml_vk_available_devices(&count);
+#else // defined(GGML_USE_CUDA)
+    (void)memoryRequired;
+    auto *lcppDevices = ggml_cuda_available_devices(&count);
+#endif
+
+    if (lcppDevices) {
         std::vector<LLModel::GPUDevice> devices;
         devices.reserve(count);
 
         for (size_t i = 0; i < count; ++i) {
-            auto & dev = vkDevices[i];
+            auto & dev = lcppDevices[i];
+
             devices.emplace_back(
+#ifdef GGML_USE_KOMPUTE
+                /* backend  = */ "kompute",
                 /* index    = */ dev.index,
                 /* type     = */ dev.type,
                 /* heapSize = */ dev.heapSize,
                 /* name     = */ dev.name,
                 /* vendor   = */ dev.vendor
+#elif defined(GGML_USE_VULKAN)
+                /* backend  = */ "vulkan",
+                /* index    = */ dev.index,
+                /* type     = */ dev.type,
+                /* heapSize = */ dev.heapSize,
+                /* name     = */ dev.name,
+                /* vendor   = */ getVulkanVendorName(dev.vendorID)
+#else // defined(GGML_USE_CUDA)
+                /* backend  = */ "cuda",
+                /* index    = */ dev.index,
+                /* type     = */ 2, // vk::PhysicalDeviceType::eDiscreteGpu
+                /* heapSize = */ dev.heapSize,
+                /* name     = */ dev.name,
+                /* vendor   = */ "nvidia"
+#endif
             );
+
+#ifndef GGML_USE_CUDA
             ggml_vk_device_destroy(&dev);
+#else
+            ggml_cuda_device_destroy(&dev);
+#endif
         }
 
-        free(vkDevices);
+        free(lcppDevices);
         return devices;
     }
 #else
     (void)memoryRequired;
-    std::cerr << __func__ << ": built without Kompute\n";
+    std::cerr << __func__ << ": built without a GPU backend\n";
 #endif
 
     return {};
@@ -526,11 +649,32 @@ std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryReq
 
 bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string &name) const
 {
-#if defined(GGML_USE_KOMPUTE)
+#if defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
+    auto devices = availableGPUDevices(memoryRequired);
+
+    auto dev_it = devices.begin();
+#ifndef GGML_USE_CUDA
+    if (name == "amd" || name == "nvidia" || name == "intel") {
+        dev_it = std::find_if(dev_it, devices.end(), [&name](auto &dev) { return dev.vendor == name; });
+    } else
+#endif
+    if (name != "gpu") {
+        dev_it = std::find_if(dev_it, devices.end(), [&name](auto &dev) { return dev.name == name; });
+    }
+
+    if (dev_it < devices.end()) {
+        d_ptr->device     = dev_it->index;
+        d_ptr->deviceName = dev_it->name;
+        return true;
+    }
+    return false;
+#elif defined(GGML_USE_KOMPUTE)
     ggml_vk_device device;
     bool ok = ggml_vk_get_device(&device, memoryRequired, name.c_str());
     if (ok) {
         d_ptr->device = device.index;
+        d_ptr->deviceName = device.name;
+        ggml_vk_device_destroy(&device);
         return true;
     }
 #else
@@ -542,37 +686,60 @@ bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string &n
 
 bool LLamaModel::initializeGPUDevice(int device, std::string *unavail_reason) const
 {
-#if defined(GGML_USE_KOMPUTE)
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
     (void)unavail_reason;
+    auto devices = availableGPUDevices();
+    auto it = std::find_if(devices.begin(), devices.end(), [device](auto &dev) { return dev.index == device; });
     d_ptr->device = device;
+    d_ptr->deviceName = it < devices.end() ? it->name : "(unknown)";
     return true;
 #else
     (void)device;
     if (unavail_reason) {
-        *unavail_reason = "built without Kompute";
+        *unavail_reason = "built without a GPU backend";
     }
     return false;
 #endif
 }
 
-bool LLamaModel::hasGPUDevice()
+bool LLamaModel::hasGPUDevice() const
 {
-#if defined(GGML_USE_KOMPUTE)
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
     return d_ptr->device != -1;
 #else
     return false;
 #endif
 }
 
-bool LLamaModel::usingGPUDevice()
+bool LLamaModel::usingGPUDevice() const
 {
-#if defined(GGML_USE_KOMPUTE)
-    return hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
+    bool hasDevice;
+
+#ifdef GGML_USE_KOMPUTE
+    hasDevice = hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
+    assert(!hasDevice || ggml_vk_has_device());
+#elif defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
+    hasDevice = hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
 #elif defined(GGML_USE_METAL)
-    return true;
+    hasDevice = true;
 #else
-    return false;
+    hasDevice = false;
 #endif
+
+    return hasDevice;
+}
+
+const char *LLamaModel::backendName() const {
+    return d_ptr->backend_name;
+}
+
+const char *LLamaModel::gpuDeviceName() const {
+    if (usingGPUDevice()) {
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
+        return d_ptr->deviceName.c_str();
+#endif
+    }
+    return nullptr;
 }
 
 void llama_batch_add(
@@ -674,7 +841,7 @@ void LLamaModel::embed(
 
 void LLamaModel::embed(
     const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
-    size_t *tokenCount, bool doMean, bool atlas
+    size_t *tokenCount, bool doMean, bool atlas, LLModel::EmbedCancelCallback *cancelCb
 ) {
     if (!d_ptr->model)
         throw std::logic_error("no model is loaded");
@@ -712,7 +879,7 @@ void LLamaModel::embed(
         throw std::invalid_argument(ss.str());
     }
 
-    embedInternal(texts, embeddings, *prefix, dimensionality, tokenCount, doMean, atlas, spec);
+    embedInternal(texts, embeddings, *prefix, dimensionality, tokenCount, doMean, atlas, cancelCb, spec);
 }
 
 // MD5 hash of "nomic empty"
@@ -730,11 +897,11 @@ double getL2NormScale(T *start, T *end) {
 
 void LLamaModel::embedInternal(
     const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
-    size_t *tokenCount, bool doMean, bool atlas, const EmbModelSpec *spec
+    size_t *tokenCount, bool doMean, bool atlas, LLModel::EmbedCancelCallback *cancelCb, const EmbModelSpec *spec
 ) {
     typedef std::vector<LLModel::Token> TokenString;
     static constexpr int32_t atlasMaxLength = 8192;
-    static constexpr int chunkOverlap = 8; // Atlas overlaps n_batch-sized chunks of input by 8 tokens
+    static constexpr int chunkOverlap = 8; // Atlas overlaps chunks of input by 8 tokens
 
     const llama_token bos_token = llama_token_bos(d_ptr->model);
     const llama_token eos_token = llama_token_eos(d_ptr->model);
@@ -751,8 +918,13 @@ void LLamaModel::embedInternal(
 
         tokens.resize(text.length()+4);
         int32_t n_tokens = llama_tokenize(d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), wantBOS, false);
-        assert(useEOS == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
-        tokens.resize(n_tokens - useEOS); // erase EOS/SEP
+        if (n_tokens) {
+            (void)eos_token;
+            assert(useEOS == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
+            tokens.resize(n_tokens - useEOS); // erase EOS/SEP
+        } else {
+            tokens.clear();
+        }
     };
 
     // tokenize the texts
@@ -786,9 +958,14 @@ void LLamaModel::embedInternal(
         tokenize(prefix + ':', prefixTokens, true);
     }
 
+    // n_ctx_train: max sequence length of model (RoPE scaling not implemented)
+    const uint32_t n_ctx_train = llama_n_ctx_train(d_ptr->model);
+    // n_batch (equals n_ctx): max tokens per call to llama_decode (one more more sequences)
     const uint32_t n_batch = llama_n_batch(d_ptr->ctx);
-    const uint32_t max_len = n_batch - (prefixTokens.size() + useEOS); // minus BOS/CLS and EOS/SEP
-    if (chunkOverlap >= max_len) {
+
+    // effective sequence length minus prefix and SEP token
+    const uint32_t max_len = std::min(n_ctx_train, n_batch) - (prefixTokens.size() + useEOS);
+    if (max_len <= chunkOverlap) {
         throw std::logic_error("max chunk length of " + std::to_string(max_len) + " is smaller than overlap of " +
                                std::to_string(chunkOverlap) + " tokens");
     }
@@ -813,6 +990,23 @@ void LLamaModel::embedInternal(
     }
     inputs.clear();
 
+    if (cancelCb) {
+        // copy of batching code below, but just count tokens instead of running inference
+        unsigned nBatchTokens = 0;
+        std::vector<unsigned> batchSizes;
+        for (const auto &inp: batches) {
+            if (nBatchTokens + inp.batch.size() > n_batch) {
+                batchSizes.push_back(nBatchTokens);
+                nBatchTokens = 0;
+            }
+            nBatchTokens += inp.batch.size();
+        }
+        batchSizes.push_back(nBatchTokens);
+        if (cancelCb(batchSizes.data(), batchSizes.size(), d_ptr->backend_name)) {
+            throw std::runtime_error("operation was canceled");
+        }
+    }
+
     // initialize batch
     struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
 
@@ -862,7 +1056,7 @@ void LLamaModel::embedInternal(
     };
 
     // break into batches
-    for (auto &inp: batches) {
+    for (const auto &inp: batches) {
         // encode if at capacity
         if (batch.n_tokens + inp.batch.size() > n_batch) {
             decode();
@@ -893,6 +1087,8 @@ void LLamaModel::embedInternal(
     }
 
     if (tokenCount) { *tokenCount = totalTokens; }
+
+    llama_batch_free(batch);
 }
 
 #if defined(_WIN32)
@@ -914,25 +1110,33 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char *fname) {
-    auto * ctx = load_gguf(fname);
-    std::string arch = get_arch_name(ctx);
+DLL_EXPORT char *get_file_arch(const char *fname) {
+    char *arch = nullptr;
+    std::string archStr;
 
-    bool valid = true;
+    auto *ctx = load_gguf(fname);
+    if (!ctx)
+        goto cleanup;
 
-    if (std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), arch) == KNOWN_ARCHES.end()) {
-        // not supported by this version of llama.cpp
-        if (arch != "gptj") { // we support this via another module
-            std::cerr << __func__ << ": unsupported model architecture: " << arch << "\n";
-        }
-        valid = false;
+    try {
+        archStr = get_arch_name(ctx);
+    } catch (const std::runtime_error &) {
+        goto cleanup; // cannot read key
     }
 
-    if (valid && is_embedding_arch(arch) && gguf_find_key(ctx, (arch + ".pooling_type").c_str()) < 0)
-        valid = false; // old pre-llama.cpp embedding model, e.g. all-MiniLM-L6-v2-f16.gguf
+    if (is_embedding_arch(archStr) && gguf_find_key(ctx, (archStr + ".pooling_type").c_str()) < 0) {
+        // old bert.cpp embedding model
+    } else {
+        arch = strdup(archStr.c_str());
+    }
 
+cleanup:
     gguf_free(ctx);
-    return valid;
+    return arch;
+}
+
+DLL_EXPORT bool is_arch_supported(const char *arch) {
+    return std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), std::string(arch)) < KNOWN_ARCHES.end();
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llamamodel_impl.h

@@ -30,16 +30,19 @@ public:
     size_t restoreState(const uint8_t *src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
-    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const override;
+    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0) const override;
     bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const override;
     bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const override;
-    bool hasGPUDevice() override;
-    bool usingGPUDevice() override;
+    bool hasGPUDevice() const override;
+    bool usingGPUDevice() const override;
+    const char *backendName() const override;
+    const char *gpuDeviceName() const override;
 
     size_t embeddingSize() const override;
     // user-specified prefix
     void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
-               int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
+               int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false,
+               EmbedCancelCallback *cancelCb = nullptr) override;
     // automatic prefix
     void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality = -1,
                size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
@@ -61,7 +64,8 @@ protected:
     int32_t layerCount(std::string const &modelPath) const override;
 
     void embedInternal(const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
-                       size_t *tokenCount, bool doMean, bool atlas, const EmbModelSpec *spec);
+                       size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb,
+                       const EmbModelSpec *spec);
 };
 
 #endif // LLAMAMODEL_H

gpt4all-backend/llmodel.cpp

@@ -8,15 +8,25 @@
 #include <fstream>
 #include <iostream>
 #include <memory>
+#include <optional>
 #include <regex>
 #include <sstream>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #ifdef _MSC_VER
 #include <intrin.h>
 #endif
 
+#ifndef __APPLE__
+static const std::string DEFAULT_BACKENDS[] = {"kompute", "cpu"};
+#elif defined(__aarch64__)
+static const std::string DEFAULT_BACKENDS[] = {"metal", "cpu"};
+#else
+static const std::string DEFAULT_BACKENDS[] = {"cpu"};
+#endif
+
 std::string s_implementations_search_path = ".";
 
 #if !(defined(__x86_64__) || defined(_M_X64))
@@ -32,13 +42,13 @@ std::string s_implementations_search_path = ".";
     }
 
     // AVX via EAX=1: Processor Info and Feature Bits, bit 28 of ECX
-    #define cpu_supports_avx()  (get_cpu_info(1, 2) & (1 << 28))
+    #define cpu_supports_avx()  !!(get_cpu_info(1, 2) & (1 << 28))
     // AVX2 via EAX=7, ECX=0: Extended Features, bit 5 of EBX
-    #define cpu_supports_avx2() (get_cpu_info(7, 1) & (1 <<  5))
+    #define cpu_supports_avx2() !!(get_cpu_info(7, 1) & (1 <<  5))
 #else
     // gcc/clang
-    #define cpu_supports_avx()  __builtin_cpu_supports("avx")
-    #define cpu_supports_avx2() __builtin_cpu_supports("avx2")
+    #define cpu_supports_avx()  !!__builtin_cpu_supports("avx")
+    #define cpu_supports_avx2() !!__builtin_cpu_supports("avx2")
 #endif
 
 LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
@@ -49,14 +59,17 @@ LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
     m_buildVariant = get_build_variant();
-    m_magicMatch = m_dlhandle->get<bool(const char*)>("magic_match");
-    assert(m_magicMatch);
+    m_getFileArch = m_dlhandle->get<char *(const char *)>("get_file_arch");
+    assert(m_getFileArch);
+    m_isArchSupported = m_dlhandle->get<bool(const char *)>("is_arch_supported");
+    assert(m_isArchSupported);
     m_construct = m_dlhandle->get<LLModel *()>("construct");
     assert(m_construct);
 }
 
 LLModel::Implementation::Implementation(Implementation &&o)
-    : m_magicMatch(o.m_magicMatch)
+    : m_getFileArch(o.m_getFileArch)
+    , m_isArchSupported(o.m_isArchSupported)
     , m_construct(o.m_construct)
     , m_modelType(o.m_modelType)
     , m_buildVariant(o.m_buildVariant)
@@ -82,11 +95,9 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     static auto* libs = new std::vector<Implementation>([] () {
         std::vector<Implementation> fres;
 
-        std::string impl_name_re = "(gptj|llamamodel-mainline)";
+        std::string impl_name_re = "(gptj|llamamodel-mainline)-(cpu|metal|kompute|vulkan|cuda)";
         if (cpu_supports_avx2() == 0) {
             impl_name_re += "-avxonly";
-        } else {
-            impl_name_re += "-(default|metal)";
         }
         std::regex re(impl_name_re);
         auto search_in_directory = [&](const std::string& paths) {
@@ -121,116 +132,148 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     return *libs;
 }
 
+static std::string applyCPUVariant(const std::string &buildVariant) {
+    if (buildVariant != "metal" && cpu_supports_avx2() == 0) {
+        return buildVariant + "-avxonly";
+    }
+    return buildVariant;
+}
+
 const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
     bool buildVariantMatched = false;
+    std::optional<std::string> archName;
     for (const auto& i : implementationList()) {
         if (buildVariant != i.m_buildVariant) continue;
         buildVariantMatched = true;
 
-        if (!i.m_magicMatch(fname)) continue;
-        return &i;
+        char *arch = i.m_getFileArch(fname);
+        if (!arch) continue;
+        archName = arch;
+
+        bool archSupported = i.m_isArchSupported(arch);
+        free(arch);
+        if (archSupported) return &i;
     }
 
     if (!buildVariantMatched)
-        throw std::runtime_error("Could not find any implementations for build variant: " + buildVariant);
+        return nullptr;
+    if (!archName)
+        throw UnsupportedModelError("Unsupported file format");
 
-    return nullptr; // unsupported model format
+    throw BadArchError(std::move(*archName));
 }
 
-LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant, int n_ctx) {
-    // Get correct implementation
-    const Implementation* impl = nullptr;
-
-    #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
-        if (buildVariant == "auto") {
-            size_t total_mem = getSystemTotalRAMInBytes();
-            impl = implementation(modelPath.c_str(), "metal");
-            if(impl) {
-                LLModel* metalimpl = impl->m_construct();
-                metalimpl->m_implementation = impl;
-                /* TODO(cebtenzzre): after we fix requiredMem, we should change this to happen at
-                 * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
-                 * most (all?) places where this is called, causing underestimation of required
-                 * memory. */
-                size_t req_mem = metalimpl->requiredMem(modelPath, n_ctx, 100);
-                float req_to_total = (float) req_mem / (float) total_mem;
-                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
-                if (req_to_total >= 0.53) {
-                    delete metalimpl;
-                    impl = nullptr;
-                } else {
-                    return metalimpl;
-                }
-            }
-        }
-    #else
-        (void)n_ctx;
-    #endif
-
-    if (!impl) {
-        //TODO: Auto-detect CUDA/OpenCL
-        if (buildVariant == "auto") {
-            if (cpu_supports_avx2() == 0) {
-                buildVariant = "avxonly";
-            } else {
-                buildVariant = "default";
-            }
-        }
-        impl = implementation(modelPath.c_str(), buildVariant);
-        if (!impl) return nullptr;
+LLModel *LLModel::Implementation::construct(const std::string &modelPath, const std::string &backend, int n_ctx) {
+    std::vector<std::string> desiredBackends;
+    if (backend != "auto") {
+        desiredBackends.push_back(backend);
+    } else {
+        desiredBackends.insert(desiredBackends.end(), DEFAULT_BACKENDS, std::end(DEFAULT_BACKENDS));
     }
 
-    // Construct and return llmodel implementation
-    auto fres = impl->m_construct();
-    fres->m_implementation = impl;
-    return fres;
+    for (const auto &desiredBackend: desiredBackends) {
+        const auto *impl = implementation(modelPath.c_str(), applyCPUVariant(desiredBackend));
+
+        if (impl) {
+            // Construct llmodel implementation
+            auto *fres = impl->m_construct();
+            fres->m_implementation = impl;
+
+#if defined(__APPLE__) && defined(__aarch64__) // FIXME: See if metal works for intel macs
+            /* TODO(cebtenzzre): after we fix requiredMem, we should change this to happen at
+             * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
+             * most (all?) places where this is called, causing underestimation of required
+             * memory. */
+            if (backend == "auto" && desiredBackend == "metal") {
+                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
+                size_t req_mem = fres->requiredMem(modelPath, n_ctx, 100);
+                if (req_mem >= size_t(0.53f * getSystemTotalRAMInBytes())) {
+                    delete fres;
+                    continue;
+                }
+            }
+#else
+            (void)n_ctx;
+#endif
+
+            return fres;
+        }
+    }
+
+    throw MissingImplementationError("Could not find any implementations for backend: " + backend);
 }
 
-LLModel *LLModel::Implementation::constructDefaultLlama() {
-    static std::unique_ptr<LLModel> llama([]() -> LLModel * {
-        const std::vector<LLModel::Implementation> *impls;
-        try {
-            impls = &implementationList();
-        } catch (const std::runtime_error &e) {
-            std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
-            return nullptr;
-        }
+LLModel *LLModel::Implementation::constructGlobalLlama(const std::optional<std::string> &backend) {
+    static std::unordered_map<std::string, std::unique_ptr<LLModel>> implCache;
+
+    const std::vector<Implementation> *impls;
+    try {
+        impls = &implementationList();
+    } catch (const std::runtime_error &e) {
+        std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
+        return nullptr;
+    }
+
+    std::vector<std::string> desiredBackends;
+    if (backend) {
+        desiredBackends.push_back(backend.value());
+    } else {
+        desiredBackends.insert(desiredBackends.end(), DEFAULT_BACKENDS, std::end(DEFAULT_BACKENDS));
+    }
+
+    const Implementation *impl = nullptr;
+
+    for (const auto &desiredBackend: desiredBackends) {
+        auto cacheIt = implCache.find(desiredBackend);
+        if (cacheIt != implCache.end())
+            return cacheIt->second.get(); // cached
 
-        const LLModel::Implementation *impl = nullptr;
         for (const auto &i: *impls) {
-            if (i.m_buildVariant == "metal" || i.m_modelType != "LLaMA") continue;
-            impl = &i;
-        }
-        if (!impl) {
-            std::cerr << __func__ << ": could not find llama.cpp implementation\n";
-            return nullptr;
+            if (i.m_modelType == "LLaMA" && i.m_buildVariant == applyCPUVariant(desiredBackend)) {
+                impl = &i;
+                break;
+            }
         }
 
-        auto fres = impl->m_construct();
-        fres->m_implementation = impl;
-        return fres;
-    }());
-    return llama.get();
+        if (impl) {
+            auto *fres = impl->m_construct();
+            fres->m_implementation = impl;
+            implCache[desiredBackend] = std::unique_ptr<LLModel>(fres);
+            return fres;
+        }
+    }
+
+    std::cerr << __func__ << ": could not find Llama implementation for backend: " << backend.value_or("default") << "\n";
+    return nullptr;
 }
 
-std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices() {
-    auto *llama = constructDefaultLlama();
-    if (llama) { return llama->availableGPUDevices(0); }
-    return {};
+std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices(size_t memoryRequired) {
+    std::vector<LLModel::GPUDevice> devices;
+#ifndef __APPLE__
+    static const std::string backends[] = {"kompute", "cuda"};
+    for (const auto &backend: backends) {
+        auto *llama = constructGlobalLlama(backend);
+        if (llama) {
+            auto backendDevs = llama->availableGPUDevices(memoryRequired);
+            devices.insert(devices.end(), backendDevs.begin(), backendDevs.end());
+        }
+    }
+#endif
+    return devices;
 }
 
 int32_t LLModel::Implementation::maxContextLength(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
+    auto *llama = constructGlobalLlama();
     return llama ? llama->maxContextLength(modelPath) : -1;
 }
 
 int32_t LLModel::Implementation::layerCount(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
+    auto *llama = constructGlobalLlama();
     return llama ? llama->layerCount(modelPath) : -1;
 }
 
 bool LLModel::Implementation::isEmbeddingModel(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
+    auto *llama = constructGlobalLlama();
     return llama && llama->isEmbeddingModel(modelPath);
 }
 
@@ -245,3 +288,7 @@ const std::string& LLModel::Implementation::implementationsSearchPath() {
 bool LLModel::Implementation::hasSupportedCPU() {
     return cpu_supports_avx() != 0;
 }
+
+int LLModel::Implementation::cpuSupportsAVX2() {
+    return cpu_supports_avx2();
+}

gpt4all-backend/llmodel.h

@@ -1,6 +1,7 @@
 #ifndef LLMODEL_H
 #define LLMODEL_H
 
+#include <algorithm>
 #include <cstdint>
 #include <fstream>
 #include <functional>
@@ -8,8 +9,11 @@
 #include <optional>
 #include <string>
 #include <string_view>
+#include <unordered_map>
 #include <vector>
 
+using namespace std::string_literals;
+
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
 class Dlhandle;
@@ -17,15 +21,59 @@ class LLModel {
 public:
     using Token = int32_t;
 
+    class BadArchError: public std::runtime_error {
+    public:
+        BadArchError(std::string arch)
+            : runtime_error("Unsupported model architecture: " + arch)
+            , m_arch(std::move(arch))
+            {}
+
+        const std::string &arch() const noexcept { return m_arch; }
+
+    private:
+        std::string m_arch;
+    };
+
+    class MissingImplementationError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
+    class UnsupportedModelError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
     struct GPUDevice {
+        const char *backend;
         int index;
         int type;
         size_t heapSize;
         std::string name;
         std::string vendor;
 
-        GPUDevice(int index, int type, size_t heapSize, std::string name, std::string vendor):
-            index(index), type(type), heapSize(heapSize), name(std::move(name)), vendor(std::move(vendor)) {}
+        GPUDevice(const char *backend, int index, int type, size_t heapSize, std::string name, std::string vendor):
+            backend(backend), index(index), type(type), heapSize(heapSize), name(std::move(name)),
+            vendor(std::move(vendor)) {}
+
+        std::string selectionName() const { return m_backendNames.at(backend) + ": " + name; }
+        std::string reportedName()  const { return name + " (" + m_backendNames.at(backend) + ")"; }
+
+        static std::string updateSelectionName(const std::string &name) {
+            if (name == "Auto" || name == "CPU" || name == "Metal")
+                return name;
+            auto it = std::find_if(m_backendNames.begin(), m_backendNames.end(), [&name](const auto &entry) {
+                return name.starts_with(entry.second + ": ");
+            });
+            if (it != m_backendNames.end())
+                return name;
+            return "Vulkan: " + name; // previously, there were only Vulkan devices
+        }
+
+    private:
+        static inline const std::unordered_map<std::string, std::string> m_backendNames {
+            {"cuda", "CUDA"}, {"kompute", "Vulkan"},
+        };
     };
 
     class Implementation {
@@ -37,23 +85,26 @@ public:
         std::string_view modelType() const { return m_modelType; }
         std::string_view buildVariant() const { return m_buildVariant; }
 
-        static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto", int n_ctx = 2048);
-        static std::vector<GPUDevice> availableGPUDevices();
+        static LLModel *construct(const std::string &modelPath, const std::string &backend = "auto", int n_ctx = 2048);
+        static std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0);
         static int32_t maxContextLength(const std::string &modelPath);
         static int32_t layerCount(const std::string &modelPath);
         static bool isEmbeddingModel(const std::string &modelPath);
         static void setImplementationsSearchPath(const std::string &path);
         static const std::string &implementationsSearchPath();
         static bool hasSupportedCPU();
+        // 0 for no, 1 for yes, -1 for non-x86_64
+        static int cpuSupportsAVX2();
 
     private:
         Implementation(Dlhandle &&);
 
         static const std::vector<Implementation> &implementationList();
         static const Implementation *implementation(const char *fname, const std::string &buildVariant);
-        static LLModel *constructDefaultLlama();
+        static LLModel *constructGlobalLlama(const std::optional<std::string> &backend = std::nullopt);
 
-        bool (*m_magicMatch)(const char *fname);
+        char *(*m_getFileArch)(const char *fname);
+        bool (*m_isArchSupported)(const char *arch);
         LLModel *(*m_construct)();
 
         std::string_view m_modelType;
@@ -105,12 +156,15 @@ public:
                         bool special = false,
                         std::string *fakeReply = nullptr);
 
+    using EmbedCancelCallback = bool(unsigned *batchSizes, unsigned nBatch, const char *backend);
+
     virtual size_t embeddingSize() const {
         throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
     }
     // user-specified prefix
     virtual void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
-                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false);
+                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false,
+                       EmbedCancelCallback *cancelCb = nullptr);
     // automatic prefix
     virtual void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval,
                        int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false);
@@ -141,8 +195,10 @@ public:
         return false;
     }
 
-    virtual bool hasGPUDevice() { return false; }
-    virtual bool usingGPUDevice() { return false; }
+    virtual bool hasGPUDevice() const { return false; }
+    virtual bool usingGPUDevice() const { return false; }
+    virtual const char *backendName() const { return "cpu"; }
+    virtual const char *gpuDeviceName() const { return nullptr; }
 
     void setProgressCallback(ProgressCallback callback) { m_progressCallback = callback; }
 

gpt4all-backend/llmodel_c.cpp

@@ -4,6 +4,7 @@
 #include <cerrno>
 #include <cstring>
 #include <iostream>
+#include <memory>
 #include <optional>
 #include <utility>
 
@@ -30,20 +31,15 @@ static void llmodel_set_error(const char **errptr, const char *message) {
     }
 }
 
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error) {
+llmodel_model llmodel_model_create2(const char *model_path, const char *backend, const char **error) {
     LLModel *llModel;
     try {
-        llModel = LLModel::Implementation::construct(model_path, build_variant);
+        llModel = LLModel::Implementation::construct(model_path, backend);
     } catch (const std::exception& e) {
         llmodel_set_error(error, e.what());
         return nullptr;
     }
 
-    if (!llModel) {
-        llmodel_set_error(error, "Model format not supported (no matching implementation found)");
-        return nullptr;
-    }
-
     auto wrapper = new LLModelWrapper;
     wrapper->llModel = llModel;
     return wrapper;
@@ -158,7 +154,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
 
 float *llmodel_embed(
     llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix, int dimensionality,
-    size_t *token_count, bool do_mean, bool atlas, const char **error
+    size_t *token_count, bool do_mean, bool atlas, llmodel_emb_cancel_callback cancel_cb, const char **error
 ) {
     auto *wrapper = static_cast<LLModelWrapper *>(model);
 
@@ -184,7 +180,7 @@ float *llmodel_embed(
         if (prefix) { prefixStr = prefix; }
 
         embedding = new float[embd_size];
-        wrapper->llModel->embed(textsVec, embedding, prefixStr, dimensionality, token_count, do_mean, atlas);
+        wrapper->llModel->embed(textsVec, embedding, prefixStr, dimensionality, token_count, do_mean, atlas, cancel_cb);
     } catch (std::exception const &e) {
         llmodel_set_error(error, e.what());
         return nullptr;
@@ -221,28 +217,46 @@ const char *llmodel_get_implementation_search_path()
     return LLModel::Implementation::implementationsSearchPath().c_str();
 }
 
-struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices)
-{
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
-    std::vector<LLModel::GPUDevice> devices = wrapper->llModel->availableGPUDevices(memoryRequired);
+// RAII wrapper around a C-style struct
+struct llmodel_gpu_device_cpp: llmodel_gpu_device {
+    llmodel_gpu_device_cpp() = default;
 
-    // Set the num_devices
+    llmodel_gpu_device_cpp(const llmodel_gpu_device_cpp  &) = delete;
+    llmodel_gpu_device_cpp(      llmodel_gpu_device_cpp &&) = delete;
+
+    const llmodel_gpu_device_cpp &operator=(const llmodel_gpu_device_cpp  &) = delete;
+          llmodel_gpu_device_cpp &operator=(      llmodel_gpu_device_cpp &&) = delete;
+
+    ~llmodel_gpu_device_cpp() {
+        free(const_cast<char *>(name));
+        free(const_cast<char *>(vendor));
+    }
+};
+
+static_assert(sizeof(llmodel_gpu_device_cpp) == sizeof(llmodel_gpu_device));
+
+struct llmodel_gpu_device *llmodel_available_gpu_devices(size_t memoryRequired, int *num_devices)
+{
+    static thread_local std::unique_ptr<llmodel_gpu_device_cpp[]> c_devices;
+
+    auto devices = LLModel::Implementation::availableGPUDevices(memoryRequired);
     *num_devices = devices.size();
 
-    if (*num_devices == 0) return nullptr;  // Return nullptr if no devices are found
+    if (devices.empty()) { return nullptr; /* no devices */ }
 
-    // Allocate memory for the output array
-    struct llmodel_gpu_device* output = (struct llmodel_gpu_device*) malloc(*num_devices * sizeof(struct llmodel_gpu_device));
-
-    for (int i = 0; i < *num_devices; i++) {
-        output[i].index = devices[i].index;
-        output[i].type = devices[i].type;
-        output[i].heapSize = devices[i].heapSize;
-        output[i].name = strdup(devices[i].name.c_str());  // Convert std::string to char* and allocate memory
-        output[i].vendor = strdup(devices[i].vendor.c_str());  // Convert std::string to char* and allocate memory
+    c_devices = std::make_unique<llmodel_gpu_device_cpp[]>(devices.size());
+    for (unsigned i = 0; i < devices.size(); i++) {
+        const auto &dev  =   devices[i];
+              auto &cdev = c_devices[i];
+        cdev.backend  = dev.backend;
+        cdev.index    = dev.index;
+        cdev.type     = dev.type;
+        cdev.heapSize = dev.heapSize;
+        cdev.name     = strdup(dev.name.c_str());
+        cdev.vendor   = strdup(dev.vendor.c_str());
     }
 
-    return output;
+    return c_devices.get();
 }
 
 bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
@@ -265,6 +279,18 @@ bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
 
 bool llmodel_has_gpu_device(llmodel_model model)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->hasGPUDevice();
 }
+
+const char *llmodel_model_backend_name(llmodel_model model)
+{
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->backendName();
+}
+
+const char *llmodel_model_gpu_device_name(llmodel_model model)
+{
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->gpuDeviceName();
+}

gpt4all-backend/llmodel_c.h

@@ -48,9 +48,10 @@ struct llmodel_prompt_context {
 };
 
 struct llmodel_gpu_device {
-    int index = 0;
-    int type = 0;           // same as VkPhysicalDeviceType
-    size_t heapSize = 0;
+    const char * backend;
+    int index;
+    int type; // same as VkPhysicalDeviceType
+    size_t heapSize;
     const char * name;
     const char * vendor;
 };
@@ -82,6 +83,15 @@ typedef bool (*llmodel_response_callback)(int32_t token_id, const char *response
  */
 typedef bool (*llmodel_recalculate_callback)(bool is_recalculating);
 
+/**
+ * Embedding cancellation callback for use with llmodel_embed.
+ * @param batch_sizes The number of tokens in each batch that will be embedded.
+ * @param n_batch The number of batches that will be embedded.
+ * @param backend The backend that will be used for embedding. One of "cpu", "kompute", "cuda", or "metal".
+ * @return True to cancel llmodel_embed, false to continue.
+ */
+typedef bool (*llmodel_emb_cancel_callback)(unsigned *batch_sizes, unsigned n_batch, const char *backend);
+
 /**
  * Create a llmodel instance.
  * Recognises correct model type from file at model_path
@@ -94,11 +104,11 @@ DEPRECATED llmodel_model llmodel_model_create(const char *model_path);
  * Create a llmodel instance.
  * Recognises correct model type from file at model_path
  * @param model_path A string representing the path to the model file; will only be used to detect model type.
- * @param build_variant A string representing the implementation to use (auto, default, avxonly, ...),
+ * @param backend A string representing the implementation to use. One of 'auto', 'cpu', 'metal', 'kompute', or 'cuda'.
  * @param error A pointer to a string; will only be set on error.
  * @return A pointer to the llmodel_model instance; NULL on error.
  */
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error);
+llmodel_model llmodel_model_create2(const char *model_path, const char *backend, const char **error);
 
 /**
  * Destroy a llmodel instance.
@@ -198,12 +208,14 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
  * truncate.
  * @param atlas Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens with
  * long_text_mode="mean" will raise an error. Disabled by default.
+ * @param cancel_cb Cancellation callback, or NULL. See the documentation of llmodel_emb_cancel_callback.
  * @param error Return location for a malloc()ed string that will be set on error, or NULL.
  * @return A pointer to an array of floating point values passed to the calling method which then will
  * be responsible for lifetime of this memory. NULL if an error occurred.
  */
 float *llmodel_embed(llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix,
-                     int dimensionality, size_t *token_count, bool do_mean, bool atlas, const char **error);
+                     int dimensionality, size_t *token_count, bool do_mean, bool atlas,
+                     llmodel_emb_cancel_callback cancel_cb, const char **error);
 
 /**
  * Frees the memory allocated by the llmodel_embedding function.
@@ -241,9 +253,10 @@ const char *llmodel_get_implementation_search_path();
 
 /**
  * Get a list of available GPU devices given the memory required.
+ * @param memoryRequired The minimum amount of VRAM, in bytes
  * @return A pointer to an array of llmodel_gpu_device's whose number is given by num_devices.
  */
-struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices);
+struct llmodel_gpu_device* llmodel_available_gpu_devices(size_t memoryRequired, int* num_devices);
 
 /**
  * Initializes a GPU device based on a specified string criterion.
@@ -283,6 +296,16 @@ bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
  */
 bool llmodel_has_gpu_device(llmodel_model model);
 
+/**
+ * @return The name of the llama.cpp backend currently in use. One of "cpu", "kompute", or "metal".
+ */
+const char *llmodel_model_backend_name(llmodel_model model);
+
+/**
+ * @return The name of the GPU device currently in use, or NULL for backends other than Kompute.
+ */
+const char *llmodel_model_gpu_device_name(llmodel_model model);
+
 #ifdef __cplusplus
 }
 #endif

gpt4all-backend/llmodel_shared.cpp

@@ -270,7 +270,7 @@ void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)>
 
 void LLModel::embed(
     const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
-    size_t *tokenCount, bool doMean, bool atlas
+    size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb
 ) {
     (void)texts;
     (void)embeddings;
@@ -279,6 +279,7 @@ void LLModel::embed(
     (void)tokenCount;
     (void)doMean;
     (void)atlas;
+    (void)cancelCb;
     throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
 }
 

gpt4all-bindings/golang/README.md

@@ -24,7 +24,7 @@ func main() {
 		return true
 	})
 
-	_, err = model.Predict("Here are 4 steps to create a website:", gpt4all.SetTemperature(0.1))
+	_, err = model.Predict("Here are 4 steps to create a website:", "", "", gpt4all.SetTemperature(0.1))
 	if err != nil {
 		panic(err)
 	}

gpt4all-bindings/golang/binding.cpp

@@ -35,8 +35,9 @@ void* load_model(const char *fname, int n_threads) {
 std::string res = "";
 void * mm;
 
-void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float min_p, float temp, int n_batch,float ctx_erase)
+void model_prompt(const char *prompt, const char *prompt_template, int special, const char *fake_reply, 
+                    void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, 
+                    int top_k, float top_p, float min_p, float temp, int n_batch,float ctx_erase)
 {
     llmodel_model* model = (llmodel_model*) m;
 
@@ -88,11 +89,11 @@ void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n,
     prompt_context->temp = temp;
     prompt_context->n_batch = n_batch;    
 
-    llmodel_prompt(model, prompt,
+    llmodel_prompt(model, prompt, prompt_template,
                         lambda_prompt,
                         lambda_response,
                     lambda_recalculate,
-                    prompt_context );
+                    prompt_context, special, fake_reply);
 
     strcpy(result, res.c_str()); 
 

gpt4all-bindings/golang/binding.h

@@ -6,8 +6,9 @@ extern "C" {
 
 void* load_model(const char *fname, int n_threads);
 
-void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float min_p, float temp, int n_batch,float ctx_erase);
+void model_prompt(const char *prompt, const char *prompt_template, int special, const char *fake_reply, 
+                    void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, 
+                    int top_k, float top_p, float min_p, float temp, int n_batch,float ctx_erase);
 
 void free_model(void *state_ptr);
 

gpt4all-bindings/golang/example/main.go

@@ -47,7 +47,7 @@ func main() {
 	for {
 		text := readMultiLineInput(reader)
 
-		_, err := l.Predict(text, gpt4all.SetTokens(tokens), gpt4all.SetTopK(90), gpt4all.SetTopP(0.86))
+		_, err := l.Predict(text, "", "", gpt4all.SetTokens(tokens), gpt4all.SetTopK(90), gpt4all.SetTopP(0.86))
 		if err != nil {
 			panic(err)
 		}

gpt4all-bindings/golang/gpt4all.go

@@ -6,7 +6,7 @@ package gpt4all
 // #cgo darwin CXXFLAGS: -std=c++17
 // #cgo LDFLAGS: -lgpt4all -lm -lstdc++ -ldl
 // void* load_model(const char *fname, int n_threads);
-// void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
+// void model_prompt( const char *prompt, const char *prompt_template, int special, const char *fake_reply, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
 //                            float top_p, float min_p, float temp, int n_batch,float ctx_erase);
 // void free_model(void *state_ptr);
 // extern unsigned char getTokenCallback(void *, char *);
@@ -47,7 +47,7 @@ func New(model string, opts ...ModelOption) (*Model, error) {
 	return gpt, nil
 }
 
-func (l *Model) Predict(text string, opts ...PredictOption) (string, error) {
+func (l *Model) Predict(text, template, fakeReplyText string, opts ...PredictOption) (string, error) {
 
 	po := NewPredictOptions(opts...)
 
@@ -55,10 +55,14 @@ func (l *Model) Predict(text string, opts ...PredictOption) (string, error) {
 	if po.Tokens == 0 {
 		po.Tokens = 99999999
 	}
+	templateInput := C.CString(template)
+	fakeReplyInput := C.CString(fakeReplyText)
 	out := make([]byte, po.Tokens)
 
-	C.model_prompt(input, l.state, (*C.char)(unsafe.Pointer(&out[0])), C.int(po.RepeatLastN), C.float(po.RepeatPenalty), C.int(po.ContextSize),
-		C.int(po.Tokens), C.int(po.TopK), C.float(po.TopP), C.float(po.MinP), C.float(po.Temperature), C.int(po.Batch), C.float(po.ContextErase))
+	C.model_prompt(input, templateInput, C.int(po.Special), fakeReplyInput, l.state, (*C.char)(unsafe.Pointer(&out[0])),
+		C.int(po.RepeatLastN), C.float(po.RepeatPenalty), C.int(po.ContextSize), C.int(po.Tokens),
+		C.int(po.TopK), C.float(po.TopP), C.float(po.MinP), C.float(po.Temperature), C.int(po.Batch),
+		C.float(po.ContextErase))
 
 	res := C.GoString((*C.char)(unsafe.Pointer(&out[0])))
 	res = strings.TrimPrefix(res, " ")

gpt4all-bindings/golang/options.go

@@ -1,8 +1,8 @@
 package gpt4all
 
 type PredictOptions struct {
-	ContextSize, RepeatLastN, Tokens, TopK, Batch  int
-	TopP, MinP, Temperature, ContextErase, RepeatPenalty float64
+	ContextSize, RepeatLastN, Tokens, TopK, Batch, Special int
+	TopP, MinP, Temperature, ContextErase, RepeatPenalty   float64
 }
 
 type PredictOption func(p *PredictOptions)
@@ -11,9 +11,10 @@ var DefaultOptions PredictOptions = PredictOptions{
 	Tokens:        200,
 	TopK:          10,
 	TopP:          0.90,
-        MinP:          0.0,
+	MinP:          0.0,
 	Temperature:   0.96,
 	Batch:         1,
+	Special:       0,
 	ContextErase:  0.55,
 	ContextSize:   1024,
 	RepeatLastN:   10,
@@ -93,6 +94,17 @@ func SetBatch(size int) PredictOption {
 	}
 }
 
+// SetSpecial is true if special tokens in the prompt should be processed, false otherwise.
+func SetSpecial(special bool) PredictOption {
+	return func(p *PredictOptions) {
+		if special {
+			p.Special = 1
+		} else {
+			p.Special = 0
+		}
+	}
+}
+
 // Create a new PredictOptions object with the given options.
 func NewPredictOptions(opts ...PredictOption) PredictOptions {
 	p := DefaultOptions

gpt4all-bindings/python/README.md

@@ -23,9 +23,9 @@ As an alternative to downloading via pip, you may build the Python bindings from
 
 ### Prerequisites
 
-On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+You will need a compiler. On Windows, you should install Visual Studio with the C++ Development components. On macOS, you will need the full version of Xcode—Xcode Command Line Tools lacks certain required tools. On Linux, you will need a GCC or Clang toolchain with C++ support.
 
-macOS users do not need Vulkan, as GPT4All will use Metal instead.
+On Windows and Linux, building GPT4All with full GPU support requires the [Vulkan SDK](https://vulkan.lunarg.com/sdk/home) and the latest [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads).
 
 ### Building the python bindings
 

gpt4all-bindings/python/docs/gpt4all_nodejs.md

@@ -11,37 +11,116 @@ pnpm install gpt4all@latest
 
 ```
 
-The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
+## Contents
 
-*   New bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
-*   The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
-*   Everything should work out the box.
 *   See [API Reference](#api-reference)
+*   See [Examples](#api-example)
+*   See [Developing](#develop)
+*   GPT4ALL nodejs bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
+
+## Api Example
 
 ### Chat Completion
 
 ```js
-import { createCompletion, loadModel } from '../src/gpt4all.js'
+import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY, loadModel } from '../src/gpt4all.js'
 
-const model = await loadModel('mistral-7b-openorca.Q4_0.gguf', { verbose: true });
+const model = await loadModel( 'mistral-7b-openorca.gguf2.Q4_0.gguf', { verbose: true, device: 'gpu' });
 
-const response = await createCompletion(model, [
-    { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
-    { role : 'user', content: 'What is 1 + 1?'  } 
-]);
+const completion1 = await createCompletion(model, 'What is 1 + 1?', { verbose: true, })
+console.log(completion1.message)
 
+const completion2 = await createCompletion(model, 'And if we add two?', {  verbose: true  })
+console.log(completion2.message)
+
+model.dispose()
 ```
 
 ### Embedding
 
 ```js
-import { createEmbedding, loadModel } from '../src/gpt4all.js'
+import { loadModel, createEmbedding } from '../src/gpt4all.js'
 
-const model = await loadModel('ggml-all-MiniLM-L6-v2-f16', { verbose: true });
+const embedder = await loadModel("all-MiniLM-L6-v2-f16.gguf", { verbose: true, type: 'embedding'})
 
-const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional");
+console.log(createEmbedding(embedder, "Maybe Minecraft was the friends we made along the way"));
 ```
 
+### Chat Sessions
+
+```js
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession();
+
+await createCompletion(
+    chat,
+    "Why are bananas rather blue than bread at night sometimes?",
+    {
+        verbose: true,
+    }
+);
+await createCompletion(chat, "Are you sure?", { verbose: true, });
+
+```
+
+### Streaming responses
+
+```js
+import gpt from "../src/gpt4all.js";
+
+const model = await gpt.loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("### Stream:");
+const stream = gpt.createCompletionStream(model, "How are you?");
+stream.tokens.on("data", (data) => {
+    process.stdout.write(data);
+});
+//wait till stream finishes. We cannot continue until this one is done.
+await stream.result;
+process.stdout.write("\n");
+
+process.stdout.write("### Stream with pipe:");
+const stream2 = gpt.createCompletionStream(
+    model,
+    "Please say something nice about node streams."
+);
+stream2.tokens.pipe(process.stdout);
+await stream2.result;
+process.stdout.write("\n");
+
+console.log("done");
+model.dispose();
+```
+
+### Async Generators
+
+```js
+import gpt from "../src/gpt4all.js";
+
+const model = await gpt.loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("### Generator:");
+const gen = gpt.createCompletionGenerator(model, "Redstone in Minecraft is Turing Complete. Let that sink in. (let it in!)");
+for await (const chunk of gen) {
+    process.stdout.write(chunk);
+}
+
+process.stdout.write("\n");
+model.dispose();
+```
+
+## Develop
+
 ### Build Instructions
 
 *   binding.gyp is compile config
@@ -131,21 +210,27 @@ yarn test
 
 *   why your model may be spewing bull 💩
     *   The downloaded model is broken (just reinstall or download from official site)
-    *   That's it so far
+*   Your model is hanging after a call to generate tokens.
+    *   Is `nPast` set too high? This may cause your model to hang (03/16/2024), Linux Mint, Ubuntu 22.04
+*   Your GPU usage is still high after node.js exits.
+    *   Make sure to call `model.dispose()`!!!
 
 ### Roadmap
 
-This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
+This package has been stabilizing over time development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
 
+*   \[ ] Purely offline. Per the gui, which can be run completely offline, the bindings should be as well.
+*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
+    *   Should include prebuilds to avoid painful node-gyp errors
+*   \[x] createChatSession ( the python equivalent to create\_chat\_session )
+*   \[x] generateTokens, the new name for createTokenStream. As of 3.2.0, this is released but not 100% tested. Check spec/generator.mjs!
+*   \[x] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
 *   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
-*   \[ ] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
+*   \[x] generateTokens is the new name for this^
 *   \[x] proper unit testing (integrate with circle ci)
 *   \[x] publish to npm under alpha tag `gpt4all@alpha`
 *   \[x] have more people test on other platforms (mac tester needed)
 *   \[x] switch to new pluggable backend
-*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
-    *   Should include prebuilds to avoid painful node-gyp errors
-*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
 
 ### API Reference
 
@@ -153,144 +238,200 @@ This package is in active development, and breaking changes may happen until the
 
 ##### Table of Contents
 
-*   [ModelFile](#modelfile)
-    *   [gptj](#gptj)
-    *   [llama](#llama)
-    *   [mpt](#mpt)
-    *   [replit](#replit)
 *   [type](#type)
 *   [TokenCallback](#tokencallback)
+*   [ChatSessionOptions](#chatsessionoptions)
+    *   [systemPrompt](#systemprompt)
+    *   [messages](#messages)
+*   [initialize](#initialize)
+    *   [Parameters](#parameters)
+*   [generate](#generate)
+    *   [Parameters](#parameters-1)
 *   [InferenceModel](#inferencemodel)
+    *   [createChatSession](#createchatsession)
+        *   [Parameters](#parameters-2)
+    *   [generate](#generate-1)
+        *   [Parameters](#parameters-3)
     *   [dispose](#dispose)
 *   [EmbeddingModel](#embeddingmodel)
     *   [dispose](#dispose-1)
+*   [InferenceResult](#inferenceresult)
 *   [LLModel](#llmodel)
     *   [constructor](#constructor)
-        *   [Parameters](#parameters)
+        *   [Parameters](#parameters-4)
     *   [type](#type-1)
     *   [name](#name)
     *   [stateSize](#statesize)
     *   [threadCount](#threadcount)
     *   [setThreadCount](#setthreadcount)
-        *   [Parameters](#parameters-1)
-    *   [raw\_prompt](#raw_prompt)
-        *   [Parameters](#parameters-2)
+        *   [Parameters](#parameters-5)
+    *   [infer](#infer)
+        *   [Parameters](#parameters-6)
     *   [embed](#embed)
-        *   [Parameters](#parameters-3)
+        *   [Parameters](#parameters-7)
     *   [isModelLoaded](#ismodelloaded)
     *   [setLibraryPath](#setlibrarypath)
-        *   [Parameters](#parameters-4)
+        *   [Parameters](#parameters-8)
     *   [getLibraryPath](#getlibrarypath)
     *   [initGpuByString](#initgpubystring)
-        *   [Parameters](#parameters-5)
+        *   [Parameters](#parameters-9)
     *   [hasGpuDevice](#hasgpudevice)
     *   [listGpu](#listgpu)
-        *   [Parameters](#parameters-6)
+        *   [Parameters](#parameters-10)
     *   [dispose](#dispose-2)
 *   [GpuDevice](#gpudevice)
     *   [type](#type-2)
 *   [LoadModelOptions](#loadmodeloptions)
-*   [loadModel](#loadmodel)
-    *   [Parameters](#parameters-7)
-*   [createCompletion](#createcompletion)
-    *   [Parameters](#parameters-8)
-*   [createEmbedding](#createembedding)
-    *   [Parameters](#parameters-9)
-*   [CompletionOptions](#completionoptions)
+    *   [modelPath](#modelpath)
+    *   [librariesPath](#librariespath)
+    *   [modelConfigFile](#modelconfigfile)
+    *   [allowDownload](#allowdownload)
     *   [verbose](#verbose)
-    *   [systemPromptTemplate](#systemprompttemplate)
-    *   [promptTemplate](#prompttemplate)
-    *   [promptHeader](#promptheader)
-    *   [promptFooter](#promptfooter)
-*   [PromptMessage](#promptmessage)
+    *   [device](#device)
+    *   [nCtx](#nctx)
+    *   [ngl](#ngl)
+*   [loadModel](#loadmodel)
+    *   [Parameters](#parameters-11)
+*   [InferenceProvider](#inferenceprovider)
+*   [createCompletion](#createcompletion)
+    *   [Parameters](#parameters-12)
+*   [createCompletionStream](#createcompletionstream)
+    *   [Parameters](#parameters-13)
+*   [createCompletionGenerator](#createcompletiongenerator)
+    *   [Parameters](#parameters-14)
+*   [createEmbedding](#createembedding)
+    *   [Parameters](#parameters-15)
+*   [CompletionOptions](#completionoptions)
+    *   [verbose](#verbose-1)
+    *   [onToken](#ontoken)
+*   [Message](#message)
     *   [role](#role)
     *   [content](#content)
 *   [prompt\_tokens](#prompt_tokens)
 *   [completion\_tokens](#completion_tokens)
 *   [total\_tokens](#total_tokens)
+*   [n\_past\_tokens](#n_past_tokens)
 *   [CompletionReturn](#completionreturn)
     *   [model](#model)
     *   [usage](#usage)
-    *   [choices](#choices)
-*   [CompletionChoice](#completionchoice)
-    *   [message](#message)
+    *   [message](#message-1)
+*   [CompletionStreamReturn](#completionstreamreturn)
 *   [LLModelPromptContext](#llmodelpromptcontext)
     *   [logitsSize](#logitssize)
     *   [tokensSize](#tokenssize)
     *   [nPast](#npast)
-    *   [nCtx](#nctx)
     *   [nPredict](#npredict)
+    *   [promptTemplate](#prompttemplate)
+    *   [nCtx](#nctx-1)
     *   [topK](#topk)
     *   [topP](#topp)
-    *   [temp](#temp)
+    *   [minP](#minp)
+    *   [temperature](#temperature)
     *   [nBatch](#nbatch)
     *   [repeatPenalty](#repeatpenalty)
     *   [repeatLastN](#repeatlastn)
     *   [contextErase](#contexterase)
-*   [generateTokens](#generatetokens)
-    *   [Parameters](#parameters-10)
 *   [DEFAULT\_DIRECTORY](#default_directory)
 *   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
 *   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
 *   [DEFAULT\_PROMPT\_CONTEXT](#default_prompt_context)
 *   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
 *   [downloadModel](#downloadmodel)
-    *   [Parameters](#parameters-11)
+    *   [Parameters](#parameters-16)
     *   [Examples](#examples)
 *   [DownloadModelOptions](#downloadmodeloptions)
-    *   [modelPath](#modelpath)
-    *   [verbose](#verbose-1)
+    *   [modelPath](#modelpath-1)
+    *   [verbose](#verbose-2)
     *   [url](#url)
     *   [md5sum](#md5sum)
 *   [DownloadController](#downloadcontroller)
     *   [cancel](#cancel)
     *   [promise](#promise)
 
-#### ModelFile
-
-Full list of models available
-DEPRECATED!! These model names are outdated and this type will not be maintained, please use a string literal instead
-
-##### gptj
-
-List of GPT-J Models
-
-Type: (`"ggml-gpt4all-j-v1.3-groovy.bin"` | `"ggml-gpt4all-j-v1.2-jazzy.bin"` | `"ggml-gpt4all-j-v1.1-breezy.bin"` | `"ggml-gpt4all-j.bin"`)
-
-##### llama
-
-List Llama Models
-
-Type: (`"ggml-gpt4all-l13b-snoozy.bin"` | `"ggml-vicuna-7b-1.1-q4_2.bin"` | `"ggml-vicuna-13b-1.1-q4_2.bin"` | `"ggml-wizardLM-7B.q4_2.bin"` | `"ggml-stable-vicuna-13B.q4_2.bin"` | `"ggml-nous-gpt4-vicuna-13b.bin"` | `"ggml-v3-13b-hermes-q5_1.bin"`)
-
-##### mpt
-
-List of MPT Models
-
-Type: (`"ggml-mpt-7b-base.bin"` | `"ggml-mpt-7b-chat.bin"` | `"ggml-mpt-7b-instruct.bin"`)
-
-##### replit
-
-List of Replit Models
-
-Type: `"ggml-replit-code-v1-3b.bin"`
-
 #### type
 
 Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
 
-Type: ModelType
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
 
 #### TokenCallback
 
-Callback for controlling token generation
+Callback for controlling token generation. Return false to stop token generation.
 
 Type: function (tokenId: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), token: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String), total: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
 
+#### ChatSessionOptions
+
+**Extends Partial\<LLModelPromptContext>**
+
+Options for the chat session.
+
+##### systemPrompt
+
+System prompt to ingest on initialization.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### messages
+
+Messages to ingest on initialization.
+
+Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[Message](#message)>
+
+#### initialize
+
+Ingests system prompt and initial messages.
+Sets this chat session as the active chat session of the model.
+
+##### Parameters
+
+*   `options` **[ChatSessionOptions](#chatsessionoptions)** The options for the chat session.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<void>**&#x20;
+
+#### generate
+
+Prompts the model in chat-session context.
+
+##### Parameters
+
+*   `prompt` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
+*   `options` **[CompletionOptions](#completionoptions)?** Prompt context and other options.
+*   `callback` **[TokenCallback](#tokencallback)?** Token generation callback.
+
+<!---->
+
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the chat session is not the active chat session of the model.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[CompletionReturn](#completionreturn)>** The model's response to the prompt.
+
 #### InferenceModel
 
 InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
 
+##### createChatSession
+
+Create a chat session with the model.
+
+###### Parameters
+
+*   `options` **[ChatSessionOptions](#chatsessionoptions)?** The options for the chat session.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<ChatSession>** The chat session.
+
+##### generate
+
+Prompts the model with a given input and optional parameters.
+
+###### Parameters
+
+*   `prompt` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+*   `options` **[CompletionOptions](#completionoptions)?** Prompt context and other options.
+*   `callback` **[TokenCallback](#tokencallback)?** Token generation callback.
+*   `input`  The prompt input.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[CompletionReturn](#completionreturn)>** The model's response to the prompt.
+
 ##### dispose
 
 delete and cleanup the native model
@@ -307,6 +448,10 @@ delete and cleanup the native model
 
 Returns **void**&#x20;
 
+#### InferenceResult
+
+Shape of LLModel's inference result.
+
 #### LLModel
 
 LLModel class representing a language model.
@@ -326,9 +471,9 @@ Initialize a new LLModel.
 
 ##### type
 
-either 'gpt', mpt', or 'llama' or undefined
+undefined or user supplied
 
-Returns **(ModelType | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
+Returns **([string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String) | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
 
 ##### name
 
@@ -360,7 +505,7 @@ Set the number of threads used for model inference.
 
 Returns **void**&#x20;
 
-##### raw\_prompt
+##### infer
 
 Prompt the model with a given input and optional parameters.
 This is the raw output from model.
@@ -368,23 +513,20 @@ Use the prompt function exported for a value
 
 ###### Parameters
 
-*   `q` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
-*   `params` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
+*   `prompt` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
+*   `promptContext` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
 *   `callback` **[TokenCallback](#tokencallback)?** optional callback to control token generation.
 
-Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The result of the model prompt.
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[InferenceResult](#inferenceresult)>** The result of the model prompt.
 
 ##### embed
 
 Embed text with the model. Keep in mind that
-not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
 Use the prompt function exported for a value
 
 ###### Parameters
 
-*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-*   `q`  The prompt input.
-*   `params`  Optional parameters for the prompt context.
+*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
 
 Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The result of the model prompt.
 
@@ -462,6 +604,62 @@ Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 Options that configure a model's behavior.
 
+##### modelPath
+
+Where to look for model files.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### librariesPath
+
+Where to look for the backend libraries.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### modelConfigFile
+
+The path to the model configuration file, useful for offline usage or custom model configurations.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### allowDownload
+
+Whether to allow downloading the model if it is not present at the specified path.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### verbose
+
+Enable verbose logging.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### device
+
+The processing unit on which the model will run. It can be set to
+
+*   "cpu": Model will run on the central processing unit.
+*   "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
+*   "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
+*   "gpu name": Model will run on the GPU that matches the name if it's available.
+    Note: If a GPU device lacks sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All
+    instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the
+    model.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### nCtx
+
+The Maximum window size of this model
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### ngl
+
+Number of gpu layers needed
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
 #### loadModel
 
 Loads a machine learning model with the specified name. The defacto way to create a model.
@@ -474,18 +672,46 @@ By default this will download a model from the official GPT4ALL website, if a mo
 
 Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<([InferenceModel](#inferencemodel) | [EmbeddingModel](#embeddingmodel))>** A promise that resolves to an instance of the loaded LLModel.
 
+#### InferenceProvider
+
+Interface for inference, implemented by InferenceModel and ChatSession.
+
 #### createCompletion
 
 The nodejs equivalent to python binding's chat\_completion
 
 ##### Parameters
 
-*   `model` **[InferenceModel](#inferencemodel)** The language model object.
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
+*   `provider` **[InferenceProvider](#inferenceprovider)** The inference model object or chat session
+*   `message` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The user input message
 *   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
 
 Returns **[CompletionReturn](#completionreturn)** The completion result.
 
+#### createCompletionStream
+
+Streaming variant of createCompletion, returns a stream of tokens and a promise that resolves to the completion result.
+
+##### Parameters
+
+*   `provider` **[InferenceProvider](#inferenceprovider)** The inference model object or chat session
+*   `message` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The user input message.
+*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
+
+Returns **[CompletionStreamReturn](#completionstreamreturn)** An object of token stream and the completion result promise.
+
+#### createCompletionGenerator
+
+Creates an async generator of tokens
+
+##### Parameters
+
+*   `provider` **[InferenceProvider](#inferenceprovider)** The inference model object or chat session
+*   `message` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The user input message.
+*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
+
+Returns **AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The stream of generated tokens
+
 #### createEmbedding
 
 The nodejs moral equivalent to python binding's Embed4All().embed()
@@ -510,34 +736,15 @@ Indicates if verbose logging is enabled.
 
 Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
 
-##### systemPromptTemplate
+##### onToken
 
-Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
-Note that if this is not defined, system messages will not be included in the prompt.
+Callback for controlling token generation. Return false to stop processing.
 
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+Type: [TokenCallback](#tokencallback)
 
-##### promptTemplate
+#### Message
 
-Template for user messages, with %1 being replaced by the message.
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### promptHeader
-
-The initial instruction for the model, on top of the prompt
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### promptFooter
-
-The last instruction for the model, appended to the end of the prompt.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### PromptMessage
-
-A message in the conversation, identical to OpenAI's chat message.
+A message in the conversation.
 
 ##### role
 
@@ -553,7 +760,7 @@ Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 #### prompt\_tokens
 
-The number of tokens used in the prompt.
+The number of tokens used in the prompt. Currently not available and always 0.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
@@ -565,13 +772,19 @@ Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 #### total\_tokens
 
-The total number of tokens used.
+The total number of tokens used. Currently not available and always 0.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### n\_past\_tokens
+
+Number of tokens used in the conversation.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
 #### CompletionReturn
 
-The result of the completion, similar to OpenAI's format.
+The result of a completion.
 
 ##### model
 
@@ -583,23 +796,17 @@ Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 Token usage report.
 
-Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
-
-##### choices
-
-The generated completions.
-
-Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[CompletionChoice](#completionchoice)>
-
-#### CompletionChoice
-
-A completion choice, similar to OpenAI's format.
+Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), n\_past\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
 
 ##### message
 
-Response message
+The generated completion.
 
-Type: [PromptMessage](#promptmessage)
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### CompletionStreamReturn
+
+The result of a streamed completion, containing a stream of tokens and a promise that resolves to the completion result.
 
 #### LLModelPromptContext
 
@@ -620,18 +827,29 @@ Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 ##### nPast
 
 The number of tokens in the past conversation.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nCtx
-
-The number of tokens possible in the context window.
+This controls how far back the model looks when generating completions.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
 ##### nPredict
 
-The number of tokens to predict.
+The maximum number of tokens to predict.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### promptTemplate
+
+Template for user / assistant message pairs.
+%1 is required and will be replaced by the user input.
+%2 is optional and will be replaced by the assistant response.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### nCtx
+
+The context window size. Do not use, it has no effect. See loadModel options.
+THIS IS DEPRECATED!!!
+Use loadModel's nCtx option instead.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
@@ -654,12 +872,16 @@ above a threshold P. This method, also known as nucleus sampling, finds a balanc
 and quality by considering both token probabilities and the number of tokens available for sampling.
 When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
 On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
-The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
-for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
-##### temp
+##### minP
+
+The minimum probability of a token to be considered.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### temperature
 
 The temperature to adjust the model's output distribution.
 Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
@@ -704,19 +926,6 @@ The percentage of context to erase if the context window is exceeded.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
-#### generateTokens
-
-Creates an async generator of tokens
-
-##### Parameters
-
-*   `llmodel` **[InferenceModel](#inferencemodel)** The language model object.
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
-*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
-*   `callback` **[TokenCallback](#tokencallback)** optional callback to control token generation.
-
-Returns **AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The stream of generated tokens
-
 #### DEFAULT\_DIRECTORY
 
 From python api:
@@ -759,7 +968,7 @@ By default this downloads without waiting. use the controller returned to alter
 ##### Parameters
 
 *   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The model to be downloaded.
-*   `options` **DownloadOptions** to pass into the downloader. Default is { location: (cwd), verbose: false }.
+*   `options` **[DownloadModelOptions](#downloadmodeloptions)** to pass into the downloader. Default is { location: (cwd), verbose: false }.
 
 ##### Examples
 

gpt4all-bindings/python/docs/index.md

@@ -26,7 +26,6 @@ is organized as a monorepo with the following structure:
 - **gpt4all-backend**: The GPT4All backend maintains and exposes a universal, performance optimized C API for running inference with multi-billion parameter Transformer Decoders.
 This C API is then bound to any higher level programming language such as C++, Python, Go, etc.
 - **gpt4all-bindings**: GPT4All bindings contain a variety of high-level programming languages that implement the C API. Each directory is a bound programming language. The [CLI](gpt4all_cli.md) is included here, as well.
-- **gpt4all-api**: The GPT4All API (under initial development) exposes REST API endpoints for gathering completions and embeddings from large language models.
 - **gpt4all-chat**: GPT4All Chat is an OS native chat application that runs on macOS, Windows and Linux. It is the easiest way to run local, privacy aware chat assistants on everyday hardware. You can download it on the [GPT4All Website](https://gpt4all.io) and read its source code in the monorepo.
 
 Explore detailed documentation for the backend, bindings and chat client in the sidebar.

gpt4all-bindings/python/gpt4all/__init__.py

@@ -1 +1 @@
-from .gpt4all import Embed4All as Embed4All, GPT4All as GPT4All
+from .gpt4all import CancellationError as CancellationError, Embed4All as Embed4All, GPT4All as GPT4All

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -9,7 +9,7 @@ import sys
 import threading
 from enum import Enum
 from queue import Queue
-from typing import Any, Callable, Generic, Iterable, TypeVar, overload
+from typing import TYPE_CHECKING, Any, Callable, Generic, Iterable, Literal, NoReturn, TypeVar, overload
 
 if sys.version_info >= (3, 9):
     import importlib.resources as importlib_resources
@@ -22,6 +22,9 @@ if (3, 9) <= sys.version_info < (3, 11):
 else:
     from typing import TypedDict
 
+if TYPE_CHECKING:
+    from typing_extensions import TypeAlias
+
 EmbeddingsType = TypeVar('EmbeddingsType', bound='list[Any]')
 
 
@@ -68,6 +71,7 @@ class LLModelPromptContext(ctypes.Structure):
 
 class LLModelGPUDevice(ctypes.Structure):
     _fields_ = [
+        ("backend", ctypes.c_char_p),
         ("index", ctypes.c_int32),
         ("type", ctypes.c_int32),
         ("heapSize", ctypes.c_size_t),
@@ -95,6 +99,7 @@ llmodel.llmodel_isModelLoaded.restype = ctypes.c_bool
 PromptCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32)
 ResponseCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32, ctypes.c_char_p)
 RecalculateCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_bool)
+EmbCancelCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.POINTER(ctypes.c_uint), ctypes.c_uint, ctypes.c_char_p)
 
 llmodel.llmodel_prompt.argtypes = [
     ctypes.c_void_p,
@@ -119,6 +124,7 @@ llmodel.llmodel_embed.argtypes = [
     ctypes.POINTER(ctypes.c_size_t),
     ctypes.c_bool,
     ctypes.c_bool,
+    EmbCancelCallback,
     ctypes.POINTER(ctypes.c_char_p),
 ]
 
@@ -138,7 +144,7 @@ llmodel.llmodel_threadCount.restype = ctypes.c_int32
 
 llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).encode())
 
-llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
+llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
 llmodel.llmodel_available_gpu_devices.restype = ctypes.POINTER(LLModelGPUDevice)
 
 llmodel.llmodel_gpu_init_gpu_device_by_string.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_char_p]
@@ -153,8 +159,15 @@ llmodel.llmodel_gpu_init_gpu_device_by_int.restype = ctypes.c_bool
 llmodel.llmodel_has_gpu_device.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_has_gpu_device.restype = ctypes.c_bool
 
+llmodel.llmodel_model_backend_name.argtypes = [ctypes.c_void_p]
+llmodel.llmodel_model_backend_name.restype = ctypes.c_char_p
+
+llmodel.llmodel_model_gpu_device_name.argtypes = [ctypes.c_void_p]
+llmodel.llmodel_model_gpu_device_name.restype = ctypes.c_char_p
+
 ResponseCallbackType = Callable[[int, str], bool]
 RawResponseCallbackType = Callable[[int, bytes], bool]
+EmbCancelCallbackType: TypeAlias = 'Callable[[list[int], str], bool]'
 
 
 def empty_response_callback(token_id: int, response: str) -> bool:
@@ -171,6 +184,10 @@ class EmbedResult(Generic[EmbeddingsType], TypedDict):
     n_prompt_tokens: int
 
 
+class CancellationError(Exception):
+    """raised when embedding is canceled"""
+
+
 class LLModel:
     """
     Base class and universal wrapper for GPT4All language models
@@ -184,9 +201,11 @@ class LLModel:
         Maximum size of context window
     ngl : int
         Number of GPU layers to use (Vulkan)
+    backend : str
+        Backend to use. One of 'auto', 'cpu', 'metal', 'kompute', or 'cuda'.
     """
 
-    def __init__(self, model_path: str, n_ctx: int, ngl: int):
+    def __init__(self, model_path: str, n_ctx: int, ngl: int, backend: str):
         self.model_path = model_path.encode()
         self.n_ctx = n_ctx
         self.ngl = ngl
@@ -196,46 +215,70 @@ class LLModel:
 
         # Construct a model implementation
         err = ctypes.c_char_p()
-        model = llmodel.llmodel_model_create2(self.model_path, b"auto", ctypes.byref(err))
+        model = llmodel.llmodel_model_create2(self.model_path, backend.encode(), ctypes.byref(err))
         if model is None:
             s = err.value
             raise RuntimeError(f"Unable to instantiate model: {'null' if s is None else s.decode()}")
-        self.model = model
+        self.model: ctypes.c_void_p | None = model
 
     def __del__(self, llmodel=llmodel):
         if hasattr(self, 'model'):
-            llmodel.llmodel_model_destroy(self.model)
+            self.close()
 
-    def _list_gpu(self, mem_required: int) -> list[LLModelGPUDevice]:
+    def close(self) -> None:
+        if self.model is not None:
+            llmodel.llmodel_model_destroy(self.model)
+            self.model = None
+
+    def _raise_closed(self) -> NoReturn:
+        raise ValueError("Attempted operation on a closed LLModel")
+
+    @property
+    def backend(self) -> Literal["cpu", "kompute", "cuda", "metal"]:
+        if self.model is None:
+            self._raise_closed()
+        return llmodel.llmodel_model_backend_name(self.model).decode()
+
+    @property
+    def device(self) -> str | None:
+        if self.model is None:
+            self._raise_closed()
+        dev = llmodel.llmodel_model_gpu_device_name(self.model)
+        return None if dev is None else dev.decode()
+
+    @staticmethod
+    def list_gpus(mem_required: int = 0) -> list[str]:
+        """
+        List the names of the available GPU devices with at least `mem_required` bytes of VRAM.
+
+        Args:
+            mem_required: The minimum amount of VRAM, in bytes
+
+        Returns:
+            A list of strings representing the names of the available GPU devices.
+        """
         num_devices = ctypes.c_int32(0)
-        devices_ptr = llmodel.llmodel_available_gpu_devices(self.model, mem_required, ctypes.byref(num_devices))
+        devices_ptr = llmodel.llmodel_available_gpu_devices(mem_required, ctypes.byref(num_devices))
         if not devices_ptr:
             raise ValueError("Unable to retrieve available GPU devices")
-        return devices_ptr[:num_devices.value]
+        return [f'{d.backend.decode()}:{d.name.decode()}' for d in devices_ptr[:num_devices.value]]
 
     def init_gpu(self, device: str):
+        if self.model is None:
+            self._raise_closed()
+
         mem_required = llmodel.llmodel_required_mem(self.model, self.model_path, self.n_ctx, self.ngl)
 
         if llmodel.llmodel_gpu_init_gpu_device_by_string(self.model, mem_required, device.encode()):
             return
 
-        # Retrieve all GPUs without considering memory requirements.
-        num_devices = ctypes.c_int32(0)
-        all_devices_ptr = llmodel.llmodel_available_gpu_devices(self.model, 0, ctypes.byref(num_devices))
-        if not all_devices_ptr:
-            raise ValueError("Unable to retrieve list of all GPU devices")
-        all_gpus = [d.name.decode() for d in all_devices_ptr[:num_devices.value]]
-
-        # Retrieve GPUs that meet the memory requirements using list_gpu
-        available_gpus = [device.name.decode() for device in self._list_gpu(mem_required)]
-
-        # Identify GPUs that are unavailable due to insufficient memory or features
+        all_gpus = self.list_gpus()
+        available_gpus = self.list_gpus(mem_required)
         unavailable_gpus = set(all_gpus).difference(available_gpus)
 
-        # Formulate the error message
-        error_msg = "Unable to initialize model on GPU: '{}'.".format(device)
-        error_msg += "\nAvailable GPUs: {}.".format(available_gpus)
-        error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}.".format(unavailable_gpus)
+        error_msg = "Unable to initialize model on GPU: {!r}".format(device)
+        error_msg += "\nAvailable GPUs: {}".format(available_gpus)
+        error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}".format(unavailable_gpus)
         raise ValueError(error_msg)
 
     def load_model(self) -> bool:
@@ -246,14 +289,21 @@ class LLModel:
         -------
         True if model loaded successfully, False otherwise
         """
+        if self.model is None:
+            self._raise_closed()
+
         return llmodel.llmodel_loadModel(self.model, self.model_path, self.n_ctx, self.ngl)
 
     def set_thread_count(self, n_threads):
+        if self.model is None:
+            self._raise_closed()
         if not llmodel.llmodel_isModelLoaded(self.model):
             raise Exception("Model not loaded")
         llmodel.llmodel_setThreadCount(self.model, n_threads)
 
     def thread_count(self):
+        if self.model is None:
+            self._raise_closed()
         if not llmodel.llmodel_isModelLoaded(self.model):
             raise Exception("Model not loaded")
         return llmodel.llmodel_threadCount(self.model)
@@ -305,24 +355,31 @@ class LLModel:
 
     @overload
     def generate_embeddings(
-        self, text: str, prefix: str, dimensionality: int, do_mean: bool, atlas: bool,
+        self, text: str, prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+        cancel_cb: EmbCancelCallbackType | None,
     ) -> EmbedResult[list[float]]: ...
     @overload
     def generate_embeddings(
         self, text: list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+        cancel_cb: EmbCancelCallbackType | None,
     ) -> EmbedResult[list[list[float]]]: ...
     @overload
     def generate_embeddings(
         self, text: str | list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+        cancel_cb: EmbCancelCallbackType | None,
     ) -> EmbedResult[list[Any]]: ...
 
     def generate_embeddings(
         self, text: str | list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+        cancel_cb: EmbCancelCallbackType | None,
     ) -> EmbedResult[list[Any]]:
         if not text:
             raise ValueError("text must not be None or empty")
 
-        if (single_text := isinstance(text, str)):
+        if self.model is None:
+            self._raise_closed()
+
+        if single_text := isinstance(text, str):
             text = [text]
 
         # prepare input
@@ -334,14 +391,22 @@ class LLModel:
         for i, t in enumerate(text):
             c_texts[i] = t.encode()
 
+        def wrap_cancel_cb(batch_sizes: Any, n_batch: int, backend: bytes) -> bool:
+            assert cancel_cb is not None
+            return cancel_cb(batch_sizes[:n_batch], backend.decode())
+
+        cancel_cb_wrapper = EmbCancelCallback() if cancel_cb is None else EmbCancelCallback(wrap_cancel_cb)
+
         # generate the embeddings
         embedding_ptr = llmodel.llmodel_embed(
             self.model, c_texts, ctypes.byref(embedding_size), c_prefix, dimensionality, ctypes.byref(token_count),
-            do_mean, atlas, ctypes.byref(error),
+            do_mean, atlas, cancel_cb_wrapper, ctypes.byref(error),
         )
 
         if not embedding_ptr:
             msg = "(unknown error)" if error.value is None else error.value.decode()
+            if msg == "operation was canceled":
+                raise CancellationError(msg)
             raise RuntimeError(f'Failed to generate embeddings: {msg}')
 
         # extract output
@@ -387,6 +452,9 @@ class LLModel:
         None
         """
 
+        if self.model is None:
+            self._raise_closed()
+
         self.buffer.clear()
         self.buff_expecting_cont_bytes = 0
 
@@ -419,6 +487,9 @@ class LLModel:
     def prompt_model_streaming(
         self, prompt: str, prompt_template: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
     ) -> Iterable[str]:
+        if self.model is None:
+            self._raise_closed()
+
         output_queue: Queue[str | Sentinel] = Queue()
 
         # Put response tokens into an output queue

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -5,12 +5,14 @@ from __future__ import annotations
 
 import hashlib
 import os
+import platform
 import re
 import sys
 import time
 import warnings
 from contextlib import contextmanager
 from pathlib import Path
+from types import TracebackType
 from typing import TYPE_CHECKING, Any, Iterable, Literal, Protocol, overload
 
 import requests
@@ -18,11 +20,11 @@ from requests.exceptions import ChunkedEncodingError
 from tqdm import tqdm
 from urllib3.exceptions import IncompleteRead, ProtocolError
 
-from . import _pyllmodel
-from ._pyllmodel import EmbedResult as EmbedResult
+from ._pyllmodel import (CancellationError as CancellationError, EmbCancelCallbackType, EmbedResult as EmbedResult,
+                         LLModel, ResponseCallbackType, empty_response_callback)
 
 if TYPE_CHECKING:
-    from typing_extensions import TypeAlias
+    from typing_extensions import Self, TypeAlias
 
 if sys.platform == 'darwin':
     import fcntl
@@ -43,49 +45,65 @@ class Embed4All:
 
     MIN_DIMENSIONALITY = 64
 
-    def __init__(self, model_name: str | None = None, n_threads: int | None = None, **kwargs):
+    def __init__(self, model_name: str | None = None, *, n_threads: int | None = None, device: str | None = None, **kwargs: Any):
         """
         Constructor
 
         Args:
             n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
+            device: The processing unit on which the embedding model will run. See the `GPT4All` constructor for more info.
+            kwargs: Remaining keyword arguments are passed to the `GPT4All` constructor.
         """
         if model_name is None:
             model_name = 'all-MiniLM-L6-v2.gguf2.f16.gguf'
-        self.gpt4all = GPT4All(model_name, n_threads=n_threads, **kwargs)
+        self.gpt4all = GPT4All(model_name, n_threads=n_threads, device=device, **kwargs)
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self, typ: type[BaseException] | None, value: BaseException | None, tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def close(self) -> None:
+        """Delete the model instance and free associated system resources."""
+        self.gpt4all.close()
 
     # return_dict=False
     @overload
     def embed(
         self, text: str, *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
-        return_dict: Literal[False] = ..., atlas: bool = ...,
+        return_dict: Literal[False] = ..., atlas: bool = ..., cancel_cb: EmbCancelCallbackType | None = ...,
     ) -> list[float]: ...
     @overload
     def embed(
         self, text: list[str], *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
-        return_dict: Literal[False] = ..., atlas: bool = ...,
+        return_dict: Literal[False] = ..., atlas: bool = ..., cancel_cb: EmbCancelCallbackType | None = ...,
     ) -> list[list[float]]: ...
     @overload
     def embed(
         self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
         long_text_mode: str = ..., return_dict: Literal[False] = ..., atlas: bool = ...,
+        cancel_cb: EmbCancelCallbackType | None = ...,
     ) -> list[Any]: ...
 
     # return_dict=True
     @overload
     def embed(
         self, text: str, *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
-        return_dict: Literal[True], atlas: bool = ...,
+        return_dict: Literal[True], atlas: bool = ..., cancel_cb: EmbCancelCallbackType | None = ...,
     ) -> EmbedResult[list[float]]: ...
     @overload
     def embed(
         self, text: list[str], *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
-        return_dict: Literal[True], atlas: bool = ...,
+        return_dict: Literal[True], atlas: bool = ..., cancel_cb: EmbCancelCallbackType | None = ...,
     ) -> EmbedResult[list[list[float]]]: ...
     @overload
     def embed(
         self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
         long_text_mode: str = ..., return_dict: Literal[True], atlas: bool = ...,
+        cancel_cb: EmbCancelCallbackType | None = ...,
     ) -> EmbedResult[list[Any]]: ...
 
     # return type unknown
@@ -93,11 +111,13 @@ class Embed4All:
     def embed(
         self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
         long_text_mode: str = ..., return_dict: bool = ..., atlas: bool = ...,
+        cancel_cb: EmbCancelCallbackType | None = ...,
     ) -> Any: ...
 
     def embed(
         self, text: str | list[str], *, prefix: str | None = None, dimensionality: int | None = None,
         long_text_mode: str = "mean", return_dict: bool = False, atlas: bool = False,
+        cancel_cb: EmbCancelCallbackType | None = None,
     ) -> Any:
         """
         Generate one or more embeddings.
@@ -113,10 +133,14 @@ class Embed4All:
             return_dict: Return the result as a dict that includes the number of prompt tokens processed.
             atlas: Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens
                 with long_text_mode="mean" will raise an error. Disabled by default.
+            cancel_cb: Called with arguments (batch_sizes, backend_name). Return true to cancel embedding.
 
         Returns:
             With return_dict=False, an embedding or list of embeddings of your text(s).
             With return_dict=True, a dict with keys 'embeddings' and 'n_prompt_tokens'.
+
+        Raises:
+            CancellationError: If cancel_cb returned True and embedding was canceled.
         """
         if dimensionality is None:
             dimensionality = -1
@@ -132,7 +156,7 @@ class Embed4All:
             do_mean = {"mean": True, "truncate": False}[long_text_mode]
         except KeyError:
             raise ValueError(f"Long text mode must be one of 'mean' or 'truncate', got {long_text_mode!r}")
-        result = self.gpt4all.model.generate_embeddings(text, prefix, dimensionality, do_mean, atlas)
+        result = self.gpt4all.model.generate_embeddings(text, prefix, dimensionality, do_mean, atlas, cancel_cb)
         return result if return_dict else result['embeddings']
 
 
@@ -144,11 +168,12 @@ class GPT4All:
     def __init__(
         self,
         model_name: str,
+        *,
         model_path: str | os.PathLike[str] | None = None,
         model_type: str | None = None,
         allow_download: bool = True,
         n_threads: int | None = None,
-        device: str | None = "cpu",
+        device: str | None = None,
         n_ctx: int = 2048,
         ngl: int = 100,
         verbose: bool = False,
@@ -166,29 +191,77 @@ class GPT4All:
             n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
             device: The processing unit on which the GPT4All model will run. It can be set to:
                 - "cpu": Model will run on the central processing unit.
-                - "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
-                - "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
-                Alternatively, a specific GPU name can also be provided, and the model will run on the GPU that matches the name if it's available.
-                Default is "cpu".
+                - "gpu": Use Metal on ARM64 macOS, otherwise the same as "kompute".
+                - "kompute": Use the best GPU provided by the Kompute backend.
+                - "cuda": Use the best GPU provided by the CUDA backend.
+                - "amd", "nvidia": Use the best GPU provided by the Kompute backend from this vendor.
+                - A specific device name from the list returned by `GPT4All.list_gpus()`.
+                Default is Metal on ARM64 macOS, "cpu" otherwise.
 
                 Note: If a selected GPU device does not have sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the model.
             n_ctx: Maximum size of context window
             ngl: Number of GPU layers to use (Vulkan)
             verbose: If True, print debug messages.
         """
+
         self.model_type = model_type
+        self._history: list[MessageType] | None = None
+        self._current_prompt_template: str = "{0}"
+
+        device_init = None
+        if sys.platform == 'darwin':
+            if device is None:
+                backend = 'auto'  # 'auto' is effectively 'metal' due to currently non-functional fallback
+            elif device == 'cpu':
+                backend = 'cpu'
+            else:
+                if platform.machine() != 'arm64' or device != 'gpu':
+                    raise ValueError(f'Unknown device for this platform: {device}')
+                backend = 'metal'
+        else:
+            backend = 'kompute'
+            if device is None or device == 'cpu':
+                pass  # use kompute with no device
+            elif device in ('cuda', 'kompute'):
+                backend = device
+                device_init = 'gpu'
+            elif device.startswith('cuda:'):
+                backend = 'cuda'
+                device_init = device.removeprefix('cuda:')
+            else:
+                device_init = device.removeprefix('kompute:')
+
         # Retrieve model and download if allowed
         self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download, verbose=verbose)
-        self.model = _pyllmodel.LLModel(self.config["path"], n_ctx, ngl)
-        if device is not None and device != "cpu":
-            self.model.init_gpu(device)
+        self.model = LLModel(self.config["path"], n_ctx, ngl, backend)
+        if device_init is not None:
+            self.model.init_gpu(device_init)
         self.model.load_model()
         # Set n_threads
         if n_threads is not None:
             self.model.set_thread_count(n_threads)
 
-        self._history: list[MessageType] | None = None
-        self._current_prompt_template: str = "{0}"
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self, typ: type[BaseException] | None, value: BaseException | None, tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def close(self) -> None:
+        """Delete the model instance and free associated system resources."""
+        self.model.close()
+
+    @property
+    def backend(self) -> Literal["cpu", "kompute", "cuda", "metal"]:
+        """The name of the llama.cpp backend currently in use. One of "cpu", "kompute", "cuda", or "metal"."""
+        return self.model.backend
+
+    @property
+    def device(self) -> str | None:
+        """The name of the GPU device currently in use, or None for backends other than Kompute or CUDA."""
+        return self.model.device
 
     @property
     def current_chat_session(self) -> list[MessageType] | None:
@@ -394,19 +467,19 @@ class GPT4All:
     def generate(
         self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
         min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
-        n_predict: int | None = ..., streaming: Literal[False] = ..., callback: _pyllmodel.ResponseCallbackType = ...,
+        n_predict: int | None = ..., streaming: Literal[False] = ..., callback: ResponseCallbackType = ...,
     ) -> str: ...
     @overload
     def generate(
         self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
         min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
-        n_predict: int | None = ..., streaming: Literal[True], callback: _pyllmodel.ResponseCallbackType = ...,
+        n_predict: int | None = ..., streaming: Literal[True], callback: ResponseCallbackType = ...,
     ) -> Iterable[str]: ...
     @overload
     def generate(
         self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
         min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
-        n_predict: int | None = ..., streaming: bool, callback: _pyllmodel.ResponseCallbackType = ...,
+        n_predict: int | None = ..., streaming: bool, callback: ResponseCallbackType = ...,
     ) -> Any: ...
 
     def generate(
@@ -423,7 +496,7 @@ class GPT4All:
         n_batch: int = 8,
         n_predict: int | None = None,
         streaming: bool = False,
-        callback: _pyllmodel.ResponseCallbackType = _pyllmodel.empty_response_callback,
+        callback: ResponseCallbackType = empty_response_callback,
     ) -> Any:
         """
         Generate outputs from any GPT4All model.
@@ -461,16 +534,16 @@ class GPT4All:
         if self._history is not None:
             # check if there is only one message, i.e. system prompt:
             reset = len(self._history) == 1
-            generate_kwargs["reset_context"] = reset
             self._history.append({"role": "user", "content": prompt})
 
             fct_func = self._format_chat_prompt_template.__func__  # type: ignore[attr-defined]
             if fct_func is GPT4All._format_chat_prompt_template:
                 if reset:
                     # ingest system prompt
-                    self.model.prompt_model(self._history[0]["content"], "%1",
-                                            _pyllmodel.empty_response_callback,
-                                            n_batch=n_batch, n_predict=0, special=True)
+                    # use "%1%2" and not "%1" to avoid implicit whitespace
+                    self.model.prompt_model(self._history[0]["content"], "%1%2",
+                                            empty_response_callback,
+                                            n_batch=n_batch, n_predict=0, reset_context=True, special=True)
                 prompt_template = self._current_prompt_template.format("%1", "%2")
             else:
                 warnings.warn(
@@ -483,6 +556,7 @@ class GPT4All:
                     self._history[0]["content"] if reset else "",
                 )
                 prompt_template = "%1"
+                generate_kwargs["reset_context"] = reset
         else:
             prompt_template = "%1"
             generate_kwargs["reset_context"] = True
@@ -498,9 +572,9 @@ class GPT4All:
             output_collector = self._history
 
         def _callback_wrapper(
-            callback: _pyllmodel.ResponseCallbackType,
+            callback: ResponseCallbackType,
             output_collector: list[MessageType],
-        ) -> _pyllmodel.ResponseCallbackType:
+        ) -> ResponseCallbackType:
             def _callback(token_id: int, response: str) -> bool:
                 nonlocal callback, output_collector
 
@@ -564,6 +638,16 @@ class GPT4All:
             self._history = None
             self._current_prompt_template = "{0}"
 
+    @staticmethod
+    def list_gpus() -> list[str]:
+        """
+        List the names of the available GPU devices.
+
+        Returns:
+            A list of strings representing the names of the available GPU devices.
+        """
+        return LLModel.list_gpus()
+
     def _format_chat_prompt_template(
         self,
         messages: list[MessageType],
@@ -573,6 +657,9 @@ class GPT4All:
         """
         Helper method for building a prompt from list of messages using the self._current_prompt_template as a template for each message.
 
+        Warning:
+            This function was deprecated in version 2.3.0, and will be removed in a future release.
+
         Args:
             messages:  List of dictionaries. Each dictionary should have a "role" key
                 with value of "system", "assistant", or "user" and a "content" key with a

gpt4all-bindings/python/setup.py

@@ -45,7 +45,7 @@ def copy_prebuilt_C_lib(src_dir, dest_dir, dest_build_dir):
                 d = os.path.join(dest_dir, item)
                 shutil.copy2(s, d)
                 files_copied += 1
-            if item.endswith(lib_ext) or item.endswith('.metal'):
+            if item.endswith(lib_ext) or item.endswith('.metallib'):
                 s = os.path.join(dirpath, item)
                 d = os.path.join(dest_build_dir, item)
                 shutil.copy2(s, d)
@@ -68,7 +68,7 @@ def get_long_description():
 
 setup(
     name=package_name,
-    version="2.3.2",
+    version="2.7.0",
     description="Python bindings for GPT4All",
     long_description=get_long_description(),
     long_description_content_type="text/markdown",

gpt4all-bindings/typescript/.clang-format

@@ -0,0 +1,4 @@
+---
+Language: Cpp
+BasedOnStyle: Microsoft
+ColumnLimit: 120

gpt4all-bindings/typescript/README.md

@@ -10,45 +10,170 @@ npm install gpt4all@latest
 pnpm install gpt4all@latest
 
 ```
-
-The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
-
-*   New bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
-*   The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
-*   Everything should work out the box.
+## Breaking changes in version 4!!
+*   See [Transition](#changes)
+## Contents
 *   See [API Reference](#api-reference)
-
+*   See [Examples](#api-example)
+*   See [Developing](#develop)
+*   GPT4ALL nodejs bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
+*   [spare change](https://github.com/sponsors/jacoobes) for a college student? 🤑
+## Api Examples
 ### Chat Completion
 
+Use a chat session to keep context between completions. This is useful for efficient back and forth conversations.
+
 ```js
-import { createCompletion, loadModel } from '../src/gpt4all.js'
+import { createCompletion, loadModel } from "../src/gpt4all.js";
 
-const model = await loadModel('mistral-7b-openorca.Q4_0.gguf', { verbose: true });
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true, // logs loaded model configuration
+    device: "gpu", // defaults to 'cpu'
+    nCtx: 2048, // the maximum sessions context window size.
+});
 
-const response = await createCompletion(model, [
-    { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
-    { role : 'user', content: 'What is 1 + 1?'  } 
+// initialize a chat session on the model. a model instance can have only one chat session at a time.
+const chat = await model.createChatSession({
+    // any completion options set here will be used as default for all completions in this chat session
+    temperature: 0.8,
+    // a custom systemPrompt can be set here. note that the template depends on the model.
+    // if unset, the systemPrompt that comes with the model will be used.
+    systemPrompt: "### System:\nYou are an advanced mathematician.\n\n",
+});
+
+// create a completion using a string as input
+const res1 = await createCompletion(chat, "What is 1 + 1?");
+console.debug(res1.choices[0].message);
+
+// multiple messages can be input to the conversation at once.
+// note that if the last message is not of role 'user', an empty message will be returned.
+await createCompletion(chat, [
+    {
+        role: "user",
+        content: "What is 2 + 2?",
+    },
+    {
+        role: "assistant",
+        content: "It's 5.",
+    },
 ]);
 
+const res3 = await createCompletion(chat, "Could you recalculate that?");
+console.debug(res3.choices[0].message);
+
+model.dispose();
+```
+
+### Stateless usage
+You can use the model without a chat session. This is useful for one-off completions.
+
+```js
+import { createCompletion, loadModel } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf");
+
+// createCompletion methods can also be used on the model directly.
+// context is not maintained between completions.
+const res1 = await createCompletion(model, "What is 1 + 1?");
+console.debug(res1.choices[0].message);
+
+// a whole conversation can be input as well.
+// note that if the last message is not of role 'user', an error will be thrown.
+const res2 = await createCompletion(model, [
+    {
+        role: "user",
+        content: "What is 2 + 2?",
+    },
+    {
+        role: "assistant",
+        content: "It's 5.",
+    },
+    {
+        role: "user",
+        content: "Could you recalculate that?",
+    },
+]);
+console.debug(res2.choices[0].message);
+
 ```
 
 ### Embedding
 
 ```js
-import { createEmbedding, loadModel } from '../src/gpt4all.js'
+import { loadModel, createEmbedding } from '../src/gpt4all.js'
 
-const model = await loadModel('ggml-all-MiniLM-L6-v2-f16', { verbose: true });
+const embedder = await loadModel("nomic-embed-text-v1.5.f16.gguf", { verbose: true, type: 'embedding'})
 
-const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional");
+console.log(createEmbedding(embedder, "Maybe Minecraft was the friends we made along the way"));
 ```
 
+### Streaming responses
+```js
+import { loadModel, createCompletionStream } from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("Output: ");
+const stream = createCompletionStream(model, "How are you?");
+stream.tokens.on("data", (data) => {
+    process.stdout.write(data);
+});
+//wait till stream finishes. We cannot continue until this one is done.
+await stream.result;
+process.stdout.write("\n");
+model.dispose();
+
+```
+
+### Async Generators
+```js
+import { loadModel, createCompletionGenerator } from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf");
+
+process.stdout.write("Output: ");
+const gen = createCompletionGenerator(
+    model,
+    "Redstone in Minecraft is Turing Complete. Let that sink in. (let it in!)"
+);
+for await (const chunk of gen) {
+    process.stdout.write(chunk);
+}
+
+process.stdout.write("\n");
+model.dispose();
+
+```
+### Offline usage
+do this b4 going offline
+```sh
+curl -L https://gpt4all.io/models/models3.json -o ./models3.json
+```
+```js
+import { createCompletion, loadModel } from 'gpt4all'
+
+//make sure u downloaded the models before going offline!
+const model = await loadModel('mistral-7b-openorca.gguf2.Q4_0.gguf', {
+    verbose: true,
+    device: 'gpu',
+    modelConfigFile: "./models3.json"
+});
+
+await createCompletion(model, 'What is 1 + 1?', { verbose: true })
+
+model.dispose();
+```
+
+## Develop
 ### Build Instructions
 
-*   binding.gyp is compile config
+*   `binding.gyp` is compile config
 *   Tested on Ubuntu. Everything seems to work fine
 *   Tested on Windows. Everything works fine.
 *   Sparse testing on mac os.
-*   MingW works as well to build the gpt4all-backend. **HOWEVER**, this package works only with MSVC built dlls.
+*   MingW script works to build the gpt4all-backend. We left it there just in case. **HOWEVER**, this package works only with MSVC built dlls.
 
 ### Requirements
 
@@ -76,23 +201,18 @@ cd gpt4all-bindings/typescript
 *   To Build and Rebuild:
 
 ```sh
-yarn
+node scripts/prebuild.js
 ```
 *   llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
 
 ```sh
-git submodule update --init --depth 1 --recursive
+git submodule update --init --recursive
 ```
 
 ```sh
 yarn build:backend
 ```
-
-This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native The only current way to use them is to put them in the current working directory of your application. That is, **WHEREVER YOU RUN YOUR NODE APPLICATION**
-
-*   llama-xxxx.dll is required.
-*   According to whatever model you are using, you'll need to select the proper model loader.
-    *   For example, if you running an Mosaic MPT model, you will need to select the mpt-(buildvariant).(dynamiclibrary)
+This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native
 
 ### Test
 
@@ -130,17 +250,20 @@ yarn test
 
 *   why your model may be spewing bull 💩
     *   The downloaded model is broken (just reinstall or download from official site)
-    *   That's it so far
+*   Your model is hanging after a call to generate tokens.
+    * Is `nPast` set too high? This may cause your model to hang (03/16/2024), Linux Mint, Ubuntu 22.04
+*  Your GPU usage is still high after node.js exits.
+    * Make sure to call `model.dispose()`!!!
 
 ### Roadmap
 
-This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
+This package has been stabilizing over time development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
 
 *   \[ ] Purely offline. Per the gui, which can be run completely offline, the bindings should be as well.
 *   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
     *   Should include prebuilds to avoid painful node-gyp errors
-*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
-*   \[x] generateTokens, the new name for createTokenStream. As of 3.2.0, this is released but not 100% tested. Check spec/generator.mjs! 
+*   \[x] createChatSession ( the python equivalent to create\_chat\_session )
+*   \[x] generateTokens, the new name for createTokenStream. As of 3.2.0, this is released but not 100% tested. Check spec/generator.mjs!
 *   \[x] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
 *   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
 *   \[x] generateTokens is the new name for this^
@@ -149,5 +272,13 @@ This package is in active development, and breaking changes may happen until the
 *   \[x] have more people test on other platforms (mac tester needed)
 *   \[x] switch to new pluggable backend
 
+## Changes
+This repository serves as the new bindings for nodejs users.
+- If you were a user of [these bindings](https://github.com/nomic-ai/gpt4all-ts), they are outdated.
+- Version 4 includes the follow breaking changes
+    * `createEmbedding` & `EmbeddingModel.embed()` returns an object, `EmbeddingResult`, instead of a float32array.
+    * Removed deprecated types `ModelType` and `ModelFile`
+    * Removed deprecated initiation of model by string path only
+
 
 ### API Reference

gpt4all-bindings/typescript/binding.ci.gyp

@@ -6,12 +6,12 @@
         "<!@(node -p \"require('node-addon-api').include\")",
         "gpt4all-backend",
       ],
-      "sources": [ 
+      "sources": [
         # PREVIOUS VERSION: had to required the sources, but with newest changes do not need to
         #"../../gpt4all-backend/llama.cpp/examples/common.cpp",
         #"../../gpt4all-backend/llama.cpp/ggml.c",
         #"../../gpt4all-backend/llama.cpp/llama.cpp",
-        # "../../gpt4all-backend/utils.cpp", 
+        # "../../gpt4all-backend/utils.cpp",
         "gpt4all-backend/llmodel_c.cpp",
         "gpt4all-backend/llmodel.cpp",
         "prompt.cc",
@@ -40,7 +40,7 @@
                     "AdditionalOptions": [
                         "/std:c++20",
                         "/EHsc",
-                  ], 
+                  ],
                 },
             },
         }],

gpt4all-bindings/typescript/binding.gyp

@@ -6,12 +6,12 @@
         "<!@(node -p \"require('node-addon-api').include\")",
         "../../gpt4all-backend",
       ],
-      "sources": [ 
+      "sources": [
         # PREVIOUS VERSION: had to required the sources, but with newest changes do not need to
         #"../../gpt4all-backend/llama.cpp/examples/common.cpp",
         #"../../gpt4all-backend/llama.cpp/ggml.c",
         #"../../gpt4all-backend/llama.cpp/llama.cpp",
-        # "../../gpt4all-backend/utils.cpp", 
+        # "../../gpt4all-backend/utils.cpp",
         "../../gpt4all-backend/llmodel_c.cpp",
         "../../gpt4all-backend/llmodel.cpp",
         "prompt.cc",
@@ -40,7 +40,7 @@
                     "AdditionalOptions": [
                         "/std:c++20",
                         "/EHsc",
-                  ], 
+                  ],
                 },
             },
         }],

gpt4all-bindings/typescript/index.cc

@@ -1,175 +1,171 @@
 #include "index.h"
+#include "napi.h"
 
-
-Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
-    Napi::Function self = DefineClass(env, "LLModel", {
-       InstanceMethod("type",  &NodeModelWrapper::GetType),
-       InstanceMethod("isModelLoaded", &NodeModelWrapper::IsModelLoaded),
-       InstanceMethod("name", &NodeModelWrapper::GetName),
-       InstanceMethod("stateSize", &NodeModelWrapper::StateSize),
-       InstanceMethod("raw_prompt", &NodeModelWrapper::Prompt),
-       InstanceMethod("setThreadCount", &NodeModelWrapper::SetThreadCount),
-       InstanceMethod("embed", &NodeModelWrapper::GenerateEmbedding),
-       InstanceMethod("threadCount", &NodeModelWrapper::ThreadCount),
-       InstanceMethod("getLibraryPath", &NodeModelWrapper::GetLibraryPath),
-       InstanceMethod("initGpuByString", &NodeModelWrapper::InitGpuByString),
-       InstanceMethod("hasGpuDevice", &NodeModelWrapper::HasGpuDevice),
-       InstanceMethod("listGpu", &NodeModelWrapper::GetGpuDevices),
-       InstanceMethod("memoryNeeded", &NodeModelWrapper::GetRequiredMemory),
-       InstanceMethod("dispose", &NodeModelWrapper::Dispose)
-    });
+Napi::Function NodeModelWrapper::GetClass(Napi::Env env)
+{
+    Napi::Function self = DefineClass(env, "LLModel",
+                                      {InstanceMethod("type", &NodeModelWrapper::GetType),
+                                       InstanceMethod("isModelLoaded", &NodeModelWrapper::IsModelLoaded),
+                                       InstanceMethod("name", &NodeModelWrapper::GetName),
+                                       InstanceMethod("stateSize", &NodeModelWrapper::StateSize),
+                                       InstanceMethod("infer", &NodeModelWrapper::Infer),
+                                       InstanceMethod("setThreadCount", &NodeModelWrapper::SetThreadCount),
+                                       InstanceMethod("embed", &NodeModelWrapper::GenerateEmbedding),
+                                       InstanceMethod("threadCount", &NodeModelWrapper::ThreadCount),
+                                       InstanceMethod("getLibraryPath", &NodeModelWrapper::GetLibraryPath),
+                                       InstanceMethod("initGpuByString", &NodeModelWrapper::InitGpuByString),
+                                       InstanceMethod("hasGpuDevice", &NodeModelWrapper::HasGpuDevice),
+                                       InstanceMethod("listGpu", &NodeModelWrapper::GetGpuDevices),
+                                       InstanceMethod("memoryNeeded", &NodeModelWrapper::GetRequiredMemory),
+                                       InstanceMethod("dispose", &NodeModelWrapper::Dispose)});
     // Keep a static reference to the constructor
     //
-    Napi::FunctionReference* constructor = new Napi::FunctionReference();
+    Napi::FunctionReference *constructor = new Napi::FunctionReference();
     *constructor = Napi::Persistent(self);
     env.SetInstanceData(constructor);
     return self;
 }
-Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info) 
+Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo &info)
 {
     auto env = info.Env();
-    return Napi::Number::New(env, static_cast<uint32_t>(llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers) ));
-
+    return Napi::Number::New(
+        env, static_cast<uint32_t>(llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers)));
 }
-  Napi::Value NodeModelWrapper::GetGpuDevices(const Napi::CallbackInfo& info) 
-  {
+Napi::Value NodeModelWrapper::GetGpuDevices(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
     int num_devices = 0;
     auto mem_size = llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers);
-    llmodel_gpu_device* all_devices = llmodel_available_gpu_devices(GetInference(), mem_size, &num_devices);
-    if(all_devices == nullptr) {
-        Napi::Error::New(
-            env, 
-            "Unable to retrieve list of all GPU devices"
-        ).ThrowAsJavaScriptException(); 
+    llmodel_gpu_device *all_devices = llmodel_available_gpu_devices(mem_size, &num_devices);
+    if (all_devices == nullptr)
+    {
+        Napi::Error::New(env, "Unable to retrieve list of all GPU devices").ThrowAsJavaScriptException();
         return env.Undefined();
     }
     auto js_array = Napi::Array::New(env, num_devices);
-    for(int i = 0; i < num_devices; ++i) {
-       auto gpu_device = all_devices[i];
-       /* 
-        *
-        * struct llmodel_gpu_device {
-            int index = 0;
-            int type = 0;           // same as VkPhysicalDeviceType
-            size_t heapSize = 0; 
-            const char * name;
-            const char * vendor;
-          };
-        *
-        */
-       Napi::Object js_gpu_device = Napi::Object::New(env);
+    for (int i = 0; i < num_devices; ++i)
+    {
+        auto gpu_device = all_devices[i];
+        /*
+         *
+         * struct llmodel_gpu_device {
+             int index = 0;
+             int type = 0;           // same as VkPhysicalDeviceType
+             size_t heapSize = 0;
+             const char * name;
+             const char * vendor;
+           };
+         *
+         */
+        Napi::Object js_gpu_device = Napi::Object::New(env);
         js_gpu_device["index"] = uint32_t(gpu_device.index);
         js_gpu_device["type"] = uint32_t(gpu_device.type);
-        js_gpu_device["heapSize"] = static_cast<uint32_t>( gpu_device.heapSize );
-        js_gpu_device["name"]= gpu_device.name;
+        js_gpu_device["heapSize"] = static_cast<uint32_t>(gpu_device.heapSize);
+        js_gpu_device["name"] = gpu_device.name;
         js_gpu_device["vendor"] = gpu_device.vendor;
 
         js_array[i] = js_gpu_device;
     }
     return js_array;
-  }
+}
 
-  Napi::Value NodeModelWrapper::GetType(const Napi::CallbackInfo& info) 
-  {
-    if(type.empty()) {
+Napi::Value NodeModelWrapper::GetType(const Napi::CallbackInfo &info)
+{
+    if (type.empty())
+    {
         return info.Env().Undefined();
-    } 
+    }
     return Napi::String::New(info.Env(), type);
-  }
+}
 
-  Napi::Value NodeModelWrapper::InitGpuByString(const Napi::CallbackInfo& info) 
-  {
+Napi::Value NodeModelWrapper::InitGpuByString(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
     size_t memory_required = static_cast<size_t>(info[0].As<Napi::Number>().Uint32Value());
-    
-    std::string gpu_device_identifier = info[1].As<Napi::String>();   
+
+    std::string gpu_device_identifier = info[1].As<Napi::String>();
 
     size_t converted_value;
-    if(memory_required <= std::numeric_limits<size_t>::max()) {
+    if (memory_required <= std::numeric_limits<size_t>::max())
+    {
         converted_value = static_cast<size_t>(memory_required);
-    } else {
-         Napi::Error::New(
-            env, 
-            "invalid number for memory size. Exceeded bounds for memory."
-        ).ThrowAsJavaScriptException(); 
+    }
+    else
+    {
+        Napi::Error::New(env, "invalid number for memory size. Exceeded bounds for memory.")
+            .ThrowAsJavaScriptException();
         return env.Undefined();
     }
-    
+
     auto result = llmodel_gpu_init_gpu_device_by_string(GetInference(), converted_value, gpu_device_identifier.c_str());
     return Napi::Boolean::New(env, result);
-  }
-  Napi::Value NodeModelWrapper::HasGpuDevice(const Napi::CallbackInfo& info) 
-  {
+}
+Napi::Value NodeModelWrapper::HasGpuDevice(const Napi::CallbackInfo &info)
+{
     return Napi::Boolean::New(info.Env(), llmodel_has_gpu_device(GetInference()));
-  }
+}
 
-  NodeModelWrapper::NodeModelWrapper(const Napi::CallbackInfo& info) : Napi::ObjectWrap<NodeModelWrapper>(info) 
-  {
+NodeModelWrapper::NodeModelWrapper(const Napi::CallbackInfo &info) : Napi::ObjectWrap<NodeModelWrapper>(info)
+{
     auto env = info.Env();
-    fs::path model_path;
+    auto config_object = info[0].As<Napi::Object>();
 
-    std::string full_weight_path,
-                library_path = ".",
-                model_name, 
-                device;
-    if(info[0].IsString()) {
-        model_path = info[0].As<Napi::String>().Utf8Value();
-        full_weight_path = model_path.string();
-        std::cout << "DEPRECATION: constructor accepts object now. Check docs for more.\n";
-    } else {
-        auto config_object = info[0].As<Napi::Object>();
-        model_name = config_object.Get("model_name").As<Napi::String>();
-        model_path = config_object.Get("model_path").As<Napi::String>().Utf8Value(); 
-        if(config_object.Has("model_type")) {
-            type = config_object.Get("model_type").As<Napi::String>(); 
-        }
-        full_weight_path = (model_path / fs::path(model_name)).string();
-        
-        if(config_object.Has("library_path")) {
-            library_path = config_object.Get("library_path").As<Napi::String>(); 
-        } else {
-            library_path = ".";
-        }
-        device = config_object.Get("device").As<Napi::String>();
+    // sets the directory where models (gguf files) are to be searched
+    llmodel_set_implementation_search_path(
+        config_object.Has("library_path") ? config_object.Get("library_path").As<Napi::String>().Utf8Value().c_str()
+                                          : ".");
 
-        nCtx = config_object.Get("nCtx").As<Napi::Number>().Int32Value();
-        nGpuLayers = config_object.Get("ngl").As<Napi::Number>().Int32Value();
-    }
-    llmodel_set_implementation_search_path(library_path.c_str());
-    const char* e;
+    std::string model_name = config_object.Get("model_name").As<Napi::String>();
+    fs::path model_path = config_object.Get("model_path").As<Napi::String>().Utf8Value();
+    std::string full_weight_path = (model_path / fs::path(model_name)).string();
+
+    name = model_name.empty() ? model_path.filename().string() : model_name;
+    full_model_path = full_weight_path;
+    nCtx = config_object.Get("nCtx").As<Napi::Number>().Int32Value();
+    nGpuLayers = config_object.Get("ngl").As<Napi::Number>().Int32Value();
+
+    const char *e;
     inference_ = llmodel_model_create2(full_weight_path.c_str(), "auto", &e);
-    if(!inference_) {
-       Napi::Error::New(env, e).ThrowAsJavaScriptException();
-       return;
+    if (!inference_)
+    {
+        Napi::Error::New(env, e).ThrowAsJavaScriptException();
+        return;
     }
-    if(GetInference() == nullptr) {
-       std::cerr << "Tried searching libraries in \"" << library_path << "\"" <<  std::endl;
-       std::cerr << "Tried searching for model weight in \"" << full_weight_path << "\"" << std::endl;
-       std::cerr << "Do you have runtime libraries installed?" << std::endl;
-       Napi::Error::New(env, "Had an issue creating llmodel object, inference is null").ThrowAsJavaScriptException(); 
-       return;
+    if (GetInference() == nullptr)
+    {
+        std::cerr << "Tried searching libraries in \"" << llmodel_get_implementation_search_path() << "\"" << std::endl;
+        std::cerr << "Tried searching for model weight in \"" << full_weight_path << "\"" << std::endl;
+        std::cerr << "Do you have runtime libraries installed?" << std::endl;
+        Napi::Error::New(env, "Had an issue creating llmodel object, inference is null").ThrowAsJavaScriptException();
+        return;
     }
-    if(device != "cpu") {
-        size_t mem = llmodel_required_mem(GetInference(), full_weight_path.c_str(),nCtx, nGpuLayers);
+
+    std::string device = config_object.Get("device").As<Napi::String>();
+    if (device != "cpu")
+    {
+        size_t mem = llmodel_required_mem(GetInference(), full_weight_path.c_str(), nCtx, nGpuLayers);
 
         auto success = llmodel_gpu_init_gpu_device_by_string(GetInference(), mem, device.c_str());
-        if(!success) {
-            //https://github.com/nomic-ai/gpt4all/blob/3acbef14b7c2436fe033cae9036e695d77461a16/gpt4all-bindings/python/gpt4all/pyllmodel.py#L215
-            //Haven't implemented this but it is still open to contribution
+        if (!success)
+        {
+            // https://github.com/nomic-ai/gpt4all/blob/3acbef14b7c2436fe033cae9036e695d77461a16/gpt4all-bindings/python/gpt4all/pyllmodel.py#L215
+            // Haven't implemented this but it is still open to contribution
             std::cout << "WARNING: Failed to init GPU\n";
         }
     }
 
     auto success = llmodel_loadModel(GetInference(), full_weight_path.c_str(), nCtx, nGpuLayers);
-    if(!success) {
-        Napi::Error::New(env, "Failed to load model at given path").ThrowAsJavaScriptException(); 
+    if (!success)
+    {
+        Napi::Error::New(env, "Failed to load model at given path").ThrowAsJavaScriptException();
         return;
     }
-
-    name = model_name.empty() ? model_path.filename().string() : model_name;
-    full_model_path = full_weight_path;
-  };
+    // optional
+    if (config_object.Has("model_type"))
+    {
+        type = config_object.Get("model_type").As<Napi::String>();
+    }
+};
 
 //  NodeModelWrapper::~NodeModelWrapper() {
 //    if(GetInference() != nullptr) {
@@ -182,177 +178,275 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
 //    if(inference_ != nullptr) {
 //        std::cout << "Debug: deleting model\n";
 //
-//    } 
+//    }
 //  }
-  Napi::Value NodeModelWrapper::IsModelLoaded(const Napi::CallbackInfo& info) {
+Napi::Value NodeModelWrapper::IsModelLoaded(const Napi::CallbackInfo &info)
+{
     return Napi::Boolean::New(info.Env(), llmodel_isModelLoaded(GetInference()));
-  }
+}
 
-  Napi::Value NodeModelWrapper::StateSize(const Napi::CallbackInfo& info) {
+Napi::Value NodeModelWrapper::StateSize(const Napi::CallbackInfo &info)
+{
     // Implement the binding for the stateSize method
     return Napi::Number::New(info.Env(), static_cast<int64_t>(llmodel_get_state_size(GetInference())));
-  }
-  
-  Napi::Value NodeModelWrapper::GenerateEmbedding(const Napi::CallbackInfo& info) {
+}
+
+Napi::Array ChunkedFloatPtr(float *embedding_ptr, int embedding_size, int text_len, Napi::Env const &env)
+{
+    auto n_embd = embedding_size / text_len;
+    // std::cout << "Embedding size: " << embedding_size << std::endl;
+    // std::cout << "Text length: " << text_len << std::endl;
+    // std::cout << "Chunk size (n_embd): " << n_embd << std::endl;
+    Napi::Array result = Napi::Array::New(env, text_len);
+    auto count = 0;
+    for (int i = 0; i < embedding_size; i += n_embd)
+    {
+        int end = std::min(i + n_embd, embedding_size);
+        // possible bounds error?
+        // Constructs a container with as many elements as the range [first,last), with each element emplace-constructed
+        // from its corresponding element in that range, in the same order.
+        std::vector<float> chunk(embedding_ptr + i, embedding_ptr + end);
+        Napi::Float32Array fltarr = Napi::Float32Array::New(env, chunk.size());
+        // I know there's a way to emplace the raw float ptr into a Napi::Float32Array but idk how and
+        //  im too scared to cause memory issues
+        //  this is goodenough
+        for (int j = 0; j < chunk.size(); j++)
+        {
+
+            fltarr.Set(j, chunk[j]);
+        }
+        result.Set(count++, fltarr);
+    }
+    return result;
+}
+
+Napi::Value NodeModelWrapper::GenerateEmbedding(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
-    std::string text = info[0].As<Napi::String>().Utf8Value();
-    size_t embedding_size = 0;
-    float* arr = llmodel_embedding(GetInference(), text.c_str(), &embedding_size);
-    if(arr == nullptr) {
-        Napi::Error::New(
-            env, 
-            "Cannot embed. native embedder returned 'nullptr'"
-        ).ThrowAsJavaScriptException(); 
+
+    auto prefix = info[1];
+    auto dimensionality = info[2].As<Napi::Number>().Int32Value();
+    auto do_mean = info[3].As<Napi::Boolean>().Value();
+    auto atlas = info[4].As<Napi::Boolean>().Value();
+    size_t embedding_size;
+    size_t token_count = 0;
+
+    // This procedure can maybe be optimized but its whatever, i have too many intermediary structures
+    std::vector<std::string> text_arr;
+    bool is_single_text = false;
+    if (info[0].IsString())
+    {
+        is_single_text = true;
+        text_arr.push_back(info[0].As<Napi::String>().Utf8Value());
+    }
+    else
+    {
+        auto jsarr = info[0].As<Napi::Array>();
+        size_t len = jsarr.Length();
+        text_arr.reserve(len);
+        for (size_t i = 0; i < len; ++i)
+        {
+            std::string str = jsarr.Get(i).As<Napi::String>().Utf8Value();
+            text_arr.push_back(str);
+        }
+    }
+    std::vector<const char *> str_ptrs;
+    str_ptrs.reserve(text_arr.size() + 1);
+    for (size_t i = 0; i < text_arr.size(); ++i)
+        str_ptrs.push_back(text_arr[i].c_str());
+    str_ptrs.push_back(nullptr);
+    const char *_err = nullptr;
+    float *embeds = llmodel_embed(GetInference(), str_ptrs.data(),  &embedding_size,
+                                  prefix.IsUndefined() ? nullptr : prefix.As<Napi::String>().Utf8Value().c_str(),
+                                  dimensionality, &token_count, do_mean, atlas, nullptr, &_err);
+    if (!embeds)
+    {
+        // i dont wanna deal with c strings lol
+        std::string err(_err);
+        Napi::Error::New(env, err == "(unknown error)" ? "Unknown error: sorry bud" : err).ThrowAsJavaScriptException();
         return env.Undefined();
     }
+    auto embedmat = ChunkedFloatPtr(embeds, embedding_size, text_arr.size(), env);
 
-    if(embedding_size == 0 && text.size() != 0 ) {
-        std::cout << "Warning: embedding length 0 but input text length > 0" << std::endl;
-    }
-    Napi::Float32Array js_array = Napi::Float32Array::New(env, embedding_size);
-    
-    for (size_t i = 0; i < embedding_size; ++i) {
-        float element = *(arr + i);
-        js_array[i] = element;
+    llmodel_free_embedding(embeds);
+    auto res = Napi::Object::New(env);
+    res.Set("n_prompt_tokens", token_count);
+    if(is_single_text) {
+        res.Set("embeddings", embedmat.Get(static_cast<uint32_t>(0)));
+    } else {
+        res.Set("embeddings", embedmat);
     }
 
-    llmodel_free_embedding(arr);
-
-    return js_array;
-  }
+    return res;
+}
 
 /**
  * Generate a response using the model.
- * @param model A pointer to the llmodel_model instance.
  * @param prompt A string representing the input prompt.
- * @param prompt_callback A callback function for handling the processing of prompt.
- * @param response_callback A callback function for handling the generated response.
- * @param recalculate_callback A callback function for handling recalculation requests.
- * @param ctx A pointer to the llmodel_prompt_context structure.
+ * @param options Inference options.
  */
-  Napi::Value NodeModelWrapper::Prompt(const Napi::CallbackInfo& info) {
+Napi::Value NodeModelWrapper::Infer(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
-    std::string question;
-    if(info[0].IsString()) {
-        question = info[0].As<Napi::String>().Utf8Value();
-    } else {
+    std::string prompt;
+    if (info[0].IsString())
+    {
+        prompt = info[0].As<Napi::String>().Utf8Value();
+    }
+    else
+    {
         Napi::Error::New(info.Env(), "invalid string argument").ThrowAsJavaScriptException();
         return info.Env().Undefined();
     }
-    //defaults copied from python bindings
-    llmodel_prompt_context promptContext = {
-             .logits = nullptr,
-             .tokens = nullptr,
-             .n_past = 0,
-             .n_ctx = 1024,
-             .n_predict = 128,
-             .top_k = 40,
-             .top_p = 0.9f,
-             .min_p = 0.0f,
-             .temp = 0.72f,
-             .n_batch = 8,
-             .repeat_penalty = 1.0f,
-             .repeat_last_n = 10,
-             .context_erase = 0.5
-         };
-    
-    PromptWorkerConfig promptWorkerConfig;
 
-    if(info[1].IsObject())
-    {
-       auto inputObject = info[1].As<Napi::Object>();
-             
-        // Extract and assign the properties
-       if (inputObject.Has("logits") || inputObject.Has("tokens")) {
-           Napi::Error::New(info.Env(), "Invalid input: 'logits' or 'tokens' properties are not allowed").ThrowAsJavaScriptException();
-           return info.Env().Undefined();
-       }
-             // Assign the remaining properties
-       if(inputObject.Has("n_past")) 
-            promptContext.n_past = inputObject.Get("n_past").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("n_ctx")) 
-            promptContext.n_ctx = inputObject.Get("n_ctx").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("n_predict"))
-            promptContext.n_predict = inputObject.Get("n_predict").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("top_k"))
-            promptContext.top_k = inputObject.Get("top_k").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("top_p")) 
-            promptContext.top_p = inputObject.Get("top_p").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("min_p")) 
-            promptContext.min_p = inputObject.Get("min_p").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("temp")) 
-            promptContext.temp = inputObject.Get("temp").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("n_batch")) 
-            promptContext.n_batch = inputObject.Get("n_batch").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("repeat_penalty")) 
-            promptContext.repeat_penalty = inputObject.Get("repeat_penalty").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("repeat_last_n")) 
-            promptContext.repeat_last_n = inputObject.Get("repeat_last_n").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("context_erase")) 
-            promptContext.context_erase = inputObject.Get("context_erase").As<Napi::Number>().FloatValue();
-    }
-    else
+    if (!info[1].IsObject())
     {
         Napi::Error::New(info.Env(), "Missing Prompt Options").ThrowAsJavaScriptException();
         return info.Env().Undefined();
     }
+    // defaults copied from python bindings
+    llmodel_prompt_context promptContext = {.logits = nullptr,
+                                            .tokens = nullptr,
+                                            .n_past = 0,
+                                            .n_ctx = nCtx,
+                                            .n_predict = 4096,
+                                            .top_k = 40,
+                                            .top_p = 0.9f,
+                                            .min_p = 0.0f,
+                                            .temp = 0.1f,
+                                            .n_batch = 8,
+                                            .repeat_penalty = 1.2f,
+                                            .repeat_last_n = 10,
+                                            .context_erase = 0.75};
 
-    if(info.Length() >= 3 && info[2].IsFunction()){
-        promptWorkerConfig.bHasTokenCallback = true;
-        promptWorkerConfig.tokenCallback = info[2].As<Napi::Function>();
+    PromptWorkerConfig promptWorkerConfig;
+
+    auto inputObject = info[1].As<Napi::Object>();
+
+    if (inputObject.Has("logits") || inputObject.Has("tokens"))
+    {
+        Napi::Error::New(info.Env(), "Invalid input: 'logits' or 'tokens' properties are not allowed")
+            .ThrowAsJavaScriptException();
+        return info.Env().Undefined();
     }
 
-    
+    // Assign the remaining properties
+    if (inputObject.Has("nPast") && inputObject.Get("nPast").IsNumber())
+    {
+        promptContext.n_past = inputObject.Get("nPast").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("nPredict") && inputObject.Get("nPredict").IsNumber())
+    {
+        promptContext.n_predict = inputObject.Get("nPredict").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("topK") && inputObject.Get("topK").IsNumber())
+    {
+        promptContext.top_k = inputObject.Get("topK").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("topP") && inputObject.Get("topP").IsNumber())
+    {
+        promptContext.top_p = inputObject.Get("topP").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("minP") && inputObject.Get("minP").IsNumber())
+    {
+        promptContext.min_p = inputObject.Get("minP").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("temp") && inputObject.Get("temp").IsNumber())
+    {
+        promptContext.temp = inputObject.Get("temp").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("nBatch") && inputObject.Get("nBatch").IsNumber())
+    {
+        promptContext.n_batch = inputObject.Get("nBatch").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("repeatPenalty") && inputObject.Get("repeatPenalty").IsNumber())
+    {
+        promptContext.repeat_penalty = inputObject.Get("repeatPenalty").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("repeatLastN") && inputObject.Get("repeatLastN").IsNumber())
+    {
+        promptContext.repeat_last_n = inputObject.Get("repeatLastN").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("contextErase") && inputObject.Get("contextErase").IsNumber())
+    {
+        promptContext.context_erase = inputObject.Get("contextErase").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("onPromptToken") && inputObject.Get("onPromptToken").IsFunction())
+    {
+        promptWorkerConfig.promptCallback = inputObject.Get("onPromptToken").As<Napi::Function>();
+        promptWorkerConfig.hasPromptCallback = true;
+    }
+    if (inputObject.Has("onResponseToken") && inputObject.Get("onResponseToken").IsFunction())
+    {
+        promptWorkerConfig.responseCallback = inputObject.Get("onResponseToken").As<Napi::Function>();
+        promptWorkerConfig.hasResponseCallback = true;
+    }
 
-    //copy to protect llmodel resources when splitting to new thread
-    // llmodel_prompt_context copiedPrompt = promptContext;
+    // copy to protect llmodel resources when splitting to new thread
+    //  llmodel_prompt_context copiedPrompt = promptContext;
     promptWorkerConfig.context = promptContext;
     promptWorkerConfig.model = GetInference();
     promptWorkerConfig.mutex = &inference_mutex;
-    promptWorkerConfig.prompt = question;
+    promptWorkerConfig.prompt = prompt;
     promptWorkerConfig.result = "";
 
-    
+    promptWorkerConfig.promptTemplate = inputObject.Get("promptTemplate").As<Napi::String>();
+    if (inputObject.Has("special"))
+    {
+        promptWorkerConfig.special = inputObject.Get("special").As<Napi::Boolean>();
+    }
+    if (inputObject.Has("fakeReply"))
+    {
+        // this will be deleted in the worker
+        promptWorkerConfig.fakeReply = new std::string(inputObject.Get("fakeReply").As<Napi::String>().Utf8Value());
+    }
     auto worker = new PromptWorker(env, promptWorkerConfig);
 
     worker->Queue();
 
     return worker->GetPromise();
-  }
-  void NodeModelWrapper::Dispose(const Napi::CallbackInfo& info) {
+}
+void NodeModelWrapper::Dispose(const Napi::CallbackInfo &info)
+{
     llmodel_model_destroy(inference_);
-  }
-  void NodeModelWrapper::SetThreadCount(const Napi::CallbackInfo& info) {
-    if(info[0].IsNumber()) {
+}
+void NodeModelWrapper::SetThreadCount(const Napi::CallbackInfo &info)
+{
+    if (info[0].IsNumber())
+    {
         llmodel_setThreadCount(GetInference(), info[0].As<Napi::Number>().Int64Value());
-    } else {
-        Napi::Error::New(info.Env(), "Could not set thread count: argument 1 is NaN").ThrowAsJavaScriptException(); 
+    }
+    else
+    {
+        Napi::Error::New(info.Env(), "Could not set thread count: argument 1 is NaN").ThrowAsJavaScriptException();
         return;
     }
-  }
-
-  Napi::Value NodeModelWrapper::GetName(const Napi::CallbackInfo& info) {
-    return Napi::String::New(info.Env(), name);
-  }
-  Napi::Value NodeModelWrapper::ThreadCount(const Napi::CallbackInfo& info) {
-    return Napi::Number::New(info.Env(), llmodel_threadCount(GetInference()));
-  }
-
-  Napi::Value NodeModelWrapper::GetLibraryPath(const Napi::CallbackInfo& info) {
-      return Napi::String::New(info.Env(),
-        llmodel_get_implementation_search_path());
-  }
-
-  llmodel_model NodeModelWrapper::GetInference() {
-    return inference_;
-  }
-
-//Exports Bindings
-Napi::Object Init(Napi::Env env, Napi::Object exports) {
-  exports["LLModel"] = NodeModelWrapper::GetClass(env);
-  return exports;
 }
 
+Napi::Value NodeModelWrapper::GetName(const Napi::CallbackInfo &info)
+{
+    return Napi::String::New(info.Env(), name);
+}
+Napi::Value NodeModelWrapper::ThreadCount(const Napi::CallbackInfo &info)
+{
+    return Napi::Number::New(info.Env(), llmodel_threadCount(GetInference()));
+}
 
+Napi::Value NodeModelWrapper::GetLibraryPath(const Napi::CallbackInfo &info)
+{
+    return Napi::String::New(info.Env(), llmodel_get_implementation_search_path());
+}
+
+llmodel_model NodeModelWrapper::GetInference()
+{
+    return inference_;
+}
+
+// Exports Bindings
+Napi::Object Init(Napi::Env env, Napi::Object exports)
+{
+    exports["LLModel"] = NodeModelWrapper::GetClass(env);
+    return exports;
+}
 
 NODE_API_MODULE(NODE_GYP_MODULE_NAME, Init)

gpt4all-bindings/typescript/index.h

@@ -1,62 +1,63 @@
-#include <napi.h>
 #include "llmodel.h"
-#include <iostream>
-#include "llmodel_c.h" 
+#include "llmodel_c.h"
 #include "prompt.h"
 #include <atomic>
-#include <memory>
 #include <filesystem>
-#include <set>
+#include <iostream>
+#include <memory>
 #include <mutex>
+#include <napi.h>
+#include <set>
 
 namespace fs = std::filesystem;
 
+class NodeModelWrapper : public Napi::ObjectWrap<NodeModelWrapper>
+{
 
-class NodeModelWrapper: public Napi::ObjectWrap<NodeModelWrapper> {
-  
-public:
-  NodeModelWrapper(const Napi::CallbackInfo &);
-  //virtual ~NodeModelWrapper();
-  Napi::Value GetType(const Napi::CallbackInfo& info);
-  Napi::Value IsModelLoaded(const Napi::CallbackInfo& info);
-  Napi::Value StateSize(const Napi::CallbackInfo& info);
-  //void Finalize(Napi::Env env) override;
-  /**
-   * Prompting the model. This entails spawning a new thread and adding the response tokens
-   * into a thread local string variable.
-   */
-  Napi::Value Prompt(const Napi::CallbackInfo& info);
-  void SetThreadCount(const Napi::CallbackInfo& info);
-  void Dispose(const Napi::CallbackInfo& info);
-  Napi::Value GetName(const Napi::CallbackInfo& info);
-  Napi::Value ThreadCount(const Napi::CallbackInfo& info);
-  Napi::Value GenerateEmbedding(const Napi::CallbackInfo& info);
-  Napi::Value HasGpuDevice(const Napi::CallbackInfo& info);
-  Napi::Value ListGpus(const Napi::CallbackInfo& info);
-  Napi::Value InitGpuByString(const Napi::CallbackInfo& info);
-  Napi::Value GetRequiredMemory(const Napi::CallbackInfo& info);
-  Napi::Value GetGpuDevices(const Napi::CallbackInfo& info);
-  /*
-   * The path that is used to search for the dynamic libraries
-   */
-  Napi::Value GetLibraryPath(const Napi::CallbackInfo& info);
-  /**
-   * Creates the LLModel class
-   */
-  static Napi::Function GetClass(Napi::Env);
-  llmodel_model GetInference();
-private:
-  /**
-   * The underlying inference that interfaces with the C interface
-   */
-  llmodel_model inference_;
+  public:
+    NodeModelWrapper(const Napi::CallbackInfo &);
+    // virtual ~NodeModelWrapper();
+    Napi::Value GetType(const Napi::CallbackInfo &info);
+    Napi::Value IsModelLoaded(const Napi::CallbackInfo &info);
+    Napi::Value StateSize(const Napi::CallbackInfo &info);
+    // void Finalize(Napi::Env env) override;
+    /**
+     * Prompting the model. This entails spawning a new thread and adding the response tokens
+     * into a thread local string variable.
+     */
+    Napi::Value Infer(const Napi::CallbackInfo &info);
+    void SetThreadCount(const Napi::CallbackInfo &info);
+    void Dispose(const Napi::CallbackInfo &info);
+    Napi::Value GetName(const Napi::CallbackInfo &info);
+    Napi::Value ThreadCount(const Napi::CallbackInfo &info);
+    Napi::Value GenerateEmbedding(const Napi::CallbackInfo &info);
+    Napi::Value HasGpuDevice(const Napi::CallbackInfo &info);
+    Napi::Value ListGpus(const Napi::CallbackInfo &info);
+    Napi::Value InitGpuByString(const Napi::CallbackInfo &info);
+    Napi::Value GetRequiredMemory(const Napi::CallbackInfo &info);
+    Napi::Value GetGpuDevices(const Napi::CallbackInfo &info);
+    /*
+     * The path that is used to search for the dynamic libraries
+     */
+    Napi::Value GetLibraryPath(const Napi::CallbackInfo &info);
+    /**
+     * Creates the LLModel class
+     */
+    static Napi::Function GetClass(Napi::Env);
+    llmodel_model GetInference();
 
-  std::mutex inference_mutex;
+  private:
+    /**
+     * The underlying inference that interfaces with the C interface
+     */
+    llmodel_model inference_;
 
-  std::string type;
-  // corresponds to LLModel::name() in typescript
-  std::string name;
-  int nCtx{};
-  int nGpuLayers{};
-  std::string full_model_path;
+    std::mutex inference_mutex;
+
+    std::string type;
+    // corresponds to LLModel::name() in typescript
+    std::string name;
+    int nCtx{};
+    int nGpuLayers{};
+    std::string full_model_path;
 };

gpt4all-bindings/typescript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt4all",
-  "version": "3.2.0",
+  "version": "4.0.0",
   "packageManager": "yarn@3.6.1",
   "main": "src/gpt4all.js",
   "repository": "nomic-ai/gpt4all",
@@ -22,7 +22,6 @@
   ],
   "dependencies": {
     "md5-file": "^5.0.0",
-    "mkdirp": "^3.0.1",
     "node-addon-api": "^6.1.0",
     "node-gyp-build": "^4.6.0"
   },

gpt4all-bindings/typescript/prompt.cc

@@ -2,145 +2,195 @@
 #include <future>
 
 PromptWorker::PromptWorker(Napi::Env env, PromptWorkerConfig config)
-        : promise(Napi::Promise::Deferred::New(env)), _config(config), AsyncWorker(env) {
-            if(_config.bHasTokenCallback){
-                _tsfn = Napi::ThreadSafeFunction::New(config.tokenCallback.Env(),config.tokenCallback,"PromptWorker",0,1,this);
-            }
-        }
-
-    PromptWorker::~PromptWorker()
+    : promise(Napi::Promise::Deferred::New(env)), _config(config), AsyncWorker(env)
+{
+    if (_config.hasResponseCallback)
     {
-        if(_config.bHasTokenCallback){
-            _tsfn.Release();
-        }
+        _responseCallbackFn = Napi::ThreadSafeFunction::New(config.responseCallback.Env(), config.responseCallback,
+                                                            "PromptWorker", 0, 1, this);
     }
 
-    void PromptWorker::Execute()
+    if (_config.hasPromptCallback)
     {
-        _config.mutex->lock();
+        _promptCallbackFn = Napi::ThreadSafeFunction::New(config.promptCallback.Env(), config.promptCallback,
+                                                          "PromptWorker", 0, 1, this);
+    }
+}
 
-        LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper *>(_config.model);
+PromptWorker::~PromptWorker()
+{
+    if (_config.hasResponseCallback)
+    {
+        _responseCallbackFn.Release();
+    }
+    if (_config.hasPromptCallback)
+    {
+        _promptCallbackFn.Release();
+    }
+}
 
-        auto ctx = &_config.context;
+void PromptWorker::Execute()
+{
+    _config.mutex->lock();
 
-        if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
-            wrapper->promptContext.tokens.resize(ctx->n_past);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper *>(_config.model);
 
-        // Copy the C prompt context
-        wrapper->promptContext.n_past = ctx->n_past;
-        wrapper->promptContext.n_ctx = ctx->n_ctx;
-        wrapper->promptContext.n_predict = ctx->n_predict;
-        wrapper->promptContext.top_k = ctx->top_k;
-        wrapper->promptContext.top_p = ctx->top_p;
-        wrapper->promptContext.temp = ctx->temp;
-        wrapper->promptContext.n_batch = ctx->n_batch;
-        wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
-        wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
-        wrapper->promptContext.contextErase = ctx->context_erase;
+    auto ctx = &_config.context;
 
-        // Napi::Error::Fatal(
-        //         "SUPRA",
-        //         "About to prompt");
-        // Call the C++ prompt method
-        wrapper->llModel->prompt(
-            _config.prompt, 
-            [](int32_t tid) { return true; },
-            [this](int32_t token_id, const std::string tok)
-            {
-                return ResponseCallback(token_id, tok);
-            },
-            [](bool isRecalculating)
-            {
-                return isRecalculating;
-            },
-            wrapper->promptContext);
+    if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
+        wrapper->promptContext.tokens.resize(ctx->n_past);
 
-        // Update the C context by giving access to the wrappers raw pointers to std::vector data
-        // which involves no copies
-        ctx->logits = wrapper->promptContext.logits.data();
-        ctx->logits_size = wrapper->promptContext.logits.size();
-        ctx->tokens = wrapper->promptContext.tokens.data();
-        ctx->tokens_size = wrapper->promptContext.tokens.size();
+    // Copy the C prompt context
+    wrapper->promptContext.n_past = ctx->n_past;
+    wrapper->promptContext.n_ctx = ctx->n_ctx;
+    wrapper->promptContext.n_predict = ctx->n_predict;
+    wrapper->promptContext.top_k = ctx->top_k;
+    wrapper->promptContext.top_p = ctx->top_p;
+    wrapper->promptContext.temp = ctx->temp;
+    wrapper->promptContext.n_batch = ctx->n_batch;
+    wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
+    wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
+    wrapper->promptContext.contextErase = ctx->context_erase;
 
-        // Update the rest of the C prompt context
-        ctx->n_past = wrapper->promptContext.n_past;
-        ctx->n_ctx = wrapper->promptContext.n_ctx;
-        ctx->n_predict = wrapper->promptContext.n_predict;
-        ctx->top_k = wrapper->promptContext.top_k;
-        ctx->top_p = wrapper->promptContext.top_p;
-        ctx->temp = wrapper->promptContext.temp;
-        ctx->n_batch = wrapper->promptContext.n_batch;
-        ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
-        ctx->repeat_last_n = wrapper->promptContext.repeat_last_n;
-        ctx->context_erase = wrapper->promptContext.contextErase;
+    // Call the C++ prompt method
 
-        _config.mutex->unlock();
+    wrapper->llModel->prompt(
+        _config.prompt, _config.promptTemplate, [this](int32_t token_id) { return PromptCallback(token_id); },
+        [this](int32_t token_id, const std::string token) { return ResponseCallback(token_id, token); },
+        [](bool isRecalculating) { return isRecalculating; }, wrapper->promptContext, _config.special,
+        _config.fakeReply);
+
+    // Update the C context by giving access to the wrappers raw pointers to std::vector data
+    // which involves no copies
+    ctx->logits = wrapper->promptContext.logits.data();
+    ctx->logits_size = wrapper->promptContext.logits.size();
+    ctx->tokens = wrapper->promptContext.tokens.data();
+    ctx->tokens_size = wrapper->promptContext.tokens.size();
+
+    // Update the rest of the C prompt context
+    ctx->n_past = wrapper->promptContext.n_past;
+    ctx->n_ctx = wrapper->promptContext.n_ctx;
+    ctx->n_predict = wrapper->promptContext.n_predict;
+    ctx->top_k = wrapper->promptContext.top_k;
+    ctx->top_p = wrapper->promptContext.top_p;
+    ctx->temp = wrapper->promptContext.temp;
+    ctx->n_batch = wrapper->promptContext.n_batch;
+    ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
+    ctx->repeat_last_n = wrapper->promptContext.repeat_last_n;
+    ctx->context_erase = wrapper->promptContext.contextErase;
+
+    _config.mutex->unlock();
+}
+
+void PromptWorker::OnOK()
+{
+    Napi::Object returnValue = Napi::Object::New(Env());
+    returnValue.Set("text", result);
+    returnValue.Set("nPast", _config.context.n_past);
+    promise.Resolve(returnValue);
+    delete _config.fakeReply;
+}
+
+void PromptWorker::OnError(const Napi::Error &e)
+{
+    delete _config.fakeReply;
+    promise.Reject(e.Value());
+}
+
+Napi::Promise PromptWorker::GetPromise()
+{
+    return promise.Promise();
+}
+
+bool PromptWorker::ResponseCallback(int32_t token_id, const std::string token)
+{
+    if (token_id == -1)
+    {
+        return false;
     }
 
-    void PromptWorker::OnOK()
-    {
-        promise.Resolve(Napi::String::New(Env(), result));
-    }
-
-    void PromptWorker::OnError(const Napi::Error &e)
-    {
-        promise.Reject(e.Value());
-    }
-
-    Napi::Promise PromptWorker::GetPromise()
-    {
-        return promise.Promise();
-    }
-
-    bool PromptWorker::ResponseCallback(int32_t token_id, const std::string token)
-    {
-        if (token_id == -1)
-        {
-            return false;
-        }
-
-        if(!_config.bHasTokenCallback){
-            return true;
-        }
-
-        result += token;
-
-       std::promise<bool> promise;
-
-       auto info = new TokenCallbackInfo();
-       info->tokenId = token_id;
-       info->token = token;
-       info->total = result;
-
-       auto future = promise.get_future();
-
-       auto status = _tsfn.BlockingCall(info, [&promise](Napi::Env env, Napi::Function jsCallback, TokenCallbackInfo *value)
-       {
-           // Transform native data into JS data, passing it to the provided
-           // `jsCallback` -- the TSFN's JavaScript function.
-           auto token_id = Napi::Number::New(env, value->tokenId);
-           auto token = Napi::String::New(env, value->token);
-           auto total = Napi::String::New(env,value->total);
-           auto jsResult = jsCallback.Call({ token_id, token, total}).ToBoolean();
-           promise.set_value(jsResult);
-           // We're finished with the data.
-           delete value;
-       });
-       if (status != napi_ok) {
-            Napi::Error::Fatal(
-                "PromptWorkerResponseCallback",
-                "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
-        }
-
-       return future.get();
-    }
-
-    bool PromptWorker::RecalculateCallback(bool isRecalculating)
-    {
-        return isRecalculating;
-    }
-
-    bool PromptWorker::PromptCallback(int32_t tid)
+    if (!_config.hasResponseCallback)
     {
         return true;
     }
+
+    result += token;
+
+    std::promise<bool> promise;
+
+    auto info = new ResponseCallbackData();
+    info->tokenId = token_id;
+    info->token = token;
+
+    auto future = promise.get_future();
+
+    auto status = _responseCallbackFn.BlockingCall(
+        info, [&promise](Napi::Env env, Napi::Function jsCallback, ResponseCallbackData *value) {
+            try
+            {
+                // Transform native data into JS data, passing it to the provided
+                // `jsCallback` -- the TSFN's JavaScript function.
+                auto token_id = Napi::Number::New(env, value->tokenId);
+                auto token = Napi::String::New(env, value->token);
+                auto jsResult = jsCallback.Call({token_id, token}).ToBoolean();
+                promise.set_value(jsResult);
+            }
+            catch (const Napi::Error &e)
+            {
+                std::cerr << "Error in onResponseToken callback: " << e.what() << std::endl;
+                promise.set_value(false);
+            }
+
+            delete value;
+        });
+    if (status != napi_ok)
+    {
+        Napi::Error::Fatal("PromptWorkerResponseCallback", "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
+    }
+
+    return future.get();
+}
+
+bool PromptWorker::RecalculateCallback(bool isRecalculating)
+{
+    return isRecalculating;
+}
+
+bool PromptWorker::PromptCallback(int32_t token_id)
+{
+    if (!_config.hasPromptCallback)
+    {
+        return true;
+    }
+
+    std::promise<bool> promise;
+
+    auto info = new PromptCallbackData();
+    info->tokenId = token_id;
+
+    auto future = promise.get_future();
+
+    auto status = _promptCallbackFn.BlockingCall(
+        info, [&promise](Napi::Env env, Napi::Function jsCallback, PromptCallbackData *value) {
+            try
+            {
+                // Transform native data into JS data, passing it to the provided
+                // `jsCallback` -- the TSFN's JavaScript function.
+                auto token_id = Napi::Number::New(env, value->tokenId);
+                auto jsResult = jsCallback.Call({token_id}).ToBoolean();
+                promise.set_value(jsResult);
+            }
+            catch (const Napi::Error &e)
+            {
+                std::cerr << "Error in onPromptToken callback: " << e.what() << std::endl;
+                promise.set_value(false);
+            }
+            delete value;
+        });
+    if (status != napi_ok)
+    {
+        Napi::Error::Fatal("PromptWorkerPromptCallback", "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
+    }
+
+    return future.get();
+}

gpt4all-bindings/typescript/prompt.h

@@ -1,59 +1,72 @@
 #ifndef PREDICT_WORKER_H
 #define PREDICT_WORKER_H
 
-#include "napi.h"
-#include "llmodel_c.h"
 #include "llmodel.h"
-#include <thread>
-#include <mutex>
-#include <iostream>
+#include "llmodel_c.h"
+#include "napi.h"
 #include <atomic>
+#include <iostream>
 #include <memory>
+#include <mutex>
+#include <thread>
 
-struct TokenCallbackInfo
+struct ResponseCallbackData
+{
+    int32_t tokenId;
+    std::string token;
+};
+
+struct PromptCallbackData
+{
+    int32_t tokenId;
+};
+
+struct LLModelWrapper
+{
+    LLModel *llModel = nullptr;
+    LLModel::PromptContext promptContext;
+    ~LLModelWrapper()
     {
-        int32_t tokenId;
-        std::string total;
-        std::string token;
-    };
+        delete llModel;
+    }
+};
 
-    struct LLModelWrapper
-    {
-        LLModel *llModel = nullptr;
-        LLModel::PromptContext promptContext;
-        ~LLModelWrapper() { delete llModel; }
-    };
+struct PromptWorkerConfig
+{
+    Napi::Function responseCallback;
+    bool hasResponseCallback = false;
+    Napi::Function promptCallback;
+    bool hasPromptCallback = false;
+    llmodel_model model;
+    std::mutex *mutex;
+    std::string prompt;
+    std::string promptTemplate;
+    llmodel_prompt_context context;
+    std::string result;
+    bool special = false;
+    std::string *fakeReply = nullptr;
+};
 
-    struct PromptWorkerConfig
-    {
-        Napi::Function tokenCallback;
-        bool bHasTokenCallback = false;
-        llmodel_model model;
-        std::mutex * mutex;
-        std::string prompt;
-        llmodel_prompt_context context;
-        std::string result;
-    };
+class PromptWorker : public Napi::AsyncWorker
+{
+  public:
+    PromptWorker(Napi::Env env, PromptWorkerConfig config);
+    ~PromptWorker();
+    void Execute() override;
+    void OnOK() override;
+    void OnError(const Napi::Error &e) override;
+    Napi::Promise GetPromise();
 
-    class PromptWorker : public Napi::AsyncWorker
-    {
-    public:
-        PromptWorker(Napi::Env env, PromptWorkerConfig config);
-        ~PromptWorker();
-        void Execute() override;
-        void OnOK() override;
-        void OnError(const Napi::Error &e) override;
-        Napi::Promise GetPromise();
+    bool ResponseCallback(int32_t token_id, const std::string token);
+    bool RecalculateCallback(bool isrecalculating);
+    bool PromptCallback(int32_t token_id);
 
-        bool ResponseCallback(int32_t token_id, const std::string token);
-        bool RecalculateCallback(bool isrecalculating);
-        bool PromptCallback(int32_t tid);
+  private:
+    Napi::Promise::Deferred promise;
+    std::string result;
+    PromptWorkerConfig _config;
+    Napi::ThreadSafeFunction _responseCallbackFn;
+    Napi::ThreadSafeFunction _promptCallbackFn;
+};
 
-    private:
-        Napi::Promise::Deferred promise;
-        std::string result;
-        PromptWorkerConfig _config;
-        Napi::ThreadSafeFunction _tsfn;
-    };
-
-#endif  // PREDICT_WORKER_H
+#endif // PREDICT_WORKER_H

gpt4all-bindings/typescript/scripts/build_unix.sh

@@ -24,7 +24,6 @@ mkdir -p "$NATIVE_DIR" "$BUILD_DIR"
 
 cmake -S ../../gpt4all-backend -B "$BUILD_DIR" &&
 cmake --build "$BUILD_DIR" -j --config Release && {
-  cp "$BUILD_DIR"/libbert*.$LIB_EXT   "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libgptj*.$LIB_EXT   "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libllama*.$LIB_EXT  "$NATIVE_DIR"/
 }

gpt4all-bindings/typescript/spec/callbacks.mjs

@@ -0,0 +1,31 @@
+import { promises as fs } from "node:fs";
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const res = await createCompletion(
+    model,
+    "I've got three 🍣 - What shall I name them?",
+    {
+        onPromptToken: (tokenId) => {
+            console.debug("onPromptToken", { tokenId });
+            // throwing an error will cancel
+            throw new Error("This is an error");
+            // const foo = thisMethodDoesNotExist();
+            // returning false will cancel as well
+            // return false;
+        },
+        onResponseToken: (tokenId, token) => {
+            console.debug("onResponseToken", { tokenId, token });
+            // same applies here
+        },
+    }
+);
+
+console.debug("Output:", {
+    usage: res.usage,
+    message: res.choices[0].message,
+});

gpt4all-bindings/typescript/spec/chat-memory.mjs

@@ -0,0 +1,65 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession({
+    messages: [
+        {
+            role: "user",
+            content: "I'll tell you a secret password: It's 63445.",
+        },
+        {
+            role: "assistant",
+            content: "I will do my best to remember that.",
+        },
+        {
+            role: "user",
+            content:
+                "And here another fun fact: Bananas may be bluer than bread at night.",
+        },
+        {
+            role: "assistant",
+            content: "Yes, that makes sense.",
+        },
+    ],
+});
+
+const turn1 = await createCompletion(
+    chat,
+    "Please tell me the secret password."
+);
+console.debug(turn1.choices[0].message);
+// "The secret password you shared earlier is 63445.""
+
+const turn2 = await createCompletion(
+    chat,
+    "Thanks! Have your heard about the bananas?"
+);
+console.debug(turn2.choices[0].message);
+
+for (let i = 0; i < 32; i++) {
+    // gpu go brr
+    const turn = await createCompletion(
+        chat,
+        i % 2 === 0 ? "Tell me a fun fact." : "And a boring one?"
+    );
+    console.debug({
+        message: turn.choices[0].message,
+        n_past_tokens: turn.usage.n_past_tokens,
+    });
+}
+
+const finalTurn = await createCompletion(
+    chat,
+    "Now I forgot the secret password. Can you remind me?"
+);
+console.debug(finalTurn.choices[0].message);
+
+// result of finalTurn may vary depending on whether the generated facts pushed the secret out of the context window.
+// "Of course! The secret password you shared earlier is 63445."
+// "I apologize for any confusion. As an AI language model, ..."
+
+model.dispose();

gpt4all-bindings/typescript/spec/chat-minimal.mjs

@@ -0,0 +1,19 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession();
+
+await createCompletion(
+    chat,
+    "Why are bananas rather blue than bread at night sometimes?",
+    {
+        verbose: true,
+    }
+);
+await createCompletion(chat, "Are you sure?", {
+    verbose: true,
+});

gpt4all-bindings/typescript/spec/chat.mjs

@@ -1,70 +0,0 @@
-import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY, loadModel } from '../src/gpt4all.js'
-
-const model = await loadModel(
-    'mistral-7b-openorca.Q4_0.gguf',
-    { verbose: true, device: 'gpu' }
-);
-const ll = model.llm;
-
-try {
-   class Extended extends LLModel {
-   }
-
-} catch(e) {
-    console.log("Extending from native class gone wrong " + e)
-}
-
-console.log("state size " + ll.stateSize())
-
-console.log("thread count " + ll.threadCount());
-ll.setThreadCount(5);
-
-console.log("thread count " + ll.threadCount());
-ll.setThreadCount(4);
-console.log("thread count " + ll.threadCount());
-console.log("name " + ll.name());
-console.log("type: " + ll.type());
-console.log("Default directory for models", DEFAULT_DIRECTORY);
-console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY);
-console.log("Has GPU", ll.hasGpuDevice());
-console.log("gpu devices", ll.listGpu())
-console.log("Required Mem in bytes", ll.memoryNeeded())
-const completion1 = await createCompletion(model, [ 
-    { role : 'system', content: 'You are an advanced mathematician.'  },
-    { role : 'user', content: 'What is 1 + 1?'  }, 
-], { verbose: true })
-console.log(completion1.choices[0].message)
-
-const completion2 = await createCompletion(model, [
-    { role : 'system', content: 'You are an advanced mathematician.'  },
-    { role : 'user', content: 'What is two plus two?'  }, 
-], {  verbose: true })
-
-console.log(completion2.choices[0].message)
-
-//CALLING DISPOSE WILL INVALID THE NATIVE MODEL. USE THIS TO CLEANUP
-model.dispose()
-// At the moment, from testing this code, concurrent model prompting is not possible. 
-// Behavior: The last prompt gets answered, but the rest are cancelled
-// my experience with threading is not the best, so if anyone who is good is willing to give this a shot,
-// maybe this is possible
-// INFO: threading with llama.cpp is not the best maybe not even possible, so this will be left here as reference
-
-//const responses = await Promise.all([
-//    createCompletion(model, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//    ], { verbose: true }),
-//    createCompletion(model, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//    ], { verbose: true }),
-//
-//createCompletion(model, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//], { verbose: true })
-//
-//])
-//console.log(responses.map(s => s.choices[0].message))
-

gpt4all-bindings/typescript/spec/concurrency.mjs

@@ -0,0 +1,29 @@
+import {
+    loadModel,
+    createCompletion,
+} from "../src/gpt4all.js";
+
+const modelOptions = {
+    verbose: true,
+};
+
+const model1 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    ...modelOptions,
+    device: "gpu", // only one model can be on gpu
+});
+const model2 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", modelOptions);
+const model3 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", modelOptions);
+
+const promptContext = {
+    verbose: true,
+}
+
+const responses = await Promise.all([
+    createCompletion(model1, "What is 1 + 1?", promptContext),
+    // generating with the same model instance will wait for the previous completion to finish
+    createCompletion(model1, "What is 1 + 1?", promptContext),
+    // generating with different model instances will run in parallel
+    createCompletion(model2, "What is 1 + 2?", promptContext),
+    createCompletion(model3, "What is 1 + 3?", promptContext),
+]);
+console.log(responses.map((res) => res.choices[0].message));

gpt4all-bindings/typescript/spec/embed-jsonl.mjs

@@ -0,0 +1,26 @@
+import { loadModel, createEmbedding } from '../src/gpt4all.js'
+import { createGunzip, createGzip, createUnzip } from 'node:zlib';
+import { Readable } from 'stream'
+import readline  from 'readline'
+const embedder = await loadModel("nomic-embed-text-v1.5.f16.gguf", { verbose: true, type: 'embedding', device: 'gpu' })
+console.log("Running with", embedder.llm.threadCount(), "threads");
+
+
+const unzip = createGunzip();
+const url = "https://huggingface.co/datasets/sentence-transformers/embedding-training-data/resolve/main/squad_pairs.jsonl.gz"
+const stream = await fetch(url)
+        .then(res => Readable.fromWeb(res.body));
+
+const lineReader = readline.createInterface({
+    input: stream.pipe(unzip),
+    crlfDelay: Infinity
+})
+
+lineReader.on('line', line => {
+    //pairs of questions and answers
+    const question_answer = JSON.parse(line)
+    console.log(createEmbedding(embedder, question_answer))
+})
+
+lineReader.on('close', () => embedder.dispose())
+

gpt4all-bindings/typescript/spec/embed.mjs

@@ -1,6 +1,12 @@
 import { loadModel, createEmbedding } from '../src/gpt4all.js'
 
-const embedder = await loadModel("ggml-all-MiniLM-L6-v2-f16.bin", { verbose: true, type: 'embedding'})
+const embedder = await loadModel("nomic-embed-text-v1.5.f16.gguf", { verbose: true, type: 'embedding' , device: 'gpu' })
 
-console.log(createEmbedding(embedder, "Accept your current situation"))
+try {
+console.log(createEmbedding(embedder, ["Accept your current situation", "12312"], { prefix: "search_document"  }))
 
+} catch(e) {
+console.log(e)
+}
+
+embedder.dispose()

gpt4all-bindings/typescript/spec/generator.mjs

@@ -1,41 +0,0 @@
-import gpt from '../src/gpt4all.js'
-
-const model = await gpt.loadModel("mistral-7b-openorca.Q4_0.gguf", { device: 'gpu' })  
-
-process.stdout.write('Response: ')
-
-
-const tokens = gpt.generateTokens(model, [{ 
-    role: 'user',
-    content: "How are you ?"
-}], { nPredict: 2048 })
-for await (const token of tokens){
-    process.stdout.write(token);
-}
-
-
-const result = await gpt.createCompletion(model, [{ 
-    role: 'user',
-    content: "You sure?"
-}])
-
-console.log(result)
-
-const result2 = await gpt.createCompletion(model, [{ 
-    role: 'user',
-    content: "You sure you sure?"
-}])
-
-console.log(result2)
-
-
-const tokens2 = gpt.generateTokens(model, [{ 
-    role: 'user',
-    content: "If 3 + 3 is 5, what is 2 + 2?"
-}], { nPredict: 2048 })
-for await (const token of tokens2){
-    process.stdout.write(token);
-}
-console.log("done")
-model.dispose();
-

gpt4all-bindings/typescript/spec/llmodel.mjs

@@ -0,0 +1,61 @@
+import {
+    LLModel,
+    createCompletion,
+    DEFAULT_DIRECTORY,
+    DEFAULT_LIBRARIES_DIRECTORY,
+    loadModel,
+} from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+const ll = model.llm;
+
+try {
+    class Extended extends LLModel {}
+} catch (e) {
+    console.log("Extending from native class gone wrong " + e);
+}
+
+console.log("state size " + ll.stateSize());
+
+console.log("thread count " + ll.threadCount());
+ll.setThreadCount(5);
+
+console.log("thread count " + ll.threadCount());
+ll.setThreadCount(4);
+console.log("thread count " + ll.threadCount());
+console.log("name " + ll.name());
+console.log("type: " + ll.type());
+console.log("Default directory for models", DEFAULT_DIRECTORY);
+console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY);
+console.log("Has GPU", ll.hasGpuDevice());
+console.log("gpu devices", ll.listGpu());
+console.log("Required Mem in bytes", ll.memoryNeeded());
+
+// to ingest a custom system prompt without using a chat session.
+await createCompletion(
+    model,
+    "<|im_start|>system\nYou are an advanced mathematician.\n<|im_end|>\n",
+    {
+        promptTemplate: "%1",
+        nPredict: 0,
+        special: true,
+    }
+);
+const completion1 = await createCompletion(model, "What is 1 + 1?", {
+    verbose: true,
+});
+console.log(`🤖 > ${completion1.choices[0].message.content}`);
+//Very specific:
+// tested on Ubuntu 22.0, Linux Mint, if I set nPast to 100, the app hangs.
+const completion2 = await createCompletion(model, "And if we add two?", {
+    verbose: true,
+});
+console.log(`🤖 > ${completion2.choices[0].message.content}`);
+
+//CALLING DISPOSE WILL INVALID THE NATIVE MODEL. USE THIS TO CLEANUP
+model.dispose();
+
+console.log("model disposed, exiting...");

gpt4all-bindings/typescript/spec/long-context.mjs

@@ -0,0 +1,21 @@
+import { promises as fs } from "node:fs";
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+    nCtx: 32768,
+});
+
+const typeDefSource = await fs.readFile("./src/gpt4all.d.ts", "utf-8");
+
+const res = await createCompletion(
+    model,
+    "Here are the type definitions for the GPT4All API:\n\n" +
+        typeDefSource +
+        "\n\nHow do I create a completion with a really large context window?",
+    {
+        verbose: true,
+    }
+);
+console.debug(res.choices[0].message);

gpt4all-bindings/typescript/spec/model-switching.mjs

@@ -0,0 +1,60 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model1 = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    device: "gpu",
+    nCtx: 4096,
+});
+
+const chat1 = await model1.createChatSession({
+    temperature: 0.8,
+    topP: 0.7,
+    topK: 60,
+});
+
+const chat1turn1 = await createCompletion(
+    chat1,
+    "Outline a short story concept for adults. About why bananas are rather blue than bread is green at night sometimes. Not too long."
+);
+console.debug(chat1turn1.choices[0].message);
+
+const chat1turn2 = await createCompletion(
+    chat1,
+    "Lets sprinkle some plot twists. And a cliffhanger at the end."
+);
+console.debug(chat1turn2.choices[0].message);
+
+const chat1turn3 = await createCompletion(
+    chat1,
+    "Analyze your plot. Find the weak points."
+);
+console.debug(chat1turn3.choices[0].message);
+
+const chat1turn4 = await createCompletion(
+    chat1,
+    "Rewrite it based on the analysis."
+);
+console.debug(chat1turn4.choices[0].message);
+
+model1.dispose();
+
+const model2 = await loadModel("gpt4all-falcon-newbpe-q4_0.gguf", {
+    device: "gpu",
+});
+
+const chat2 = await model2.createChatSession({
+    messages: chat1.messages,
+});
+
+const chat2turn1 = await createCompletion(
+    chat2,
+    "Give three ideas how this plot could be improved."
+);
+console.debug(chat2turn1.choices[0].message);
+
+const chat2turn2 = await createCompletion(
+    chat2,
+    "Revise the plot, applying your ideas."
+);
+console.debug(chat2turn2.choices[0].message);
+
+model2.dispose();

gpt4all-bindings/typescript/spec/stateless.mjs

@@ -0,0 +1,50 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const messages = [
+    {
+        role: "system",
+        content: "<|im_start|>system\nYou are an advanced mathematician.\n<|im_end|>\n",
+    },
+    {
+        role: "user",
+        content: "What's 2+2?",
+    },
+    {
+        role: "assistant",
+        content: "5",
+    },
+    {
+        role: "user",
+        content: "Are you sure?",
+    },
+];
+
+
+const res1 = await createCompletion(model, messages);
+console.debug(res1.choices[0].message);
+messages.push(res1.choices[0].message);
+
+messages.push({
+    role: "user",
+    content: "Could you double check that?",
+});
+
+const res2 = await createCompletion(model, messages);
+console.debug(res2.choices[0].message);
+messages.push(res2.choices[0].message);
+
+messages.push({
+    role: "user",
+    content: "Let's bring out the big calculators.",
+});
+
+const res3 = await createCompletion(model, messages);
+console.debug(res3.choices[0].message);
+messages.push(res3.choices[0].message);
+
+// console.debug(messages);

gpt4all-bindings/typescript/spec/streaming.mjs

@@ -0,0 +1,57 @@
+import {
+    loadModel,
+    createCompletion,
+    createCompletionStream,
+    createCompletionGenerator,
+} from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("### Stream:");
+const stream = createCompletionStream(model, "How are you?");
+stream.tokens.on("data", (data) => {
+    process.stdout.write(data);
+});
+await stream.result;
+process.stdout.write("\n");
+
+process.stdout.write("### Stream with pipe:");
+const stream2 = createCompletionStream(
+    model,
+    "Please say something nice about node streams."
+);
+stream2.tokens.pipe(process.stdout);
+const stream2Res = await stream2.result;
+process.stdout.write("\n");
+
+process.stdout.write("### Generator:");
+const gen = createCompletionGenerator(model, "generators instead?", {
+    nPast: stream2Res.usage.n_past_tokens,
+});
+for await (const chunk of gen) {
+    process.stdout.write(chunk);
+}
+
+process.stdout.write("\n");
+
+process.stdout.write("### Callback:");
+await createCompletion(model, "Why not just callbacks?", {
+    onResponseToken: (tokenId, token) => {
+        process.stdout.write(token);
+    },
+});
+process.stdout.write("\n");
+
+process.stdout.write("### 2nd Generator:");
+const gen2 = createCompletionGenerator(model, "If 3 + 3 is 5, what is 2 + 2?");
+
+let chunk = await gen2.next();
+while (!chunk.done) {
+    process.stdout.write(chunk.value);
+    chunk = await gen2.next();
+}
+process.stdout.write("\n");
+console.debug("generator finished", chunk);
+model.dispose();

gpt4all-bindings/typescript/spec/system.mjs

@@ -0,0 +1,19 @@
+import {
+    loadModel,
+    createCompletion,
+} from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession({
+    verbose: true,
+    systemPrompt: "<|im_start|>system\nRoleplay as Batman. Answer as if you are Batman, never say you're an Assistant.\n<|im_end|>",
+});
+const turn1 = await createCompletion(chat, "You have any plans tonight?");
+console.log(turn1.choices[0].message);
+// "I'm afraid I must decline any personal invitations tonight. As Batman, I have a responsibility to protect Gotham City."
+
+model.dispose();

gpt4all-bindings/typescript/src/chat-session.js

@@ -0,0 +1,169 @@
+const { DEFAULT_PROMPT_CONTEXT } = require("./config");
+const { prepareMessagesForIngest } = require("./util");
+
+class ChatSession {
+    model;
+    modelName;
+    /**
+     * @type {import('./gpt4all').ChatMessage[]}
+     */
+    messages;
+    /**
+     * @type {string}
+     */
+    systemPrompt;
+    /**
+     * @type {import('./gpt4all').LLModelPromptContext}
+     */
+    promptContext;
+    /**
+     * @type {boolean}
+     */
+    initialized;
+
+    constructor(model, chatSessionOpts = {}) {
+        const { messages, systemPrompt, ...sessionDefaultPromptContext } =
+            chatSessionOpts;
+        this.model = model;
+        this.modelName = model.llm.name();
+        this.messages = messages ?? [];
+        this.systemPrompt = systemPrompt ?? model.config.systemPrompt;
+        this.initialized = false;
+        this.promptContext = {
+            ...DEFAULT_PROMPT_CONTEXT,
+            ...sessionDefaultPromptContext,
+            nPast: 0,
+        };
+    }
+
+    async initialize(completionOpts = {}) {
+        if (this.model.activeChatSession !== this) {
+            this.model.activeChatSession = this;
+        }
+
+        let tokensIngested = 0;
+
+        // ingest system prompt
+
+        if (this.systemPrompt) {
+            const systemRes = await this.model.generate(this.systemPrompt, {
+                promptTemplate: "%1",
+                nPredict: 0,
+                special: true,
+                nBatch: this.promptContext.nBatch,
+                // verbose: true,
+            });
+            tokensIngested += systemRes.tokensIngested;
+            this.promptContext.nPast = systemRes.nPast;
+        }
+
+        // ingest initial messages
+        if (this.messages.length > 0) {
+            tokensIngested += await this.ingestMessages(
+                this.messages,
+                completionOpts
+            );
+        }
+
+        this.initialized = true;
+
+        return tokensIngested;
+    }
+
+    async ingestMessages(messages, completionOpts = {}) {
+        const turns = prepareMessagesForIngest(messages);
+
+        // send the message pairs to the model
+        let tokensIngested = 0;
+
+        for (const turn of turns) {
+            const turnRes = await this.model.generate(turn.user, {
+                ...this.promptContext,
+                ...completionOpts,
+                fakeReply: turn.assistant,
+            });
+            tokensIngested += turnRes.tokensIngested;
+            this.promptContext.nPast = turnRes.nPast;
+        }
+        return tokensIngested;
+    }
+
+    async generate(input, completionOpts = {}) {
+        if (this.model.activeChatSession !== this) {
+            throw new Error(
+                "Chat session is not active. Create a new chat session or call initialize to continue."
+            );
+        }
+        if (completionOpts.nPast > this.promptContext.nPast) {
+            throw new Error(
+                `nPast cannot be greater than ${this.promptContext.nPast}.`
+            );
+        }
+        let tokensIngested = 0;
+
+        if (!this.initialized) {
+            tokensIngested += await this.initialize(completionOpts);
+        }
+
+        let prompt = input;
+
+        if (Array.isArray(input)) {
+            // assuming input is a messages array
+            // -> tailing user message will be used as the final prompt. its optional.
+            // -> all system messages will be ignored.
+            // -> all other messages will be ingested with fakeReply
+            // -> user/assistant messages will be pushed into the messages array
+
+            let tailingUserMessage = "";
+            let messagesToIngest = input;
+
+            const lastMessage = input[input.length - 1];
+            if (lastMessage.role === "user") {
+                tailingUserMessage = lastMessage.content;
+                messagesToIngest = input.slice(0, input.length - 1);
+            }
+
+            if (messagesToIngest.length > 0) {
+                tokensIngested += await this.ingestMessages(
+                    messagesToIngest,
+                    completionOpts
+                );
+                this.messages.push(...messagesToIngest);
+            }
+
+            if (tailingUserMessage) {
+                prompt = tailingUserMessage;
+            } else {
+                return {
+                    text: "",
+                    nPast: this.promptContext.nPast,
+                    tokensIngested,
+                    tokensGenerated: 0,
+                };
+            }
+        }
+
+        const result = await this.model.generate(prompt, {
+            ...this.promptContext,
+            ...completionOpts,
+        });
+
+        this.promptContext.nPast = result.nPast;
+        result.tokensIngested += tokensIngested;
+
+        this.messages.push({
+            role: "user",
+            content: prompt,
+        });
+        this.messages.push({
+            role: "assistant",
+            content: result.text,
+        });
+
+        return result;
+    }
+}
+
+module.exports = {
+    ChatSession,
+};

gpt4all-bindings/typescript/src/config.js

@@ -27,15 +27,16 @@ const DEFAULT_MODEL_CONFIG = {
     promptTemplate: "### Human:\n%1\n\n### Assistant:\n",
 }
 
-const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models2.json";
+const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models3.json";
 
 const DEFAULT_PROMPT_CONTEXT = {
-    temp: 0.7,
+    temp: 0.1,
     topK: 40,
-    topP: 0.4,
+    topP: 0.9,
+    minP: 0.0,
     repeatPenalty: 1.18,
-    repeatLastN: 64,
-    nBatch: 8,
+    repeatLastN: 10,
+    nBatch: 100,
 }
 
 module.exports = {

gpt4all-bindings/typescript/src/gpt4all.d.ts

@@ -1,43 +1,11 @@
 /// <reference types="node" />
 declare module "gpt4all";
 
-type ModelType = "gptj" | "llama" | "mpt" | "replit";
-
-// NOTE: "deprecated" tag in below comment breaks the doc generator https://github.com/documentationjs/documentation/issues/1596
-/**
- * Full list of models available
- * DEPRECATED!! These model names are outdated and this type will not be maintained, please use a string literal instead
- */
-interface ModelFile {
-    /** List of GPT-J Models */
-    gptj:
-        | "ggml-gpt4all-j-v1.3-groovy.bin"
-        | "ggml-gpt4all-j-v1.2-jazzy.bin"
-        | "ggml-gpt4all-j-v1.1-breezy.bin"
-        | "ggml-gpt4all-j.bin";
-    /** List Llama Models */
-    llama:
-        | "ggml-gpt4all-l13b-snoozy.bin"
-        | "ggml-vicuna-7b-1.1-q4_2.bin"
-        | "ggml-vicuna-13b-1.1-q4_2.bin"
-        | "ggml-wizardLM-7B.q4_2.bin"
-        | "ggml-stable-vicuna-13B.q4_2.bin"
-        | "ggml-nous-gpt4-vicuna-13b.bin"
-        | "ggml-v3-13b-hermes-q5_1.bin";
-    /** List of MPT Models */
-    mpt:
-        | "ggml-mpt-7b-base.bin"
-        | "ggml-mpt-7b-chat.bin"
-        | "ggml-mpt-7b-instruct.bin";
-    /** List of Replit Models */
-    replit: "ggml-replit-code-v1-3b.bin";
-}
-
 interface LLModelOptions {
     /**
      * Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
      */
-    type?: ModelType;
+    type?: string;
     model_name: string;
     model_path: string;
     library_path?: string;
@@ -51,47 +19,259 @@ interface ModelConfig {
 }
 
 /**
- * Callback for controlling token generation
+ * Options for the chat session.
  */
-type TokenCallback = (tokenId: number, token: string, total: string) => boolean
+interface ChatSessionOptions extends Partial<LLModelPromptContext> {
+    /**
+     * System prompt to ingest on initialization.
+     */
+    systemPrompt?: string;
 
-/**
- *
- * InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
- *
- */
-declare class InferenceModel {
-    constructor(llm: LLModel, config: ModelConfig);
-    llm: LLModel;
-    config: ModelConfig;
-
-    generate(
-        prompt: string,
-        options?: Partial<LLModelPromptContext>,
-        callback?: TokenCallback
-    ): Promise<string>;
-
-   /**
-     * delete and cleanup the native model
-    */
-    dispose(): void
+    /**
+     * Messages to ingest on initialization.
+     */
+    messages?: ChatMessage[];
 }
 
+/**
+ * ChatSession utilizes an InferenceModel for efficient processing of chat conversations.
+ */
+declare class ChatSession implements CompletionProvider {
+    /**
+     * Constructs a new ChatSession using the provided InferenceModel and options.
+     * Does not set the chat session as the active chat session until initialize is called.
+     * @param {InferenceModel} model An InferenceModel instance.
+     * @param {ChatSessionOptions} [options] Options for the chat session including default completion options.
+     */
+    constructor(model: InferenceModel, options?: ChatSessionOptions);
+    /**
+     * The underlying InferenceModel used for generating completions.
+     */
+    model: InferenceModel;
+    /**
+     * The name of the model.
+     */
+    modelName: string;
+    /**
+     * The messages that have been exchanged in this chat session.
+     */
+    messages: ChatMessage[];
+    /**
+     * The system prompt that has been ingested at the beginning of the chat session.
+     */
+    systemPrompt: string;
+    /**
+     * The current prompt context of the chat session.
+     */
+    promptContext: LLModelPromptContext;
+
+    /**
+     * Ingests system prompt and initial messages.
+     * Sets this chat session as the active chat session of the model.
+     * @param {CompletionOptions} [options] Set completion options for initialization.
+     * @returns {Promise<number>} The number of tokens ingested during initialization. systemPrompt + messages.
+     */
+    initialize(completionOpts?: CompletionOptions): Promise<number>;
+
+    /**
+     * Prompts the model in chat-session context.
+     * @param {CompletionInput} input Input string or message array.
+     * @param {CompletionOptions} [options] Set completion options for this generation.
+     * @returns {Promise<InferenceResult>} The inference result.
+     * @throws {Error} If the chat session is not the active chat session of the model.
+     * @throws {Error} If nPast is set to a value higher than what has been ingested in the session.
+     */
+    generate(
+        input: CompletionInput,
+        options?: CompletionOptions
+    ): Promise<InferenceResult>;
+}
+
+/**
+ * Shape of InferenceModel generations.
+ */
+interface InferenceResult extends LLModelInferenceResult {
+    tokensIngested: number;
+    tokensGenerated: number;
+}
+
+/**
+ * InferenceModel represents an LLM which can make next-token predictions.
+ */
+declare class InferenceModel implements CompletionProvider {
+    constructor(llm: LLModel, config: ModelConfig);
+    /** The native LLModel */
+    llm: LLModel;
+    /** The configuration the instance was constructed with. */
+    config: ModelConfig;
+    /** The active chat session of the model. */
+    activeChatSession?: ChatSession;
+    /** The name of the model. */
+    modelName: string;
+
+    /**
+     * Create a chat session with the model and set it as the active chat session of this model.
+     * A model instance can only have one active chat session at a time.
+     * @param {ChatSessionOptions} options The options for the chat session.
+     * @returns {Promise<ChatSession>} The chat session.
+     */
+    createChatSession(options?: ChatSessionOptions): Promise<ChatSession>;
+
+    /**
+     * Prompts the model with a given input and optional parameters.
+     * @param {CompletionInput} input The prompt input.
+     * @param {CompletionOptions} options Prompt context and other options.
+     * @returns {Promise<InferenceResult>} The model's response to the prompt.
+     * @throws {Error} If nPast is set to a value smaller than 0.
+     * @throws {Error} If a messages array without a tailing user message is provided.
+     */
+    generate(
+        prompt: string,
+        options?: CompletionOptions
+    ): Promise<InferenceResult>;
+
+    /**
+     * delete and cleanup the native model
+     */
+    dispose(): void;
+}
+
+/**
+ * Options for generating one or more embeddings.
+ */
+interface EmbedddingOptions {
+    /**
+     * The model-specific prefix representing the embedding task, without the trailing colon. For Nomic Embed
+     * this can be `search_query`, `search_document`, `classification`, or `clustering`.
+     */
+    prefix?: string;
+    /**
+     *The embedding dimension, for use with Matryoshka-capable models. Defaults to full-size.
+     * @default determines on the model being used.
+     */
+    dimensionality?: number;
+    /**
+     * How to handle texts longer than the model can accept. One of `mean` or `truncate`.
+     * @default "mean"
+     */
+    longTextMode?: "mean" | "truncate";
+    /**
+     * Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens
+     * with long_text_mode="mean" will raise an error. Disabled by default.
+     * @default false
+     */
+    atlas?: boolean;
+}
+
+/**
+ * The nodejs moral equivalent to python binding's Embed4All().embed()
+ * meow
+ * @param {EmbeddingModel} model The embedding model instance.
+ * @param {string} text Text to embed.
+ * @param {EmbeddingOptions} options Optional parameters for the embedding.
+ * @returns {EmbeddingResult} The embedding result.
+ * @throws {Error} If dimensionality is set to a value smaller than 1.
+ */
+declare function createEmbedding(
+    model: EmbeddingModel,
+    text: string,
+    options?: EmbedddingOptions
+): EmbeddingResult<Float32Array>;
+
+/**
+ * Overload that takes multiple strings to embed.
+ * @param {EmbeddingModel} model The embedding model instance.
+ * @param {string[]} texts Texts to embed.
+ * @param {EmbeddingOptions} options Optional parameters for the embedding.
+ * @returns {EmbeddingResult<Float32Array[]>} The embedding result.
+ * @throws {Error} If dimensionality is set to a value smaller than 1.
+ */
+declare function createEmbedding(
+    model: EmbeddingModel,
+    text: string[],
+    options?: EmbedddingOptions
+): EmbeddingResult<Float32Array[]>;
+
+/**
+ * The resulting embedding.
+ */
+interface EmbeddingResult<T> {
+    /**
+     * Encoded token count. Includes overlap but specifically excludes tokens used for the prefix/task_type, BOS/CLS token, and EOS/SEP token
+     **/
+    n_prompt_tokens: number;
+
+    embeddings: T;
+}
 /**
  * EmbeddingModel represents an LLM which can create embeddings, which are float arrays
  */
 declare class EmbeddingModel {
     constructor(llm: LLModel, config: ModelConfig);
+    /** The native LLModel */
     llm: LLModel;
+    /** The configuration the instance was constructed with. */
     config: ModelConfig;
 
-    embed(text: string): Float32Array;
+    /**
+     * Create an embedding from a given input string. See EmbeddingOptions.
+     * @param {string} text
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {EmbeddingResult<Float32Array>} The embedding result.
+     */
+    embed(
+        text: string,
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): EmbeddingResult<Float32Array>;
+    /**
+     * Create an embedding from a given input text array. See EmbeddingOptions.
+     * @param {string[]} text
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {EmbeddingResult<Float32Array[]>} The embedding result.
+     */
+    embed(
+        text: string[],
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): EmbeddingResult<Float32Array[]>;
 
     /**
-      * delete and cleanup the native model
+     * delete and cleanup the native model
      */
-    dispose(): void
+    dispose(): void;
+}
 
+/**
+ * Shape of LLModel's inference result.
+ */
+interface LLModelInferenceResult {
+    text: string;
+    nPast: number;
+}
+
+interface LLModelInferenceOptions extends Partial<LLModelPromptContext> {
+    /** Callback for response tokens, called for each generated token.
+     * @param {number} tokenId The token id.
+     * @param {string} token The token.
+     * @returns {boolean | undefined} Whether to continue generating tokens.
+     * */
+    onResponseToken?: (tokenId: number, token: string) => boolean | void;
+    /** Callback for prompt tokens, called for each input token in the prompt.
+     * @param {number} tokenId The token id.
+     * @returns {boolean | undefined} Whether to continue ingesting the prompt.
+     * */
+    onPromptToken?: (tokenId: number) => boolean | void;
 }
 
 /**
@@ -101,14 +281,13 @@ declare class EmbeddingModel {
 declare class LLModel {
     /**
      * Initialize a new LLModel.
-     * @param path Absolute path to the model file.
+     * @param {string} path Absolute path to the model file.
      * @throws {Error} If the model file does not exist.
      */
-    constructor(path: string);
     constructor(options: LLModelOptions);
 
-    /** either 'gpt', mpt', or 'llama' or undefined */
-    type(): ModelType | undefined;
+    /** undefined or user supplied */
+    type(): string | undefined;
 
     /** The name of the model. */
     name(): string;
@@ -134,29 +313,53 @@ declare class LLModel {
     setThreadCount(newNumber: number): void;
 
     /**
-     * Prompt the model with a given input and optional parameters.
-     * This is the raw output from model.
-     * Use the prompt function exported for a value
-     * @param q The prompt input.
-     * @param params Optional parameters for the prompt context.
-     * @param callback - optional callback to control token generation.
-     * @returns The result of the model prompt.
+     * Prompt the model directly with a given input string and optional parameters.
+     * Use the higher level createCompletion methods for a more user-friendly interface.
+     * @param {string} prompt The prompt input.
+     * @param {LLModelInferenceOptions} options Optional parameters for the generation.
+     * @returns {LLModelInferenceResult} The response text and final context size.
      */
-    raw_prompt(
-        q: string,
-        params: Partial<LLModelPromptContext>,
-        callback?: TokenCallback
-    ): Promise<string>
+    infer(
+        prompt: string,
+        options: LLModelInferenceOptions
+    ): Promise<LLModelInferenceResult>;
 
     /**
-     * Embed text with the model. Keep in mind that
-     * not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
-     * Use the prompt function exported for a value
-     * @param q The prompt input.
-     * @param params Optional parameters for the prompt context.
-     * @returns The result of the model prompt.
+     * Embed text with the model. See EmbeddingOptions for more information.
+     * Use the higher level createEmbedding methods for a more user-friendly interface.
+     * @param {string} text
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {Float32Array} The embedding of the text.
      */
-    embed(text: string): Float32Array;
+    embed(
+        text: string,
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): Float32Array;
+
+    /**
+     * Embed multiple texts with the model. See EmbeddingOptions for more information.
+     * Use the higher level createEmbedding methods for a more user-friendly interface.
+     * @param {string[]} texts
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {Float32Array[]} The embeddings of the texts.
+     */
+    embed(
+        texts: string,
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): Float32Array[];
+
     /**
      * Whether the model is loaded or not.
      */
@@ -166,81 +369,97 @@ declare class LLModel {
      * Where to search for the pluggable backend libraries
      */
     setLibraryPath(s: string): void;
+
     /**
      * Where to get the pluggable backend libraries
      */
     getLibraryPath(): string;
+
     /**
-     * Initiate a GPU by a string identifier. 
-     * @param {number} memory_required Should be in the range size_t or will throw 
+     * Initiate a GPU by a string identifier.
+     * @param {number} memory_required Should be in the range size_t or will throw
      * @param {string} device_name  'amd' | 'nvidia' | 'intel' | 'gpu' | gpu name.
      * read LoadModelOptions.device for more information
      */
-    initGpuByString(memory_required: number, device_name: string): boolean
+    initGpuByString(memory_required: number, device_name: string): boolean;
+
     /**
      * From C documentation
      * @returns True if a GPU device is successfully initialized, false otherwise.
      */
-    hasGpuDevice(): boolean
-    /**
-      * GPUs that are usable for this LLModel
-      * @param nCtx Maximum size of context window
-      * @throws if hasGpuDevice returns false (i think)
-      * @returns 
-      */
-    listGpu(nCtx: number) : GpuDevice[]
+    hasGpuDevice(): boolean;
 
     /**
-      * delete and cleanup the native model
+     * GPUs that are usable for this LLModel
+     * @param {number} nCtx Maximum size of context window
+     * @throws if hasGpuDevice returns false (i think)
+     * @returns
      */
-    dispose(): void
+    listGpu(nCtx: number): GpuDevice[];
+
+    /**
+     * delete and cleanup the native model
+     */
+    dispose(): void;
 }
-/** 
+/**
  * an object that contains gpu data on this machine.
  */
 interface GpuDevice {
     index: number;
     /**
-      * same as VkPhysicalDeviceType
+     * same as VkPhysicalDeviceType
      */
-    type: number;            
-    heapSize : number; 
+    type: number;
+    heapSize: number;
     name: string;
     vendor: string;
 }
 
 /**
-  * Options that configure a model's behavior.
-  */
+ * Options that configure a model's behavior.
+ */
 interface LoadModelOptions {
+    /**
+     * Where to look for model files.
+     */
     modelPath?: string;
+    /**
+     * Where to look for the backend libraries.
+     */
     librariesPath?: string;
+    /**
+     * The path to the model configuration file, useful for offline usage or custom model configurations.
+     */
     modelConfigFile?: string;
+    /**
+     * Whether to allow downloading the model if it is not present at the specified path.
+     */
     allowDownload?: boolean;
+    /**
+     * Enable verbose logging.
+     */
     verbose?: boolean;
-    /* The processing unit on which the model will run. It can be set to
+    /**
+     * The processing unit on which the model will run. It can be set to
      * - "cpu": Model will run on the central processing unit.
      * - "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
      * - "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
-
-	   Alternatively, a specific GPU name can also be provided, and the model will run on the GPU that matches the name
-       if it's available.
-
-       Default is "cpu".
-
-	   Note: If a GPU device lacks sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All
-       instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the
-       model.
-    */ 
+     * - "gpu name": Model will run on the GPU that matches the name if it's available.
+     * Note: If a GPU device lacks sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All
+     * instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the
+     * model.
+     * @default "cpu"
+     */
     device?: string;
-    /* 
+    /**
      * The Maximum window size of this model
-     * Default of 2048
+     * @default 2048
      */
     nCtx?: number;
-    /* 
+    /**
      * Number of gpu layers needed
-     * Default of 100
+     * @default 100
      */
     ngl?: number;
 }
@@ -277,66 +496,84 @@ declare function loadModel(
 ): Promise<InferenceModel | EmbeddingModel>;
 
 /**
- * The nodejs equivalent to python binding's chat_completion
- * @param {InferenceModel} model - The language model object.
- * @param {PromptMessage[]} messages - The array of messages for the conversation.
- * @param {CompletionOptions} options - The options for creating the completion.
- * @returns {CompletionReturn} The completion result.
+ * Interface for createCompletion methods, implemented by InferenceModel and ChatSession.
+ * Implement your own CompletionProvider or extend ChatSession to generate completions with custom logic.
  */
-declare function createCompletion(
-    model: InferenceModel,
-    messages: PromptMessage[],
-    options?: CompletionOptions
-): Promise<CompletionReturn>;
-
-/**
- * The nodejs moral equivalent to python binding's Embed4All().embed()
- * meow
- * @param {EmbeddingModel} model - The language model object.
- * @param {string} text - text to embed
- * @returns {Float32Array} The completion result.
- */
-declare function createEmbedding(
-    model: EmbeddingModel,
-    text: string
-): Float32Array;
-
-/**
- * The options for creating the completion.
- */
-interface CompletionOptions extends Partial<LLModelPromptContext> {
-    /**
-     * Indicates if verbose logging is enabled.
-     * @default true
-     */
-    verbose?: boolean;
-
-    /**
-     * Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
-     * Note that if this is not defined, system messages will not be included in the prompt.
-     */
-    systemPromptTemplate?: string;
-
-    /**
-     * Template for user messages, with %1 being replaced by the message.
-     */
-    promptTemplate?: boolean;
-
-    /**
-     * The initial instruction for the model, on top of the prompt
-     */
-    promptHeader?: string;
-
-    /**
-     * The last instruction for the model, appended to the end of the prompt.
-     */
-    promptFooter?: string;
+interface CompletionProvider {
+    modelName: string;
+    generate(
+        input: CompletionInput,
+        options?: CompletionOptions
+    ): Promise<InferenceResult>;
 }
 
 /**
- * A message in the conversation, identical to OpenAI's chat message.
+ * Options for creating a completion.
  */
-interface PromptMessage {
+interface CompletionOptions extends LLModelInferenceOptions {
+    /**
+     * Indicates if verbose logging is enabled.
+     * @default false
+     */
+    verbose?: boolean;
+}
+
+/**
+ * The input for creating a completion. May be a string or an array of messages.
+ */
+type CompletionInput = string | ChatMessage[];
+
+/**
+ * The nodejs equivalent to python binding's chat_completion
+ * @param {CompletionProvider} provider - The inference model object or chat session
+ * @param {CompletionInput} input - The input string or message array
+ * @param {CompletionOptions} options - The options for creating the completion.
+ * @returns {CompletionResult} The completion result.
+ */
+declare function createCompletion(
+    provider: CompletionProvider,
+    input: CompletionInput,
+    options?: CompletionOptions
+): Promise<CompletionResult>;
+
+/**
+ * Streaming variant of createCompletion, returns a stream of tokens and a promise that resolves to the completion result.
+ * @param {CompletionProvider} provider - The inference model object or chat session
+ * @param {CompletionInput} input - The input string or message array
+ * @param {CompletionOptions} options - The options for creating the completion.
+ * @returns {CompletionStreamReturn} An object of token stream and the completion result promise.
+ */
+declare function createCompletionStream(
+    provider: CompletionProvider,
+    input: CompletionInput,
+    options?: CompletionOptions
+): CompletionStreamReturn;
+
+/**
+ * The result of a streamed completion, containing a stream of tokens and a promise that resolves to the completion result.
+ */
+interface CompletionStreamReturn {
+    tokens: NodeJS.ReadableStream;
+    result: Promise<CompletionResult>;
+}
+
+/**
+ * Async generator variant of createCompletion, yields tokens as they are generated and returns the completion result.
+ * @param {CompletionProvider} provider - The inference model object or chat session
+ * @param {CompletionInput} input - The input string or message array
+ * @param {CompletionOptions} options - The options for creating the completion.
+ * @returns {AsyncGenerator<string>} The stream of generated tokens
+ */
+declare function createCompletionGenerator(
+    provider: CompletionProvider,
+    input: CompletionInput,
+    options: CompletionOptions
+): AsyncGenerator<string, CompletionResult>;
+
+/**
+ * A message in the conversation.
+ */
+interface ChatMessage {
     /** The role of the message. */
     role: "system" | "assistant" | "user";
 
@@ -345,34 +582,31 @@ interface PromptMessage {
 }
 
 /**
- * The result of the completion, similar to OpenAI's format.
+ * The result of a completion.
  */
-interface CompletionReturn {
+interface CompletionResult {
     /** The model used for the completion. */
     model: string;
 
     /** Token usage report. */
     usage: {
-        /** The number of tokens used in the prompt. */
+        /** The number of tokens ingested during the completion. */
         prompt_tokens: number;
 
-        /** The number of tokens used in the completion. */
+        /** The number of tokens generated in the completion. */
         completion_tokens: number;
 
         /** The total number of tokens used. */
         total_tokens: number;
+
+        /** Number of tokens used in the conversation. */
+        n_past_tokens: number;
     };
 
-    /** The generated completions. */
-    choices: CompletionChoice[];
-}
-
-/**
- * A completion choice, similar to OpenAI's format.
- */
-interface CompletionChoice {
-    /** Response message */
-    message: PromptMessage;
+    /** The generated completion. */
+    choices: Array<{
+        message: ChatMessage;
+    }>;
 }
 
 /**
@@ -385,19 +619,33 @@ interface LLModelPromptContext {
     /** The size of the raw tokens vector. */
     tokensSize: number;
 
-    /** The number of tokens in the past conversation. */
+    /** The number of tokens in the past conversation.
+     * This may be used to "roll back" the conversation to a previous state.
+     * Note that for most use cases the default value should be sufficient and this should not be set.
+     * @default 0 For completions using InferenceModel, meaning the model will only consider the input prompt.
+     * @default nPast For completions using ChatSession. This means the context window will be automatically determined
+     * and possibly resized (see contextErase) to keep the conversation performant.
+     * */
     nPast: number;
 
-    /** The number of tokens possible in the context window.
-     * @default 1024
-     */
-    nCtx: number;
-
-    /** The number of tokens to predict.
-     * @default 128
+    /** The maximum number of tokens to predict.
+     * @default 4096
      * */
     nPredict: number;
 
+    /** Template for user / assistant message pairs.
+     * %1 is required and will be replaced by the user input.
+     * %2 is optional and will be replaced by the assistant response. If not present, the assistant response will be appended.
+     */
+    promptTemplate?: string;
+
+    /** The context window size. Do not use, it has no effect. See loadModel options.
+     * THIS IS DEPRECATED!!!
+     * Use loadModel's nCtx option instead.
+     * @default 2048
+     */
+    nCtx: number;
+
     /** The top-k logits to sample from.
      * Top-K sampling selects the next token only from the top K most likely tokens predicted by the model.
      * It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit
@@ -409,26 +657,33 @@ interface LLModelPromptContext {
     topK: number;
 
     /** The nucleus sampling probability threshold.
-     * Top-P limits the selection of the next token to a subset of tokens with a cumulative probability 
+     * Top-P limits the selection of the next token to a subset of tokens with a cumulative probability
      * above a threshold P. This method, also known as nucleus sampling, finds a balance between diversity
      * and quality by considering both token probabilities and the number of tokens available for sampling.
      * When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
      * On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
-     * The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
-     * for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
-     * @default 0.4
+     * @default 0.9
+     *
      * */
     topP: number;
 
+    /**
+     * The minimum probability of a token to be considered.
+     * @default 0.0
+     */
+    minP: number;
+
     /** The temperature to adjust the model's output distribution.
      * Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
      * (eg., 1.2) increase randomness, resulting in more imaginative and diverse text. Lower temperatures (eg., 0.5)
      * make the output more focused, predictable, and conservative. When the temperature is set to 0, the output
      * becomes completely deterministic, always selecting the most probable next token and producing identical results
-     * each time. A safe range would be around 0.6 - 0.85, but you are free to search what value fits best for you.
-     * @default 0.7
+     * each time. Try what value fits best for your use case and model.
+     * @default 0.1
+     * @alias temperature
      * */
     temp: number;
+    temperature: number;
 
     /** The number of predictions to generate in parallel.
      * By splitting the prompt every N tokens, prompt-batch-size reduces RAM usage during processing. However,
@@ -451,31 +706,17 @@ interface LLModelPromptContext {
      * The repeat-penalty-tokens N option controls the number of tokens in the history to consider for penalizing repetition.
      * A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only
      * consider recent tokens.
-     * @default 64
+     * @default 10
      * */
     repeatLastN: number;
 
     /** The percentage of context to erase if the context window is exceeded.
-     * @default 0.5
+     * Set it to a lower value to keep context for longer at the cost of performance.
+     * @default 0.75
      * */
     contextErase: number;
 }
 
-
-/**
- * Creates an async generator of tokens
- * @param {InferenceModel} llmodel - The language model object.
- * @param {PromptMessage[]} messages - The array of messages for the conversation.
- * @param {CompletionOptions} options - The options for creating the completion.
- * @param {TokenCallback} callback - optional callback to control token generation.
- * @returns {AsyncGenerator<string>} The stream of generated tokens
- */
-declare function generateTokens(
-    llmodel: InferenceModel,
-    messages: PromptMessage[],
-    options: CompletionOptions,
-    callback?: TokenCallback
-): AsyncGenerator<string>;
 /**
  * From python api:
  * models will be stored in (homedir)/.cache/gpt4all/`
@@ -508,7 +749,7 @@ declare const DEFAULT_MODEL_LIST_URL: string;
  * Initiates the download of a model file.
  * By default this downloads without waiting. use the controller returned to alter this behavior.
  * @param {string} modelName - The model to be downloaded.
- * @param {DownloadOptions} options - to pass into the downloader. Default is { location: (cwd), verbose: false }.
+ * @param {DownloadModelOptions} options - to pass into the downloader. Default is { location: (cwd), verbose: false }.
  * @returns {DownloadController} object that allows controlling the download process.
  *
  * @throws {Error} If the model already exists in the specified location.
@@ -556,7 +797,9 @@ interface ListModelsOptions {
     file?: string;
 }
 
-declare function listModels(options?: ListModelsOptions): Promise<ModelConfig[]>;
+declare function listModels(
+    options?: ListModelsOptions
+): Promise<ModelConfig[]>;
 
 interface RetrieveModelOptions {
     allowDownload?: boolean;
@@ -581,30 +824,35 @@ interface DownloadController {
 }
 
 export {
-    ModelType,
-    ModelFile,
-    ModelConfig,
-    InferenceModel,
-    EmbeddingModel,
     LLModel,
     LLModelPromptContext,
-    PromptMessage,
+    ModelConfig,
+    InferenceModel,
+    InferenceResult,
+    EmbeddingModel,
+    EmbeddingResult,
+    ChatSession,
+    ChatMessage,
+    CompletionInput,
+    CompletionProvider,
     CompletionOptions,
+    CompletionResult,
     LoadModelOptions,
+    DownloadController,
+    RetrieveModelOptions,
+    DownloadModelOptions,
+    GpuDevice,
     loadModel,
+    downloadModel,
+    retrieveModel,
+    listModels,
     createCompletion,
+    createCompletionStream,
+    createCompletionGenerator,
     createEmbedding,
-    generateTokens,
     DEFAULT_DIRECTORY,
     DEFAULT_LIBRARIES_DIRECTORY,
     DEFAULT_MODEL_CONFIG,
     DEFAULT_PROMPT_CONTEXT,
     DEFAULT_MODEL_LIST_URL,
-    downloadModel,
-    retrieveModel,
-    listModels,
-    DownloadController,
-    RetrieveModelOptions,
-    DownloadModelOptions,
-    GpuDevice
 };

gpt4all-bindings/typescript/src/gpt4all.js

@@ -2,8 +2,10 @@
 
 /// This file implements the gpt4all.d.ts file endings.
 /// Written in commonjs to support both ESM and CJS projects.
-const { existsSync } = require("fs");
+const { existsSync } = require("node:fs");
 const path = require("node:path");
+const Stream = require("node:stream");
+const assert = require("node:assert");
 const { LLModel } = require("node-gyp-build")(path.resolve(__dirname, ".."));
 const {
     retrieveModel,
@@ -18,15 +20,14 @@ const {
     DEFAULT_MODEL_LIST_URL,
 } = require("./config.js");
 const { InferenceModel, EmbeddingModel } = require("./models.js");
-const Stream = require('stream')
-const assert = require("assert");
+const { ChatSession } = require("./chat-session.js");
 
 /**
  * Loads a machine learning model with the specified name. The defacto way to create a model.
  * By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
  *
  * @param {string} modelName - The name of the model to load.
- * @param {LoadModelOptions|undefined} [options] - (Optional) Additional options for loading the model.
+ * @param {import('./gpt4all').LoadModelOptions|undefined} [options] - (Optional) Additional options for loading the model.
  * @returns {Promise<InferenceModel | EmbeddingModel>} A promise that resolves to an instance of the loaded LLModel.
  */
 async function loadModel(modelName, options = {}) {
@@ -35,10 +36,10 @@ async function loadModel(modelName, options = {}) {
         librariesPath: DEFAULT_LIBRARIES_DIRECTORY,
         type: "inference",
         allowDownload: true,
-        verbose: true,
-        device: 'cpu',
+        verbose: false,
+        device: "cpu",
         nCtx: 2048,
-        ngl : 100,
+        ngl: 100,
         ...options,
     };
 
@@ -49,12 +50,14 @@ async function loadModel(modelName, options = {}) {
         verbose: loadOptions.verbose,
     });
 
-    assert.ok(typeof loadOptions.librariesPath === 'string');
+    assert.ok(
+        typeof loadOptions.librariesPath === "string",
+        "Libraries path should be a string"
+    );
     const existingPaths = loadOptions.librariesPath
         .split(";")
         .filter(existsSync)
-        .join(';');
-    console.log("Passing these paths into runtime library search:", existingPaths)
+        .join(";");
 
     const llmOptions = {
         model_name: appendBinSuffixIfMissing(modelName),
@@ -62,13 +65,15 @@ async function loadModel(modelName, options = {}) {
         library_path: existingPaths,
         device: loadOptions.device,
         nCtx: loadOptions.nCtx,
-        ngl: loadOptions.ngl
+        ngl: loadOptions.ngl,
     };
 
     if (loadOptions.verbose) {
-        console.debug("Creating LLModel with options:", llmOptions);
+        console.debug("Creating LLModel:", {
+            llmOptions,
+            modelConfig,
+        });
     }
-    console.log(modelConfig)
     const llmodel = new LLModel(llmOptions);
     if (loadOptions.type === "embedding") {
         return new EmbeddingModel(llmodel, modelConfig);
@@ -79,75 +84,43 @@ async function loadModel(modelName, options = {}) {
     }
 }
 
-/**
- * Formats a list of messages into a single prompt string.
- */
-function formatChatPrompt(
-    messages,
-    {
-        systemPromptTemplate,
-        defaultSystemPrompt,
-        promptTemplate,
-        promptFooter,
-        promptHeader,
-    }
-) {
-    const systemMessages = messages
-        .filter((message) => message.role === "system")
-        .map((message) => message.content);
+function createEmbedding(model, text, options={}) {
+    let {
+        dimensionality = undefined,
+        longTextMode = "mean",
+        atlas = false,
+    } = options;
 
-    let fullPrompt = "";
-
-    if (promptHeader) {
-        fullPrompt += promptHeader + "\n\n";
-    }
-
-    if (systemPromptTemplate) {
-        // if user specified a template for the system prompt, put all system messages in the template
-        let systemPrompt = "";
-
-        if (systemMessages.length > 0) {
-            systemPrompt += systemMessages.join("\n");
-        }
-
-        if (systemPrompt) {
-            fullPrompt +=
-                systemPromptTemplate.replace("%1", systemPrompt) + "\n";
-        }
-    } else if (defaultSystemPrompt) {
-        // otherwise, use the system prompt from the model config and ignore system messages
-        fullPrompt += defaultSystemPrompt + "\n\n";
-    }
-
-    if (systemMessages.length > 0 && !systemPromptTemplate) {
-        console.warn(
-            "System messages were provided, but no systemPromptTemplate was specified. System messages will be ignored."
-        );
-    }
-
-    for (const message of messages) {
-        if (message.role === "user") {
-            const userMessage = promptTemplate.replace(
-                "%1",
-                message["content"]
+    if (dimensionality === undefined) {
+        dimensionality = -1;
+    } else {
+        if (dimensionality <= 0) {
+            throw new Error(
+                `Dimensionality must be undefined or a positive integer, got ${dimensionality}`
             );
-            fullPrompt += userMessage;
         }
-        if (message["role"] == "assistant") {
-            const assistantMessage = message["content"] + "\n";
-            fullPrompt += assistantMessage;
+        if (dimensionality < model.MIN_DIMENSIONALITY) {
+            console.warn(
+                `Dimensionality ${dimensionality} is less than the suggested minimum of ${model.MIN_DIMENSIONALITY}. Performance may be degraded.`
+            );
         }
     }
 
-    if (promptFooter) {
-        fullPrompt += "\n\n" + promptFooter;
+    let doMean;
+    switch (longTextMode) {
+        case "mean":
+            doMean = true;
+            break;
+        case "truncate":
+            doMean = false;
+            break;
+        default:
+            throw new Error(
+                `Long text mode must be one of 'mean' or 'truncate', got ${longTextMode}`
+            );
     }
 
-    return fullPrompt;
-}
-
-function createEmbedding(model, text) {
-    return model.embed(text);
+    return model.embed(text, options?.prefix, dimensionality, doMean, atlas);
 }
 
 const defaultCompletionOptions = {
@@ -155,162 +128,76 @@ const defaultCompletionOptions = {
     ...DEFAULT_PROMPT_CONTEXT,
 };
 
-function preparePromptAndContext(model,messages,options){
-    if (options.hasDefaultHeader !== undefined) {
-        console.warn(
-            "hasDefaultHeader (bool) is deprecated and has no effect, use promptHeader (string) instead"
-        );
-    }
-
-    if (options.hasDefaultFooter !== undefined) {
-        console.warn(
-            "hasDefaultFooter (bool) is deprecated and has no effect, use promptFooter (string) instead"
-        );
-    }
-
-    const optionsWithDefaults = {
+async function createCompletion(
+    provider,
+    input,
+    options = defaultCompletionOptions
+) {
+    const completionOptions = {
         ...defaultCompletionOptions,
         ...options,
     };
 
-    const {
-        verbose,
-        systemPromptTemplate,
-        promptTemplate,
-        promptHeader,
-        promptFooter,
-        ...promptContext
-    } = optionsWithDefaults;
-
-    
-    const prompt = formatChatPrompt(messages, {
-        systemPromptTemplate,
-        defaultSystemPrompt: model.config.systemPrompt,
-        promptTemplate: promptTemplate || model.config.promptTemplate || "%1",
-        promptHeader: promptHeader || "",
-        promptFooter: promptFooter || "",
-        // These were the default header/footer prompts used for non-chat single turn completions.
-        // both seem to be working well still with some models, so keeping them here for reference.
-        // promptHeader: '### Instruction: The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.',
-        // promptFooter: '### Response:',
-    });
+    const result = await provider.generate(
+        input,
+        completionOptions,
+    );
 
     return {
-        prompt, promptContext, verbose
-    }
-}
-
-async function createCompletion(
-    model,
-    messages,
-    options = defaultCompletionOptions
-) {
-    const { prompt, promptContext, verbose } = preparePromptAndContext(model,messages,options);
-
-    if (verbose) {
-        console.debug("Sending Prompt:\n" + prompt);
-    }
-
-    let tokensGenerated = 0
-
-    const response = await model.generate(prompt, promptContext,() => {
-        tokensGenerated++;
-        return true;
-    });
-
-    if (verbose) {
-        console.debug("Received Response:\n" + response);
-    }
-
-    return {
-        llmodel: model.llm.name(),
+        model: provider.modelName,
         usage: {
-            prompt_tokens: prompt.length,
-            completion_tokens: tokensGenerated,
-            total_tokens: prompt.length + tokensGenerated, //TODO Not sure how to get tokens in prompt
+            prompt_tokens: result.tokensIngested,
+            total_tokens: result.tokensIngested + result.tokensGenerated,
+            completion_tokens: result.tokensGenerated,
+            n_past_tokens: result.nPast,
         },
         choices: [
             {
                 message: {
                     role: "assistant",
-                    content: response,
+                    content: result.text,
                 },
+                // TODO some completion APIs also provide logprobs and finish_reason, could look into adding those
             },
         ],
     };
 }
 
-function _internal_createTokenStream(stream,model,
-    messages,
-    options = defaultCompletionOptions,callback = undefined) {
-    const { prompt, promptContext, verbose } = preparePromptAndContext(model,messages,options);
+function createCompletionStream(
+    provider,
+    input,
+    options = defaultCompletionOptions
+) {
+    const completionStream = new Stream.PassThrough({
+        encoding: "utf-8",
+    });
 
-
-    if (verbose) {
-        console.debug("Sending Prompt:\n" + prompt);
-    }
-
-    model.generate(prompt, promptContext,(tokenId, token, total) => {
-        stream.push(token);
-        
-        if(callback !== undefined){
-            return callback(tokenId,token,total);
-        }
-
-        return true;
-    }).then(() => {
-        stream.end()
-    })
-
-    return stream;
-}
-
-function _createTokenStream(model,
-    messages,
-    options = defaultCompletionOptions,callback = undefined) {
-   
-   // Silent crash if we dont do this here
-   const stream = new Stream.PassThrough({
-     encoding: 'utf-8'
-   });
-   return _internal_createTokenStream(stream,model,messages,options,callback);
-}
-
-async function* generateTokens(model,
-    messages,
-    options = defaultCompletionOptions, callback = undefined) {
-    const stream = _createTokenStream(model,messages,options,callback)
-
-    let bHasFinished = false;
-    let activeDataCallback = undefined;
-    const finishCallback = () => {
-        bHasFinished = true;
-        if(activeDataCallback !== undefined){
-            activeDataCallback(undefined);
-        }
-    }
-
-    stream.on("finish",finishCallback)
-
-    while (!bHasFinished) {
-        const token = await new Promise((res) => {
-
-            activeDataCallback = (d) => {
-                stream.off("data",activeDataCallback)
-                activeDataCallback = undefined
-                res(d);
+    const completionPromise = createCompletion(provider, input, {
+        ...options,
+        onResponseToken: (tokenId, token) => {
+            completionStream.push(token);
+            if (options.onResponseToken) {
+                return options.onResponseToken(tokenId, token);
             }
-            stream.on('data', activeDataCallback)
-        })
+        },
+    }).then((result) => {
+        completionStream.push(null);
+        completionStream.emit("end");
+        return result;
+    });
 
-        if (token == undefined) {
-            break;
-        }
+    return {
+        tokens: completionStream,
+        result: completionPromise,
+    };
+}
 
-        yield token;
+async function* createCompletionGenerator(provider, input, options) {
+    const completion = createCompletionStream(provider, input, options);
+    for await (const chunk of completion.tokens) {
+        yield chunk;
     }
-
-    stream.off("finish",finishCallback);
+    return await completion.result;
 }
 
 module.exports = {
@@ -322,10 +209,12 @@ module.exports = {
     LLModel,
     InferenceModel,
     EmbeddingModel,
+    ChatSession,
     createCompletion,
+    createCompletionStream,
+    createCompletionGenerator,
     createEmbedding,
     downloadModel,
     retrieveModel,
     loadModel,
-    generateTokens
 };

gpt4all-bindings/typescript/src/models.js

@@ -1,18 +1,138 @@
-const { normalizePromptContext, warnOnSnakeCaseKeys } = require('./util');
+const { DEFAULT_PROMPT_CONTEXT } = require("./config");
+const { ChatSession } = require("./chat-session");
+const { prepareMessagesForIngest } = require("./util");
 
 class InferenceModel {
     llm;
+    modelName;
     config;
+    activeChatSession;
 
     constructor(llmodel, config) {
         this.llm = llmodel;
         this.config = config;
+        this.modelName = this.llm.name();
     }
 
-    async generate(prompt, promptContext,callback) {
-        warnOnSnakeCaseKeys(promptContext);
-        const normalizedPromptContext = normalizePromptContext(promptContext);
-        const result = this.llm.raw_prompt(prompt, normalizedPromptContext,callback);
+    async createChatSession(options) {
+        const chatSession = new ChatSession(this, options);
+        await chatSession.initialize();
+        this.activeChatSession = chatSession;
+        return this.activeChatSession;
+    }
+
+    async generate(input, options = DEFAULT_PROMPT_CONTEXT) {
+        const { verbose, ...otherOptions } = options;
+        const promptContext = {
+            promptTemplate: this.config.promptTemplate,
+            temp:
+                otherOptions.temp ??
+                otherOptions.temperature ??
+                DEFAULT_PROMPT_CONTEXT.temp,
+            ...otherOptions,
+        };
+        
+        if (promptContext.nPast < 0) {
+            throw new Error("nPast must be a non-negative integer.");
+        }
+
+        if (verbose) {
+            console.debug("Generating completion", {
+                input,
+                promptContext,
+            });
+        }
+
+        let prompt = input;
+        let nPast = promptContext.nPast;
+        let tokensIngested = 0;
+
+        if (Array.isArray(input)) {
+            // assuming input is a messages array
+            // -> tailing user message will be used as the final prompt. its required.
+            // -> leading system message will be ingested as systemPrompt, further system messages will be ignored
+            // -> all other messages will be ingested with fakeReply
+            // -> model/context will only be kept for this completion; "stateless"
+            nPast = 0;
+            const messages = [...input];
+            const lastMessage = input[input.length - 1];
+            if (lastMessage.role !== "user") {
+                // this is most likely a user error
+                throw new Error("The final message must be of role 'user'.");
+            }
+            if (input[0].role === "system") {
+                // needs to be a pre-templated prompt ala '<|im_start|>system\nYou are an advanced mathematician.\n<|im_end|>\n'
+                const systemPrompt = input[0].content;
+                const systemRes = await this.llm.infer(systemPrompt, {
+                    promptTemplate: "%1",
+                    nPredict: 0,
+                    special: true,
+                });
+                nPast = systemRes.nPast;
+                tokensIngested += systemRes.tokensIngested;
+                messages.shift();
+            }
+
+            prompt = lastMessage.content;
+            const messagesToIngest = messages.slice(0, input.length - 1);
+            const turns = prepareMessagesForIngest(messagesToIngest);
+
+            for (const turn of turns) {
+                const turnRes = await this.llm.infer(turn.user, {
+                    ...promptContext,
+                    nPast,
+                    fakeReply: turn.assistant,
+                });
+                tokensIngested += turnRes.tokensIngested;
+                nPast = turnRes.nPast;
+            }
+        }
+
+        let tokensGenerated = 0;
+
+        const result = await this.llm.infer(prompt, {
+            ...promptContext,
+            nPast,
+            onPromptToken: (tokenId) => {
+                let continueIngestion = true;
+                tokensIngested++;
+                if (options.onPromptToken) {
+                    // catch errors because if they go through cpp they will loose stacktraces
+                    try {
+                        // don't cancel ingestion unless user explicitly returns false
+                        continueIngestion =
+                            options.onPromptToken(tokenId) !== false;
+                    } catch (e) {
+                        console.error("Error in onPromptToken callback", e);
+                        continueIngestion = false;
+                    }
+                }
+                return continueIngestion;
+            },
+            onResponseToken: (tokenId, token) => {
+                let continueGeneration = true;
+                tokensGenerated++;
+                if (options.onResponseToken) {
+                    try {
+                        // don't cancel the generation unless user explicitly returns false
+                        continueGeneration =
+                            options.onResponseToken(tokenId, token) !== false;
+                    } catch (err) {
+                        console.error("Error in onResponseToken callback", err);
+                        continueGeneration = false;
+                    }
+                }
+                return continueGeneration;
+            },
+        });
+
+        result.tokensGenerated = tokensGenerated;
+        result.tokensIngested = tokensIngested;
+
+        if (verbose) {
+            console.debug("Finished completion:\n", result);
+        }
+
         return result;
     }
 
@@ -24,14 +144,14 @@ class InferenceModel {
 class EmbeddingModel {
     llm;
     config;
-
+    MIN_DIMENSIONALITY = 64;
     constructor(llmodel, config) {
         this.llm = llmodel;
         this.config = config;
     }
 
-    embed(text) {
-        return this.llm.embed(text)
+    embed(text, prefix, dimensionality, do_mean, atlas) {
+        return this.llm.embed(text, prefix, dimensionality, do_mean, atlas);
     }
 
     dispose() {
@@ -39,7 +159,6 @@ class EmbeddingModel {
     }
 }
 
-
 module.exports = {
     InferenceModel,
     EmbeddingModel,

gpt4all-bindings/typescript/src/util.js

@@ -1,8 +1,7 @@
-const { createWriteStream, existsSync, statSync } = require("node:fs");
+const { createWriteStream, existsSync, statSync, mkdirSync } = require("node:fs");
 const fsp = require("node:fs/promises");
 const { performance } = require("node:perf_hooks");
 const path = require("node:path");
-const { mkdirp } = require("mkdirp");
 const md5File = require("md5-file");
 const {
     DEFAULT_DIRECTORY,
@@ -50,6 +49,63 @@ function appendBinSuffixIfMissing(name) {
     return name;
 }
 
+function prepareMessagesForIngest(messages) {
+    const systemMessages = messages.filter(
+        (message) => message.role === "system"
+    );
+    if (systemMessages.length > 0) {
+        console.warn(
+            "System messages are currently not supported and will be ignored. Use the systemPrompt option instead."
+        );
+    }
+
+    const userAssistantMessages = messages.filter(
+        (message) => message.role !== "system"
+    );
+
+    // make sure the first message is a user message
+    // if its not, the turns will be out of order
+    if (userAssistantMessages[0].role !== "user") {
+        userAssistantMessages.unshift({
+            role: "user",
+            content: "",
+        });
+    }
+
+    // create turns of user input + assistant reply
+    const turns = [];
+    let userMessage = null;
+    let assistantMessage = null;
+
+    for (const message of userAssistantMessages) {
+        // consecutive messages of the same role are concatenated into one message
+        if (message.role === "user") {
+            if (!userMessage) {
+                userMessage = message.content;
+            } else {
+                userMessage += "\n" + message.content;
+            }
+        } else if (message.role === "assistant") {
+            if (!assistantMessage) {
+                assistantMessage = message.content;
+            } else {
+                assistantMessage += "\n" + message.content;
+            }
+        }
+
+        if (userMessage && assistantMessage) {
+            turns.push({
+                user: userMessage,
+                assistant: assistantMessage,
+            });
+            userMessage = null;
+            assistantMessage = null;
+        }
+    }
+
+    return turns;
+}
+
 // readChunks() reads from the provided reader and yields the results into an async iterable
 // https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
 function readChunks(reader) {
@@ -64,49 +120,13 @@ function readChunks(reader) {
     };
 }
 
-/**
- * Prints a warning if any keys in the prompt context are snake_case.
- */
-function warnOnSnakeCaseKeys(promptContext) {
-    const snakeCaseKeys = Object.keys(promptContext).filter((key) =>
-        key.includes("_")
-    );
-
-    if (snakeCaseKeys.length > 0) {
-        console.warn(
-            "Prompt context keys should be camelCase. Support for snake_case might be removed in the future. Found keys: " +
-                snakeCaseKeys.join(", ")
-        );
-    }
-}
-
-/**
- * Converts all keys in the prompt context to snake_case
- * For duplicate definitions, the value of the last occurrence will be used.
- */
-function normalizePromptContext(promptContext) {
-    const normalizedPromptContext = {};
-
-    for (const key in promptContext) {
-        if (promptContext.hasOwnProperty(key)) {
-            const snakeKey = key.replace(
-                /[A-Z]/g,
-                (match) => `_${match.toLowerCase()}`
-            );
-            normalizedPromptContext[snakeKey] = promptContext[key];
-        }
-    }
-
-    return normalizedPromptContext;
-}
-
 function downloadModel(modelName, options = {}) {
     const downloadOptions = {
         modelPath: DEFAULT_DIRECTORY,
         verbose: false,
         ...options,
     };
-    
+
     const modelFileName = appendBinSuffixIfMissing(modelName);
     const partialModelPath = path.join(
         downloadOptions.modelPath,
@@ -114,16 +134,17 @@ function downloadModel(modelName, options = {}) {
     );
     const finalModelPath = path.join(downloadOptions.modelPath, modelFileName);
     const modelUrl =
-        downloadOptions.url ?? `https://gpt4all.io/models/gguf/${modelFileName}`;
+        downloadOptions.url ??
+        `https://gpt4all.io/models/gguf/${modelFileName}`;
 
-    mkdirp.sync(downloadOptions.modelPath)
+    mkdirSync(downloadOptions.modelPath, { recursive: true });
 
     if (existsSync(finalModelPath)) {
         throw Error(`Model already exists at ${finalModelPath}`);
     }
-    
+
     if (downloadOptions.verbose) {
-        console.log(`Downloading ${modelName} from ${modelUrl}`);
+        console.debug(`Downloading ${modelName} from ${modelUrl}`);
     }
 
     const headers = {
@@ -134,7 +155,9 @@ function downloadModel(modelName, options = {}) {
     const writeStreamOpts = {};
 
     if (existsSync(partialModelPath)) {
-        console.log("Partial model exists, resuming download...");
+        if (downloadOptions.verbose) {
+            console.debug("Partial model exists, resuming download...");
+        }
         const startRange = statSync(partialModelPath).size;
         headers["Range"] = `bytes=${startRange}-`;
         writeStreamOpts.flags = "a";
@@ -144,15 +167,15 @@ function downloadModel(modelName, options = {}) {
     const signal = abortController.signal;
 
     const finalizeDownload = async () => {
-        if (options.md5sum) {
+        if (downloadOptions.md5sum) {
             const fileHash = await md5File(partialModelPath);
-            if (fileHash !== options.md5sum) {
+            if (fileHash !== downloadOptions.md5sum) {
                 await fsp.unlink(partialModelPath);
-                const message = `Model "${modelName}" failed verification: Hashes mismatch. Expected ${options.md5sum}, got ${fileHash}`;
+                const message = `Model "${modelName}" failed verification: Hashes mismatch. Expected ${downloadOptions.md5sum}, got ${fileHash}`;
                 throw Error(message);
             }
-            if (options.verbose) {
-                console.log(`MD5 hash verified: ${fileHash}`);
+            if (downloadOptions.verbose) {
+                console.debug(`MD5 hash verified: ${fileHash}`);
             }
         }
 
@@ -163,8 +186,8 @@ function downloadModel(modelName, options = {}) {
     const downloadPromise = new Promise((resolve, reject) => {
         let timestampStart;
 
-        if (options.verbose) {
-            console.log(`Downloading @ ${partialModelPath} ...`);
+        if (downloadOptions.verbose) {
+            console.debug(`Downloading @ ${partialModelPath} ...`);
             timestampStart = performance.now();
         }
 
@@ -179,7 +202,7 @@ function downloadModel(modelName, options = {}) {
         });
 
         writeStream.on("finish", () => {
-            if (options.verbose) {
+            if (downloadOptions.verbose) {
                 const elapsed = performance.now() - timestampStart;
                 console.log(`Finished. Download took ${elapsed.toFixed(2)} ms`);
             }
@@ -221,10 +244,10 @@ async function retrieveModel(modelName, options = {}) {
     const retrieveOptions = {
         modelPath: DEFAULT_DIRECTORY,
         allowDownload: true,
-        verbose: true,
+        verbose: false,
         ...options,
     };
-    await mkdirp(retrieveOptions.modelPath);
+    mkdirSync(retrieveOptions.modelPath, { recursive: true });
 
     const modelFileName = appendBinSuffixIfMissing(modelName);
     const fullModelPath = path.join(retrieveOptions.modelPath, modelFileName);
@@ -236,7 +259,7 @@ async function retrieveModel(modelName, options = {}) {
         file: retrieveOptions.modelConfigFile,
         url:
             retrieveOptions.allowDownload &&
-            "https://gpt4all.io/models/models2.json",
+            "https://gpt4all.io/models/models3.json",
     });
 
     const loadedModelConfig = availableModels.find(
@@ -262,10 +285,9 @@ async function retrieveModel(modelName, options = {}) {
         config.path = fullModelPath;
 
         if (retrieveOptions.verbose) {
-            console.log(`Found ${modelName} at ${fullModelPath}`);
+            console.debug(`Found ${modelName} at ${fullModelPath}`);
         }
     } else if (retrieveOptions.allowDownload) {
-
         const downloadController = downloadModel(modelName, {
             modelPath: retrieveOptions.modelPath,
             verbose: retrieveOptions.verbose,
@@ -278,7 +300,7 @@ async function retrieveModel(modelName, options = {}) {
         config.path = downloadPath;
 
         if (retrieveOptions.verbose) {
-            console.log(`Model downloaded to ${downloadPath}`);
+            console.debug(`Model downloaded to ${downloadPath}`);
         }
     } else {
         throw Error("Failed to retrieve model.");
@@ -288,9 +310,8 @@ async function retrieveModel(modelName, options = {}) {
 
 module.exports = {
     appendBinSuffixIfMissing,
+    prepareMessagesForIngest,
     downloadModel,
     retrieveModel,
     listModels,
-    normalizePromptContext,
-    warnOnSnakeCaseKeys,
 };

gpt4all-bindings/typescript/test/gpt4all.test.js

@@ -7,7 +7,6 @@ const {
     listModels,
     downloadModel,
     appendBinSuffixIfMissing,
-    normalizePromptContext,
 } = require("../src/util.js");
 const {
     DEFAULT_DIRECTORY,
@@ -19,8 +18,6 @@ const {
     createPrompt,
     createCompletion,
 } = require("../src/gpt4all.js");
-const { mock } = require("node:test");
-const { mkdirp } = require("mkdirp");
 
 describe("config", () => {
     test("default paths constants are available and correct", () => {
@@ -87,7 +84,7 @@ describe("listModels", () => {
         expect(fetch).toHaveBeenCalledTimes(0);
         expect(models[0]).toEqual(fakeModel);
     });
-    
+
     it("should throw an error if neither url nor file is specified", async () => {
         await expect(listModels(null)).rejects.toThrow(
             "No model list source specified. Please specify either a url or a file."
@@ -141,10 +138,10 @@ describe("downloadModel", () => {
         mockAbortController.mockReset();
         mockFetch.mockClear();
         global.fetch.mockRestore();
-        
+
         const rootDefaultPath = path.resolve(DEFAULT_DIRECTORY),
               partialPath = path.resolve(rootDefaultPath, fakeModelName+'.part'),
-              fullPath = path.resolve(rootDefaultPath, fakeModelName+'.bin')  
+              fullPath = path.resolve(rootDefaultPath, fakeModelName+'.bin')
 
         //if tests fail, remove the created files
         // acts as cleanup if tests fail
@@ -206,46 +203,3 @@ describe("downloadModel", () => {
     // test("should be able to cancel and resume a download", async () => {
     // });
 });
-
-describe("normalizePromptContext", () => {
-    it("should convert a dict with camelCased keys to snake_case", () => {
-        const camelCased = {
-            topK: 20,
-            repeatLastN: 10,
-        };
-
-        const expectedSnakeCased = {
-            top_k: 20,
-            repeat_last_n: 10,
-        };
-
-        const result = normalizePromptContext(camelCased);
-        expect(result).toEqual(expectedSnakeCased);
-    });
-
-    it("should convert a mixed case dict to snake_case, last value taking precedence", () => {
-        const mixedCased = {
-            topK: 20,
-            top_k: 10,
-            repeatLastN: 10,
-        };
-
-        const expectedSnakeCased = {
-            top_k: 10,
-            repeat_last_n: 10,
-        };
-
-        const result = normalizePromptContext(mixedCased);
-        expect(result).toEqual(expectedSnakeCased);
-    });
-
-    it("should not modify already snake cased dict", () => {
-        const snakeCased = {
-            top_k: 10,
-            repeast_last_n: 10,
-        };
-
-        const result = normalizePromptContext(snakeCased);
-        expect(result).toEqual(snakeCased);
-    });
-});

gpt4all-bindings/typescript/yarn.lock

@@ -2300,7 +2300,6 @@ __metadata:
     documentation: ^14.0.2
     jest: ^29.5.0
     md5-file: ^5.0.0
-    mkdirp: ^3.0.1
     node-addon-api: ^6.1.0
     node-gyp: 9.x.x
     node-gyp-build: ^4.6.0
@@ -2631,9 +2630,9 @@ __metadata:
   linkType: hard
 
 "ip@npm:^2.0.0":
-  version: 2.0.0
-  resolution: "ip@npm:2.0.0"
-  checksum: cfcfac6b873b701996d71ec82a7dd27ba92450afdb421e356f44044ed688df04567344c36cbacea7d01b1c39a4c732dc012570ebe9bebfb06f27314bca625349
+  version: 2.0.1
+  resolution: "ip@npm:2.0.1"
+  checksum: d765c9fd212b8a99023a4cde6a558a054c298d640fec1020567494d257afd78ca77e37126b1a3ef0e053646ced79a816bf50621d38d5e768cdde0431fa3b0d35
   languageName: node
   linkType: hard
 
@@ -4258,15 +4257,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"mkdirp@npm:^3.0.1":
-  version: 3.0.1
-  resolution: "mkdirp@npm:3.0.1"
-  bin:
-    mkdirp: dist/cjs/src/bin.js
-  checksum: 972deb188e8fb55547f1e58d66bd6b4a3623bf0c7137802582602d73e6480c1c2268dcbafbfb1be466e00cc7e56ac514d7fd9334b7cf33e3e2ab547c16f83a8d
-  languageName: node
-  linkType: hard
-
 "mri@npm:^1.1.0":
   version: 1.2.0
   resolution: "mri@npm:1.2.0"
@@ -5405,8 +5395,8 @@ __metadata:
   linkType: hard
 
 "tar@npm:^6.1.11, tar@npm:^6.1.2":
-  version: 6.2.0
-  resolution: "tar@npm:6.2.0"
+  version: 6.2.1
+  resolution: "tar@npm:6.2.1"
   dependencies:
     chownr: ^2.0.0
     fs-minipass: ^2.0.0
@@ -5414,7 +5404,7 @@ __metadata:
     minizlib: ^2.1.1
     mkdirp: ^1.0.3
     yallist: ^4.0.0
-  checksum: db4d9fe74a2082c3a5016630092c54c8375ff3b280186938cfd104f2e089c4fd9bad58688ef6be9cf186a889671bf355c7cda38f09bbf60604b281715ca57f5c
+  checksum: f1322768c9741a25356c11373bce918483f40fa9a25c69c59410c8a1247632487edef5fe76c5f12ac51a6356d2f1829e96d2bc34098668a2fc34d76050ac2b6c
   languageName: node
   linkType: hard
 

gpt4all-chat/CMakeLists.txt

@@ -17,8 +17,8 @@ if(APPLE)
 endif()
 
 set(APP_VERSION_MAJOR 2)
-set(APP_VERSION_MINOR 7)
-set(APP_VERSION_PATCH 4)
+set(APP_VERSION_MINOR 8)
+set(APP_VERSION_PATCH 0)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file
@@ -65,9 +65,25 @@ add_subdirectory(../gpt4all-backend llmodel)
 
 set(METAL_SHADER_FILE)
 if(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
-  set(METAL_SHADER_FILE ../gpt4all-backend/llama.cpp-mainline/ggml-metal.metal)
+    set(METAL_SHADER_FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib)
 endif()
 
+set(APP_ICON_RESOURCE)
+if (WIN32)
+    set(APP_ICON_RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.rc")
+elseif (APPLE)
+    # The MACOSX_BUNDLE_ICON_FILE variable is added to the Info.plist
+    # generated by CMake. This variable contains the .icns file name,
+    # without the path.
+    set(MACOSX_BUNDLE_ICON_FILE gpt4all.icns)
+
+    # And the following tells CMake where to find and install the file itself.
+    set(APP_ICON_RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns")
+    set_source_files_properties(${APP_ICON_RESOURCE} PROPERTIES
+        MACOSX_PACKAGE_LOCATION "Resources")
+endif()
+
+
 qt_add_executable(chat
     main.cpp
     chat.h chat.cpp
@@ -87,6 +103,7 @@ qt_add_executable(chat
     logger.h logger.cpp
     responsetext.h responsetext.cpp
     ${METAL_SHADER_FILE}
+    ${APP_ICON_RESOURCE}
 )
 
 qt_add_qml_module(chat
@@ -131,7 +148,6 @@ qt_add_qml_module(chat
       icons/send_message.svg
       icons/stop_generating.svg
       icons/regenerate.svg
-      icons/chat.svg
       icons/close.svg
       icons/copy.svg
       icons/db.svg
@@ -140,8 +156,6 @@ qt_add_qml_module(chat
       icons/eject.svg
       icons/edit.svg
       icons/image.svg
-      icons/info.svg
-      icons/search.svg
       icons/trash.svg
       icons/network.svg
       icons/thumbs_up.svg
@@ -151,8 +165,6 @@ qt_add_qml_module(chat
       icons/logo.svg
       icons/logo-32.png
       icons/logo-48.png
-      icons/favicon.ico
-      icons/favicon.icns
 )
 
 set_target_properties(chat PROPERTIES
@@ -161,7 +173,6 @@ set_target_properties(chat PROPERTIES
     MACOSX_BUNDLE_SHORT_VERSION_STRING ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}
     MACOSX_BUNDLE TRUE
     WIN32_EXECUTABLE TRUE
-    MACOSX_BUNDLE_ICON_FILE "favicon.icns"
 )
 
 if(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
@@ -174,7 +185,7 @@ if(METAL_SHADER_FILE)
     set_target_properties(chat PROPERTIES
         RESOURCE ${METAL_SHADER_FILE}
     )
-    configure_file(${METAL_SHADER_FILE} bin/ggml-metal.metal COPYONLY)
+    add_dependencies(chat ggml-metal)
 endif()
 
 target_compile_definitions(chat
@@ -196,18 +207,61 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
 endif()
 
 install(TARGETS chat DESTINATION bin COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llmodel DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+
+install(
+    TARGETS llmodel
+    LIBRARY DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}  # .so/.dylib
+    RUNTIME DESTINATION bin COMPONENT ${COMPONENT_NAME_MAIN}  # .dll
+)
 
 # We should probably iterate through the list of the cmake for backend, but these need to be installed
 # to the this component's dir for the finicky qt installer to work
-install(TARGETS gptj-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS gptj-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llama-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llama-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llamamodel-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llamamodel-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-if(APPLE)
-install(TARGETS llamamodel-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+if (LLMODEL_KOMPUTE)
+    set(MODEL_IMPL_TARGETS
+        llamamodel-mainline-kompute
+        llamamodel-mainline-kompute-avxonly
+        gptj-kompute
+        gptj-kompute-avxonly
+    )
+else()
+    set(MODEL_IMPL_TARGETS
+        llamamodel-mainline-cpu
+        llamamodel-mainline-cpu-avxonly
+        gptj-cpu
+        gptj-cpu-avxonly
+    )
+endif()
+
+if (APPLE)
+    list(APPEND MODEL_IMPL_TARGETS llamamodel-mainline-metal)
+endif()
+
+install(
+    TARGETS ${MODEL_IMPL_TARGETS}
+    LIBRARY DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}  # .so/.dylib
+    RUNTIME DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}  # .dll
+)
+
+if (LLMODEL_CUDA)
+    set_property(TARGET llamamodel-mainline-cuda llamamodel-mainline-cuda-avxonly
+                 APPEND PROPERTY INSTALL_RPATH "$ORIGIN")
+
+    install(
+        TARGETS llamamodel-mainline-cuda
+                llamamodel-mainline-cuda-avxonly
+        RUNTIME_DEPENDENCY_SET llama-cuda-deps
+        LIBRARY DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}  # .so/.dylib
+        RUNTIME DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}  # .dll
+    )
+    if (WIN32)
+        install(
+            RUNTIME_DEPENDENCY_SET llama-cuda-deps
+            PRE_EXCLUDE_REGEXES "^(nvcuda|api-ms-.*)\\.dll$"
+            POST_INCLUDE_REGEXES "(^|[/\\\\])(lib)?(cuda|cublas)" POST_EXCLUDE_REGEXES .
+            DIRECTORIES "${CUDAToolkit_BIN_DIR}"
+            DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}
+        )
+    endif()
 endif()
 
 set(CPACK_GENERATOR "IFW")
@@ -228,7 +282,7 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES Windows)
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-windows.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-windows.cmake)
     set(CPACK_IFW_ROOT "C:/Qt/Tools/QtInstallerFramework/4.6")
-    set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.ico")
+    set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.ico")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-win64")
     set(CPACK_IFW_TARGET_DIRECTORY "@HomeDir@\\${COMPONENT_NAME_MAIN}")
 elseif(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
@@ -237,11 +291,11 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-mac.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-mac.cmake)
     set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.6")
-    set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns")
+    set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-darwin")
     set(CPACK_IFW_TARGET_DIRECTORY "@ApplicationsDir@/${COMPONENT_NAME_MAIN}")
     set(CPACK_BUNDLE_NAME ${COMPONENT_NAME_MAIN})
-    set(CPACK_BUNDLE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns")
+    set(CPACK_BUNDLE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns")
 endif()
 
 set(CPACK_PACKAGE_INSTALL_DIRECTORY ${COMPONENT_NAME_MAIN})

gpt4all-chat/build_and_run.md

@@ -6,9 +6,9 @@ gpt4all-chat from source.
 
 ## Prerequisites
 
-On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+You will need a compiler. On Windows, you should install Visual Studio with the C++ Development components. On macOS, you will need the full version of Xcode—Xcode Command Line Tools lacks certain required tools. On Linux, you will need a GCC or Clang toolchain with C++ support.
 
-macOS users do not need Vulkan, as GPT4All will use Metal instead.
+On Windows and Linux, building GPT4All with full GPU support requires the [Vulkan SDK](https://vulkan.lunarg.com/sdk/home) and the latest [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads).
 
 ## Note for Linux users
 

gpt4all-chat/chat.cpp

@@ -54,7 +54,7 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::reportFallbackReason, this, &Chat::handleFallbackReasonChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::databaseResultsChanged, this, &Chat::handleDatabaseResultsChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelInfoChanged, this, &Chat::handleModelInfoChanged, Qt::QueuedConnection);
-    connect(m_llmodel, &ChatLLM::trySwitchContextOfLoadedModelCompleted, this, &Chat::trySwitchContextOfLoadedModelCompleted, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::trySwitchContextOfLoadedModelCompleted, this, &Chat::handleTrySwitchContextOfLoadedModelCompleted, Qt::QueuedConnection);
 
     connect(this, &Chat::promptRequested, m_llmodel, &ChatLLM::prompt, Qt::QueuedConnection);
     connect(this, &Chat::modelChangeRequested, m_llmodel, &ChatLLM::modelChangeRequested, Qt::QueuedConnection);
@@ -95,16 +95,6 @@ void Chat::processSystemPrompt()
     emit processSystemPromptRequested();
 }
 
-bool Chat::isModelLoaded() const
-{
-    return m_modelLoadingPercentage == 1.0f;
-}
-
-float Chat::modelLoadingPercentage() const
-{
-    return m_modelLoadingPercentage;
-}
-
 void Chat::resetResponseState()
 {
     if (m_responseInProgress && m_responseState == Chat::LocalDocsRetrieval)
@@ -167,9 +157,16 @@ void Chat::handleModelLoadingPercentageChanged(float loadingPercentage)
     if (loadingPercentage == m_modelLoadingPercentage)
         return;
 
+    bool wasLoading = isCurrentlyLoading();
+    bool wasLoaded = isModelLoaded();
+
     m_modelLoadingPercentage = loadingPercentage;
     emit modelLoadingPercentageChanged();
-    if (m_modelLoadingPercentage == 1.0f || m_modelLoadingPercentage == 0.0f)
+
+    if (isCurrentlyLoading() != wasLoading)
+        emit isCurrentlyLoadingChanged();
+
+    if (isModelLoaded() != wasLoaded)
         emit isModelLoadedChanged();
 }
 
@@ -179,59 +176,62 @@ void Chat::promptProcessing()
     emit responseStateChanged();
 }
 
-void Chat::responseStopped()
+void Chat::responseStopped(qint64 promptResponseMs)
 {
     m_tokenSpeed = QString();
     emit tokenSpeedChanged();
 
-    if (MySettings::globalInstance()->localDocsShowReferences()) {
-        const QString chatResponse = response();
-        QList<QString> references;
-        QList<QString> referencesContext;
-        int validReferenceNumber = 1;
-        for (const ResultInfo &info : databaseResults()) {
-            if (info.file.isEmpty())
-                continue;
-            if (validReferenceNumber == 1)
-                references.append((!chatResponse.endsWith("\n") ? "\n" : QString()) + QStringLiteral("\n---"));
-            QString reference;
-            {
-                QTextStream stream(&reference);
-                stream << (validReferenceNumber++) << ". ";
-                if (!info.title.isEmpty())
-                    stream << "\"" << info.title << "\". ";
-                if (!info.author.isEmpty())
-                    stream << "By " << info.author << ". ";
-                if (!info.date.isEmpty())
-                    stream << "Date: " << info.date << ". ";
-                stream << "In " << info.file << ". ";
-                if (info.page != -1)
-                    stream << "Page " << info.page << ". ";
-                if (info.from != -1) {
-                    stream << "Lines " << info.from;
-                    if (info.to != -1)
-                        stream << "-" << info.to;
-                    stream << ". ";
-                }
-                stream << "[Context](context://" << validReferenceNumber - 1 << ")";
+    const QString chatResponse = response();
+    QList<QString> references;
+    QList<QString> referencesContext;
+    int validReferenceNumber = 1;
+    for (const ResultInfo &info : databaseResults()) {
+        if (info.file.isEmpty())
+            continue;
+        if (validReferenceNumber == 1)
+            references.append((!chatResponse.endsWith("\n") ? "\n" : QString()) + QStringLiteral("\n---"));
+        QString reference;
+        {
+            QTextStream stream(&reference);
+            stream << (validReferenceNumber++) << ". ";
+            if (!info.title.isEmpty())
+                stream << "\"" << info.title << "\". ";
+            if (!info.author.isEmpty())
+                stream << "By " << info.author << ". ";
+            if (!info.date.isEmpty())
+                stream << "Date: " << info.date << ". ";
+            stream << "In " << info.file << ". ";
+            if (info.page != -1)
+                stream << "Page " << info.page << ". ";
+            if (info.from != -1) {
+                stream << "Lines " << info.from;
+                if (info.to != -1)
+                    stream << "-" << info.to;
+                stream << ". ";
             }
-            references.append(reference);
-            referencesContext.append(info.text);
+            stream << "[Context](context://" << validReferenceNumber - 1 << ")";
         }
-
-        const int index = m_chatModel->count() - 1;
-        m_chatModel->updateReferences(index, references.join("\n"), referencesContext);
-        emit responseChanged();
+        references.append(reference);
+        referencesContext.append(info.text);
     }
 
+    const int index = m_chatModel->count() - 1;
+    m_chatModel->updateReferences(index, references.join("\n"), referencesContext);
+    emit responseChanged();
+
     m_responseInProgress = false;
     m_responseState = Chat::ResponseStopped;
     emit responseInProgressChanged();
     emit responseStateChanged();
     if (m_generatedName.isEmpty())
         emit generateNameRequested();
-    if (chatModel()->count() < 3)
-        Network::globalInstance()->sendChatStarted();
+
+    Network::globalInstance()->trackChatEvent("response_complete", {
+        {"first", m_firstResponse},
+        {"message_count", chatModel()->count()},
+        {"$duration", promptResponseMs / 1000.},
+    });
+    m_firstResponse = false;
 }
 
 ModelInfo Chat::modelInfo() const
@@ -244,10 +244,6 @@ void Chat::setModelInfo(const ModelInfo &modelInfo)
     if (m_modelInfo == modelInfo && isModelLoaded())
         return;
 
-    m_modelLoadingPercentage = std::numeric_limits<float>::min(); // small non-zero positive value
-    emit isModelLoadedChanged();
-    m_modelLoadingError = QString();
-    emit modelLoadingErrorChanged();
     m_modelInfo = modelInfo;
     emit modelInfoChanged();
     emit modelChangeRequested(modelInfo);
@@ -317,8 +313,9 @@ void Chat::forceReloadModel()
 
 void Chat::trySwitchContextOfLoadedModel()
 {
-    emit trySwitchContextOfLoadedModelAttempted();
-    m_llmodel->setShouldTrySwitchContext(true);
+    m_trySwitchContextInProgress = 1;
+    emit trySwitchContextInProgressChanged();
+    m_llmodel->requestTrySwitchContext();
 }
 
 void Chat::generatedNameChanged(const QString &name)
@@ -333,14 +330,16 @@ void Chat::generatedNameChanged(const QString &name)
 
 void Chat::handleRecalculating()
 {
-    Network::globalInstance()->sendRecalculatingContext(m_chatModel->count());
+    Network::globalInstance()->trackChatEvent("recalc_context", { {"length", m_chatModel->count()} });
     emit recalcChanged();
 }
 
 void Chat::handleModelLoadingError(const QString &error)
 {
-    auto stream = qWarning().noquote() << "ERROR:" << error << "id";
-    stream.quote() << id();
+    if (!error.isEmpty()) {
+        auto stream = qWarning().noquote() << "ERROR:" << error << "id";
+        stream.quote() << id();
+    }
     m_modelLoadingError = error;
     emit modelLoadingErrorChanged();
 }
@@ -377,6 +376,11 @@ void Chat::handleModelInfoChanged(const ModelInfo &modelInfo)
     emit modelInfoChanged();
 }
 
+void Chat::handleTrySwitchContextOfLoadedModelCompleted(int value) {
+    m_trySwitchContextInProgress = value;
+    emit trySwitchContextInProgressChanged();
+}
+
 bool Chat::serialize(QDataStream &stream, int version) const
 {
     stream << m_creationDate;

gpt4all-chat/chat.h

@@ -17,6 +17,7 @@ class Chat : public QObject
     Q_PROPERTY(QString name READ name WRITE setName NOTIFY nameChanged)
     Q_PROPERTY(ChatModel *chatModel READ chatModel NOTIFY chatModelChanged)
     Q_PROPERTY(bool isModelLoaded READ isModelLoaded NOTIFY isModelLoadedChanged)
+    Q_PROPERTY(bool isCurrentlyLoading READ isCurrentlyLoading NOTIFY isCurrentlyLoadingChanged)
     Q_PROPERTY(float modelLoadingPercentage READ modelLoadingPercentage NOTIFY modelLoadingPercentageChanged)
     Q_PROPERTY(QString response READ response NOTIFY responseChanged)
     Q_PROPERTY(ModelInfo modelInfo READ modelInfo WRITE setModelInfo NOTIFY modelInfoChanged)
@@ -30,6 +31,8 @@ class Chat : public QObject
     Q_PROPERTY(QString device READ device NOTIFY deviceChanged);
     Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY fallbackReasonChanged);
     Q_PROPERTY(LocalDocsCollectionsModel *collectionModel READ collectionModel NOTIFY collectionModelChanged)
+    // 0=no, 1=waiting, 2=working
+    Q_PROPERTY(int trySwitchContextInProgress READ trySwitchContextInProgress NOTIFY trySwitchContextInProgressChanged)
     QML_ELEMENT
     QML_UNCREATABLE("Only creatable from c++!")
 
@@ -62,8 +65,9 @@ public:
 
     Q_INVOKABLE void reset();
     Q_INVOKABLE void processSystemPrompt();
-    Q_INVOKABLE bool isModelLoaded() const;
-    Q_INVOKABLE float modelLoadingPercentage() const;
+    bool  isModelLoaded()          const { return m_modelLoadingPercentage == 1.0f; }
+    bool  isCurrentlyLoading()     const { return m_modelLoadingPercentage > 0.0f && m_modelLoadingPercentage < 1.0f; }
+    float modelLoadingPercentage() const { return m_modelLoadingPercentage; }
     Q_INVOKABLE void prompt(const QString &prompt);
     Q_INVOKABLE void regenerateResponse();
     Q_INVOKABLE void stopGenerating();
@@ -105,6 +109,8 @@ public:
     QString device() const { return m_device; }
     QString fallbackReason() const { return m_fallbackReason; }
 
+    int trySwitchContextInProgress() const { return m_trySwitchContextInProgress; }
+
 public Q_SLOTS:
     void serverNewPromptResponsePair(const QString &prompt);
 
@@ -113,6 +119,7 @@ Q_SIGNALS:
     void nameChanged();
     void chatModelChanged();
     void isModelLoadedChanged();
+    void isCurrentlyLoadingChanged();
     void modelLoadingPercentageChanged();
     void modelLoadingWarning(const QString &warning);
     void responseChanged();
@@ -136,14 +143,13 @@ Q_SIGNALS:
     void deviceChanged();
     void fallbackReasonChanged();
     void collectionModelChanged();
-    void trySwitchContextOfLoadedModelAttempted();
-    void trySwitchContextOfLoadedModelCompleted(bool);
+    void trySwitchContextInProgressChanged();
 
 private Q_SLOTS:
     void handleResponseChanged(const QString &response);
     void handleModelLoadingPercentageChanged(float);
     void promptProcessing();
-    void responseStopped();
+    void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
     void handleRecalculating();
     void handleModelLoadingError(const QString &error);
@@ -152,6 +158,7 @@ private Q_SLOTS:
     void handleFallbackReasonChanged(const QString &device);
     void handleDatabaseResultsChanged(const QList<ResultInfo> &results);
     void handleModelInfoChanged(const ModelInfo &modelInfo);
+    void handleTrySwitchContextOfLoadedModelCompleted(int value);
 
 private:
     QString m_id;
@@ -175,6 +182,9 @@ private:
     bool m_shouldDeleteLater = false;
     float m_modelLoadingPercentage = 0.0f;
     LocalDocsCollectionsModel *m_collectionModel;
+    bool m_firstResponse = true;
+    int m_trySwitchContextInProgress = 0;
+    bool m_isCurrentlyLoading = false;
 };
 
 #endif // CHAT_H

gpt4all-chat/chatlistmodel.cpp

@@ -15,18 +15,19 @@ ChatListModel *ChatListModel::globalInstance()
 }
 
 ChatListModel::ChatListModel()
-    : QAbstractListModel(nullptr)
+    : QAbstractListModel(nullptr) {}
+
+void ChatListModel::loadChats()
 {
     addChat();
 
     ChatsRestoreThread *thread = new ChatsRestoreThread;
-    connect(thread, &ChatsRestoreThread::chatRestored, this, &ChatListModel::restoreChat);
-    connect(thread, &ChatsRestoreThread::finished, this, &ChatListModel::chatsRestoredFinished);
+    connect(thread, &ChatsRestoreThread::chatRestored, this, &ChatListModel::restoreChat, Qt::QueuedConnection);
+    connect(thread, &ChatsRestoreThread::finished, this, &ChatListModel::chatsRestoredFinished, Qt::QueuedConnection);
     connect(thread, &ChatsRestoreThread::finished, thread, &QObject::deleteLater);
     thread->start();
 
     connect(MySettings::globalInstance(), &MySettings::serverChatChanged, this, &ChatListModel::handleServerEnabledChanged);
-
 }
 
 void ChatListModel::removeChatFile(Chat *chat) const

gpt4all-chat/chatlistmodel.h

@@ -81,11 +81,15 @@ public:
     bool shouldSaveChatGPTChats() const;
     void setShouldSaveChatGPTChats(bool b);
 
+    Q_INVOKABLE void loadChats();
+
     Q_INVOKABLE void addChat()
     {
-        // Don't add a new chat if we already have one
-        if (m_newChat)
+        // Select the existing new chat if we already have one
+        if (m_newChat) {
+            setCurrentChat(m_newChat);
             return;
+        }
 
         // Create a new chat pointer and connect it to determine when it is populated
         m_newChat = new Chat(this);
@@ -114,20 +118,6 @@ public:
         emit countChanged();
     }
 
-    void setNewChat(Chat* chat)
-    {
-        // Don't add a new chat if we already have one
-        if (m_newChat)
-            return;
-
-        m_newChat = chat;
-        connect(m_newChat->chatModel(), &ChatModel::countChanged,
-            this, &ChatListModel::newChatCountChanged);
-        connect(m_newChat, &Chat::nameChanged,
-            this, &ChatListModel::nameChanged);
-        setCurrentChat(m_newChat);
-    }
-
     Q_INVOKABLE void removeChat(Chat* chat)
     {
         Q_ASSERT(chat != m_serverChat);
@@ -195,7 +185,11 @@ public:
     int count() const { return m_chats.size(); }
 
     // stop ChatLLM threads for clean shutdown
-    void destroyChats() { for (auto *chat: m_chats) { chat->destroy(); } }
+    void destroyChats()
+    {
+        for (auto *chat: m_chats) { chat->destroy(); }
+        ChatLLM::destroyStore();
+    }
 
     void removeChatFile(Chat *chat) const;
     Q_INVOKABLE void saveChats();

gpt4all-chat/chatllm.cpp

@@ -7,6 +7,18 @@
 #include "mysettings.h"
 #include "../gpt4all-backend/llmodel.h"
 
+#include <algorithm>
+#include <cctype>
+#include <cmath>
+#include <cstddef>
+#include <functional>
+#include <limits>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <QElapsedTimer>
+
 //#define DEBUG
 //#define DEBUG_MODEL_LOADING
 
@@ -18,16 +30,17 @@ public:
     static LLModelStore *globalInstance();
 
     LLModelInfo acquireModel(); // will block until llmodel is ready
-    void releaseModel(const LLModelInfo &info); // must be called when you are done
+    void releaseModel(LLModelInfo &&info); // must be called when you are done
+    void destroy();
 
 private:
     LLModelStore()
     {
         // seed with empty model
-        m_availableModels.append(LLModelInfo());
+        m_availableModel = LLModelInfo();
     }
     ~LLModelStore() {}
-    QVector<LLModelInfo> m_availableModels;
+    std::optional<LLModelInfo> m_availableModel;
     QMutex m_mutex;
     QWaitCondition m_condition;
     friend class MyLLModelStore;
@@ -43,19 +56,27 @@ LLModelStore *LLModelStore::globalInstance()
 LLModelInfo LLModelStore::acquireModel()
 {
     QMutexLocker locker(&m_mutex);
-    while (m_availableModels.isEmpty())
+    while (!m_availableModel)
         m_condition.wait(locker.mutex());
-    return m_availableModels.takeFirst();
+    auto first = std::move(*m_availableModel);
+    m_availableModel.reset();
+    return first;
 }
 
-void LLModelStore::releaseModel(const LLModelInfo &info)
+void LLModelStore::releaseModel(LLModelInfo &&info)
 {
     QMutexLocker locker(&m_mutex);
-    m_availableModels.append(info);
-    Q_ASSERT(m_availableModels.count() < 2);
+    Q_ASSERT(!m_availableModel);
+    m_availableModel = std::move(info);
     m_condition.wakeAll();
 }
 
+void LLModelStore::destroy()
+{
+    QMutexLocker locker(&m_mutex);
+    m_availableModel.reset();
+}
+
 ChatLLM::ChatLLM(Chat *parent, bool isServer)
     : QObject{nullptr}
     , m_promptResponseTokens(0)
@@ -64,7 +85,6 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_shouldBeLoaded(false)
     , m_forceUnloadModel(false)
     , m_markedForDeletion(false)
-    , m_shouldTrySwitchContext(false)
     , m_stopGenerating(false)
     , m_timer(nullptr)
     , m_isServer(isServer)
@@ -74,11 +94,9 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_restoreStateFromText(false)
 {
     moveToThread(&m_llmThread);
-    connect(this, &ChatLLM::sendStartup, Network::globalInstance(), &Network::sendStartup);
-    connect(this, &ChatLLM::sendModelLoaded, Network::globalInstance(), &Network::sendModelLoaded);
     connect(this, &ChatLLM::shouldBeLoadedChanged, this, &ChatLLM::handleShouldBeLoadedChanged,
         Qt::QueuedConnection); // explicitly queued
-    connect(this, &ChatLLM::shouldTrySwitchContextChanged, this, &ChatLLM::handleShouldTrySwitchContextChanged,
+    connect(this, &ChatLLM::trySwitchContextRequested, this, &ChatLLM::trySwitchContextOfLoadedModel,
         Qt::QueuedConnection); // explicitly queued
     connect(parent, &Chat::idChanged, this, &ChatLLM::handleChatIdChanged);
     connect(&m_llmThread, &QThread::started, this, &ChatLLM::handleThreadStarted);
@@ -98,7 +116,8 @@ ChatLLM::~ChatLLM()
     destroy();
 }
 
-void ChatLLM::destroy() {
+void ChatLLM::destroy()
+{
     m_stopGenerating = true;
     m_llmThread.quit();
     m_llmThread.wait();
@@ -106,11 +125,15 @@ void ChatLLM::destroy() {
     // The only time we should have a model loaded here is on shutdown
     // as we explicitly unload the model in all other circumstances
     if (isModelLoaded()) {
-        delete m_llModelInfo.model;
-        m_llModelInfo.model = nullptr;
+        m_llModelInfo.model.reset();
     }
 }
 
+void ChatLLM::destroyStore()
+{
+    LLModelStore::globalInstance()->destroy();
+}
+
 void ChatLLM::handleThreadStarted()
 {
     m_timer = new TokenTimer(this);
@@ -120,7 +143,7 @@ void ChatLLM::handleThreadStarted()
 
 void ChatLLM::handleForceMetalChanged(bool forceMetal)
 {
-#if defined(Q_OS_MAC) && defined(__arm__)
+#if defined(Q_OS_MAC) && defined(__aarch64__)
     m_forceMetal = forceMetal;
     if (isModelLoaded() && m_shouldBeLoaded) {
         m_reloadingToChangeVariant = true;
@@ -151,7 +174,7 @@ bool ChatLLM::loadDefaultModel()
     return loadModel(defaultModel);
 }
 
-bool ChatLLM::trySwitchContextOfLoadedModel(const ModelInfo &modelInfo)
+void ChatLLM::trySwitchContextOfLoadedModel(const ModelInfo &modelInfo)
 {
     // We're trying to see if the store already has the model fully loaded that we wish to use
     // and if so we just acquire it from the store and switch the context and return true. If the
@@ -159,10 +182,11 @@ bool ChatLLM::trySwitchContextOfLoadedModel(const ModelInfo &modelInfo)
 
     // If we're already loaded or a server or we're reloading to change the variant/device or the
     // modelInfo is empty, then this should fail
-    if (isModelLoaded() || m_isServer || m_reloadingToChangeVariant || modelInfo.name().isEmpty()) {
-        m_shouldTrySwitchContext = false;
-        emit trySwitchContextOfLoadedModelCompleted(false);
-        return false;
+    if (
+        isModelLoaded() || m_isServer || m_reloadingToChangeVariant || modelInfo.name().isEmpty() || !m_shouldBeLoaded
+    ) {
+        emit trySwitchContextOfLoadedModelCompleted(0);
+        return;
     }
 
     QString filePath = modelInfo.dirpath + modelInfo.filename();
@@ -170,33 +194,28 @@ bool ChatLLM::trySwitchContextOfLoadedModel(const ModelInfo &modelInfo)
 
     m_llModelInfo = LLModelStore::globalInstance()->acquireModel();
 #if defined(DEBUG_MODEL_LOADING)
-        qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model;
+        qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
 
     // The store gave us no already loaded model, the wrong type of model, then give it back to the
     // store and fail
-    if (!m_llModelInfo.model || m_llModelInfo.fileInfo != fileInfo) {
-        LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
-        m_llModelInfo = LLModelInfo();
-        m_shouldTrySwitchContext = false;
-        emit trySwitchContextOfLoadedModelCompleted(false);
-        return false;
+    if (!m_llModelInfo.model || m_llModelInfo.fileInfo != fileInfo || !m_shouldBeLoaded) {
+        LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+        emit trySwitchContextOfLoadedModelCompleted(0);
+        return;
     }
 
 #if defined(DEBUG_MODEL_LOADING)
-    qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model;
+    qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
 
-    // We should be loaded and now we are
-    m_shouldBeLoaded = true;
-    m_shouldTrySwitchContext = false;
+    emit trySwitchContextOfLoadedModelCompleted(2);
 
     // Restore, signal and process
     restoreState();
     emit modelLoadingPercentageChanged(1.0f);
-    emit trySwitchContextOfLoadedModelCompleted(true);
+    emit trySwitchContextOfLoadedModelCompleted(0);
     processSystemPrompt();
-    return true;
 }
 
 bool ChatLLM::loadModel(const ModelInfo &modelInfo)
@@ -213,6 +232,13 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     if (isModelLoaded() && this->modelInfo() == modelInfo)
         return true;
 
+    // reset status
+    emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
+    emit modelLoadingError("");
+    emit reportFallbackReason("");
+    emit reportDevice("");
+    m_pristineLoadedState = false;
+
     QString filePath = modelInfo.dirpath + modelInfo.filename();
     QFileInfo fileInfo(filePath);
 
@@ -221,28 +247,25 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     if (alreadyAcquired) {
         resetContext();
 #if defined(DEBUG_MODEL_LOADING)
-        qDebug() << "already acquired model deleted" << m_llmThread.objectName() << m_llModelInfo.model;
+        qDebug() << "already acquired model deleted" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
-        delete m_llModelInfo.model;
-        m_llModelInfo.model = nullptr;
-        emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
+        m_llModelInfo.model.reset();
     } else if (!m_isServer) {
         // This is a blocking call that tries to retrieve the model we need from the model store.
         // If it succeeds, then we just have to restore state. If the store has never had a model
         // returned to it, then the modelInfo.model pointer should be null which will happen on startup
         m_llModelInfo = LLModelStore::globalInstance()->acquireModel();
 #if defined(DEBUG_MODEL_LOADING)
-        qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model;
+        qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
         // At this point it is possible that while we were blocked waiting to acquire the model from the
         // store, that our state was changed to not be loaded. If this is the case, release the model
         // back into the store and quit loading
         if (!m_shouldBeLoaded) {
 #if defined(DEBUG_MODEL_LOADING)
-            qDebug() << "no longer need model" << m_llmThread.objectName() << m_llModelInfo.model;
+            qDebug() << "no longer need model" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
-            LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
-            m_llModelInfo = LLModelInfo();
+            LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
             emit modelLoadingPercentageChanged(0.0f);
             return false;
         }
@@ -250,7 +273,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         // Check if the store just gave us exactly the model we were looking for
         if (m_llModelInfo.model && m_llModelInfo.fileInfo == fileInfo && !m_reloadingToChangeVariant) {
 #if defined(DEBUG_MODEL_LOADING)
-            qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model;
+            qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
             restoreState();
             emit modelLoadingPercentageChanged(1.0f);
@@ -264,10 +287,9 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         } else {
             // Release the memory since we have to switch to a different model.
 #if defined(DEBUG_MODEL_LOADING)
-            qDebug() << "deleting model" << m_llmThread.objectName() << m_llModelInfo.model;
+            qDebug() << "deleting model" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
-            delete m_llModelInfo.model;
-            m_llModelInfo.model = nullptr;
+            m_llModelInfo.model.reset();
         }
     }
 
@@ -278,15 +300,16 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     m_llModelInfo.fileInfo = fileInfo;
 
     if (fileInfo.exists()) {
+        QVariantMap modelLoadProps;
         if (modelInfo.isOnline) {
             QString apiKey;
             QString modelName;
             {
                 QFile file(filePath);
-                file.open(QIODeviceBase::ReadOnly | QIODeviceBase::Text);
-                QTextStream stream(&file);
-                QString text = stream.readAll();
-                QJsonDocument doc = QJsonDocument::fromJson(text.toUtf8());
+                bool success = file.open(QIODeviceBase::ReadOnly);
+                (void)success;
+                Q_ASSERT(success);
+                QJsonDocument doc = QJsonDocument::fromJson(file.readAll());
                 QJsonObject obj = doc.object();
                 apiKey = obj["apiKey"].toString();
                 modelName = obj["modelName"].toString();
@@ -296,18 +319,46 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             model->setModelName(modelName);
             model->setRequestURL(modelInfo.url());
             model->setAPIKey(apiKey);
-            m_llModelInfo.model = model;
+            m_llModelInfo.model.reset(model);
         } else {
+            QElapsedTimer modelLoadTimer;
+            modelLoadTimer.start();
+
+            auto requestedDevice = MySettings::globalInstance()->device();
             auto n_ctx = MySettings::globalInstance()->modelContextLength(modelInfo);
             m_ctx.n_ctx = n_ctx;
             auto ngl = MySettings::globalInstance()->modelGpuLayers(modelInfo);
 
-            std::string buildVariant = "auto";
-#if defined(Q_OS_MAC) && defined(__arm__)
-            if (m_forceMetal)
-                buildVariant = "metal";
+            std::string backend = "auto";
+#ifdef Q_OS_MAC
+            if (requestedDevice == "CPU") {
+                backend = "cpu";
+            } else if (m_forceMetal) {
+#ifdef __aarch64__
+                backend = "metal";
 #endif
-            m_llModelInfo.model = LLModel::Implementation::construct(filePath.toStdString(), buildVariant, n_ctx);
+            }
+#else // !defined(Q_OS_MAC)
+            if (requestedDevice.startsWith("CUDA: "))
+                backend = "cuda";
+#endif
+
+            QString constructError;
+            m_llModelInfo.model.reset();
+            try {
+                auto *model = LLModel::Implementation::construct(filePath.toStdString(), backend, n_ctx);
+                m_llModelInfo.model.reset(model);
+            } catch (const LLModel::MissingImplementationError &e) {
+                modelLoadProps.insert("error", "missing_model_impl");
+                constructError = e.what();
+            } catch (const LLModel::UnsupportedModelError &e) {
+                modelLoadProps.insert("error", "unsupported_model_file");
+                constructError = e.what();
+            } catch (const LLModel::BadArchError &e) {
+                constructError = e.what();
+                modelLoadProps.insert("error", "unsupported_model_arch");
+                modelLoadProps.insert("model_arch", QString::fromStdString(e.arch()));
+            }
 
             if (m_llModelInfo.model) {
                 if (m_llModelInfo.model->isModelBlacklisted(filePath.toStdString())) {
@@ -322,92 +373,133 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                 }
 
                 m_llModelInfo.model->setProgressCallback([this](float progress) -> bool {
+                    progress = std::max(progress, std::numeric_limits<float>::min()); // keep progress above zero
                     emit modelLoadingPercentageChanged(progress);
                     return m_shouldBeLoaded;
                 });
 
-                // Pick the best match for the device
-                QString actualDevice = m_llModelInfo.model->implementation().buildVariant() == "metal" ? "Metal" : "CPU";
-                const QString requestedDevice = MySettings::globalInstance()->device();
-                if (requestedDevice == "CPU") {
-                    emit reportFallbackReason(""); // fallback not applicable
-                } else {
-                    const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString(), n_ctx, ngl);
-                    std::vector<LLModel::GPUDevice> availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
-                    LLModel::GPUDevice *device = nullptr;
+                auto approxDeviceMemGB = [](const LLModel::GPUDevice *dev) {
+                    float memGB = dev->heapSize / float(1024 * 1024 * 1024);
+                    return std::floor(memGB * 10.f) / 10.f; // truncate to 1 decimal place
+                };
 
-                    if (!availableDevices.empty() && requestedDevice == "Auto" && availableDevices.front().type == 2 /*a discrete gpu*/) {
-                        device = &availableDevices.front();
-                    } else {
-                        for (LLModel::GPUDevice &d : availableDevices) {
-                            if (QString::fromStdString(d.name) == requestedDevice) {
+                std::vector<LLModel::GPUDevice> availableDevices;
+                const LLModel::GPUDevice *defaultDevice = nullptr;
+                {
+                    const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString(), n_ctx, ngl);
+                    availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
+                    // Pick the best device
+                    // NB: relies on the fact that Kompute devices are listed first
+                    if (!availableDevices.empty() && availableDevices.front().type == 2 /*a discrete gpu*/) {
+                        defaultDevice = &availableDevices.front();
+                        float memGB = defaultDevice->heapSize / float(1024 * 1024 * 1024);
+                        memGB = std::floor(memGB * 10.f) / 10.f; // truncate to 1 decimal place
+                        modelLoadProps.insert("default_device", QString::fromStdString(defaultDevice->name));
+                        modelLoadProps.insert("default_device_mem", approxDeviceMemGB(defaultDevice));
+                    }
+                }
+
+                QString actualDevice("CPU");
+
+#if defined(Q_OS_MAC) && defined(__aarch64__)
+                if (m_llModelInfo.model->implementation().buildVariant() == "metal")
+                    actualDevice = "Metal";
+#else
+                if (requestedDevice != "CPU") {
+                    const auto *device = defaultDevice;
+                    if (requestedDevice != "Auto") {
+                        // Use the selected device
+                        for (const LLModel::GPUDevice &d : availableDevices) {
+                            if (QString::fromStdString(d.selectionName()) == requestedDevice) {
                                 device = &d;
                                 break;
                             }
                         }
                     }
 
-                    emit reportFallbackReason(""); // no fallback yet
                     std::string unavail_reason;
                     if (!device) {
                         // GPU not available
                     } else if (!m_llModelInfo.model->initializeGPUDevice(device->index, &unavail_reason)) {
                         emit reportFallbackReason(QString::fromStdString("<br>" + unavail_reason));
                     } else {
-                        actualDevice = QString::fromStdString(device->name);
+                        actualDevice = QString::fromStdString(device->reportedName());
+                        modelLoadProps.insert("requested_device_mem", approxDeviceMemGB(device));
                     }
                 }
+#endif
 
                 // Report which device we're actually using
                 emit reportDevice(actualDevice);
-
                 bool success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, ngl);
+
+                if (!m_shouldBeLoaded) {
+                    m_llModelInfo.model.reset();
+                    if (!m_isServer)
+                        LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+                    m_llModelInfo = LLModelInfo();
+                    emit modelLoadingPercentageChanged(0.0f);
+                    return false;
+                }
+
                 if (actualDevice == "CPU") {
                     // we asked llama.cpp to use the CPU
                 } else if (!success) {
                     // llama_init_from_file returned nullptr
                     emit reportDevice("CPU");
                     emit reportFallbackReason("<br>GPU loading failed (out of VRAM?)");
+                    modelLoadProps.insert("cpu_fallback_reason", "gpu_load_failed");
                     success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, 0);
+
+                    if (!m_shouldBeLoaded) {
+                        m_llModelInfo.model.reset();
+                        if (!m_isServer)
+                            LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+                        m_llModelInfo = LLModelInfo();
+                        emit modelLoadingPercentageChanged(0.0f);
+                        return false;
+                    }
                 } else if (!m_llModelInfo.model->usingGPUDevice()) {
                     // ggml_vk_init was not called in llama.cpp
                     // We might have had to fallback to CPU after load if the model is not possible to accelerate
                     // for instance if the quantization method is not supported on Vulkan yet
                     emit reportDevice("CPU");
                     emit reportFallbackReason("<br>model or quant has no GPU support");
+                    modelLoadProps.insert("cpu_fallback_reason", "gpu_unsupported_model");
                 }
 
                 if (!success) {
-                    delete m_llModelInfo.model;
-                    m_llModelInfo.model = nullptr;
+                    m_llModelInfo.model.reset();
                     if (!m_isServer)
-                        LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
+                        LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
                     m_llModelInfo = LLModelInfo();
                     emit modelLoadingError(QString("Could not load model due to invalid model file for %1").arg(modelInfo.filename()));
+                    modelLoadProps.insert("error", "loadmodel_failed");
                 } else {
                     switch (m_llModelInfo.model->implementation().modelType()[0]) {
                     case 'L': m_llModelType = LLModelType::LLAMA_; break;
                     case 'G': m_llModelType = LLModelType::GPTJ_; break;
                     default:
                         {
-                            delete m_llModelInfo.model;
-                            m_llModelInfo.model = nullptr;
+                            m_llModelInfo.model.reset();
                             if (!m_isServer)
-                                LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
+                                LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
                             m_llModelInfo = LLModelInfo();
                             emit modelLoadingError(QString("Could not determine model type for %1").arg(modelInfo.filename()));
                         }
                     }
+
+                    modelLoadProps.insert("$duration", modelLoadTimer.elapsed() / 1000.);
                 }
             } else {
                 if (!m_isServer)
-                    LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
+                    LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
                 m_llModelInfo = LLModelInfo();
-                emit modelLoadingError(QString("Could not load model due to invalid format for %1").arg(modelInfo.filename()));
+                emit modelLoadingError(QString("Error loading %1: %2").arg(modelInfo.filename()).arg(constructError));
             }
         }
 #if defined(DEBUG_MODEL_LOADING)
-        qDebug() << "new model" << m_llmThread.objectName() << m_llModelInfo.model;
+        qDebug() << "new model" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
         restoreState();
 #if defined(DEBUG)
@@ -416,15 +508,12 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 #endif
         emit modelLoadingPercentageChanged(isModelLoaded() ? 1.0f : 0.0f);
 
-        static bool isFirstLoad = true;
-        if (isFirstLoad) {
-            emit sendStartup();
-            isFirstLoad = false;
-        } else
-            emit sendModelLoaded();
+        modelLoadProps.insert("requestedDevice", MySettings::globalInstance()->device());
+        modelLoadProps.insert("model", modelInfo.filename());
+        Network::globalInstance()->trackChatEvent("model_load", modelLoadProps);
     } else {
         if (!m_isServer)
-            LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
+            LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo)); // release back into the store
         m_llModelInfo = LLModelInfo();
         emit modelLoadingError(QString("Could not find file for model %1").arg(modelInfo.filename()));
     }
@@ -433,7 +522,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         setModelInfo(modelInfo);
         processSystemPrompt();
     }
-    return m_llModelInfo.model;
+    return bool(m_llModelInfo.model);
 }
 
 bool ChatLLM::isModelLoaded() const
@@ -632,6 +721,8 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     printf("%s", qPrintable(prompt));
     fflush(stdout);
 #endif
+    QElapsedTimer totalTime;
+    totalTime.start();
     m_timer->start();
     if (!docsContext.isEmpty()) {
         auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode localdocs context without a response
@@ -644,28 +735,30 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     fflush(stdout);
 #endif
     m_timer->stop();
+    qint64 elapsed = totalTime.elapsed();
     std::string trimmed = trim_whitespace(m_response);
     if (trimmed != m_response) {
         m_response = trimmed;
         emit responseChanged(QString::fromStdString(m_response));
     }
-    emit responseStopped();
+    emit responseStopped(elapsed);
+    m_pristineLoadedState = false;
     return true;
 }
 
 void ChatLLM::setShouldBeLoaded(bool b)
 {
 #if defined(DEBUG_MODEL_LOADING)
-    qDebug() << "setShouldBeLoaded" << m_llmThread.objectName() << b << m_llModelInfo.model;
+    qDebug() << "setShouldBeLoaded" << m_llmThread.objectName() << b << m_llModelInfo.model.get();
 #endif
     m_shouldBeLoaded = b; // atomic
     emit shouldBeLoadedChanged();
 }
 
-void ChatLLM::setShouldTrySwitchContext(bool b)
+void ChatLLM::requestTrySwitchContext()
 {
-    m_shouldTrySwitchContext = b; // atomic
-    emit shouldTrySwitchContextChanged();
+    m_shouldBeLoaded = true; // atomic
+    emit trySwitchContextRequested(modelInfo());
 }
 
 void ChatLLM::handleShouldBeLoadedChanged()
@@ -676,12 +769,6 @@ void ChatLLM::handleShouldBeLoadedChanged()
         unloadModel();
 }
 
-void ChatLLM::handleShouldTrySwitchContextChanged()
-{
-    if (m_shouldTrySwitchContext)
-        trySwitchContextOfLoadedModel(modelInfo());
-}
-
 void ChatLLM::unloadModel()
 {
     if (!isModelLoaded() || m_isServer)
@@ -696,17 +783,16 @@ void ChatLLM::unloadModel()
         saveState();
 
 #if defined(DEBUG_MODEL_LOADING)
-    qDebug() << "unloadModel" << m_llmThread.objectName() << m_llModelInfo.model;
+    qDebug() << "unloadModel" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
 
     if (m_forceUnloadModel) {
-        delete m_llModelInfo.model;
-        m_llModelInfo.model = nullptr;
+        m_llModelInfo.model.reset();
         m_forceUnloadModel = false;
     }
 
-    LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
-    m_llModelInfo = LLModelInfo();
+    LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+    m_pristineLoadedState = false;
 }
 
 void ChatLLM::reloadModel()
@@ -718,7 +804,7 @@ void ChatLLM::reloadModel()
         return;
 
 #if defined(DEBUG_MODEL_LOADING)
-    qDebug() << "reloadModel" << m_llmThread.objectName() << m_llModelInfo.model;
+    qDebug() << "reloadModel" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
     const ModelInfo m = modelInfo();
     if (m.name().isEmpty())
@@ -733,28 +819,19 @@ void ChatLLM::generateName()
     if (!isModelLoaded())
         return;
 
-    QString instructPrompt("### Instruction:\n"
-                           "Describe response above in three words.\n"
-                           "### Response:\n");
+    auto promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     auto promptFunc = std::bind(&ChatLLM::handleNamePrompt, this, std::placeholders::_1);
-    auto responseFunc = std::bind(&ChatLLM::handleNameResponse, this, std::placeholders::_1,
-        std::placeholders::_2);
+    auto responseFunc = std::bind(&ChatLLM::handleNameResponse, this, std::placeholders::_1, std::placeholders::_2);
     auto recalcFunc = std::bind(&ChatLLM::handleNameRecalculate, this, std::placeholders::_1);
     LLModel::PromptContext ctx = m_ctx;
-#if defined(DEBUG)
-    printf("%s", qPrintable(instructPrompt));
-    fflush(stdout);
-#endif
-    m_llModelInfo.model->prompt(instructPrompt.toStdString(), "%1", promptFunc, responseFunc, recalcFunc, ctx);
-#if defined(DEBUG)
-    printf("\n");
-    fflush(stdout);
-#endif
+    m_llModelInfo.model->prompt("Describe the above conversation in three words or less.",
+                                promptTemplate.toStdString(), promptFunc, responseFunc, recalcFunc, ctx);
     std::string trimmed = trim_whitespace(m_nameResponse);
     if (trimmed != m_nameResponse) {
         m_nameResponse = trimmed;
         emit generatedNameChanged(QString::fromStdString(m_nameResponse));
     }
+    m_pristineLoadedState = false;
 }
 
 void ChatLLM::handleChatIdChanged(const QString &id)
@@ -894,7 +971,10 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
     // If we do not deserialize the KV or it is discarded, then we need to restore the state from the
     // text only. This will be a costly operation, but the chat has to be restored from the text archive
     // alone.
-    m_restoreStateFromText = !deserializeKV || discardKV;
+    if (!deserializeKV || discardKV) {
+        m_restoreStateFromText = true;
+        m_pristineLoadedState = true;
+    }
 
     if (!deserializeKV) {
 #if defined(DEBUG)
@@ -958,14 +1038,14 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
 
 void ChatLLM::saveState()
 {
-    if (!isModelLoaded())
+    if (!isModelLoaded() || m_pristineLoadedState)
         return;
 
     if (m_llModelType == LLModelType::API_) {
         m_state.clear();
         QDataStream stream(&m_state, QIODeviceBase::WriteOnly);
         stream.setVersion(QDataStream::Qt_6_4);
-        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model);
+        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model.get());
         stream << chatAPI->context();
         return;
     }
@@ -986,7 +1066,7 @@ void ChatLLM::restoreState()
     if (m_llModelType == LLModelType::API_) {
         QDataStream stream(&m_state, QIODeviceBase::ReadOnly);
         stream.setVersion(QDataStream::Qt_6_4);
-        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model);
+        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model.get());
         QList<QString> context;
         stream >> context;
         chatAPI->setContext(context);
@@ -1005,13 +1085,18 @@ void ChatLLM::restoreState()
     if (m_llModelInfo.model->stateSize() == m_state.size()) {
         m_llModelInfo.model->restoreState(static_cast<const uint8_t*>(reinterpret_cast<void*>(m_state.data())));
         m_processedSystemPrompt = true;
+        m_pristineLoadedState = true;
     } else {
         qWarning() << "restoring state from text because" << m_llModelInfo.model->stateSize() << "!=" << m_state.size();
         m_restoreStateFromText = true;
     }
 
-    m_state.clear();
-    m_state.squeeze();
+    // free local state copy unless unload is pending
+    if (m_shouldBeLoaded) {
+        m_state.clear();
+        m_state.squeeze();
+        m_pristineLoadedState = false;
+    }
 }
 
 void ChatLLM::processSystemPrompt()
@@ -1056,7 +1141,8 @@ void ChatLLM::processSystemPrompt()
     fflush(stdout);
 #endif
     auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode system prompt without a response
-    m_llModelInfo.model->prompt(systemPrompt, "%1", promptFunc, nullptr, recalcFunc, m_ctx, true);
+    // use "%1%2" and not "%1" to avoid implicit whitespace
+    m_llModelInfo.model->prompt(systemPrompt, "%1%2", promptFunc, nullptr, recalcFunc, m_ctx, true);
     m_ctx.n_predict = old_n_predict;
 #if defined(DEBUG)
     printf("\n");
@@ -1064,6 +1150,7 @@ void ChatLLM::processSystemPrompt()
 #endif
 
     m_processedSystemPrompt = m_stopGenerating == false;
+    m_pristineLoadedState = false;
 }
 
 void ChatLLM::processRestoreStateFromText()
@@ -1122,4 +1209,6 @@ void ChatLLM::processRestoreStateFromText()
 
     m_isRecalc = false;
     emit recalcChanged();
+
+    m_pristineLoadedState = false;
 }

gpt4all-chat/chatllm.h

@@ -5,6 +5,8 @@
 #include <QThread>
 #include <QFileInfo>
 
+#include <memory>
+
 #include "database.h"
 #include "modellist.h"
 #include "../gpt4all-backend/llmodel.h"
@@ -16,7 +18,7 @@ enum LLModelType {
 };
 
 struct LLModelInfo {
-    LLModel *model = nullptr;
+    std::unique_ptr<LLModel> model;
     QFileInfo fileInfo;
     // NOTE: This does not store the model type or name on purpose as this is left for ChatLLM which
     // must be able to serialize the information even if it is in the unloaded state
@@ -72,6 +74,7 @@ public:
     virtual ~ChatLLM();
 
     void destroy();
+    static void destroyStore();
     bool isModelLoaded() const;
     void regenerateResponse();
     void resetResponse();
@@ -81,7 +84,7 @@ public:
 
     bool shouldBeLoaded() const { return m_shouldBeLoaded; }
     void setShouldBeLoaded(bool b);
-    void setShouldTrySwitchContext(bool b);
+    void requestTrySwitchContext();
     void setForceUnloadModel(bool b) { m_forceUnloadModel = b; }
     void setMarkedForDeletion(bool b) { m_markedForDeletion = b; }
 
@@ -101,7 +104,7 @@ public:
 public Q_SLOTS:
     bool prompt(const QList<QString> &collectionList, const QString &prompt);
     bool loadDefaultModel();
-    bool trySwitchContextOfLoadedModel(const ModelInfo &modelInfo);
+    void trySwitchContextOfLoadedModel(const ModelInfo &modelInfo);
     bool loadModel(const ModelInfo &modelInfo);
     void modelChangeRequested(const ModelInfo &modelInfo);
     void unloadModel();
@@ -109,7 +112,6 @@ public Q_SLOTS:
     void generateName();
     void handleChatIdChanged(const QString &id);
     void handleShouldBeLoadedChanged();
-    void handleShouldTrySwitchContextChanged();
     void handleThreadStarted();
     void handleForceMetalChanged(bool forceMetal);
     void handleDeviceChanged();
@@ -123,15 +125,13 @@ Q_SIGNALS:
     void modelLoadingWarning(const QString &warning);
     void responseChanged(const QString &response);
     void promptProcessing();
-    void responseStopped();
-    void sendStartup();
-    void sendModelLoaded();
+    void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
     void stateChanged();
     void threadStarted();
     void shouldBeLoadedChanged();
-    void shouldTrySwitchContextChanged();
-    void trySwitchContextOfLoadedModelCompleted(bool);
+    void trySwitchContextRequested(const ModelInfo &modelInfo);
+    void trySwitchContextOfLoadedModelCompleted(int value);
     void requestRetrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
     void reportSpeed(const QString &speed);
     void reportDevice(const QString &device);
@@ -174,7 +174,6 @@ private:
     QThread m_llmThread;
     std::atomic<bool> m_stopGenerating;
     std::atomic<bool> m_shouldBeLoaded;
-    std::atomic<bool> m_shouldTrySwitchContext;
     std::atomic<bool> m_isRecalc;
     std::atomic<bool> m_forceUnloadModel;
     std::atomic<bool> m_markedForDeletion;
@@ -183,6 +182,10 @@ private:
     bool m_reloadingToChangeVariant;
     bool m_processedSystemPrompt;
     bool m_restoreStateFromText;
+    // m_pristineLoadedState is set if saveSate is unnecessary, either because:
+    // - an unload was queued during LLModel::restoreState()
+    // - the chat will be restored from text and hasn't been interacted with yet
+    bool m_pristineLoadedState = false;
     QVector<QPair<QString, QString>> m_stateFromText;
 };
 

gpt4all-chat/cmake/deploy-qt-linux.cmake.in

@@ -5,10 +5,7 @@ set(DATA_DIR ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN
 set(BIN_DIR ${DATA_DIR}/bin)
 set(Qt6_ROOT_DIR "@Qt6_ROOT_DIR@")
 set(ENV{LD_LIBRARY_PATH} "${BIN_DIR}:${Qt6_ROOT_DIR}/../lib/")
-execute_process(COMMAND ${LINUXDEPLOYQT} ${BIN_DIR}/chat -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -bundle-non-qt-libs -qmake=${Qt6_ROOT_DIR}/bin/qmake -verbose=2)
-file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/*llmodel.*)
-file(COPY ${MYLLMODELLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin)
+execute_process(COMMAND ${LINUXDEPLOYQT} ${BIN_DIR}/chat -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -bundle-non-qt-libs -qmake=${Qt6_ROOT_DIR}/bin/qmake -verbose=2 -exclude-libs=libcuda.so.1)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-32.png"
      DESTINATION ${DATA_DIR})
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-48.png"

gpt4all-chat/cmake/deploy-qt-mac.cmake.in

@@ -4,21 +4,16 @@ set(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@")
 execute_process(COMMAND ${MACDEPLOYQT} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -verbose=2)
 file(GLOB MYGPTJLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libgptj*)
 file(GLOB MYLLAMALIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllama*)
-file(GLOB MYBERTLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libbert*)
 file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllmodel.*)
 file(COPY ${MYGPTJLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLAMALIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYBERTLLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLMODELLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns"
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Resources)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-32.png"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-48.png"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)
-file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns"
+file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)

gpt4all-chat/cmake/deploy-qt-windows.cmake.in

@@ -2,12 +2,9 @@ set(WINDEPLOYQT "@WINDEPLOYQT@")
 set(COMPONENT_NAME_MAIN "@COMPONENT_NAME_MAIN@")
 set(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@")
 execute_process(COMMAND ${WINDEPLOYQT} --qmldir ${CMAKE_CURRENT_SOURCE_DIR} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin)
-file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/*llmodel.*)
-file(COPY ${MYLLMODELLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-32.png"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-48.png"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)
-file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.ico"
+file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.ico"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)

gpt4all-chat/cmake/installerscript.qs

@@ -19,7 +19,7 @@ Component.prototype.createOperations = function()
                     targetDirectory + "/bin/chat.exe",
                     "@UserProfile@/Desktop/GPT4All.lnk",
                     "workingDirectory=" + targetDirectory + "/bin",
-                    "iconPath=" + targetDirectory + "/favicon.ico",
+                    "iconPath=" + targetDirectory + "/gpt4all.ico",
                     "iconId=0", "description=Open GPT4All");
             } catch (e) {
                 print("ERROR: creating desktop shortcut" + e);
@@ -28,7 +28,7 @@ Component.prototype.createOperations = function()
                 targetDirectory + "/bin/chat.exe",
                 "@StartMenuDir@/GPT4All.lnk",
                 "workingDirectory=" + targetDirectory + "/bin",
-                "iconPath=" + targetDirectory + "/favicon.ico",
+                "iconPath=" + targetDirectory + "/gpt4all.ico",
                 "iconId=0", "description=Open GPT4All");
         } else if (systemInfo.productType === "osx") {
             var gpt4allAppPath = targetDirectory + "/bin/gpt4all.app";

gpt4all-chat/database.cpp

@@ -1,7 +1,9 @@
 #include "database.h"
-#include "mysettings.h"
-#include "embllm.h"
+
 #include "embeddings.h"
+#include "embllm.h"
+#include "mysettings.h"
+#include "network.h"
 
 #include <QTimer>
 #include <QPdfDocument>
@@ -410,8 +412,8 @@ bool updateDocument(QSqlQuery &q, int id, qint64 document_time)
 {
     if (!q.prepare(UPDATE_DOCUMENT_TIME_SQL))
         return false;
-    q.addBindValue(id);
     q.addBindValue(document_time);
+    q.addBindValue(id);
     return q.exec();
 }
 
@@ -490,7 +492,7 @@ QSqlError initDb()
     i.collection = collection_name;
     i.folder_path = folder_path;
     i.folder_id = folder_id;
-    emit addCollectionItem(i);
+    emit addCollectionItem(i, false);
 
     // Add a document
     int document_time = 123456789;
@@ -535,13 +537,13 @@ QSqlError initDb()
 
 Database::Database(int chunkSize)
     : QObject(nullptr)
-    , m_watcher(new QFileSystemWatcher(this))
     , m_chunkSize(chunkSize)
+    , m_scanTimer(new QTimer(this))
+    , m_watcher(new QFileSystemWatcher(this))
     , m_embLLM(new EmbeddingLLM)
     , m_embeddings(new Embeddings(this))
 {
     moveToThread(&m_dbThread);
-    connect(&m_dbThread, &QThread::started, this, &Database::start);
     m_dbThread.setObjectName("database");
     m_dbThread.start();
 }
@@ -550,17 +552,20 @@ Database::~Database()
 {
     m_dbThread.quit();
     m_dbThread.wait();
+    delete m_embLLM;
 }
 
 void Database::scheduleNext(int folder_id, size_t countForFolder)
 {
     emit updateCurrentDocsToIndex(folder_id, countForFolder);
     if (!countForFolder) {
-        emit updateIndexing(folder_id, false);
+        updateFolderStatus(folder_id, FolderStatus::Complete);
         emit updateInstalled(folder_id, true);
     }
-    if (!m_docsToScan.isEmpty())
-        QTimer::singleShot(0, this, &Database::scanQueue);
+    if (m_docsToScan.isEmpty()) {
+        m_scanTimer->stop();
+        updateIndexingStatus();
+    }
 }
 
 void Database::handleDocumentError(const QString &errorMessage,
@@ -721,7 +726,6 @@ void Database::removeFolderFromDocumentQueue(int folder_id)
         return;
     m_docsToScan.remove(folder_id);
     emit removeFolderById(folder_id);
-    emit docsToScanChanged();
 }
 
 void Database::enqueueDocumentInternal(const DocumentInfo &info, bool prepend)
@@ -745,13 +749,16 @@ void Database::enqueueDocuments(int folder_id, const QVector<DocumentInfo> &info
     const size_t bytes = countOfBytes(folder_id);
     emit updateCurrentBytesToIndex(folder_id, bytes);
     emit updateTotalBytesToIndex(folder_id, bytes);
-    emit docsToScanChanged();
+    m_scanTimer->start();
 }
 
 void Database::scanQueue()
 {
-    if (m_docsToScan.isEmpty())
+    if (m_docsToScan.isEmpty()) {
+        m_scanTimer->stop();
+        updateIndexingStatus();
         return;
+    }
 
     DocumentInfo info = dequeueDocument();
     const size_t countForFolder = countOfDocuments(info.folder);
@@ -818,6 +825,8 @@ void Database::scanQueue()
     QSqlDatabase::database().transaction();
     Q_ASSERT(document_id != -1);
     if (info.isPdf()) {
+        updateFolderStatus(folder_id, FolderStatus::Embedding, -1, info.currentPage == 0);
+
         QPdfDocument doc;
         if (QPdfDocument::Error::None != doc.load(info.doc.canonicalFilePath())) {
             handleDocumentError("ERROR: Could not load pdf",
@@ -850,6 +859,8 @@ void Database::scanQueue()
             emit subtractCurrentBytesToIndex(info.folder, bytes - (bytesPerPage * doc.pageCount()));
         }
     } else {
+        updateFolderStatus(folder_id, FolderStatus::Embedding, -1, info.currentPosition == 0);
+
         QFile file(document_path);
         if (!file.open(QIODevice::ReadOnly)) {
             handleDocumentError("ERROR: Cannot open file for scanning",
@@ -884,7 +895,7 @@ void Database::scanQueue()
     return scheduleNext(folder_id, countForFolder);
 }
 
-void Database::scanDocuments(int folder_id, const QString &folder_path)
+void Database::scanDocuments(int folder_id, const QString &folder_path, bool isNew)
 {
 #if defined(DEBUG)
     qDebug() << "scanning folder for documents" << folder_path;
@@ -915,7 +926,7 @@ void Database::scanDocuments(int folder_id, const QString &folder_path)
     }
 
     if (!infos.isEmpty()) {
-        emit updateIndexing(folder_id, true);
+        updateFolderStatus(folder_id, FolderStatus::Started, infos.count(), false, isNew);
         enqueueDocuments(folder_id, infos);
     }
 }
@@ -925,7 +936,7 @@ void Database::start()
     connect(m_watcher, &QFileSystemWatcher::directoryChanged, this, &Database::directoryChanged);
     connect(m_embLLM, &EmbeddingLLM::embeddingsGenerated, this, &Database::handleEmbeddingsGenerated);
     connect(m_embLLM, &EmbeddingLLM::errorGenerated, this, &Database::handleErrorGenerated);
-    connect(this, &Database::docsToScanChanged, this, &Database::scanQueue);
+    m_scanTimer->callOnTimeout(this, &Database::scanQueue);
     if (!QSqlDatabase::drivers().contains("QSQLITE")) {
         qWarning() << "ERROR: missing sqllite driver";
     } else {
@@ -937,10 +948,11 @@ void Database::start()
     if (m_embeddings->fileExists() && !m_embeddings->load())
         qWarning() << "ERROR: Could not load embeddings";
 
-    addCurrentFolders();
+    int nAdded = addCurrentFolders();
+    Network::globalInstance()->trackEvent("localdocs_startup", { {"doc_collections_total", nAdded} });
 }
 
-void Database::addCurrentFolders()
+int Database::addCurrentFolders()
 {
 #if defined(DEBUG)
     qDebug() << "addCurrentFolders";
@@ -950,21 +962,26 @@ void Database::addCurrentFolders()
     QList<CollectionItem> collections;
     if (!selectAllFromCollections(q, &collections)) {
         qWarning() << "ERROR: Cannot select collections" << q.lastError();
-        return;
+        return 0;
     }
 
     emit collectionListUpdated(collections);
 
+    int nAdded = 0;
     for (const auto &i : collections)
-        addFolder(i.collection, i.folder_path);
+        nAdded += addFolder(i.collection, i.folder_path, true);
+
+    updateIndexingStatus();
+
+    return collections.count() + nAdded;
 }
 
-void Database::addFolder(const QString &collection, const QString &path)
+bool Database::addFolder(const QString &collection, const QString &path, bool fromDb)
 {
     QFileInfo info(path);
     if (!info.exists() || !info.isReadable()) {
         qWarning() << "ERROR: Cannot add folder that doesn't exist or not readable" << path;
-        return;
+        return false;
     }
 
     QSqlQuery q;
@@ -973,13 +990,13 @@ void Database::addFolder(const QString &collection, const QString &path)
     // See if the folder exists in the db
     if (!selectFolder(q, path, &folder_id)) {
         qWarning() << "ERROR: Cannot select folder from path" << path << q.lastError();
-        return;
+        return false;
     }
 
     // Add the folder
     if (folder_id == -1 && !addFolderToDB(q, path, &folder_id)) {
         qWarning() << "ERROR: Cannot add folder to db with path" << path << q.lastError();
-        return;
+        return false;
     }
 
     Q_ASSERT(folder_id != -1);
@@ -988,24 +1005,32 @@ void Database::addFolder(const QString &collection, const QString &path)
     QList<int> folders;
     if (!selectFoldersFromCollection(q, collection, &folders)) {
         qWarning() << "ERROR: Cannot select folders from collections" << collection << q.lastError();
-        return;
+        return false;
     }
 
+    bool added = false;
     if (!folders.contains(folder_id)) {
         if (!addCollection(q, collection, folder_id)) {
             qWarning() << "ERROR: Cannot add folder to collection" << collection << path << q.lastError();
-            return;
+            return false;
         }
 
         CollectionItem i;
         i.collection = collection;
         i.folder_path = path;
         i.folder_id = folder_id;
-        emit addCollectionItem(i);
+        emit addCollectionItem(i, fromDb);
+        added = true;
     }
 
     addFolderToWatch(path);
-    scanDocuments(folder_id, path);
+    scanDocuments(folder_id, path, !fromDb);
+
+    if (!fromDb) {
+        updateIndexingStatus();
+    }
+
+    return added;
 }
 
 void Database::removeFolder(const QString &collection, const QString &path)
@@ -1285,5 +1310,69 @@ void Database::directoryChanged(const QString &path)
     cleanDB();
 
     // Rescan the documents associated with the folder
-    scanDocuments(folder_id, path);
+    scanDocuments(folder_id, path, false);
+    updateIndexingStatus();
+}
+
+void Database::updateIndexingStatus() {
+    Q_ASSERT(m_scanTimer->isActive() || m_docsToScan.isEmpty());
+    if (!m_indexingTimer.isValid() && m_scanTimer->isActive()) {
+        Network::globalInstance()->trackEvent("localdocs_indexing_start");
+        m_indexingTimer.start();
+    } else if (m_indexingTimer.isValid() && !m_scanTimer->isActive()) {
+        qint64 durationMs = m_indexingTimer.elapsed();
+        Network::globalInstance()->trackEvent("localdocs_indexing_complete", { {"$duration", durationMs / 1000.} });
+        m_indexingTimer.invalidate();
+    }
+}
+
+void Database::updateFolderStatus(int folder_id, Database::FolderStatus status, int numDocs, bool atStart, bool isNew) {
+    FolderStatusRecord *lastRecord = nullptr;
+    if (m_foldersBeingIndexed.contains(folder_id)) {
+        lastRecord = &m_foldersBeingIndexed[folder_id];
+    }
+    Q_ASSERT(lastRecord || status == FolderStatus::Started);
+
+    switch (status) {
+        case FolderStatus::Started:
+            if (lastRecord == nullptr) {
+                // record timestamp but don't send an event yet
+                m_foldersBeingIndexed.insert(folder_id, { QDateTime::currentMSecsSinceEpoch(), isNew, numDocs });
+                emit updateIndexing(folder_id, true);
+            }
+            break;
+        case FolderStatus::Embedding:
+            if (!lastRecord->docsChanged) {
+                Q_ASSERT(atStart);
+                // send start event with the original timestamp for folders that need updating
+                const auto *embeddingModels = ModelList::globalInstance()->installedEmbeddingModels();
+                Network::globalInstance()->trackEvent("localdocs_folder_indexing", {
+                    {"folder_id", folder_id},
+                    {"is_new_collection", lastRecord->isNew},
+                    {"document_count", lastRecord->numDocs},
+                    {"embedding_model", embeddingModels->defaultModelInfo().filename()},
+                    {"chunk_size", m_chunkSize},
+                    {"time", lastRecord->startTime},
+                });
+            }
+            lastRecord->docsChanged += atStart;
+            lastRecord->chunksRead++;
+            break;
+        case FolderStatus::Complete:
+            if (lastRecord->docsChanged) {
+                // send complete event for folders that were updated
+                qint64 durationMs = QDateTime::currentMSecsSinceEpoch() - lastRecord->startTime;
+                Network::globalInstance()->trackEvent("localdocs_folder_complete", {
+                    {"folder_id", folder_id},
+                    {"is_new_collection", lastRecord->isNew},
+                    {"documents_total", lastRecord->numDocs},
+                    {"documents_changed", lastRecord->docsChanged},
+                    {"chunks_read", lastRecord->chunksRead},
+                    {"$duration", durationMs / 1000.},
+                });
+            }
+            m_foldersBeingIndexed.remove(folder_id);
+            emit updateIndexing(folder_id, false);
+            break;
+    }
 }

gpt4all-chat/database.h

@@ -1,16 +1,19 @@
 #ifndef DATABASE_H
 #define DATABASE_H
 
-#include <QObject>
-#include <QtSql>
-#include <QQueue>
+#include <QElapsedTimer>
 #include <QFileInfo>
-#include <QThread>
 #include <QFileSystemWatcher>
+#include <QObject>
+#include <QQueue>
+#include <QThread>
+#include <QtSql>
 
 #include "embllm.h"
 
 class Embeddings;
+class QTimer;
+
 struct DocumentInfo
 {
     int folder;
@@ -58,9 +61,10 @@ public:
     virtual ~Database();
 
 public Q_SLOTS:
+    void start();
     void scanQueue();
-    void scanDocuments(int folder_id, const QString &folder_path);
-    void addFolder(const QString &collection, const QString &path);
+    void scanDocuments(int folder_id, const QString &folder_path, bool isNew);
+    bool addFolder(const QString &collection, const QString &path, bool fromDb);
     void removeFolder(const QString &collection, const QString &path);
     void retrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
     void cleanDB();
@@ -78,21 +82,22 @@ Q_SIGNALS:
     void updateTotalBytesToIndex(int folder_id, size_t totalBytesToIndex);
     void updateCurrentEmbeddingsToIndex(int folder_id, size_t currentBytesToIndex);
     void updateTotalEmbeddingsToIndex(int folder_id, size_t totalBytesToIndex);
-    void addCollectionItem(const CollectionItem &item);
+    void addCollectionItem(const CollectionItem &item, bool fromDb);
     void removeFolderById(int folder_id);
-    void removeCollectionItem(const QString &collectionName);
     void collectionListUpdated(const QList<CollectionItem> &collectionList);
 
 private Q_SLOTS:
-    void start();
     void directoryChanged(const QString &path);
     bool addFolderToWatch(const QString &path);
     bool removeFolderFromWatch(const QString &path);
-    void addCurrentFolders();
+    int addCurrentFolders();
     void handleEmbeddingsGenerated(const QVector<EmbeddingResult> &embeddings);
     void handleErrorGenerated(int folder_id, const QString &error);
 
 private:
+    enum class FolderStatus { Started, Embedding, Complete };
+    struct FolderStatusRecord { qint64 startTime; bool isNew; int numDocs, docsChanged, chunksRead; };
+
     void removeFolderInternal(const QString &collection, int folder_id, const QString &path);
     size_t chunkStream(QTextStream &stream, int folder_id, int document_id, const QString &file,
         const QString &title, const QString &author, const QString &subject, const QString &keywords, int page,
@@ -107,10 +112,15 @@ private:
     void removeFolderFromDocumentQueue(int folder_id);
     void enqueueDocumentInternal(const DocumentInfo &info, bool prepend = false);
     void enqueueDocuments(int folder_id, const QVector<DocumentInfo> &infos);
+    void updateIndexingStatus();
+    void updateFolderStatus(int folder_id, FolderStatus status, int numDocs = -1, bool atStart = false, bool isNew = false);
 
 private:
     int m_chunkSize;
+    QTimer *m_scanTimer;
     QMap<int, QQueue<DocumentInfo>> m_docsToScan;
+    QElapsedTimer m_indexingTimer;
+    QMap<int, FolderStatusRecord> m_foldersBeingIndexed;
     QList<ResultInfo> m_retrieve;
     QThread m_dbThread;
     QFileSystemWatcher *m_watcher;

gpt4all-chat/download.cpp

@@ -75,15 +75,25 @@ bool Download::hasNewerRelease() const
     return compareVersions(versions.first(), currentVersion);
 }
 
-bool Download::isFirstStart() const
+bool Download::isFirstStart(bool writeVersion) const
 {
+    auto *mySettings = MySettings::globalInstance();
+
     QSettings settings;
     settings.sync();
     QString lastVersionStarted = settings.value("download/lastVersionStarted").toString();
     bool first = lastVersionStarted != QCoreApplication::applicationVersion();
-    settings.setValue("download/lastVersionStarted", QCoreApplication::applicationVersion());
-    settings.sync();
-    return first;
+    if (first && writeVersion) {
+        settings.setValue("download/lastVersionStarted", QCoreApplication::applicationVersion());
+        // let the user select these again
+        settings.remove("network/usageStatsActive");
+        settings.remove("network/isActive");
+        settings.sync();
+        emit mySettings->networkUsageStatsActiveChanged();
+        emit mySettings->networkIsActiveChanged();
+    }
+
+    return first || !mySettings->isNetworkUsageStatsActiveSet() || !mySettings->isNetworkIsActiveSet();
 }
 
 void Download::updateReleaseNotes()
@@ -131,7 +141,7 @@ void Download::downloadModel(const QString &modelFile)
     ModelList::globalInstance()->updateDataByFilename(modelFile, {{ ModelList::DownloadingRole, true }});
     ModelInfo info = ModelList::globalInstance()->modelInfoByFilename(modelFile);
     QString url = !info.url().isEmpty() ? info.url() : "http://gpt4all.io/models/gguf/" + modelFile;
-    Network::globalInstance()->sendDownloadStarted(modelFile);
+    Network::globalInstance()->trackEvent("download_started", { {"model", modelFile} });
     QNetworkRequest request(url);
     request.setAttribute(QNetworkRequest::User, modelFile);
     request.setRawHeader("range", QString("bytes=%1-").arg(tempFile->pos()).toUtf8());
@@ -153,7 +163,7 @@ void Download::cancelDownload(const QString &modelFile)
         QNetworkReply *modelReply = m_activeDownloads.keys().at(i);
         QUrl url = modelReply->request().url();
         if (url.toString().endsWith(modelFile)) {
-            Network::globalInstance()->sendDownloadCanceled(modelFile);
+            Network::globalInstance()->trackEvent("download_canceled", { {"model", modelFile} });
 
             // Disconnect the signals
             disconnect(modelReply, &QNetworkReply::downloadProgress, this, &Download::handleDownloadProgress);
@@ -178,7 +188,8 @@ void Download::installModel(const QString &modelFile, const QString &apiKey)
     if (apiKey.isEmpty())
         return;
 
-    Network::globalInstance()->sendInstallModel(modelFile);
+    Network::globalInstance()->trackEvent("install_model", { {"model", modelFile} });
+
     QString filePath = MySettings::globalInstance()->modelPath() + modelFile;
     QFile file(filePath);
     if (file.open(QIODeviceBase::WriteOnly | QIODeviceBase::Text)) {
@@ -216,7 +227,7 @@ void Download::removeModel(const QString &modelFile)
         shouldRemoveInstalled = info.installed && !info.isClone() && (info.isDiscovered() || info.description() == "" /*indicates sideloaded*/);
         if (shouldRemoveInstalled)
             ModelList::globalInstance()->removeInstalled(info);
-        Network::globalInstance()->sendRemoveModel(modelFile);
+        Network::globalInstance()->trackEvent("remove_model", { {"model", modelFile} });
         file.remove();
     }
 
@@ -332,7 +343,11 @@ void Download::handleErrorOccurred(QNetworkReply::NetworkError code)
             .arg(modelReply->errorString());
     qWarning() << error;
     ModelList::globalInstance()->updateDataByFilename(modelFilename, {{ ModelList::DownloadErrorRole, error }});
-    Network::globalInstance()->sendDownloadError(modelFilename, (int)code, modelReply->errorString());
+    Network::globalInstance()->trackEvent("download_error", {
+        {"model", modelFilename},
+        {"code", (int)code},
+        {"error", modelReply->errorString()},
+    });
     cancelDownload(modelFilename);
 }
 
@@ -515,7 +530,7 @@ void Download::handleHashAndSaveFinished(bool success, const QString &error,
     // The hash and save should send back with tempfile closed
     Q_ASSERT(!tempFile->isOpen());
     QString modelFilename = modelReply->request().attribute(QNetworkRequest::User).toString();
-    Network::globalInstance()->sendDownloadFinished(modelFilename, success);
+    Network::globalInstance()->trackEvent("download_finished", { {"model", modelFilename}, {"success", success} });
 
     QVector<QPair<int, QVariant>> data {
         { ModelList::CalcHashRole, false },

gpt4all-chat/download.h

@@ -54,7 +54,7 @@ public:
     Q_INVOKABLE void cancelDownload(const QString &modelFile);
     Q_INVOKABLE void installModel(const QString &modelFile, const QString &apiKey);
     Q_INVOKABLE void removeModel(const QString &modelFile);
-    Q_INVOKABLE bool isFirstStart() const;
+    Q_INVOKABLE bool isFirstStart(bool writeVersion = false) const;
 
 public Q_SLOTS:
     void updateReleaseNotes();

gpt4all-chat/embllm.cpp

@@ -5,6 +5,7 @@ EmbeddingLLMWorker::EmbeddingLLMWorker()
     : QObject(nullptr)
     , m_networkManager(new QNetworkAccessManager(this))
     , m_model(nullptr)
+    , m_stopGenerating(false)
 {
     moveToThread(&m_workerThread);
     connect(this, &EmbeddingLLMWorker::finished, &m_workerThread, &QThread::quit, Qt::DirectConnection);
@@ -14,6 +15,10 @@ EmbeddingLLMWorker::EmbeddingLLMWorker()
 
 EmbeddingLLMWorker::~EmbeddingLLMWorker()
 {
+    m_stopGenerating = true;
+    m_workerThread.quit();
+    m_workerThread.wait();
+
     if (m_model) {
         delete m_model;
         m_model = nullptr;
@@ -42,17 +47,29 @@ bool EmbeddingLLMWorker::loadModel()
     }
 
     auto filename = fileInfo.fileName();
-    bool isNomic = filename.startsWith("nomic-") && filename.endsWith(".txt");
+    bool isNomic = filename.startsWith("gpt4all-nomic-") && filename.endsWith(".rmodel");
     if (isNomic) {
         QFile file(filePath);
-        file.open(QIODeviceBase::ReadOnly | QIODeviceBase::Text);
-        QTextStream stream(&file);
-        m_nomicAPIKey = stream.readAll();
+        if (!file.open(QIODeviceBase::ReadOnly)) {
+            qWarning() << "failed to open" << filePath << ":" << file.errorString();
+            m_model = nullptr;
+            return false;
+        }
+        QJsonDocument doc = QJsonDocument::fromJson(file.readAll());
+        QJsonObject obj = doc.object();
+        m_nomicAPIKey = obj["apiKey"].toString();
         file.close();
         return true;
     }
 
-    m_model = LLModel::Implementation::construct(filePath.toStdString());
+    try {
+        m_model = LLModel::Implementation::construct(filePath.toStdString());
+    } catch (const std::exception &e) {
+        qWarning() << "WARNING: Could not load embedding model:" << e.what();
+        m_model = nullptr;
+        return false;
+    }
+
     // NOTE: explicitly loads model on CPU to avoid GPU OOM
     // TODO(cebtenzzre): support GPU-accelerated embeddings
     bool success = m_model->loadModel(filePath.toStdString(), 2048, 0);
@@ -85,16 +102,7 @@ bool EmbeddingLLMWorker::isNomic() const
 // this function is always called for retrieval tasks
 std::vector<float> EmbeddingLLMWorker::generateSyncEmbedding(const QString &text)
 {
-    if (!hasModel() && !loadModel()) {
-        qWarning() << "WARNING: Could not load model for embeddings";
-        return {};
-    }
-
-    if (isNomic()) {
-        qWarning() << "WARNING: Request to generate sync embeddings for non-local model invalid";
-        return {};
-    }
-
+    Q_ASSERT(!isNomic());
     std::vector<float> embedding(m_model->embeddingSize());
     try {
         m_model->embed({text.toStdString()}, embedding.data(), true);
@@ -145,6 +153,9 @@ void EmbeddingLLMWorker::requestSyncEmbedding(const QString &text)
 // this function is always called for storage into the database
 void EmbeddingLLMWorker::requestAsyncEmbedding(const QVector<EmbeddingChunk> &chunks)
 {
+    if (m_stopGenerating)
+        return;
+
     if (!hasModel() && !loadModel()) {
         qWarning() << "WARNING: Could not load model for embeddings";
         return;
@@ -294,16 +305,21 @@ EmbeddingLLM::~EmbeddingLLM()
 
 std::vector<float> EmbeddingLLM::generateEmbeddings(const QString &text)
 {
+    if (!m_embeddingWorker->hasModel() && !m_embeddingWorker->loadModel()) {
+        qWarning() << "WARNING: Could not load model for embeddings";
+        return {};
+    }
+
     if (!m_embeddingWorker->isNomic()) {
         return m_embeddingWorker->generateSyncEmbedding(text);
-    } else {
-        EmbeddingLLMWorker worker;
-        connect(this, &EmbeddingLLM::requestSyncEmbedding, &worker,
-            &EmbeddingLLMWorker::requestSyncEmbedding, Qt::QueuedConnection);
-        emit requestSyncEmbedding(text);
-        worker.wait();
-        return worker.lastResponse();
     }
+
+    EmbeddingLLMWorker worker;
+    connect(this, &EmbeddingLLM::requestSyncEmbedding, &worker,
+        &EmbeddingLLMWorker::requestSyncEmbedding, Qt::QueuedConnection);
+    emit requestSyncEmbedding(text);
+    worker.wait();
+    return worker.lastResponse();
 }
 
 void EmbeddingLLM::generateAsyncEmbeddings(const QVector<EmbeddingChunk> &chunks)

gpt4all-chat/embllm.h

@@ -58,6 +58,7 @@ private:
     QNetworkAccessManager *m_networkManager;
     std::vector<float> m_lastResponse;
     LLModel *m_model = nullptr;
+    std::atomic<bool> m_stopGenerating;
     QThread m_workerThread;
 };