cmake: fix Metal build after #2310 (#2350)

I don't understand why this is needed, but it works.

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/continue_config.yml

@@ -97,7 +97,9 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Installing Qt
           command: |
@@ -121,6 +123,7 @@ jobs:
             set -eo pipefail
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
             export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
+            export PATH=$PATH:/usr/local/cuda/bin
             mkdir build
             cd build
             mkdir upload
@@ -162,6 +165,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Build
           command: |
@@ -218,7 +226,9 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Installing Qt
           command: |
@@ -235,6 +245,7 @@ jobs:
           name: Build
           command: |
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
+            export PATH=$PATH:/usr/local/cuda/bin
             ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S gpt4all-chat -B build
             ~/Qt/Tools/CMake/bin/cmake --build build --target all
 
@@ -269,6 +280,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Build
           command: |
@@ -394,12 +410,15 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk
+            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
             pip install setuptools wheel cmake
       - run:
           name: Build C library
           command: |
+            export PATH=$PATH:/usr/local/cuda/bin
             git submodule update --init --recursive
             cd gpt4all-backend
             cmake -B build
@@ -459,6 +478,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Install dependencies
           command:
@@ -530,11 +554,14 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list            
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk
+            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Build Libraries
           command: |
+            export PATH=$PATH:/usr/local/cuda/bin
             cd gpt4all-backend
             mkdir -p runtimes/build
             cd runtimes/build
@@ -599,6 +626,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Install dependencies
           command: |
@@ -642,6 +674,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Install dependencies
           command: |

LICENSE_SOM.txt

@@ -1,30 +0,0 @@
-Software for Open Models License (SOM)
-Version 1.0 dated August 30th, 2023
-
-This license governs use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
-
-This license is intended to encourage open release of models created, modified, processed, or otherwise used via the Software under open licensing terms, and should be interpreted in light of that intent.
-
-1. Definitions
-The “Licensor” is the person or entity who is making the Software available under this license. “Software” is the software made available by Licensor under this license.
-A “Model” is the output of a machine learning algorithm, and excludes the Software.
-“Model Source Materials” must include the Model and model weights, and may include any input data, input data descriptions, documentation or training descriptions for the Model.
-“Open Licensing Terms” means: (a) any open source license approved by the Open Source Initiative, or (b) any other terms that make the Model Source Materials publicly available free of charge, and allow recipients to use, modify and distribute the Model Source Materials. Terms described in (b) may include reasonable restrictions such as non-commercial or non-production limitations, or require use in compliance with law.
-
-2. Grant of Rights. Subject to the conditions and limitations in section 3:
-(A) Copyright Grant. Licensor grants you a non-exclusive, worldwide, royalty-free copyright license to copy, modify, and distribute the Software and any modifications of the Software you create under this license. The foregoing license includes without limitation the right to create, modify, and use Models using this Software.
-
-(B) Patent Grant. Licensor grants you a non-exclusive, worldwide, royalty-free license, under any patents owned or controlled by Licensor, to make, have made, use, sell, offer for sale, import, or otherwise exploit the Software.  No license is granted to patent rights that are not embodied in the operation of the Software in the form provided by Licensor.
-
-3. Conditions and Limitations
-(A) Model Licensing and Access. If you use the Software to create, modify, process, or otherwise use any Model, including usage to create inferences with a Model, whether or not you make the Model available to others, you must make that Model Source Materials publicly available under Open Licensing Terms. 
-
-(B) No Re-Licensing. If you redistribute the Software, or modifications to the Software made under the license granted above, you must make it available only under the terms of this license. You may offer additional terms such as warranties, maintenance and support, but You, and not Licensor, are responsible for performing such terms.
-
-(C) No Trademark License. This license does not grant you rights to use the Licensor’s name, logo, or trademarks.
-
-(D) If you assert in writing a claim against any person or entity alleging that the use of the Software infringes any patent, all of your licenses to the Software under Section 2 end automatically as of the date you asserted the claim.
-
-(E) If you distribute any portion of the Software, you must retain all copyright, patent, trademark, and attribution notices that are present in the Software, and you must include a copy of this license.
-
-(F) The Software is licensed “as-is.” You bear the entire risk of using it. Licensor gives You no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws that this license cannot change. To the extent permitted under your local laws, the Licensor disclaims and excludes the implied warranties of merchantability, fitness for a particular purpose and non-infringement. To the extent this disclaimer is unlawful, you, and not Licensor, are responsible for any liability.

README.md

@@ -1,82 +1,73 @@
 <h1 align="center">GPT4All</h1>
-
-<p align="center">Open-source large language models that run locally on your CPU and nearly any GPU</p>
-
-
+<p align="center">Privacy-oriented software for chatting with large language models that run on your own computer.</p>
 <p align="center">
-Join the <a href="https://discord.gg/tyc74KNVK3?event=1227642051294658621">GPT4All 2024 Roadmap Townhall</a> on April 18, 2024 at 12pm EST
+  <a href="https://gpt4all.io">Official Website</a> &bull; <a href="https://docs.gpt4all.io">Documentation</a> &bull; <a href="https://discord.gg/mGZE39AS3e">Discord</a>
 </p>
-
 <p align="center">
-<a href="https://gpt4all.io">GPT4All Website and Models</a> • <a href="https://docs.gpt4all.io">GPT4All Documentation</a> • <a href="https://discord.gg/mGZE39AS3e">Discord</a>
+  Official Download Links: <a href="https://gpt4all.io/installers/gpt4all-installer-win64.exe">Windows</a> &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-darwin.dmg">macOS</a> &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-linux.run">Ubuntu</a>
 </p>
-
-
 <p align="center">
-<a href="https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html">🦜️🔗 Official Langchain Backend</a> 
+  <b>NEW:</b> <a href="https://forms.nomic.ai/gpt4all-release-notes-signup">Subscribe to our mailing list</a> for updates and news!
 </p>
-
 <p align="center">
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
 </p>
-
 <p align="center">
- <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg?&logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNSIgaGVpZ2h0PSI0IiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxwYXRoIGQ9Ik00LjQzIDEuODgyYTEuNDQgMS40NCAwIDAgMS0uMDk4LjQyNmMtLjA1LjEyMy0uMTE1LjIzLS4xOTIuMzIyLS4wNzUuMDktLjE2LjE2NS0uMjU1LjIyNmExLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxMmMtLjA5OS4wMTItLjE5Mi4wMTQtLjI3OS4wMDZsLTEuNTkzLS4xNHYtLjQwNmgxLjY1OGMuMDkuMDAxLjE3LS4xNjkuMjQ2LS4xOTFhLjYwMy42MDMgMCAwIDAgLjItLjEwNi41MjkuNTI5IDAgMCAwIC4xMzgtLjE3LjY1NC42NTQgMCAwIDAgLjA2NS0uMjRsLjAyOC0uMzJhLjkzLjkzIDAgMCAwLS4wMzYtLjI0OS41NjcuNTY3IDAgMCAwLS4xMDMtLjIuNTAyLjUwMiAwIDAgMC0uMTY4LS4xMzguNjA4LjYwOCAwIDAgMC0uMjQtLjA2N0wyLjQzNy43MjkgMS42MjUuNjcxYS4zMjIuMzIyIDAgMCAwLS4yMzIuMDU4LjM3NS4zNzUgMCAwIDAtLjExNi4yMzJsLS4xMTYgMS40NS0uMDU4LjY5Ny0uMDU4Ljc1NEwuNzA1IDRsLS4zNTctLjA3OUwuNjAyLjkwNkMuNjE3LjcyNi42NjMuNTc0LjczOS40NTRhLjk1OC45NTggMCAwIDEgLjI3NC0uMjg1Ljk3MS45NzEgMCAwIDEgLjMzNy0uMTRjLjExOS0uMDI2LjIyNy0uMDM0LjMyNS0uMDI2TDMuMjMyLjE2Yy4xNTkuMDE0LjMzNi4wMy40NTkuMDgyYTEuMTczIDEuMTczIDAgMCAxIC41NDUuNDQ3Yy4wNi4wOTQuMTA5LjE5Mi4xNDQuMjkzYTEuMzkyIDEuMzkyIDAgMCAxIC4wNzguNThsLS4wMjkuMzJaIiBmaWxsPSIjRjI3NzdBIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+Cjwvc3ZnPgo=" alt="phorm.ai"></a>
+ <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg" alt="phorm.ai"></a>
 </p>
 
 <p align="center">
-  <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
+  <img width="auto" height="400" src="https://github.com/nomic-ai/gpt4all/assets/14168726/495fce3e-769b-4e5a-a394-99f072ac4d29">
 </p>
 <p align="center">
-Run on an M1 macOS Device (not sped up!)
+Run on an M2 MacBook Pro (not sped up!)
 </p>
 
-## GPT4All: An ecosystem of open-source on-edge large language models.
 
-GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and any GPU. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
+## About GPT4All
+
+GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and NVIDIA and AMD GPUs. Note that your CPU needs to support [AVX instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
 
 Learn more in the [documentation](https://docs.gpt4all.io).
 
-A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models.
+A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily deploy their own on-edge large language models.
 
-### What's New ([Issue Tracker](https://github.com/orgs/nomic-ai/projects/2))
+
+### Installation
+
+The recommended way to install GPT4All is to use one of the online installers linked above in this README, which are also available at the [GPT4All website](https://gpt4all.io/). These require an internet connection at install time, are slightly easier to use on macOS due to code signing, and provide a version of GPT4All that can check for updates.
+
+An alternative way to install GPT4All is to use one of the offline installers available on the [Releases page](https://github.com/nomic-ai/gpt4all/releases). These do not require an internet connection at install time, and can be used to install an older version of GPT4All if so desired. But using these requires acknowledging a security warning on macOS, and they provide a version of GPT4All that is unable to notify you of updates, so you should enable notifications for Releases on this repository (Watch > Custom > Releases) or sign up for announcements in our [Discord server](https://discord.gg/mGZE39AS3e).
+
+
+### What's New
 - **October 19th, 2023**: GGUF Support Launches with Support for:
     - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
     - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4\_0 and Q4\_1 quantizations in GGUF.
     - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
-- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on AMD, Intel, Samsung, Qualcomm and NVIDIA GPUs.
-- **August 15th, 2023**: GPT4All API launches allowing inference of local LLMs from docker containers.
-- **July 2023**: Stable support for LocalDocs, a GPT4All Plugin that allows you to privately and locally chat with your data.
+- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on NVIDIA and AMD GPUs.
+- **July 2023**: Stable support for LocalDocs, a feature that allows you to privately and locally chat with your data.
+- **June 28th, 2023**: [Docker-based API server] launches allowing inference of local LLMs from an OpenAI-compatible HTTP endpoint.
+
+[Docker-based API server]: https://github.com/nomic-ai/gpt4all/tree/cef74c2be20f5b697055d5b8b506861c7b997fab/gpt4all-api
 
 
-### Chat Client
-Run any GPT4All model natively on your home desktop with the auto-updating desktop chat client. See <a href="https://gpt4all.io">GPT4All Website</a> for a full list of open-source models you can run with this powerful desktop application.
+### Building From Source
 
-Direct Installer Links:
+* Follow the instructions [here](gpt4all-chat/build_and_run.md) to build the GPT4All Chat UI from source.
 
-* [macOS](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
-
-* [Windows](https://gpt4all.io/installers/gpt4all-installer-win64.exe)
-
-* [Ubuntu](https://gpt4all.io/installers/gpt4all-installer-linux.run)
-
-Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io/)
-
-### Chat Client building and running
-
-* Follow the visual instructions on the chat client [build_and_run](gpt4all-chat/build_and_run.md) page
 
 ### Bindings
 
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/java">:computer: Official Java Bindings</a>
+* :snake: <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python">Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
+* :computer: <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">Typescript Bindings</a>
+
 
 ### Integrations
 
-* 🗃️ [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
+* :parrot::link: [Langchain](https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html)
+* :card_file_box: [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
+
 
 ## Contributing
 GPT4All welcomes contributions, involvement, and discussion from the open source community!
@@ -86,6 +77,59 @@ Check project discord, with project owners, or through existing issues/PRs to av
 Please make sure to tag all of the above with relevant project identifiers or your contribution could potentially get lost.
 Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 
+
+## GPT4All 2024 Roadmap
+To contribute to the development of any of the below roadmap items, make or find the corresponding issue and cross-reference the [in-progress task](https://github.com/orgs/nomic-ai/projects/2/views/1).
+
+Each item should have an issue link below.
+
+- Chat UI Language Localization (localize UI into the native languages of users)
+    - [ ] Chinese
+    - [ ] German
+    - [ ] French
+    - [ ] Portuguese
+    - [ ] Your native language here. 
+- UI Redesign: an internal effort at Nomic to improve the UI/UX of gpt4all for all users.
+    - [ ] Design new user interface and gather community feedback
+    - [ ] Implement the new user interface and experience.
+- Installer and Update Improvements
+    - [ ] Seamless native installation and update process on OSX
+    - [ ] Seamless native installation and update process on Windows
+    - [ ] Seamless native installation and update process on Linux
+- Model discoverability improvements:
+    - [x] Support huggingface model discoverability
+    - [ ] Support Nomic hosted model discoverability
+- LocalDocs (towards a local perplexity)
+    - Multilingual LocalDocs Support
+        - [ ] Create a multilingual experience
+        - [ ] Incorporate a multilingual embedding model
+        - [ ] Specify a preferred multilingual LLM for localdocs
+    - Improved RAG techniques
+        - [ ] Query augmentation and re-writing
+        - [ ] Improved chunking and text extraction from arbitrary modalities
+            - [ ] Custom PDF extractor past the QT default (charts, tables, text)
+        - [ ] Faster indexing and local exact search with v1.5 hamming embeddings and reranking (skip ANN index construction!)
+    - Support queries like 'summarize X document'
+    - Multimodal LocalDocs support with Nomic Embed
+    - Nomic Dataset Integration with real-time LocalDocs
+        - [ ] Include an option to allow the export of private LocalDocs collections to Nomic Atlas for debugging data/chat quality
+        - [ ] Allow optional sharing of LocalDocs collections between users.
+        - [ ] Allow the import of a LocalDocs collection from an Atlas Datasets
+            - Chat with live version of Wikipedia, Chat with Pubmed, chat with the latest snapshot of world news.
+- First class Multilingual LLM Support
+    - [ ] Recommend and set a default LLM for German
+    - [ ] Recommend and set a default LLM for English
+    - [ ] Recommend and set a default LLM for Chinese
+    - [ ] Recommend and set a default LLM for Spanish
+
+- Server Mode improvements
+    - Improved UI and new requested features:
+        - [ ] Fix outstanding bugs and feature requests around networking configurations.
+        - [ ] Support Nomic Embed inferencing
+        - [ ] First class documentation
+        - [ ] Improving developer use and quality of server mode (e.g. support larger batches)
+
+
 ## Technical Reports
 
 <p align="center">
@@ -100,6 +144,7 @@ Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 <a href="https://s3.amazonaws.com/static.nomic.ai/gpt4all/2023_GPT4All_Technical_Report.pdf">:green_book: Technical Report 1: GPT4All</a>
 </p>
 
+
 ## Citation
 
 If you utilize this repository, models or data in a downstream project, please consider citing it with:

gpt4all-api/.gitignore

@@ -1,112 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-app/__pycache__/
-gpt4all_api/__pycache__/
-gpt4all_api/app/api_v1/__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# VS Code
-.vscode/
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-
-*.lock
-*.cache

gpt4all-api/.isort.cfg

@@ -1,7 +0,0 @@
-[settings]
-known_third_party=geopy,nltk,np,numpy,pandas,pysbd,fire,torch
-
-line_length=120
-include_trailing_comma=True
-multi_line_output=3
-use_parentheses=True

gpt4all-api/LICENSE

@@ -1,13 +0,0 @@
-Copyright 2023 Nomic, Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

gpt4all-api/README.md

@@ -1,90 +0,0 @@
-# GPT4All REST API
-
-NOTICE: We are considering to deprecate this API as it has become challenging to maintain and test. If you have any interest in maintaining this or would like to takeover and adopt or discuss the future of this API please speak up in the discord channel.
-
-This directory contains the source code to run and build docker images that run a FastAPI app
-for serving inference from GPT4All models. The API matches the OpenAI API spec.
-
-## Tutorial
-
-The following tutorial assumes that you have checked out this repo and cd'd into it.
-
-### Starting the app
-
-First change your working directory to `gpt4all/gpt4all-api`.
-
-Now you can build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
-```bash
-DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
-```
-
-Then, start the backend with:
-
-```bash
-docker compose up --build
-```
-
-This will run both the API and locally hosted GPU inference server. If you want to run the API without the GPU inference server, you can run:
-
-```bash
-docker compose up --build gpt4all_api
-```
-
-To run the API with the GPU inference server, you will need to include environment variables (like the `MODEL_ID`). Edit the `.env` file and run
-```bash
-docker compose --env-file .env up --build
-```
-
-
-#### Spinning up your app
-Run `docker compose up` to spin up the backend. Monitor the logs for errors in-case you forgot to set an environment variable above.
-
-
-#### Development
-Run
-
-```bash
-docker compose up --build
-```
-and edit files in the `app` directory. The api will hot-reload on changes.
-
-You can run the unit tests with
-
-```bash
-make test
-```
-
-#### Viewing API documentation
-
-Once the FastAPI ap is started you can access its documentation and test the search endpoint by going to:
-```
-localhost:80/docs
-```
-
-This documentation should match the OpenAI OpenAPI spec located at https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-#### Running inference
-```python
-import openai
-openai.api_base = "http://localhost:4891/v1"
-
-openai.api_key = "not needed for a local LLM"
-
-
-def test_completion():
-    model = "gpt4all-j-v1.3-groovy"
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model,
-        prompt=prompt,
-        max_tokens=50,
-        temperature=0.28,
-        top_p=0.95,
-        n=1,
-        echo=True,
-        stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-    print(response)
-```

gpt4all-api/docker-compose.gpu.yaml

@@ -1,24 +0,0 @@
-version: "3.8"
-
-services:
-  gpt4all_gpu:
-    image: ghcr.io/huggingface/text-generation-inference:0.9.3
-    container_name: gpt4all_gpu
-    restart: always #restart on error (usually code compilation from save during bad state)
-    environment:
-      - HUGGING_FACE_HUB_TOKEN=token
-      - USE_FLASH_ATTENTION=false
-      - MODEL_ID=''
-      - NUM_SHARD=1
-    command: --model-id $MODEL_ID --num-shard $NUM_SHARD
-    volumes:
-      - ./:/data
-    ports:
-      - "8080:80"
-    shm_size: 1g
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              capabilities: [gpu]

gpt4all-api/docker-compose.yaml

@@ -1,22 +0,0 @@
-version: "3.8"
-
-services:
-  gpt4all_api:
-    image: gpt4all_api
-    container_name: gpt4all_api
-    restart: always #restart on error (usually code compilation from save during bad state)
-    ports:
-      - "4891:4891"
-    env_file:
-      - .env
-    environment:
-      - APP_ENVIRONMENT=dev
-      - WEB_CONCURRENCY=2
-      - LOGLEVEL=debug
-      - PORT=4891
-      - model=${MODEL_BIN} # using variable from .env file
-      - inference_mode=cpu
-    volumes:
-      - './gpt4all_api/app:/app'
-      - './gpt4all_api/models:/models' # models are mounted in the container
-    command: ["/start-reload.sh"]

gpt4all-api/gpt4all_api/Dockerfile.buildkit

@@ -1,17 +0,0 @@
-# syntax=docker/dockerfile:1.0.0-experimental
-FROM tiangolo/uvicorn-gunicorn:python3.11
-
-# Put first so anytime this file changes other cached layers are invalidated.
-COPY gpt4all_api/requirements.txt /requirements.txt
-
-RUN pip install --upgrade pip
-
-# Run various pip install commands with ssh keys from host machine.
-RUN --mount=type=ssh pip install -r /requirements.txt && \
-  rm -Rf /root/.cache && rm -Rf /tmp/pip-install*
-
-# Finally, copy app and client.
-COPY gpt4all_api/app /app
-
-RUN mkdir -p /models
-

gpt4all-api/gpt4all_api/README.md

@@ -1 +0,0 @@
-# FastAPI app for serving GPT4All models

gpt4all-api/gpt4all_api/app/api_v1/api.py

@@ -1,9 +0,0 @@
-from api_v1.routes import chat, completions, engines, health
-from fastapi import APIRouter
-
-router = APIRouter()
-
-router.include_router(chat.router)
-router.include_router(completions.router)
-router.include_router(engines.router)
-router.include_router(health.router)

gpt4all-api/gpt4all_api/app/api_v1/events.py

@@ -1,29 +0,0 @@
-import logging
-
-from api_v1.settings import settings
-from fastapi import HTTPException
-from fastapi.responses import JSONResponse
-from starlette.requests import Request
-
-log = logging.getLogger(__name__)
-
-
-startup_msg_fmt = """
- Starting up GPT4All API
-"""
-
-
-async def on_http_error(request: Request, exc: HTTPException):
-    return JSONResponse({'detail': exc.detail}, status_code=exc.status_code)
-
-
-async def on_startup(app):
-    startup_msg = startup_msg_fmt.format(settings=settings)
-    log.info(startup_msg)
-
-
-def startup_event_handler(app):
-    async def start_app() -> None:
-        await on_startup(app)
-
-    return start_app

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -1,103 +0,0 @@
-import logging
-import time
-from typing import List
-from uuid import uuid4
-from fastapi import APIRouter, HTTPException
-from gpt4all import GPT4All
-from pydantic import BaseModel, Field
-from api_v1.settings import settings
-from fastapi.responses import StreamingResponse
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-class ChatCompletionMessage(BaseModel):
-    role: str
-    content: str
-
-class ChatCompletionRequest(BaseModel):
-    model: str = Field(settings.model, description='The model to generate a completion from.')
-    messages: List[ChatCompletionMessage] = Field(..., description='Messages for the chat completion.')
-    temperature: float = Field(settings.temp, description='Model temperature')
-
-class ChatCompletionChoice(BaseModel):
-    message: ChatCompletionMessage
-    index: int
-    logprobs: float
-    finish_reason: str
-
-class ChatCompletionUsage(BaseModel):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-class ChatCompletionResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[ChatCompletionChoice]
-    usage: ChatCompletionUsage
-
-router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
-
-@router.post("/completions", response_model=ChatCompletionResponse)
-async def chat_completion(request: ChatCompletionRequest):
-    '''
-    Completes a GPT4All model response based on the last message in the chat.
-    '''
-    # GPU is not implemented yet
-    if settings.inference_mode == "gpu":
-        raise HTTPException(status_code=400,
-              detail=f"Not implemented yet: Can only infer in CPU mode.")
-
-    # we only support the configured model
-    if request.model != settings.model:
-        raise HTTPException(status_code=400,
-              detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
-
-    # run only of we have a message
-    if request.messages:
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        # format system message and conversation history correctly
-        formatted_messages = ""
-        for message in request.messages:
-            formatted_messages += f"<|im_start|>{message.role}\n{message.content}<|im_end|>\n"
-
-        # the LLM will complete the response of the assistant
-        formatted_messages += "<|im_start|>assistant\n"
-        response = model.generate(
-            prompt=formatted_messages,
-            temp=request.temperature
-            )
-
-        # the LLM may continue to hallucinate the conversation, but we want only the first response
-        # so, cut off everything after first <|im_end|>
-        index = response.find("<|im_end|>")
-        response_content = response[:index].strip()
-    else:
-        response_content = "No messages received."
-
-    # Create a chat message for the response
-    response_message = ChatCompletionMessage(role="assistant", content=response_content)
-
-    # Create a choice object with the response message
-    response_choice = ChatCompletionChoice(
-        message=response_message,
-        index=0,
-        logprobs=-1.0,  # Placeholder value
-        finish_reason="length"  # Placeholder value
-    )
-
-    # Create the response object
-    chat_response = ChatCompletionResponse(
-        id=str(uuid4()),
-        created=int(time.time()),
-        model=request.model,
-        choices=[response_choice],
-        usage=ChatCompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0),  # Placeholder values
-    )
-
-    return chat_response

gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py

@@ -1,215 +0,0 @@
-import json
-from typing import List, Dict, Iterable, AsyncIterable
-import logging
-import time
-from typing import Dict, List, Union, Optional
-from uuid import uuid4
-import aiohttp
-import asyncio
-from api_v1.settings import settings
-from fastapi import APIRouter, Depends, Response, Security, status, HTTPException
-from fastapi.responses import StreamingResponse
-from gpt4all import GPT4All
-from pydantic import BaseModel, Field
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class CompletionRequest(BaseModel):
-    model: str = Field(settings.model, description='The model to generate a completion from.')
-    prompt: Union[List[str], str] = Field(..., description='The prompt to begin completing from.')
-    max_tokens: int = Field(None, description='Max tokens to generate')
-    temperature: float = Field(settings.temp, description='Model temperature')
-    top_p: Optional[float] = Field(settings.top_p, description='top_p')
-    top_k: Optional[int] = Field(settings.top_k, description='top_k')
-    n: int = Field(1, description='How many completions to generate for each prompt')
-    stream: bool = Field(False, description='Stream responses')
-    repeat_penalty: float = Field(settings.repeat_penalty, description='Repeat penalty')
-
-
-class CompletionChoice(BaseModel):
-    text: str
-    index: int
-    logprobs: float
-    finish_reason: str
-
-
-class CompletionUsage(BaseModel):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-
-class CompletionResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[CompletionChoice]
-    usage: CompletionUsage
-
-
-class CompletionStreamResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[CompletionChoice]
-
-
-router = APIRouter(prefix="/completions", tags=["Completion Endpoints"])
-
-def stream_completion(output: Iterable, base_response: CompletionStreamResponse):
-    """
-    Streams a GPT4All output to the client.
-
-    Args:
-        output: The output of GPT4All.generate(), which is an iterable of tokens.
-        base_response: The base response object, which is cloned and modified for each token.
-
-    Returns:
-        A Generator of CompletionStreamResponse objects, which are serialized to JSON Event Stream format.
-    """
-    for token in output:
-        chunk = base_response.copy()
-        chunk.choices = [dict(CompletionChoice(
-            text=token,
-            index=0,
-            logprobs=-1,
-            finish_reason=''
-        ))]
-        yield f"data: {json.dumps(dict(chunk))}\n\n"
-
-async def gpu_infer(payload, header):
-    async with aiohttp.ClientSession() as session:
-        try:
-            async with session.post(
-                settings.hf_inference_server_host, headers=header, data=json.dumps(payload)
-            ) as response:
-                resp = await response.json()
-            return resp
-
-        except aiohttp.ClientError as e:
-            # Handle client-side errors (e.g., connection error, invalid URL)
-            logger.error(f"Client error: {e}")
-        except aiohttp.ServerError as e:
-            # Handle server-side errors (e.g., internal server error)
-            logger.error(f"Server error: {e}")
-        except json.JSONDecodeError as e:
-            # Handle JSON decoding errors
-            logger.error(f"JSON decoding error: {e}")
-        except Exception as e:
-            # Handle other unexpected exceptions
-            logger.error(f"Unexpected error: {e}")
-
-@router.post("/", response_model=CompletionResponse)
-async def completions(request: CompletionRequest):
-    '''
-    Completes a GPT4All model response.
-    '''
-    if settings.inference_mode == "gpu":
-        params = request.dict(exclude={'model', 'prompt', 'max_tokens', 'n'})
-        params["max_new_tokens"] = request.max_tokens
-        params["num_return_sequences"] = request.n
-
-        header = {"Content-Type": "application/json"}
-        if isinstance(request.prompt, list):
-            tasks = []
-            for prompt in request.prompt:
-                payload = {"parameters": params}
-                payload["inputs"] = prompt
-                task = gpu_infer(payload, header)
-                tasks.append(task)
-            results = await asyncio.gather(*tasks)
-
-            choices = []
-            for response in results:
-                scores = response["scores"] if "scores" in response else -1.0
-                choices.append(
-                    dict(
-                        CompletionChoice(
-                            text=response["generated_text"], index=0, logprobs=scores, finish_reason='stop'
-                        )
-                    )
-                )
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=choices,
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
-
-        else:
-            payload = {"parameters": params}
-            # If streaming, we need to return a StreamingResponse
-            payload["inputs"] = request.prompt
-
-            resp = await gpu_infer(payload, header)
-
-            output = resp["generated_text"]
-            # this returns all logprobs
-            scores = resp["scores"] if "scores" in resp else -1.0
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(text=output, index=0, logprobs=scores, finish_reason='stop'))],
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
-
-    else:
-
-        if request.model != settings.model:
-            raise HTTPException(status_code=400,
-                                detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
-
-        if isinstance(request.prompt, list):
-            if len(request.prompt) > 1:
-                raise HTTPException(status_code=400, detail="Can only infer one inference per request in CPU mode.")
-            else:
-                request.prompt = request.prompt[0]
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        output = model.generate(prompt=request.prompt,
-                                max_tokens=request.max_tokens,
-                                streaming=request.stream,
-                                top_k=request.top_k,
-                                top_p=request.top_p,
-                                temp=request.temperature,
-                                )
-
-        # If streaming, we need to return a StreamingResponse
-        if request.stream:
-            base_chunk = CompletionStreamResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[]
-            )
-            return StreamingResponse((response for response in stream_completion(output, base_chunk)),
-                                     media_type="text/event-stream")
-        else:
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(
-                    text=output,
-                    index=0,
-                    logprobs=-1,
-                    finish_reason='stop'
-                ))],
-                usage={
-                    'prompt_tokens': 0,  # TODO how to compute this?
-                    'completion_tokens': 0,
-                    'total_tokens': 0
-                }
-            )

gpt4all-api/gpt4all_api/app/api_v1/routes/embeddings.py

@@ -1,65 +0,0 @@
-from typing import List, Union
-from fastapi import APIRouter
-from api_v1.settings import settings
-from gpt4all import Embed4All
-from pydantic import BaseModel, Field
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class EmbeddingRequest(BaseModel):
-    model: str = Field(
-        settings.model, description="The model to generate an embedding from."
-    )
-    input: Union[str, List[str], List[int], List[List[int]]] = Field(
-        ..., description="Input text to embed, encoded as a string or array of tokens."
-    )
-
-
-class EmbeddingUsage(BaseModel):
-    prompt_tokens: int = 0
-    total_tokens: int = 0
-
-
-class Embedding(BaseModel):
-    index: int = 0
-    object: str = "embedding"
-    embedding: List[float]
-
-
-class EmbeddingResponse(BaseModel):
-    object: str = "list"
-    model: str
-    data: List[Embedding]
-    usage: EmbeddingUsage
-
-
-router = APIRouter(prefix="/embeddings", tags=["Embedding Endpoints"])
-
-embedder = Embed4All()
-
-
-def get_embedding(data: EmbeddingRequest) -> EmbeddingResponse:
-    """
-    Calculates the embedding for the given input using a specified model.
-
-    Args:
-        data (EmbeddingRequest): An EmbeddingRequest object containing the input data
-        and model name.
-
-    Returns:
-        EmbeddingResponse: An EmbeddingResponse object encapsulating the calculated embedding,
-        usage info, and the model name.
-    """
-    embedding = embedder.embed(data.input)
-    return EmbeddingResponse(
-        data=[Embedding(embedding=embedding)], usage=EmbeddingUsage(), model=data.model
-    )
-
-
-@router.post("/", response_model=EmbeddingResponse)
-def embeddings(data: EmbeddingRequest):
-    """
-    Creates a GPT4All embedding
-    """
-    return get_embedding(data)

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -1,39 +0,0 @@
-import requests
-from fastapi import APIRouter, HTTPException
-from pydantic import BaseModel, Field
-from typing import List, Dict
-
-# Define the router for the engines module
-router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
-
-# Define the models for the engines module
-class ListEnginesResponse(BaseModel):
-    data: List[Dict] = Field(..., description="All available models.")
-
-class EngineResponse(BaseModel):
-    data: List[Dict] = Field(..., description="All available models.")
-
-
-# Define the routes for the engines module
-@router.get("/", response_model=ListEnginesResponse)
-async def list_engines():
-    try:
-        response = requests.get('https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json')
-        response.raise_for_status()  # This will raise an HTTPError if the HTTP request returned an unsuccessful status code
-        engines = response.json()
-        return ListEnginesResponse(data=engines)
-    except requests.RequestException as e:
-        logger.error(f"Error fetching engine list: {e}")
-        raise HTTPException(status_code=500, detail="Error fetching engine list")
-
-# Define the routes for the engines module
-@router.get("/{engine_id}", response_model=EngineResponse)
-async def retrieve_engine(engine_id: str):
-    try:
-        # Implement logic to fetch a specific engine's details
-        # This is a placeholder, replace with your actual data retrieval logic
-        engine_details = {"id": engine_id, "name": "Engine Name", "description": "Engine Description"}
-        return EngineResponse(data=[engine_details])
-    except Exception as e:
-        logger.error(f"Error fetching engine details: {e}")
-        raise HTTPException(status_code=500, detail=f"Error fetching details for engine {engine_id}")

gpt4all-api/gpt4all_api/app/api_v1/routes/health.py

@@ -1,13 +0,0 @@
-import logging
-from fastapi import APIRouter
-from fastapi.responses import JSONResponse
-
-log = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/health", tags=["Health"])
-
-
-@router.get('/', response_class=JSONResponse)
-async def health_check():
-    """Runs a health check on this instance of the API."""
-    return JSONResponse({'status': 'ok'}, headers={'Access-Control-Allow-Origin': '*'})

gpt4all-api/gpt4all_api/app/api_v1/settings.py

@@ -1,19 +0,0 @@
-from pydantic import BaseSettings
-
-
-class Settings(BaseSettings):
-    app_environment = 'dev'
-    model: str = 'ggml-mpt-7b-chat.bin'
-    gpt4all_path: str = '/models'
-    inference_mode: str = "cpu"
-    hf_inference_server_host: str = "http://gpt4all_gpu:80/generate"
-    sentry_dns: str = None
-
-    temp: float = 0.18
-    top_p: float = 1.0
-    top_k: int = 50
-    repeat_penalty: float = 1.18
-
-
-
-settings = Settings()

gpt4all-api/gpt4all_api/app/docs.py

@@ -1,3 +0,0 @@
-desc = 'GPT4All API'
-
-endpoint_paths = {'health': '/health'}

gpt4all-api/gpt4all_api/app/main.py

@@ -1,84 +0,0 @@
-import logging
-import os
-
-import docs
-from api_v1 import events
-from api_v1.api import router as v1_router
-from api_v1.settings import settings
-from fastapi import FastAPI, HTTPException, Request
-from fastapi.logger import logger as fastapi_logger
-from starlette.middleware.cors import CORSMiddleware
-
-logger = logging.getLogger(__name__)
-
-app = FastAPI(title='GPT4All API', description=docs.desc)
-
-# CORS Configuration (in-case you want to deploy)
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["GET", "POST", "OPTIONS"],
-    allow_headers=["*"],
-)
-
-logger.info('Adding v1 endpoints..')
-
-# add v1
-app.include_router(v1_router, prefix='/v1')
-app.add_event_handler('startup', events.startup_event_handler(app))
-app.add_exception_handler(HTTPException, events.on_http_error)
-
-
-@app.on_event("startup")
-async def startup():
-    global model
-    if settings.inference_mode == "cpu":
-        logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
-        from gpt4all import GPT4All
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
-    else:
-        # is it possible to do this once the server is up?
-        ## TODO block until HF inference server is up.
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
-
-
-@app.on_event("shutdown")
-async def shutdown():
-    logger.info("Shutting down API")
-
-
-if settings.sentry_dns is not None:
-    import sentry_sdk
-
-    def traces_sampler(sampling_context):
-        if 'health' in sampling_context['transaction_context']['name']:
-            return False
-
-    sentry_sdk.init(
-        dsn=settings.sentry_dns, traces_sample_rate=0.1, traces_sampler=traces_sampler, send_default_pii=False
-    )
-
-# This is needed to get logs to show up in the app
-if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
-    gunicorn_error_logger = logging.getLogger("gunicorn.error")
-    gunicorn_logger = logging.getLogger("gunicorn")
-
-    root_logger = logging.getLogger()
-    fastapi_logger.setLevel(gunicorn_logger.level)
-    fastapi_logger.handlers = gunicorn_error_logger.handlers
-    root_logger.setLevel(gunicorn_logger.level)
-
-    uvicorn_logger = logging.getLogger("uvicorn.access")
-    uvicorn_logger.handlers = gunicorn_error_logger.handlers
-else:
-    # https://github.com/tiangolo/fastapi/issues/2019
-    LOG_FORMAT2 = (
-        "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
-    )
-    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -1,93 +0,0 @@
-"""
-Use the OpenAI python API to test gpt4all models.
-"""
-from typing import List, get_args
-import os
-from dotenv import load_dotenv
-
-import openai
-
-openai.api_base = "http://localhost:4891/v1"
-openai.api_key = "not needed for a local LLM"
-
-# Load the .env file
-env_path = 'gpt4all-api/gpt4all_api/.env'
-load_dotenv(dotenv_path=env_path)
-
-# Fetch MODEL_ID from .env file
-model_id = os.getenv('MODEL_BIN', 'default_model_id')
-embedding = os.getenv('EMBEDDING', 'default_embedding_model_id')
-print (model_id)
-print (embedding)
-
-def test_completion():
-    model = model_id
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-
-def test_streaming_completion():
-    model = model_id
-    prompt = "Who is Michael Jordan?"
-    tokens = []
-    for resp in openai.Completion.create(
-            model=model,
-            prompt=prompt,
-            max_tokens=50,
-            temperature=0.28,
-            top_p=0.95,
-            n=1,
-            echo=True,
-            stream=True):
-        tokens.append(resp.choices[0].text)
-
-    assert (len(tokens) > 0)
-    assert (len("".join(tokens)) > len(prompt))
-
-# Modified test batch, problems with keyerror in response
-def test_batched_completion():
-    model = model_id  # replace with your specific model ID
-    prompt = "Who is Michael Jordan?"
-    responses = []
-
-    # Loop to create completions one at a time
-    for _ in range(3):
-        response = openai.Completion.create(
-            model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-        )
-        responses.append(response)
-
-    # Assertions to check the responses
-    for response in responses:
-        assert len(response['choices'][0]['text']) > len(prompt)
-
-    assert len(responses) == 3
-
-def test_embedding():
-    model = embedding
-    prompt = "Who is Michael Jordan?"
-    response = openai.Embedding.create(model=model, input=prompt)
-    output = response["data"][0]["embedding"]
-    args = get_args(List[float])
-
-    assert response["model"] == model
-    assert isinstance(output, list)
-    assert all(isinstance(x, args) for x in output)
-
-def test_chat_completion():
-    model = model_id
-
-    response = openai.ChatCompletion.create(
-        model=model,
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": "Knock knock."},
-            {"role": "assistant", "content": "Who's there?"},
-            {"role": "user", "content": "Orange."},
-            ]
-    )
-
-    assert response.choices[0].message.role == "assistant"
-    assert len(response.choices[0].message.content) > 0

gpt4all-api/gpt4all_api/env

@@ -1,3 +0,0 @@
-# Add your GGUF compatible model LLM here. ie: MODEL_BIN="mistral-7b-instruct-v0.1.Q4_0", rename file ".env"
-# Make sure this LLM matches the model you placed inside the models folder
-MODEL_BIN=""

gpt4all-api/gpt4all_api/models/README.md

@@ -1 +0,0 @@
-### Drop GGUF compatible models here, make sure it matches MODEL_BIN on your .env file

gpt4all-api/gpt4all_api/requirements.txt

@@ -1,13 +0,0 @@
-aiohttp>=3.6.2
-aiofiles
-pydantic>=1.4.0,<2.0.0
-requests>=2.24.0
-ujson>=2.0.2
-fastapi>=0.95.0
-Jinja2>=3.0
-gpt4all>=1.0.0
-pytest
-openai==0.28.0
-black
-isort
-python-dotenv

gpt4all-api/makefile

@@ -1,46 +0,0 @@
-ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-APP_NAME:=gpt4all_api
-PYTHON:=python3.8
-SHELL := /bin/bash
-
-all: dependencies
-
-fresh: clean dependencies
-
-testenv: clean_testenv test_build
-	docker compose -f docker-compose.yaml up --build
-
-testenv_gpu: clean_testenv test_build
-	docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up --build
-
-testenv_d: clean_testenv test_build
-	docker compose env up --build -d
-
-test:
-	docker compose exec $(APP_NAME) pytest -svv --disable-warnings -p no:cacheprovider /app/tests
-
-test_build:
-    DOCKER_BUILDKIT=1 docker build -t $(APP_NAME) --progress plain -f $(APP_NAME)/Dockerfile.buildkit .
-
-clean_testenv:
-	docker compose down -v
-
-fresh_testenv: clean_testenv testenv
-
-venv:
-	if [ ! -d $(ROOT_DIR)/venv ]; then $(PYTHON) -m venv $(ROOT_DIR)/venv; fi
-
-dependencies: venv
-	source $(ROOT_DIR)/venv/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
-
-clean: clean_testenv
-	# Remove existing environment
-	rm -rf $(ROOT_DIR)/venv;
-	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
-
-
-black:
-	source $(ROOT_DIR)/venv/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
-
-isort:
-	source $(ROOT_DIR)/venv/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)

gpt4all-backend/CMakeLists.txt

@@ -2,15 +2,23 @@ cmake_minimum_required(VERSION 3.16)
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-if(APPLE)
-  option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
-  if(BUILD_UNIVERSAL)
+if (APPLE)
+    option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
+else()
+    option(LLMODEL_KOMPUTE "llmodel: use Kompute"              ON)
+    option(LLMODEL_VULKAN  "llmodel: use Vulkan"               OFF)
+    option(LLMODEL_CUDA    "llmodel: use CUDA"                 ON)
+    option(LLMODEL_ROCM    "llmodel: use ROCm"                 OFF)
+endif()
+
+if (APPLE)
+  if (BUILD_UNIVERSAL)
     # Build a Universal binary on macOS
     # This requires that the found Qt library is compiled as Universal binaries.
     set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
   else()
     # Build for the host architecture on macOS
-    if(NOT CMAKE_OSX_ARCHITECTURES)
+    if (NOT CMAKE_OSX_ARCHITECTURES)
       set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
     endif()
   endif()
@@ -39,11 +47,35 @@ else()
     message(STATUS "Interprocedural optimization support detected")
 endif()
 
+set(DIRECTORY llama.cpp-mainline)
 include(llama.cpp.cmake)
 
-set(BUILD_VARIANTS default avxonly)
-if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-    set(BUILD_VARIANTS ${BUILD_VARIANTS} metal)
+set(BUILD_VARIANTS)
+set(GPTJ_BUILD_VARIANT cpu)
+if (APPLE)
+    list(APPEND BUILD_VARIANTS metal)
+endif()
+if (LLMODEL_KOMPUTE)
+    list(APPEND BUILD_VARIANTS kompute kompute-avxonly)
+    set(GPTJ_BUILD_VARIANT kompute)
+else()
+    list(PREPEND BUILD_VARIANTS cpu cpu-avxonly)
+endif()
+if (LLMODEL_VULKAN)
+    list(APPEND BUILD_VARIANTS vulkan vulkan-avxonly)
+endif()
+if (LLMODEL_CUDA)
+    include(CheckLanguage)
+    check_language(CUDA)
+    if (NOT CMAKE_CUDA_COMPILER)
+        message(WARNING "CUDA Toolkit not found. To build without CUDA, use -DLLMODEL_CUDA=OFF.")
+    endif()
+    enable_language(CUDA)
+    list(APPEND BUILD_VARIANTS cuda cuda-avxonly)
+endif()
+if (LLMODEL_ROCM)
+    enable_language(HIP)
+    list(APPEND BUILD_VARIANTS rocm rocm-avxonly)
 endif()
 
 set(CMAKE_VERBOSE_MAKEFILE ON)
@@ -51,24 +83,34 @@ set(CMAKE_VERBOSE_MAKEFILE ON)
 # Go through each build variant
 foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     # Determine flags
-    if (BUILD_VARIANT STREQUAL avxonly)
-        set(GPT4ALL_ALLOW_NON_AVX NO)
+    if (BUILD_VARIANT MATCHES avxonly)
+        set(GPT4ALL_ALLOW_NON_AVX OFF)
     else()
-        set(GPT4ALL_ALLOW_NON_AVX YES)
+        set(GPT4ALL_ALLOW_NON_AVX ON)
     endif()
     set(LLAMA_AVX2 ${GPT4ALL_ALLOW_NON_AVX})
     set(LLAMA_F16C ${GPT4ALL_ALLOW_NON_AVX})
     set(LLAMA_FMA  ${GPT4ALL_ALLOW_NON_AVX})
 
-    if (BUILD_VARIANT STREQUAL metal)
-        set(LLAMA_METAL YES)
-    else()
-        set(LLAMA_METAL NO)
+    set(LLAMA_METAL   OFF)
+    set(LLAMA_KOMPUTE OFF)
+    set(LLAMA_VULKAN  OFF)
+    set(LLAMA_CUDA    OFF)
+    set(LLAMA_ROCM    OFF)
+    if (BUILD_VARIANT MATCHES metal)
+        set(LLAMA_METAL   ON)
+    elseif (BUILD_VARIANT MATCHES kompute)
+        set(LLAMA_KOMPUTE ON)
+    elseif (BUILD_VARIANT MATCHES vulkan)
+        set(LLAMA_VULKAN  ON)
+    elseif (BUILD_VARIANT MATCHES cuda)
+        set(LLAMA_CUDA    ON)
+    elseif (BUILD_VARIANT MATCHES rocm)
+        set(LLAMA_HIPBLAS ON)
     endif()
 
     # Include GGML
-    set(LLAMA_K_QUANTS YES)
-    include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
+    include_ggml(-mainline-${BUILD_VARIANT})
 
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
@@ -93,11 +135,15 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    if (NOT LLAMA_METAL)
+    if (BUILD_VARIANT MATCHES ${GPTJ_BUILD_VARIANT})
         add_library(gptj-${BUILD_VARIANT} SHARED
             gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
         prepare_target(gptj llama-mainline)
     endif()
+
+    if (BUILD_VARIANT STREQUAL cuda)
+        set(CUDAToolkit_BIN_DIR ${CUDAToolkit_BIN_DIR} PARENT_SCOPE)
+    endif()
 endforeach()
 
 add_library(llmodel

gpt4all-backend/gptj.cpp

@@ -785,13 +785,15 @@ const std::vector<LLModel::Token> &GPTJ::endTokens() const
     return fres;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
+const char *get_arch_name(gguf_context *ctx_gguf) {
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    if (kid == -1)
+        throw std::runtime_error("key not found in model: general.architecture");
+
     enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
+    if (ktype != GGUF_TYPE_STRING)
+        throw std::runtime_error("key general.architecture has wrong type");
+
     return gguf_get_val_str(ctx_gguf, kid);
 }
 
@@ -814,21 +816,29 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
+DLL_EXPORT char *get_file_arch(const char *fname) {
     struct ggml_context * ctx_meta = NULL;
     struct gguf_init_params params = {
         /*.no_alloc = */ true,
         /*.ctx      = */ &ctx_meta,
     };
     gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
 
-    bool isValid = gguf_get_version(ctx_gguf) <= 3;
-    isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
+    char *arch = nullptr;
+    if (ctx_gguf && gguf_get_version(ctx_gguf) <= 3) {
+        try {
+            arch = strdup(get_arch_name(ctx_gguf));
+        } catch (const std::runtime_error &) {
+            // cannot read key -> return null
+        }
+    }
 
     gguf_free(ctx_gguf);
-    return isValid;
+    return arch;
+}
+
+DLL_EXPORT bool is_arch_supported(const char *arch) {
+    return !strcmp(arch, "gptj");
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llamamodel.cpp

@@ -22,7 +22,11 @@
 #include <llama.h>
 #include <ggml.h>
 #ifdef GGML_USE_KOMPUTE
-#include <ggml-kompute.h>
+#   include <ggml-kompute.h>
+#elif GGML_USE_VULKAN
+#   include <ggml-vulkan.h>
+#elif GGML_USE_CUDA
+#   include <ggml-cuda.h>
 #endif
 
 using namespace std::string_literals;
@@ -32,13 +36,44 @@ static constexpr int GGUF_VER_MAX = 3;
 
 static const char * const modelType_ = "LLaMA";
 
+// note: same order as LLM_ARCH_NAMES in llama.cpp
 static const std::vector<const char *> KNOWN_ARCHES {
-    "baichuan", "bert", "bloom", "codeshell", "falcon", "gemma", "gpt2", "llama", "mpt", "nomic-bert", "orion",
-    "persimmon", "phi2", "plamo", "qwen", "qwen2", "refact", "stablelm", "starcoder"
+    "llama",
+    "falcon",
+    // "grok", -- 314B parameters
+    "gpt2",
+    // "gptj", -- no inference code
+    // "gptneox", -- no inference code
+    "mpt",
+    "baichuan",
+    "starcoder",
+    // "persimmon", -- CUDA generates garbage
+    "refact",
+    "bert",
+    "nomic-bert",
+    "bloom",
+    "stablelm",
+    "qwen",
+    "qwen2",
+    "qwen2moe",
+    "phi2",
+    "phi3",
+    // "plamo", -- https://github.com/ggerganov/llama.cpp/issues/5669
+    "codeshell",
+    "orion",
+    "internlm2",
+    // "minicpm", -- CUDA generates garbage
+    "gemma",
+    "starcoder2",
+    // "mamba", -- CUDA missing SSM_CONV
+    "xverse",
+    "command-r",
+    // "dbrx", -- 16x12B parameters
+    "olmo",
 };
 
 static const std::vector<const char *> EMBEDDING_ARCHES {
-    "bert", "nomic-bert"
+    "bert", "nomic-bert",
 };
 
 static bool is_embedding_arch(const std::string &arch) {
@@ -104,13 +139,15 @@ static int llama_sample_top_p_top_k(
     return llama_sample_token(ctx, &candidates_p);
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
+const char *get_arch_name(gguf_context *ctx_gguf) {
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    if (kid == -1)
+        throw std::runtime_error("key not found in model: general.architecture");
+
     enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != (GGUF_TYPE_STRING)) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
+    if (ktype != GGUF_TYPE_STRING)
+        throw std::runtime_error("key general.architecture has wrong type");
+
     return gguf_get_val_str(ctx_gguf, kid);
 }
 
@@ -136,13 +173,20 @@ static gguf_context *load_gguf(const char *fname) {
 }
 
 static int32_t get_arch_key_u32(std::string const &modelPath, std::string const &archKey) {
+    int32_t value = -1;
+    std::string arch;
+
     auto * ctx = load_gguf(modelPath.c_str());
     if (!ctx)
-        return -1;
-    std::string arch = get_arch_name(ctx);
+        goto cleanup;
 
-    int32_t value = -1;
-    if (ctx) {
+    try {
+        arch = get_arch_name(ctx);
+    } catch (const std::runtime_error &) {
+        goto cleanup; // cannot read key
+    }
+
+    {
         auto key = arch + "." + archKey;
         int keyidx = gguf_find_key(ctx, key.c_str());
         if (keyidx != -1) {
@@ -152,6 +196,7 @@ static int32_t get_arch_key_u32(std::string const &modelPath, std::string const
         }
     }
 
+cleanup:
     gguf_free(ctx);
     return value;
 }
@@ -160,6 +205,7 @@ struct LLamaPrivate {
     const std::string modelPath;
     bool modelLoaded = false;
     int device = -1;
+    std::string deviceName;
     llama_model *model = nullptr;
     llama_context *ctx = nullptr;
     llama_model_params model_params;
@@ -244,15 +290,26 @@ bool LLamaModel::isModelBlacklisted(const std::string &modelPath) const {
 }
 
 bool LLamaModel::isEmbeddingModel(const std::string &modelPath) const {
+    bool result = false;
+    std::string arch;
+
     auto *ctx_gguf = load_gguf(modelPath.c_str());
     if (!ctx_gguf) {
         std::cerr << __func__ << ": failed to load GGUF from " <<  modelPath << "\n";
-        return false;
+        goto cleanup;
     }
 
-    std::string arch = get_arch_name(ctx_gguf);
+    try {
+        arch = get_arch_name(ctx_gguf);
+    } catch (const std::runtime_error &) {
+        goto cleanup; // cannot read key
+    }
+
+    result = is_embedding_arch(arch);
+
+cleanup:
     gguf_free(ctx_gguf);
-    return is_embedding_arch(arch);
+    return result;
 }
 
 bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
@@ -292,18 +349,19 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
 
     d_ptr->backend_name = "cpu"; // default
 
-#ifdef GGML_USE_KOMPUTE
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
     if (d_ptr->device != -1) {
         d_ptr->model_params.main_gpu = d_ptr->device;
         d_ptr->model_params.n_gpu_layers = ngl;
+        d_ptr->model_params.split_mode = LLAMA_SPLIT_MODE_NONE;
     }
 #elif defined(GGML_USE_METAL)
     (void)ngl;
 
     if (llama_verbose()) {
         std::cerr << "llama.cpp: using Metal" << std::endl;
-        d_ptr->backend_name = "metal";
     }
+    d_ptr->backend_name = "metal";
 
     // always fully offload on Metal
     // TODO(cebtenzzre): use this parameter to allow using more than 53% of system RAM to load a model
@@ -316,6 +374,7 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     if (!d_ptr->model) {
         fflush(stdout);
         d_ptr->device = -1;
+        d_ptr->deviceName.clear();
         std::cerr << "LLAMA ERROR: failed to load model from " << modelPath << std::endl;
         return false;
     }
@@ -358,17 +417,24 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
         llama_free_model(d_ptr->model);
         d_ptr->model = nullptr;
         d_ptr->device = -1;
+        d_ptr->deviceName.clear();
         return false;
     }
 
     d_ptr->end_tokens = {llama_token_eos(d_ptr->model)};
 
+    if (usingGPUDevice()) {
 #ifdef GGML_USE_KOMPUTE
-    if (usingGPUDevice() && ggml_vk_has_device()) {
-        std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
+        if (llama_verbose()) {
+            std::cerr << "llama.cpp: using Vulkan on " << d_ptr->deviceName << std::endl;
+        }
         d_ptr->backend_name = "kompute";
-    }
+#elif defined(GGML_USE_VULKAN)
+        d_ptr->backend_name = "vulkan";
+#elif defined(GGML_USE_CUDA)
+        d_ptr->backend_name = "cuda";
 #endif
+    }
 
     m_supportsEmbedding = isEmbedding;
     m_supportsCompletion = !isEmbedding;
@@ -429,7 +495,18 @@ std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::
 
 std::string LLamaModel::tokenToString(Token id) const
 {
-    return llama_token_to_piece(d_ptr->ctx, id);
+    std::vector<char> result(8, 0);
+    const int n_tokens = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), false);
+    if (n_tokens < 0) {
+        result.resize(-n_tokens);
+        int check = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), false);
+        GGML_ASSERT(check == -n_tokens);
+    }
+    else {
+        result.resize(n_tokens);
+    }
+
+    return std::string(result.data(), result.size());
 }
 
 LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
@@ -494,34 +571,77 @@ int32_t LLamaModel::layerCount(std::string const &modelPath) const
     return get_arch_key_u32(modelPath, "block_count");
 }
 
+#ifdef GGML_USE_VULKAN
+static const char *getVulkanVendorName(uint32_t vendorID) {
+    switch (vendorID) {
+        case 0x10DE: return "nvidia";
+        case 0x1002: return "amd";
+        case 0x8086: return "intel";
+        default:     return "unknown";
+    }
+}
+#endif
+
 std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryRequired) const
 {
-#ifdef GGML_USE_KOMPUTE
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
     size_t count = 0;
-    auto * vkDevices = ggml_vk_available_devices(memoryRequired, &count);
 
-    if (vkDevices) {
+#ifdef GGML_USE_KOMPUTE
+    auto *lcppDevices = ggml_vk_available_devices(memoryRequired, &count);
+#elif defined(GGML_USE_VULKAN)
+    (void)memoryRequired; // hasn't been used since GGUF was added
+    auto *lcppDevices = ggml_vk_available_devices(&count);
+#else // defined(GGML_USE_CUDA)
+    (void)memoryRequired;
+    auto *lcppDevices = ggml_cuda_available_devices(&count);
+#endif
+
+    if (lcppDevices) {
         std::vector<LLModel::GPUDevice> devices;
         devices.reserve(count);
 
         for (size_t i = 0; i < count; ++i) {
-            auto & dev = vkDevices[i];
+            auto & dev = lcppDevices[i];
+
             devices.emplace_back(
+#ifdef GGML_USE_KOMPUTE
+                /* backend  = */ "kompute",
                 /* index    = */ dev.index,
                 /* type     = */ dev.type,
                 /* heapSize = */ dev.heapSize,
                 /* name     = */ dev.name,
                 /* vendor   = */ dev.vendor
+#elif defined(GGML_USE_VULKAN)
+                /* backend  = */ "vulkan",
+                /* index    = */ dev.index,
+                /* type     = */ dev.type,
+                /* heapSize = */ dev.heapSize,
+                /* name     = */ dev.name,
+                /* vendor   = */ getVulkanVendorName(dev.vendorID)
+#else // defined(GGML_USE_CUDA)
+                /* backend  = */ "cuda",
+                /* index    = */ dev.index,
+                /* type     = */ 2, // vk::PhysicalDeviceType::eDiscreteGpu
+                /* heapSize = */ dev.heapSize,
+                /* name     = */ dev.name,
+                /* vendor   = */ "nvidia"
+#endif
             );
+
+#ifndef GGML_USE_CUDA
             ggml_vk_device_destroy(&dev);
+#else
+            ggml_cuda_device_destroy(&dev);
+#endif
         }
 
-        free(vkDevices);
+        free(lcppDevices);
         return devices;
     }
 #else
     (void)memoryRequired;
-    std::cerr << __func__ << ": built without Kompute\n";
+    std::cerr << __func__ << ": built without a GPU backend\n";
 #endif
 
     return {};
@@ -529,11 +649,32 @@ std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryReq
 
 bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string &name) const
 {
-#if defined(GGML_USE_KOMPUTE)
+#if defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
+    auto devices = availableGPUDevices(memoryRequired);
+
+    auto dev_it = devices.begin();
+#ifndef GGML_USE_CUDA
+    if (name == "amd" || name == "nvidia" || name == "intel") {
+        dev_it = std::find_if(dev_it, devices.end(), [&name](auto &dev) { return dev.vendor == name; });
+    } else
+#endif
+    if (name != "gpu") {
+        dev_it = std::find_if(dev_it, devices.end(), [&name](auto &dev) { return dev.name == name; });
+    }
+
+    if (dev_it < devices.end()) {
+        d_ptr->device     = dev_it->index;
+        d_ptr->deviceName = dev_it->name;
+        return true;
+    }
+    return false;
+#elif defined(GGML_USE_KOMPUTE)
     ggml_vk_device device;
     bool ok = ggml_vk_get_device(&device, memoryRequired, name.c_str());
     if (ok) {
         d_ptr->device = device.index;
+        d_ptr->deviceName = device.name;
+        ggml_vk_device_destroy(&device);
         return true;
     }
 #else
@@ -545,37 +686,60 @@ bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string &n
 
 bool LLamaModel::initializeGPUDevice(int device, std::string *unavail_reason) const
 {
-#if defined(GGML_USE_KOMPUTE)
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
     (void)unavail_reason;
+    auto devices = availableGPUDevices();
+    auto it = std::find_if(devices.begin(), devices.end(), [device](auto &dev) { return dev.index == device; });
     d_ptr->device = device;
+    d_ptr->deviceName = it < devices.end() ? it->name : "(unknown)";
     return true;
 #else
     (void)device;
     if (unavail_reason) {
-        *unavail_reason = "built without Kompute";
+        *unavail_reason = "built without a GPU backend";
     }
     return false;
 #endif
 }
 
-bool LLamaModel::hasGPUDevice()
+bool LLamaModel::hasGPUDevice() const
 {
-#if defined(GGML_USE_KOMPUTE)
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
     return d_ptr->device != -1;
 #else
     return false;
 #endif
 }
 
-bool LLamaModel::usingGPUDevice()
+bool LLamaModel::usingGPUDevice() const
 {
-#if defined(GGML_USE_KOMPUTE)
-    return hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
+    bool hasDevice;
+
+#ifdef GGML_USE_KOMPUTE
+    hasDevice = hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
+    assert(!hasDevice || ggml_vk_has_device());
+#elif defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
+    hasDevice = hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
 #elif defined(GGML_USE_METAL)
-    return true;
+    hasDevice = true;
 #else
-    return false;
+    hasDevice = false;
 #endif
+
+    return hasDevice;
+}
+
+const char *LLamaModel::backendName() const {
+    return d_ptr->backend_name;
+}
+
+const char *LLamaModel::gpuDeviceName() const {
+    if (usingGPUDevice()) {
+#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
+        return d_ptr->deviceName.c_str();
+#endif
+    }
+    return nullptr;
 }
 
 void llama_batch_add(
@@ -755,6 +919,7 @@ void LLamaModel::embedInternal(
         tokens.resize(text.length()+4);
         int32_t n_tokens = llama_tokenize(d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), wantBOS, false);
         if (n_tokens) {
+            (void)eos_token;
             assert(useEOS == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
             tokens.resize(n_tokens - useEOS); // erase EOS/SEP
         } else {
@@ -922,6 +1087,8 @@ void LLamaModel::embedInternal(
     }
 
     if (tokenCount) { *tokenCount = totalTokens; }
+
+    llama_batch_free(batch);
 }
 
 #if defined(_WIN32)
@@ -943,25 +1110,33 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char *fname) {
-    auto * ctx = load_gguf(fname);
-    std::string arch = get_arch_name(ctx);
+DLL_EXPORT char *get_file_arch(const char *fname) {
+    char *arch = nullptr;
+    std::string archStr;
 
-    bool valid = true;
+    auto *ctx = load_gguf(fname);
+    if (!ctx)
+        goto cleanup;
 
-    if (std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), arch) == KNOWN_ARCHES.end()) {
-        // not supported by this version of llama.cpp
-        if (arch != "gptj") { // we support this via another module
-            std::cerr << __func__ << ": unsupported model architecture: " << arch << "\n";
-        }
-        valid = false;
+    try {
+        archStr = get_arch_name(ctx);
+    } catch (const std::runtime_error &) {
+        goto cleanup; // cannot read key
     }
 
-    if (valid && is_embedding_arch(arch) && gguf_find_key(ctx, (arch + ".pooling_type").c_str()) < 0)
-        valid = false; // old pre-llama.cpp embedding model, e.g. all-MiniLM-L6-v2-f16.gguf
+    if (is_embedding_arch(archStr) && gguf_find_key(ctx, (archStr + ".pooling_type").c_str()) < 0) {
+        // old bert.cpp embedding model
+    } else {
+        arch = strdup(archStr.c_str());
+    }
 
+cleanup:
     gguf_free(ctx);
-    return valid;
+    return arch;
+}
+
+DLL_EXPORT bool is_arch_supported(const char *arch) {
+    return std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), std::string(arch)) < KNOWN_ARCHES.end();
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llamamodel_impl.h

@@ -30,11 +30,13 @@ public:
     size_t restoreState(const uint8_t *src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
-    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const override;
+    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0) const override;
     bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const override;
     bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const override;
-    bool hasGPUDevice() override;
-    bool usingGPUDevice() override;
+    bool hasGPUDevice() const override;
+    bool usingGPUDevice() const override;
+    const char *backendName() const override;
+    const char *gpuDeviceName() const override;
 
     size_t embeddingSize() const override;
     // user-specified prefix

gpt4all-backend/llmodel.cpp

@@ -8,15 +8,25 @@
 #include <fstream>
 #include <iostream>
 #include <memory>
+#include <optional>
 #include <regex>
 #include <sstream>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #ifdef _MSC_VER
 #include <intrin.h>
 #endif
 
+#ifndef __APPLE__
+static const std::string DEFAULT_BACKENDS[] = {"kompute", "cpu"};
+#elif defined(__aarch64__)
+static const std::string DEFAULT_BACKENDS[] = {"metal", "cpu"};
+#else
+static const std::string DEFAULT_BACKENDS[] = {"cpu"};
+#endif
+
 std::string s_implementations_search_path = ".";
 
 #if !(defined(__x86_64__) || defined(_M_X64))
@@ -32,13 +42,13 @@ std::string s_implementations_search_path = ".";
     }
 
     // AVX via EAX=1: Processor Info and Feature Bits, bit 28 of ECX
-    #define cpu_supports_avx()  (get_cpu_info(1, 2) & (1 << 28))
+    #define cpu_supports_avx()  !!(get_cpu_info(1, 2) & (1 << 28))
     // AVX2 via EAX=7, ECX=0: Extended Features, bit 5 of EBX
-    #define cpu_supports_avx2() (get_cpu_info(7, 1) & (1 <<  5))
+    #define cpu_supports_avx2() !!(get_cpu_info(7, 1) & (1 <<  5))
 #else
     // gcc/clang
-    #define cpu_supports_avx()  __builtin_cpu_supports("avx")
-    #define cpu_supports_avx2() __builtin_cpu_supports("avx2")
+    #define cpu_supports_avx()  !!__builtin_cpu_supports("avx")
+    #define cpu_supports_avx2() !!__builtin_cpu_supports("avx2")
 #endif
 
 LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
@@ -49,14 +59,17 @@ LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
     m_buildVariant = get_build_variant();
-    m_magicMatch = m_dlhandle->get<bool(const char*)>("magic_match");
-    assert(m_magicMatch);
+    m_getFileArch = m_dlhandle->get<char *(const char *)>("get_file_arch");
+    assert(m_getFileArch);
+    m_isArchSupported = m_dlhandle->get<bool(const char *)>("is_arch_supported");
+    assert(m_isArchSupported);
     m_construct = m_dlhandle->get<LLModel *()>("construct");
     assert(m_construct);
 }
 
 LLModel::Implementation::Implementation(Implementation &&o)
-    : m_magicMatch(o.m_magicMatch)
+    : m_getFileArch(o.m_getFileArch)
+    , m_isArchSupported(o.m_isArchSupported)
     , m_construct(o.m_construct)
     , m_modelType(o.m_modelType)
     , m_buildVariant(o.m_buildVariant)
@@ -82,11 +95,9 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     static auto* libs = new std::vector<Implementation>([] () {
         std::vector<Implementation> fres;
 
-        std::string impl_name_re = "(gptj|llamamodel-mainline)";
+        std::string impl_name_re = "(gptj|llamamodel-mainline)-(cpu|metal|kompute|vulkan|cuda)";
         if (cpu_supports_avx2() == 0) {
             impl_name_re += "-avxonly";
-        } else {
-            impl_name_re += "-(default|metal)";
         }
         std::regex re(impl_name_re);
         auto search_in_directory = [&](const std::string& paths) {
@@ -121,116 +132,148 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     return *libs;
 }
 
+static std::string applyCPUVariant(const std::string &buildVariant) {
+    if (buildVariant != "metal" && cpu_supports_avx2() == 0) {
+        return buildVariant + "-avxonly";
+    }
+    return buildVariant;
+}
+
 const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
     bool buildVariantMatched = false;
+    std::optional<std::string> archName;
     for (const auto& i : implementationList()) {
         if (buildVariant != i.m_buildVariant) continue;
         buildVariantMatched = true;
 
-        if (!i.m_magicMatch(fname)) continue;
-        return &i;
+        char *arch = i.m_getFileArch(fname);
+        if (!arch) continue;
+        archName = arch;
+
+        bool archSupported = i.m_isArchSupported(arch);
+        free(arch);
+        if (archSupported) return &i;
     }
 
     if (!buildVariantMatched)
-        throw std::runtime_error("Could not find any implementations for build variant: " + buildVariant);
+        return nullptr;
+    if (!archName)
+        throw UnsupportedModelError("Unsupported file format");
 
-    return nullptr; // unsupported model format
+    throw BadArchError(std::move(*archName));
 }
 
-LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant, int n_ctx) {
-    // Get correct implementation
-    const Implementation* impl = nullptr;
-
-    #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
-        if (buildVariant == "auto") {
-            size_t total_mem = getSystemTotalRAMInBytes();
-            impl = implementation(modelPath.c_str(), "metal");
-            if(impl) {
-                LLModel* metalimpl = impl->m_construct();
-                metalimpl->m_implementation = impl;
-                /* TODO(cebtenzzre): after we fix requiredMem, we should change this to happen at
-                 * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
-                 * most (all?) places where this is called, causing underestimation of required
-                 * memory. */
-                size_t req_mem = metalimpl->requiredMem(modelPath, n_ctx, 100);
-                float req_to_total = (float) req_mem / (float) total_mem;
-                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
-                if (req_to_total >= 0.53) {
-                    delete metalimpl;
-                    impl = nullptr;
-                } else {
-                    return metalimpl;
-                }
-            }
-        }
-    #else
-        (void)n_ctx;
-    #endif
-
-    if (!impl) {
-        //TODO: Auto-detect CUDA/OpenCL
-        if (buildVariant == "auto") {
-            if (cpu_supports_avx2() == 0) {
-                buildVariant = "avxonly";
-            } else {
-                buildVariant = "default";
-            }
-        }
-        impl = implementation(modelPath.c_str(), buildVariant);
-        if (!impl) return nullptr;
+LLModel *LLModel::Implementation::construct(const std::string &modelPath, const std::string &backend, int n_ctx) {
+    std::vector<std::string> desiredBackends;
+    if (backend != "auto") {
+        desiredBackends.push_back(backend);
+    } else {
+        desiredBackends.insert(desiredBackends.end(), DEFAULT_BACKENDS, std::end(DEFAULT_BACKENDS));
     }
 
-    // Construct and return llmodel implementation
-    auto fres = impl->m_construct();
-    fres->m_implementation = impl;
-    return fres;
+    for (const auto &desiredBackend: desiredBackends) {
+        const auto *impl = implementation(modelPath.c_str(), applyCPUVariant(desiredBackend));
+
+        if (impl) {
+            // Construct llmodel implementation
+            auto *fres = impl->m_construct();
+            fres->m_implementation = impl;
+
+#if defined(__APPLE__) && defined(__aarch64__) // FIXME: See if metal works for intel macs
+            /* TODO(cebtenzzre): after we fix requiredMem, we should change this to happen at
+             * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
+             * most (all?) places where this is called, causing underestimation of required
+             * memory. */
+            if (backend == "auto" && desiredBackend == "metal") {
+                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
+                size_t req_mem = fres->requiredMem(modelPath, n_ctx, 100);
+                if (req_mem >= size_t(0.53f * getSystemTotalRAMInBytes())) {
+                    delete fres;
+                    continue;
+                }
+            }
+#else
+            (void)n_ctx;
+#endif
+
+            return fres;
+        }
+    }
+
+    throw MissingImplementationError("Could not find any implementations for backend: " + backend);
 }
 
-LLModel *LLModel::Implementation::constructDefaultLlama() {
-    static std::unique_ptr<LLModel> llama([]() -> LLModel * {
-        const std::vector<LLModel::Implementation> *impls;
-        try {
-            impls = &implementationList();
-        } catch (const std::runtime_error &e) {
-            std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
-            return nullptr;
-        }
+LLModel *LLModel::Implementation::constructGlobalLlama(const std::optional<std::string> &backend) {
+    static std::unordered_map<std::string, std::unique_ptr<LLModel>> implCache;
+
+    const std::vector<Implementation> *impls;
+    try {
+        impls = &implementationList();
+    } catch (const std::runtime_error &e) {
+        std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
+        return nullptr;
+    }
+
+    std::vector<std::string> desiredBackends;
+    if (backend) {
+        desiredBackends.push_back(backend.value());
+    } else {
+        desiredBackends.insert(desiredBackends.end(), DEFAULT_BACKENDS, std::end(DEFAULT_BACKENDS));
+    }
+
+    const Implementation *impl = nullptr;
+
+    for (const auto &desiredBackend: desiredBackends) {
+        auto cacheIt = implCache.find(desiredBackend);
+        if (cacheIt != implCache.end())
+            return cacheIt->second.get(); // cached
 
-        const LLModel::Implementation *impl = nullptr;
         for (const auto &i: *impls) {
-            if (i.m_buildVariant == "metal" || i.m_modelType != "LLaMA") continue;
-            impl = &i;
-        }
-        if (!impl) {
-            std::cerr << __func__ << ": could not find llama.cpp implementation\n";
-            return nullptr;
+            if (i.m_modelType == "LLaMA" && i.m_buildVariant == applyCPUVariant(desiredBackend)) {
+                impl = &i;
+                break;
+            }
         }
 
-        auto fres = impl->m_construct();
-        fres->m_implementation = impl;
-        return fres;
-    }());
-    return llama.get();
+        if (impl) {
+            auto *fres = impl->m_construct();
+            fres->m_implementation = impl;
+            implCache[desiredBackend] = std::unique_ptr<LLModel>(fres);
+            return fres;
+        }
+    }
+
+    std::cerr << __func__ << ": could not find Llama implementation for backend: " << backend.value_or("default") << "\n";
+    return nullptr;
 }
 
 std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices(size_t memoryRequired) {
-    auto *llama = constructDefaultLlama();
-    if (llama) { return llama->availableGPUDevices(memoryRequired); }
-    return {};
+    std::vector<LLModel::GPUDevice> devices;
+#ifndef __APPLE__
+    static const std::string backends[] = {"kompute", "cuda"};
+    for (const auto &backend: backends) {
+        auto *llama = constructGlobalLlama(backend);
+        if (llama) {
+            auto backendDevs = llama->availableGPUDevices(memoryRequired);
+            devices.insert(devices.end(), backendDevs.begin(), backendDevs.end());
+        }
+    }
+#endif
+    return devices;
 }
 
 int32_t LLModel::Implementation::maxContextLength(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
+    auto *llama = constructGlobalLlama();
     return llama ? llama->maxContextLength(modelPath) : -1;
 }
 
 int32_t LLModel::Implementation::layerCount(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
+    auto *llama = constructGlobalLlama();
     return llama ? llama->layerCount(modelPath) : -1;
 }
 
 bool LLModel::Implementation::isEmbeddingModel(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
+    auto *llama = constructGlobalLlama();
     return llama && llama->isEmbeddingModel(modelPath);
 }
 
@@ -245,3 +288,7 @@ const std::string& LLModel::Implementation::implementationsSearchPath() {
 bool LLModel::Implementation::hasSupportedCPU() {
     return cpu_supports_avx() != 0;
 }
+
+int LLModel::Implementation::cpuSupportsAVX2() {
+    return cpu_supports_avx2();
+}

gpt4all-backend/llmodel.h

@@ -1,6 +1,7 @@
 #ifndef LLMODEL_H
 #define LLMODEL_H
 
+#include <algorithm>
 #include <cstdint>
 #include <fstream>
 #include <functional>
@@ -8,8 +9,11 @@
 #include <optional>
 #include <string>
 #include <string_view>
+#include <unordered_map>
 #include <vector>
 
+using namespace std::string_literals;
+
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
 class Dlhandle;
@@ -17,15 +21,59 @@ class LLModel {
 public:
     using Token = int32_t;
 
+    class BadArchError: public std::runtime_error {
+    public:
+        BadArchError(std::string arch)
+            : runtime_error("Unsupported model architecture: " + arch)
+            , m_arch(std::move(arch))
+            {}
+
+        const std::string &arch() const noexcept { return m_arch; }
+
+    private:
+        std::string m_arch;
+    };
+
+    class MissingImplementationError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
+    class UnsupportedModelError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
     struct GPUDevice {
+        const char *backend;
         int index;
         int type;
         size_t heapSize;
         std::string name;
         std::string vendor;
 
-        GPUDevice(int index, int type, size_t heapSize, std::string name, std::string vendor):
-            index(index), type(type), heapSize(heapSize), name(std::move(name)), vendor(std::move(vendor)) {}
+        GPUDevice(const char *backend, int index, int type, size_t heapSize, std::string name, std::string vendor):
+            backend(backend), index(index), type(type), heapSize(heapSize), name(std::move(name)),
+            vendor(std::move(vendor)) {}
+
+        std::string selectionName() const { return m_backendNames.at(backend) + ": " + name; }
+        std::string reportedName()  const { return name + " (" + m_backendNames.at(backend) + ")"; }
+
+        static std::string updateSelectionName(const std::string &name) {
+            if (name == "Auto" || name == "CPU" || name == "Metal")
+                return name;
+            auto it = std::find_if(m_backendNames.begin(), m_backendNames.end(), [&name](const auto &entry) {
+                return name.starts_with(entry.second + ": ");
+            });
+            if (it != m_backendNames.end())
+                return name;
+            return "Vulkan: " + name; // previously, there were only Vulkan devices
+        }
+
+    private:
+        static inline const std::unordered_map<std::string, std::string> m_backendNames {
+            {"cuda", "CUDA"}, {"kompute", "Vulkan"},
+        };
     };
 
     class Implementation {
@@ -37,7 +85,7 @@ public:
         std::string_view modelType() const { return m_modelType; }
         std::string_view buildVariant() const { return m_buildVariant; }
 
-        static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto", int n_ctx = 2048);
+        static LLModel *construct(const std::string &modelPath, const std::string &backend = "auto", int n_ctx = 2048);
         static std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0);
         static int32_t maxContextLength(const std::string &modelPath);
         static int32_t layerCount(const std::string &modelPath);
@@ -45,15 +93,18 @@ public:
         static void setImplementationsSearchPath(const std::string &path);
         static const std::string &implementationsSearchPath();
         static bool hasSupportedCPU();
+        // 0 for no, 1 for yes, -1 for non-x86_64
+        static int cpuSupportsAVX2();
 
     private:
         Implementation(Dlhandle &&);
 
         static const std::vector<Implementation> &implementationList();
         static const Implementation *implementation(const char *fname, const std::string &buildVariant);
-        static LLModel *constructDefaultLlama();
+        static LLModel *constructGlobalLlama(const std::optional<std::string> &backend = std::nullopt);
 
-        bool (*m_magicMatch)(const char *fname);
+        char *(*m_getFileArch)(const char *fname);
+        bool (*m_isArchSupported)(const char *arch);
         LLModel *(*m_construct)();
 
         std::string_view m_modelType;
@@ -144,8 +195,10 @@ public:
         return false;
     }
 
-    virtual bool hasGPUDevice() { return false; }
-    virtual bool usingGPUDevice() { return false; }
+    virtual bool hasGPUDevice() const { return false; }
+    virtual bool usingGPUDevice() const { return false; }
+    virtual const char *backendName() const { return "cpu"; }
+    virtual const char *gpuDeviceName() const { return nullptr; }
 
     void setProgressCallback(ProgressCallback callback) { m_progressCallback = callback; }
 

gpt4all-backend/llmodel_c.cpp

@@ -31,20 +31,15 @@ static void llmodel_set_error(const char **errptr, const char *message) {
     }
 }
 
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error) {
+llmodel_model llmodel_model_create2(const char *model_path, const char *backend, const char **error) {
     LLModel *llModel;
     try {
-        llModel = LLModel::Implementation::construct(model_path, build_variant);
+        llModel = LLModel::Implementation::construct(model_path, backend);
     } catch (const std::exception& e) {
         llmodel_set_error(error, e.what());
         return nullptr;
     }
 
-    if (!llModel) {
-        llmodel_set_error(error, "Model format not supported (no matching implementation found)");
-        return nullptr;
-    }
-
     auto wrapper = new LLModelWrapper;
     wrapper->llModel = llModel;
     return wrapper;
@@ -253,6 +248,7 @@ struct llmodel_gpu_device *llmodel_available_gpu_devices(size_t memoryRequired,
     for (unsigned i = 0; i < devices.size(); i++) {
         const auto &dev  =   devices[i];
               auto &cdev = c_devices[i];
+        cdev.backend  = dev.backend;
         cdev.index    = dev.index;
         cdev.type     = dev.type;
         cdev.heapSize = dev.heapSize;
@@ -283,6 +279,18 @@ bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
 
 bool llmodel_has_gpu_device(llmodel_model model)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->hasGPUDevice();
 }
+
+const char *llmodel_model_backend_name(llmodel_model model)
+{
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->backendName();
+}
+
+const char *llmodel_model_gpu_device_name(llmodel_model model)
+{
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->gpuDeviceName();
+}

gpt4all-backend/llmodel_c.h

@@ -48,6 +48,7 @@ struct llmodel_prompt_context {
 };
 
 struct llmodel_gpu_device {
+    const char * backend;
     int index;
     int type; // same as VkPhysicalDeviceType
     size_t heapSize;
@@ -86,7 +87,7 @@ typedef bool (*llmodel_recalculate_callback)(bool is_recalculating);
  * Embedding cancellation callback for use with llmodel_embed.
  * @param batch_sizes The number of tokens in each batch that will be embedded.
  * @param n_batch The number of batches that will be embedded.
- * @param backend The backend that will be used for embedding. One of "cpu", "kompute", or "metal".
+ * @param backend The backend that will be used for embedding. One of "cpu", "kompute", "cuda", or "metal".
  * @return True to cancel llmodel_embed, false to continue.
  */
 typedef bool (*llmodel_emb_cancel_callback)(unsigned *batch_sizes, unsigned n_batch, const char *backend);
@@ -103,11 +104,11 @@ DEPRECATED llmodel_model llmodel_model_create(const char *model_path);
  * Create a llmodel instance.
  * Recognises correct model type from file at model_path
  * @param model_path A string representing the path to the model file; will only be used to detect model type.
- * @param build_variant A string representing the implementation to use (auto, default, avxonly, ...),
+ * @param backend A string representing the implementation to use. One of 'auto', 'cpu', 'metal', 'kompute', or 'cuda'.
  * @param error A pointer to a string; will only be set on error.
  * @return A pointer to the llmodel_model instance; NULL on error.
  */
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error);
+llmodel_model llmodel_model_create2(const char *model_path, const char *backend, const char **error);
 
 /**
  * Destroy a llmodel instance.
@@ -295,6 +296,16 @@ bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
  */
 bool llmodel_has_gpu_device(llmodel_model model);
 
+/**
+ * @return The name of the llama.cpp backend currently in use. One of "cpu", "kompute", or "metal".
+ */
+const char *llmodel_model_backend_name(llmodel_model model);
+
+/**
+ * @return The name of the GPU device currently in use, or NULL for backends other than Kompute.
+ */
+const char *llmodel_model_gpu_device_name(llmodel_model model);
+
 #ifdef __cplusplus
 }
 #endif

gpt4all-bindings/golang/README.md

@@ -24,7 +24,7 @@ func main() {
 		return true
 	})
 
-	_, err = model.Predict("Here are 4 steps to create a website:", gpt4all.SetTemperature(0.1))
+	_, err = model.Predict("Here are 4 steps to create a website:", "", "", gpt4all.SetTemperature(0.1))
 	if err != nil {
 		panic(err)
 	}

gpt4all-bindings/golang/binding.cpp

@@ -35,8 +35,9 @@ void* load_model(const char *fname, int n_threads) {
 std::string res = "";
 void * mm;
 
-void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float min_p, float temp, int n_batch,float ctx_erase)
+void model_prompt(const char *prompt, const char *prompt_template, int special, const char *fake_reply, 
+                    void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, 
+                    int top_k, float top_p, float min_p, float temp, int n_batch,float ctx_erase)
 {
     llmodel_model* model = (llmodel_model*) m;
 
@@ -88,11 +89,11 @@ void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n,
     prompt_context->temp = temp;
     prompt_context->n_batch = n_batch;    
 
-    llmodel_prompt(model, prompt,
+    llmodel_prompt(model, prompt, prompt_template,
                         lambda_prompt,
                         lambda_response,
                     lambda_recalculate,
-                    prompt_context );
+                    prompt_context, special, fake_reply);
 
     strcpy(result, res.c_str()); 
 

gpt4all-bindings/golang/binding.h

@@ -6,8 +6,9 @@ extern "C" {
 
 void* load_model(const char *fname, int n_threads);
 
-void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float min_p, float temp, int n_batch,float ctx_erase);
+void model_prompt(const char *prompt, const char *prompt_template, int special, const char *fake_reply, 
+                    void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, 
+                    int top_k, float top_p, float min_p, float temp, int n_batch,float ctx_erase);
 
 void free_model(void *state_ptr);
 

gpt4all-bindings/golang/example/main.go

@@ -47,7 +47,7 @@ func main() {
 	for {
 		text := readMultiLineInput(reader)
 
-		_, err := l.Predict(text, gpt4all.SetTokens(tokens), gpt4all.SetTopK(90), gpt4all.SetTopP(0.86))
+		_, err := l.Predict(text, "", "", gpt4all.SetTokens(tokens), gpt4all.SetTopK(90), gpt4all.SetTopP(0.86))
 		if err != nil {
 			panic(err)
 		}

gpt4all-bindings/golang/gpt4all.go

@@ -6,7 +6,7 @@ package gpt4all
 // #cgo darwin CXXFLAGS: -std=c++17
 // #cgo LDFLAGS: -lgpt4all -lm -lstdc++ -ldl
 // void* load_model(const char *fname, int n_threads);
-// void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
+// void model_prompt( const char *prompt, const char *prompt_template, int special, const char *fake_reply, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
 //                            float top_p, float min_p, float temp, int n_batch,float ctx_erase);
 // void free_model(void *state_ptr);
 // extern unsigned char getTokenCallback(void *, char *);
@@ -47,7 +47,7 @@ func New(model string, opts ...ModelOption) (*Model, error) {
 	return gpt, nil
 }
 
-func (l *Model) Predict(text string, opts ...PredictOption) (string, error) {
+func (l *Model) Predict(text, template, fakeReplyText string, opts ...PredictOption) (string, error) {
 
 	po := NewPredictOptions(opts...)
 
@@ -55,10 +55,14 @@ func (l *Model) Predict(text string, opts ...PredictOption) (string, error) {
 	if po.Tokens == 0 {
 		po.Tokens = 99999999
 	}
+	templateInput := C.CString(template)
+	fakeReplyInput := C.CString(fakeReplyText)
 	out := make([]byte, po.Tokens)
 
-	C.model_prompt(input, l.state, (*C.char)(unsafe.Pointer(&out[0])), C.int(po.RepeatLastN), C.float(po.RepeatPenalty), C.int(po.ContextSize),
-		C.int(po.Tokens), C.int(po.TopK), C.float(po.TopP), C.float(po.MinP), C.float(po.Temperature), C.int(po.Batch), C.float(po.ContextErase))
+	C.model_prompt(input, templateInput, C.int(po.Special), fakeReplyInput, l.state, (*C.char)(unsafe.Pointer(&out[0])),
+		C.int(po.RepeatLastN), C.float(po.RepeatPenalty), C.int(po.ContextSize), C.int(po.Tokens),
+		C.int(po.TopK), C.float(po.TopP), C.float(po.MinP), C.float(po.Temperature), C.int(po.Batch),
+		C.float(po.ContextErase))
 
 	res := C.GoString((*C.char)(unsafe.Pointer(&out[0])))
 	res = strings.TrimPrefix(res, " ")

gpt4all-bindings/golang/options.go

@@ -1,8 +1,8 @@
 package gpt4all
 
 type PredictOptions struct {
-	ContextSize, RepeatLastN, Tokens, TopK, Batch  int
-	TopP, MinP, Temperature, ContextErase, RepeatPenalty float64
+	ContextSize, RepeatLastN, Tokens, TopK, Batch, Special int
+	TopP, MinP, Temperature, ContextErase, RepeatPenalty   float64
 }
 
 type PredictOption func(p *PredictOptions)
@@ -11,9 +11,10 @@ var DefaultOptions PredictOptions = PredictOptions{
 	Tokens:        200,
 	TopK:          10,
 	TopP:          0.90,
-        MinP:          0.0,
+	MinP:          0.0,
 	Temperature:   0.96,
 	Batch:         1,
+	Special:       0,
 	ContextErase:  0.55,
 	ContextSize:   1024,
 	RepeatLastN:   10,
@@ -93,6 +94,17 @@ func SetBatch(size int) PredictOption {
 	}
 }
 
+// SetSpecial is true if special tokens in the prompt should be processed, false otherwise.
+func SetSpecial(special bool) PredictOption {
+	return func(p *PredictOptions) {
+		if special {
+			p.Special = 1
+		} else {
+			p.Special = 0
+		}
+	}
+}
+
 // Create a new PredictOptions object with the given options.
 func NewPredictOptions(opts ...PredictOption) PredictOptions {
 	p := DefaultOptions

gpt4all-bindings/python/README.md

@@ -23,9 +23,9 @@ As an alternative to downloading via pip, you may build the Python bindings from
 
 ### Prerequisites
 
-On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+You will need a compiler. On Windows, you should install Visual Studio with the C++ Development components. On macOS, you will need the full version of Xcode—Xcode Command Line Tools lacks certain required tools. On Linux, you will need a GCC or Clang toolchain with C++ support.
 
-macOS users do not need Vulkan, as GPT4All will use Metal instead.
+On Windows and Linux, building GPT4All with full GPU support requires the [Vulkan SDK](https://vulkan.lunarg.com/sdk/home) and the latest [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads).
 
 ### Building the python bindings
 

gpt4all-bindings/python/docs/index.md

@@ -26,7 +26,6 @@ is organized as a monorepo with the following structure:
 - **gpt4all-backend**: The GPT4All backend maintains and exposes a universal, performance optimized C API for running inference with multi-billion parameter Transformer Decoders.
 This C API is then bound to any higher level programming language such as C++, Python, Go, etc.
 - **gpt4all-bindings**: GPT4All bindings contain a variety of high-level programming languages that implement the C API. Each directory is a bound programming language. The [CLI](gpt4all_cli.md) is included here, as well.
-- **gpt4all-api**: The GPT4All API (under initial development) exposes REST API endpoints for gathering completions and embeddings from large language models.
 - **gpt4all-chat**: GPT4All Chat is an OS native chat application that runs on macOS, Windows and Linux. It is the easiest way to run local, privacy aware chat assistants on everyday hardware. You can download it on the [GPT4All Website](https://gpt4all.io) and read its source code in the monorepo.
 
 Explore detailed documentation for the backend, bindings and chat client in the sidebar.

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -9,7 +9,7 @@ import sys
 import threading
 from enum import Enum
 from queue import Queue
-from typing import TYPE_CHECKING, Any, Callable, Generic, Iterable, NoReturn, TypeVar, overload
+from typing import TYPE_CHECKING, Any, Callable, Generic, Iterable, Literal, NoReturn, TypeVar, overload
 
 if sys.version_info >= (3, 9):
     import importlib.resources as importlib_resources
@@ -71,6 +71,7 @@ class LLModelPromptContext(ctypes.Structure):
 
 class LLModelGPUDevice(ctypes.Structure):
     _fields_ = [
+        ("backend", ctypes.c_char_p),
         ("index", ctypes.c_int32),
         ("type", ctypes.c_int32),
         ("heapSize", ctypes.c_size_t),
@@ -158,6 +159,12 @@ llmodel.llmodel_gpu_init_gpu_device_by_int.restype = ctypes.c_bool
 llmodel.llmodel_has_gpu_device.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_has_gpu_device.restype = ctypes.c_bool
 
+llmodel.llmodel_model_backend_name.argtypes = [ctypes.c_void_p]
+llmodel.llmodel_model_backend_name.restype = ctypes.c_char_p
+
+llmodel.llmodel_model_gpu_device_name.argtypes = [ctypes.c_void_p]
+llmodel.llmodel_model_gpu_device_name.restype = ctypes.c_char_p
+
 ResponseCallbackType = Callable[[int, str], bool]
 RawResponseCallbackType = Callable[[int, bytes], bool]
 EmbCancelCallbackType: TypeAlias = 'Callable[[list[int], str], bool]'
@@ -194,9 +201,11 @@ class LLModel:
         Maximum size of context window
     ngl : int
         Number of GPU layers to use (Vulkan)
+    backend : str
+        Backend to use. One of 'auto', 'cpu', 'metal', 'kompute', or 'cuda'.
     """
 
-    def __init__(self, model_path: str, n_ctx: int, ngl: int):
+    def __init__(self, model_path: str, n_ctx: int, ngl: int, backend: str):
         self.model_path = model_path.encode()
         self.n_ctx = n_ctx
         self.ngl = ngl
@@ -206,7 +215,7 @@ class LLModel:
 
         # Construct a model implementation
         err = ctypes.c_char_p()
-        model = llmodel.llmodel_model_create2(self.model_path, b"auto", ctypes.byref(err))
+        model = llmodel.llmodel_model_create2(self.model_path, backend.encode(), ctypes.byref(err))
         if model is None:
             s = err.value
             raise RuntimeError(f"Unable to instantiate model: {'null' if s is None else s.decode()}")
@@ -224,6 +233,19 @@ class LLModel:
     def _raise_closed(self) -> NoReturn:
         raise ValueError("Attempted operation on a closed LLModel")
 
+    @property
+    def backend(self) -> Literal["cpu", "kompute", "cuda", "metal"]:
+        if self.model is None:
+            self._raise_closed()
+        return llmodel.llmodel_model_backend_name(self.model).decode()
+
+    @property
+    def device(self) -> str | None:
+        if self.model is None:
+            self._raise_closed()
+        dev = llmodel.llmodel_model_gpu_device_name(self.model)
+        return None if dev is None else dev.decode()
+
     @staticmethod
     def list_gpus(mem_required: int = 0) -> list[str]:
         """
@@ -239,7 +261,7 @@ class LLModel:
         devices_ptr = llmodel.llmodel_available_gpu_devices(mem_required, ctypes.byref(num_devices))
         if not devices_ptr:
             raise ValueError("Unable to retrieve available GPU devices")
-        return [d.name.decode() for d in devices_ptr[:num_devices.value]]
+        return [f'{d.backend.decode()}:{d.name.decode()}' for d in devices_ptr[:num_devices.value]]
 
     def init_gpu(self, device: str):
         if self.model is None:
@@ -333,22 +355,23 @@ class LLModel:
 
     @overload
     def generate_embeddings(
-        self, text: str, prefix: str, dimensionality: int, do_mean: bool, atlas: bool, cancel_cb: EmbCancelCallbackType,
+        self, text: str, prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+        cancel_cb: EmbCancelCallbackType | None,
     ) -> EmbedResult[list[float]]: ...
     @overload
     def generate_embeddings(
         self, text: list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
-        cancel_cb: EmbCancelCallbackType,
+        cancel_cb: EmbCancelCallbackType | None,
     ) -> EmbedResult[list[list[float]]]: ...
     @overload
     def generate_embeddings(
         self, text: str | list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
-        cancel_cb: EmbCancelCallbackType,
+        cancel_cb: EmbCancelCallbackType | None,
     ) -> EmbedResult[list[Any]]: ...
 
     def generate_embeddings(
         self, text: str | list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
-        cancel_cb: EmbCancelCallbackType,
+        cancel_cb: EmbCancelCallbackType | None,
     ) -> EmbedResult[list[Any]]:
         if not text:
             raise ValueError("text must not be None or empty")
@@ -368,11 +391,11 @@ class LLModel:
         for i, t in enumerate(text):
             c_texts[i] = t.encode()
 
-        def wrap_cancel_cb(batch_sizes: ctypes.POINTER(ctypes.c_uint), n_batch: int, backend: bytes) -> bool:
+        def wrap_cancel_cb(batch_sizes: Any, n_batch: int, backend: bytes) -> bool:
             assert cancel_cb is not None
             return cancel_cb(batch_sizes[:n_batch], backend.decode())
 
-        cancel_cb_wrapper = EmbCancelCallback(0x0 if cancel_cb is None else wrap_cancel_cb)
+        cancel_cb_wrapper = EmbCancelCallback() if cancel_cb is None else EmbCancelCallback(wrap_cancel_cb)
 
         # generate the embeddings
         embedding_ptr = llmodel.llmodel_embed(

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 
 import hashlib
 import os
+import platform
 import re
 import sys
 import time
@@ -44,7 +45,7 @@ class Embed4All:
 
     MIN_DIMENSIONALITY = 64
 
-    def __init__(self, model_name: str | None = None, *, n_threads: int | None = None, device: str | None = "cpu", **kwargs: Any):
+    def __init__(self, model_name: str | None = None, *, n_threads: int | None = None, device: str | None = None, **kwargs: Any):
         """
         Constructor
 
@@ -172,7 +173,7 @@ class GPT4All:
         model_type: str | None = None,
         allow_download: bool = True,
         n_threads: int | None = None,
-        device: str | None = "cpu",
+        device: str | None = None,
         n_ctx: int = 2048,
         ngl: int = 100,
         verbose: bool = False,
@@ -190,30 +191,56 @@ class GPT4All:
             n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
             device: The processing unit on which the GPT4All model will run. It can be set to:
                 - "cpu": Model will run on the central processing unit.
-                - "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
-                - "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
+                - "gpu": Use Metal on ARM64 macOS, otherwise the same as "kompute".
+                - "kompute": Use the best GPU provided by the Kompute backend.
+                - "cuda": Use the best GPU provided by the CUDA backend.
+                - "amd", "nvidia": Use the best GPU provided by the Kompute backend from this vendor.
                 - A specific device name from the list returned by `GPT4All.list_gpus()`.
-                Default is "cpu".
+                Default is Metal on ARM64 macOS, "cpu" otherwise.
 
                 Note: If a selected GPU device does not have sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the model.
             n_ctx: Maximum size of context window
             ngl: Number of GPU layers to use (Vulkan)
             verbose: If True, print debug messages.
         """
+
         self.model_type = model_type
+        self._history: list[MessageType] | None = None
+        self._current_prompt_template: str = "{0}"
+
+        device_init = None
+        if sys.platform == 'darwin':
+            if device is None:
+                backend = 'auto'  # 'auto' is effectively 'metal' due to currently non-functional fallback
+            elif device == 'cpu':
+                backend = 'cpu'
+            else:
+                if platform.machine() != 'arm64' or device != 'gpu':
+                    raise ValueError(f'Unknown device for this platform: {device}')
+                backend = 'metal'
+        else:
+            backend = 'kompute'
+            if device is None or device == 'cpu':
+                pass  # use kompute with no device
+            elif device in ('cuda', 'kompute'):
+                backend = device
+                device_init = 'gpu'
+            elif device.startswith('cuda:'):
+                backend = 'cuda'
+                device_init = device.removeprefix('cuda:')
+            else:
+                device_init = device.removeprefix('kompute:')
+
         # Retrieve model and download if allowed
         self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download, verbose=verbose)
-        self.model = LLModel(self.config["path"], n_ctx, ngl)
-        if device is not None and device != "cpu":
-            self.model.init_gpu(device)
+        self.model = LLModel(self.config["path"], n_ctx, ngl, backend)
+        if device_init is not None:
+            self.model.init_gpu(device_init)
         self.model.load_model()
         # Set n_threads
         if n_threads is not None:
             self.model.set_thread_count(n_threads)
 
-        self._history: list[MessageType] | None = None
-        self._current_prompt_template: str = "{0}"
-
     def __enter__(self) -> Self:
         return self
 
@@ -226,6 +253,16 @@ class GPT4All:
         """Delete the model instance and free associated system resources."""
         self.model.close()
 
+    @property
+    def backend(self) -> Literal["cpu", "kompute", "cuda", "metal"]:
+        """The name of the llama.cpp backend currently in use. One of "cpu", "kompute", "cuda", or "metal"."""
+        return self.model.backend
+
+    @property
+    def device(self) -> str | None:
+        """The name of the GPU device currently in use, or None for backends other than Kompute or CUDA."""
+        return self.model.device
+
     @property
     def current_chat_session(self) -> list[MessageType] | None:
         return None if self._history is None else list(self._history)
@@ -497,16 +534,16 @@ class GPT4All:
         if self._history is not None:
             # check if there is only one message, i.e. system prompt:
             reset = len(self._history) == 1
-            generate_kwargs["reset_context"] = reset
             self._history.append({"role": "user", "content": prompt})
 
             fct_func = self._format_chat_prompt_template.__func__  # type: ignore[attr-defined]
             if fct_func is GPT4All._format_chat_prompt_template:
                 if reset:
                     # ingest system prompt
-                    self.model.prompt_model(self._history[0]["content"], "%1",
+                    # use "%1%2" and not "%1" to avoid implicit whitespace
+                    self.model.prompt_model(self._history[0]["content"], "%1%2",
                                             empty_response_callback,
-                                            n_batch=n_batch, n_predict=0, special=True)
+                                            n_batch=n_batch, n_predict=0, reset_context=True, special=True)
                 prompt_template = self._current_prompt_template.format("%1", "%2")
             else:
                 warnings.warn(
@@ -519,6 +556,7 @@ class GPT4All:
                     self._history[0]["content"] if reset else "",
                 )
                 prompt_template = "%1"
+                generate_kwargs["reset_context"] = reset
         else:
             prompt_template = "%1"
             generate_kwargs["reset_context"] = True

gpt4all-bindings/python/setup.py

@@ -45,7 +45,7 @@ def copy_prebuilt_C_lib(src_dir, dest_dir, dest_build_dir):
                 d = os.path.join(dest_dir, item)
                 shutil.copy2(s, d)
                 files_copied += 1
-            if item.endswith(lib_ext) or item.endswith('.metal'):
+            if item.endswith(lib_ext) or item.endswith('.metallib'):
                 s = os.path.join(dirpath, item)
                 d = os.path.join(dest_build_dir, item)
                 shutil.copy2(s, d)
@@ -68,7 +68,7 @@ def get_long_description():
 
 setup(
     name=package_name,
-    version="2.5.0",
+    version="2.7.0",
     description="Python bindings for GPT4All",
     long_description=get_long_description(),
     long_description_content_type="text/markdown",

gpt4all-bindings/typescript/yarn.lock

@@ -5395,8 +5395,8 @@ __metadata:
   linkType: hard
 
 "tar@npm:^6.1.11, tar@npm:^6.1.2":
-  version: 6.2.0
-  resolution: "tar@npm:6.2.0"
+  version: 6.2.1
+  resolution: "tar@npm:6.2.1"
   dependencies:
     chownr: ^2.0.0
     fs-minipass: ^2.0.0
@@ -5404,7 +5404,7 @@ __metadata:
     minizlib: ^2.1.1
     mkdirp: ^1.0.3
     yallist: ^4.0.0
-  checksum: db4d9fe74a2082c3a5016630092c54c8375ff3b280186938cfd104f2e089c4fd9bad58688ef6be9cf186a889671bf355c7cda38f09bbf60604b281715ca57f5c
+  checksum: f1322768c9741a25356c11373bce918483f40fa9a25c69c59410c8a1247632487edef5fe76c5f12ac51a6356d2f1829e96d2bc34098668a2fc34d76050ac2b6c
   languageName: node
   linkType: hard
 

gpt4all-chat/CMakeLists.txt

@@ -17,8 +17,8 @@ if(APPLE)
 endif()
 
 set(APP_VERSION_MAJOR 2)
-set(APP_VERSION_MINOR 7)
-set(APP_VERSION_PATCH 4)
+set(APP_VERSION_MINOR 8)
+set(APP_VERSION_PATCH 0)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file
@@ -65,12 +65,24 @@ add_subdirectory(../gpt4all-backend llmodel)
 
 set(METAL_SHADER_FILE)
 if(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
-  set(METAL_SHADER_FILE ../gpt4all-backend/llama.cpp-mainline/ggml-metal.metal)
+    set(METAL_SHADER_FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib)
+endif()
+
+set(APP_ICON_RESOURCE)
+if (WIN32)
+    set(APP_ICON_RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.rc")
+elseif (APPLE)
+    # The MACOSX_BUNDLE_ICON_FILE variable is added to the Info.plist
+    # generated by CMake. This variable contains the .icns file name,
+    # without the path.
+    set(MACOSX_BUNDLE_ICON_FILE gpt4all.icns)
+
+    # And the following tells CMake where to find and install the file itself.
+    set(APP_ICON_RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns")
+    set_source_files_properties(${APP_ICON_RESOURCE} PROPERTIES
+        MACOSX_PACKAGE_LOCATION "Resources")
 endif()
 
-set(APP_ICON_FILE "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns")
-set_source_files_properties(${APP_ICON_FILE} PROPERTIES
-    MACOSX_PACKAGE_LOCATION "Resources")
 
 qt_add_executable(chat
     main.cpp
@@ -91,7 +103,7 @@ qt_add_executable(chat
     logger.h logger.cpp
     responsetext.h responsetext.cpp
     ${METAL_SHADER_FILE}
-    ${APP_ICON_FILE}
+    ${APP_ICON_RESOURCE}
 )
 
 qt_add_qml_module(chat
@@ -103,11 +115,11 @@ qt_add_qml_module(chat
       qml/ChatDrawer.qml
       qml/ChatView.qml
       qml/CollectionsDialog.qml
-      qml/ModelDownloaderView.qml
+      qml/ModelDownloaderDialog.qml
       qml/NetworkDialog.qml
       qml/NewVersionDialog.qml
       qml/ThumbsDownDialog.qml
-      qml/SettingsView.qml
+      qml/SettingsDialog.qml
       qml/StartupDialog.qml
       qml/PopupDialog.qml
       qml/AboutDialog.qml
@@ -136,7 +148,6 @@ qt_add_qml_module(chat
       icons/send_message.svg
       icons/stop_generating.svg
       icons/regenerate.svg
-      icons/chat.svg
       icons/close.svg
       icons/copy.svg
       icons/db.svg
@@ -145,10 +156,6 @@ qt_add_qml_module(chat
       icons/eject.svg
       icons/edit.svg
       icons/image.svg
-      icons/info.svg
-      icons/local-docs.svg
-      icons/models.svg
-      icons/search.svg
       icons/trash.svg
       icons/network.svg
       icons/thumbs_up.svg
@@ -158,8 +165,6 @@ qt_add_qml_module(chat
       icons/logo.svg
       icons/logo-32.png
       icons/logo-48.png
-      icons/favicon.ico
-      icons/favicon.icns
 )
 
 set_target_properties(chat PROPERTIES
@@ -168,7 +173,6 @@ set_target_properties(chat PROPERTIES
     MACOSX_BUNDLE_SHORT_VERSION_STRING ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}
     MACOSX_BUNDLE TRUE
     WIN32_EXECUTABLE TRUE
-    MACOSX_BUNDLE_ICON_FILE "favicon.icns"
 )
 
 if(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
@@ -181,7 +185,7 @@ if(METAL_SHADER_FILE)
     set_target_properties(chat PROPERTIES
         RESOURCE ${METAL_SHADER_FILE}
     )
-    configure_file(${METAL_SHADER_FILE} bin/ggml-metal.metal COPYONLY)
+    add_dependencies(chat ggml-metal)
 endif()
 
 target_compile_definitions(chat
@@ -203,18 +207,61 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
 endif()
 
 install(TARGETS chat DESTINATION bin COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llmodel DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+
+install(
+    TARGETS llmodel
+    LIBRARY DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}  # .so/.dylib
+    RUNTIME DESTINATION bin COMPONENT ${COMPONENT_NAME_MAIN}  # .dll
+)
 
 # We should probably iterate through the list of the cmake for backend, but these need to be installed
 # to the this component's dir for the finicky qt installer to work
-install(TARGETS gptj-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS gptj-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llama-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llama-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llamamodel-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS llamamodel-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-if(APPLE)
-install(TARGETS llamamodel-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+if (LLMODEL_KOMPUTE)
+    set(MODEL_IMPL_TARGETS
+        llamamodel-mainline-kompute
+        llamamodel-mainline-kompute-avxonly
+        gptj-kompute
+        gptj-kompute-avxonly
+    )
+else()
+    set(MODEL_IMPL_TARGETS
+        llamamodel-mainline-cpu
+        llamamodel-mainline-cpu-avxonly
+        gptj-cpu
+        gptj-cpu-avxonly
+    )
+endif()
+
+if (APPLE)
+    list(APPEND MODEL_IMPL_TARGETS llamamodel-mainline-metal)
+endif()
+
+install(
+    TARGETS ${MODEL_IMPL_TARGETS}
+    LIBRARY DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}  # .so/.dylib
+    RUNTIME DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}  # .dll
+)
+
+if (LLMODEL_CUDA)
+    set_property(TARGET llamamodel-mainline-cuda llamamodel-mainline-cuda-avxonly
+                 APPEND PROPERTY INSTALL_RPATH "$ORIGIN")
+
+    install(
+        TARGETS llamamodel-mainline-cuda
+                llamamodel-mainline-cuda-avxonly
+        RUNTIME_DEPENDENCY_SET llama-cuda-deps
+        LIBRARY DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}  # .so/.dylib
+        RUNTIME DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}  # .dll
+    )
+    if (WIN32)
+        install(
+            RUNTIME_DEPENDENCY_SET llama-cuda-deps
+            PRE_EXCLUDE_REGEXES "^(nvcuda|api-ms-.*)\\.dll$"
+            POST_INCLUDE_REGEXES "(^|[/\\\\])(lib)?(cuda|cublas)" POST_EXCLUDE_REGEXES .
+            DIRECTORIES "${CUDAToolkit_BIN_DIR}"
+            DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}
+        )
+    endif()
 endif()
 
 set(CPACK_GENERATOR "IFW")
@@ -235,7 +282,7 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES Windows)
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-windows.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-windows.cmake)
     set(CPACK_IFW_ROOT "C:/Qt/Tools/QtInstallerFramework/4.6")
-    set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.ico")
+    set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.ico")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-win64")
     set(CPACK_IFW_TARGET_DIRECTORY "@HomeDir@\\${COMPONENT_NAME_MAIN}")
 elseif(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
@@ -244,11 +291,11 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-mac.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-mac.cmake)
     set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.6")
-    set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns")
+    set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-darwin")
     set(CPACK_IFW_TARGET_DIRECTORY "@ApplicationsDir@/${COMPONENT_NAME_MAIN}")
     set(CPACK_BUNDLE_NAME ${COMPONENT_NAME_MAIN})
-    set(CPACK_BUNDLE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns")
+    set(CPACK_BUNDLE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns")
 endif()
 
 set(CPACK_PACKAGE_INSTALL_DIRECTORY ${COMPONENT_NAME_MAIN})

gpt4all-chat/build_and_run.md

@@ -6,9 +6,9 @@ gpt4all-chat from source.
 
 ## Prerequisites
 
-On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+You will need a compiler. On Windows, you should install Visual Studio with the C++ Development components. On macOS, you will need the full version of Xcode—Xcode Command Line Tools lacks certain required tools. On Linux, you will need a GCC or Clang toolchain with C++ support.
 
-macOS users do not need Vulkan, as GPT4All will use Metal instead.
+On Windows and Linux, building GPT4All with full GPU support requires the [Vulkan SDK](https://vulkan.lunarg.com/sdk/home) and the latest [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads).
 
 ## Note for Linux users
 

gpt4all-chat/chat.cpp

@@ -54,7 +54,7 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::reportFallbackReason, this, &Chat::handleFallbackReasonChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::databaseResultsChanged, this, &Chat::handleDatabaseResultsChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelInfoChanged, this, &Chat::handleModelInfoChanged, Qt::QueuedConnection);
-    connect(m_llmodel, &ChatLLM::trySwitchContextOfLoadedModelCompleted, this, &Chat::trySwitchContextOfLoadedModelCompleted, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::trySwitchContextOfLoadedModelCompleted, this, &Chat::handleTrySwitchContextOfLoadedModelCompleted, Qt::QueuedConnection);
 
     connect(this, &Chat::promptRequested, m_llmodel, &ChatLLM::prompt, Qt::QueuedConnection);
     connect(this, &Chat::modelChangeRequested, m_llmodel, &ChatLLM::modelChangeRequested, Qt::QueuedConnection);
@@ -95,16 +95,6 @@ void Chat::processSystemPrompt()
     emit processSystemPromptRequested();
 }
 
-bool Chat::isModelLoaded() const
-{
-    return m_modelLoadingPercentage == 1.0f;
-}
-
-float Chat::modelLoadingPercentage() const
-{
-    return m_modelLoadingPercentage;
-}
-
 void Chat::resetResponseState()
 {
     if (m_responseInProgress && m_responseState == Chat::LocalDocsRetrieval)
@@ -167,9 +157,16 @@ void Chat::handleModelLoadingPercentageChanged(float loadingPercentage)
     if (loadingPercentage == m_modelLoadingPercentage)
         return;
 
+    bool wasLoading = isCurrentlyLoading();
+    bool wasLoaded = isModelLoaded();
+
     m_modelLoadingPercentage = loadingPercentage;
     emit modelLoadingPercentageChanged();
-    if (m_modelLoadingPercentage == 1.0f || m_modelLoadingPercentage == 0.0f)
+
+    if (isCurrentlyLoading() != wasLoading)
+        emit isCurrentlyLoadingChanged();
+
+    if (isModelLoaded() != wasLoaded)
         emit isModelLoadedChanged();
 }
 
@@ -179,59 +176,62 @@ void Chat::promptProcessing()
     emit responseStateChanged();
 }
 
-void Chat::responseStopped()
+void Chat::responseStopped(qint64 promptResponseMs)
 {
     m_tokenSpeed = QString();
     emit tokenSpeedChanged();
 
-    if (MySettings::globalInstance()->localDocsShowReferences()) {
-        const QString chatResponse = response();
-        QList<QString> references;
-        QList<QString> referencesContext;
-        int validReferenceNumber = 1;
-        for (const ResultInfo &info : databaseResults()) {
-            if (info.file.isEmpty())
-                continue;
-            if (validReferenceNumber == 1)
-                references.append((!chatResponse.endsWith("\n") ? "\n" : QString()) + QStringLiteral("\n---"));
-            QString reference;
-            {
-                QTextStream stream(&reference);
-                stream << (validReferenceNumber++) << ". ";
-                if (!info.title.isEmpty())
-                    stream << "\"" << info.title << "\". ";
-                if (!info.author.isEmpty())
-                    stream << "By " << info.author << ". ";
-                if (!info.date.isEmpty())
-                    stream << "Date: " << info.date << ". ";
-                stream << "In " << info.file << ". ";
-                if (info.page != -1)
-                    stream << "Page " << info.page << ". ";
-                if (info.from != -1) {
-                    stream << "Lines " << info.from;
-                    if (info.to != -1)
-                        stream << "-" << info.to;
-                    stream << ". ";
-                }
-                stream << "[Context](context://" << validReferenceNumber - 1 << ")";
+    const QString chatResponse = response();
+    QList<QString> references;
+    QList<QString> referencesContext;
+    int validReferenceNumber = 1;
+    for (const ResultInfo &info : databaseResults()) {
+        if (info.file.isEmpty())
+            continue;
+        if (validReferenceNumber == 1)
+            references.append((!chatResponse.endsWith("\n") ? "\n" : QString()) + QStringLiteral("\n---"));
+        QString reference;
+        {
+            QTextStream stream(&reference);
+            stream << (validReferenceNumber++) << ". ";
+            if (!info.title.isEmpty())
+                stream << "\"" << info.title << "\". ";
+            if (!info.author.isEmpty())
+                stream << "By " << info.author << ". ";
+            if (!info.date.isEmpty())
+                stream << "Date: " << info.date << ". ";
+            stream << "In " << info.file << ". ";
+            if (info.page != -1)
+                stream << "Page " << info.page << ". ";
+            if (info.from != -1) {
+                stream << "Lines " << info.from;
+                if (info.to != -1)
+                    stream << "-" << info.to;
+                stream << ". ";
             }
-            references.append(reference);
-            referencesContext.append(info.text);
+            stream << "[Context](context://" << validReferenceNumber - 1 << ")";
         }
-
-        const int index = m_chatModel->count() - 1;
-        m_chatModel->updateReferences(index, references.join("\n"), referencesContext);
-        emit responseChanged();
+        references.append(reference);
+        referencesContext.append(info.text);
     }
 
+    const int index = m_chatModel->count() - 1;
+    m_chatModel->updateReferences(index, references.join("\n"), referencesContext);
+    emit responseChanged();
+
     m_responseInProgress = false;
     m_responseState = Chat::ResponseStopped;
     emit responseInProgressChanged();
     emit responseStateChanged();
     if (m_generatedName.isEmpty())
         emit generateNameRequested();
-    if (chatModel()->count() < 3)
-        Network::globalInstance()->sendChatStarted();
+
+    Network::globalInstance()->trackChatEvent("response_complete", {
+        {"first", m_firstResponse},
+        {"message_count", chatModel()->count()},
+        {"$duration", promptResponseMs / 1000.},
+    });
+    m_firstResponse = false;
 }
 
 ModelInfo Chat::modelInfo() const
@@ -244,10 +244,6 @@ void Chat::setModelInfo(const ModelInfo &modelInfo)
     if (m_modelInfo == modelInfo && isModelLoaded())
         return;
 
-    m_modelLoadingPercentage = std::numeric_limits<float>::min(); // small non-zero positive value
-    emit isModelLoadedChanged();
-    m_modelLoadingError = QString();
-    emit modelLoadingErrorChanged();
     m_modelInfo = modelInfo;
     emit modelInfoChanged();
     emit modelChangeRequested(modelInfo);
@@ -317,8 +313,9 @@ void Chat::forceReloadModel()
 
 void Chat::trySwitchContextOfLoadedModel()
 {
-    emit trySwitchContextOfLoadedModelAttempted();
-    m_llmodel->setShouldTrySwitchContext(true);
+    m_trySwitchContextInProgress = 1;
+    emit trySwitchContextInProgressChanged();
+    m_llmodel->requestTrySwitchContext();
 }
 
 void Chat::generatedNameChanged(const QString &name)
@@ -333,14 +330,16 @@ void Chat::generatedNameChanged(const QString &name)
 
 void Chat::handleRecalculating()
 {
-    Network::globalInstance()->sendRecalculatingContext(m_chatModel->count());
+    Network::globalInstance()->trackChatEvent("recalc_context", { {"length", m_chatModel->count()} });
     emit recalcChanged();
 }
 
 void Chat::handleModelLoadingError(const QString &error)
 {
-    auto stream = qWarning().noquote() << "ERROR:" << error << "id";
-    stream.quote() << id();
+    if (!error.isEmpty()) {
+        auto stream = qWarning().noquote() << "ERROR:" << error << "id";
+        stream.quote() << id();
+    }
     m_modelLoadingError = error;
     emit modelLoadingErrorChanged();
 }
@@ -377,6 +376,11 @@ void Chat::handleModelInfoChanged(const ModelInfo &modelInfo)
     emit modelInfoChanged();
 }
 
+void Chat::handleTrySwitchContextOfLoadedModelCompleted(int value) {
+    m_trySwitchContextInProgress = value;
+    emit trySwitchContextInProgressChanged();
+}
+
 bool Chat::serialize(QDataStream &stream, int version) const
 {
     stream << m_creationDate;

gpt4all-chat/chat.h

@@ -17,6 +17,7 @@ class Chat : public QObject
     Q_PROPERTY(QString name READ name WRITE setName NOTIFY nameChanged)
     Q_PROPERTY(ChatModel *chatModel READ chatModel NOTIFY chatModelChanged)
     Q_PROPERTY(bool isModelLoaded READ isModelLoaded NOTIFY isModelLoadedChanged)
+    Q_PROPERTY(bool isCurrentlyLoading READ isCurrentlyLoading NOTIFY isCurrentlyLoadingChanged)
     Q_PROPERTY(float modelLoadingPercentage READ modelLoadingPercentage NOTIFY modelLoadingPercentageChanged)
     Q_PROPERTY(QString response READ response NOTIFY responseChanged)
     Q_PROPERTY(ModelInfo modelInfo READ modelInfo WRITE setModelInfo NOTIFY modelInfoChanged)
@@ -30,6 +31,8 @@ class Chat : public QObject
     Q_PROPERTY(QString device READ device NOTIFY deviceChanged);
     Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY fallbackReasonChanged);
     Q_PROPERTY(LocalDocsCollectionsModel *collectionModel READ collectionModel NOTIFY collectionModelChanged)
+    // 0=no, 1=waiting, 2=working
+    Q_PROPERTY(int trySwitchContextInProgress READ trySwitchContextInProgress NOTIFY trySwitchContextInProgressChanged)
     QML_ELEMENT
     QML_UNCREATABLE("Only creatable from c++!")
 
@@ -62,8 +65,9 @@ public:
 
     Q_INVOKABLE void reset();
     Q_INVOKABLE void processSystemPrompt();
-    Q_INVOKABLE bool isModelLoaded() const;
-    Q_INVOKABLE float modelLoadingPercentage() const;
+    bool  isModelLoaded()          const { return m_modelLoadingPercentage == 1.0f; }
+    bool  isCurrentlyLoading()     const { return m_modelLoadingPercentage > 0.0f && m_modelLoadingPercentage < 1.0f; }
+    float modelLoadingPercentage() const { return m_modelLoadingPercentage; }
     Q_INVOKABLE void prompt(const QString &prompt);
     Q_INVOKABLE void regenerateResponse();
     Q_INVOKABLE void stopGenerating();
@@ -105,6 +109,8 @@ public:
     QString device() const { return m_device; }
     QString fallbackReason() const { return m_fallbackReason; }
 
+    int trySwitchContextInProgress() const { return m_trySwitchContextInProgress; }
+
 public Q_SLOTS:
     void serverNewPromptResponsePair(const QString &prompt);
 
@@ -113,6 +119,7 @@ Q_SIGNALS:
     void nameChanged();
     void chatModelChanged();
     void isModelLoadedChanged();
+    void isCurrentlyLoadingChanged();
     void modelLoadingPercentageChanged();
     void modelLoadingWarning(const QString &warning);
     void responseChanged();
@@ -136,14 +143,13 @@ Q_SIGNALS:
     void deviceChanged();
     void fallbackReasonChanged();
     void collectionModelChanged();
-    void trySwitchContextOfLoadedModelAttempted();
-    void trySwitchContextOfLoadedModelCompleted(bool);
+    void trySwitchContextInProgressChanged();
 
 private Q_SLOTS:
     void handleResponseChanged(const QString &response);
     void handleModelLoadingPercentageChanged(float);
     void promptProcessing();
-    void responseStopped();
+    void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
     void handleRecalculating();
     void handleModelLoadingError(const QString &error);
@@ -152,6 +158,7 @@ private Q_SLOTS:
     void handleFallbackReasonChanged(const QString &device);
     void handleDatabaseResultsChanged(const QList<ResultInfo> &results);
     void handleModelInfoChanged(const ModelInfo &modelInfo);
+    void handleTrySwitchContextOfLoadedModelCompleted(int value);
 
 private:
     QString m_id;
@@ -175,6 +182,9 @@ private:
     bool m_shouldDeleteLater = false;
     float m_modelLoadingPercentage = 0.0f;
     LocalDocsCollectionsModel *m_collectionModel;
+    bool m_firstResponse = true;
+    int m_trySwitchContextInProgress = 0;
+    bool m_isCurrentlyLoading = false;
 };
 
 #endif // CHAT_H

gpt4all-chat/chatlistmodel.cpp

@@ -15,18 +15,19 @@ ChatListModel *ChatListModel::globalInstance()
 }
 
 ChatListModel::ChatListModel()
-    : QAbstractListModel(nullptr)
+    : QAbstractListModel(nullptr) {}
+
+void ChatListModel::loadChats()
 {
     addChat();
 
     ChatsRestoreThread *thread = new ChatsRestoreThread;
-    connect(thread, &ChatsRestoreThread::chatRestored, this, &ChatListModel::restoreChat);
-    connect(thread, &ChatsRestoreThread::finished, this, &ChatListModel::chatsRestoredFinished);
+    connect(thread, &ChatsRestoreThread::chatRestored, this, &ChatListModel::restoreChat, Qt::QueuedConnection);
+    connect(thread, &ChatsRestoreThread::finished, this, &ChatListModel::chatsRestoredFinished, Qt::QueuedConnection);
     connect(thread, &ChatsRestoreThread::finished, thread, &QObject::deleteLater);
     thread->start();
 
     connect(MySettings::globalInstance(), &MySettings::serverChatChanged, this, &ChatListModel::handleServerEnabledChanged);
-
 }
 
 void ChatListModel::removeChatFile(Chat *chat) const

gpt4all-chat/chatlistmodel.h

@@ -81,11 +81,15 @@ public:
     bool shouldSaveChatGPTChats() const;
     void setShouldSaveChatGPTChats(bool b);
 
+    Q_INVOKABLE void loadChats();
+
     Q_INVOKABLE void addChat()
     {
-        // Don't add a new chat if we already have one
-        if (m_newChat)
+        // Select the existing new chat if we already have one
+        if (m_newChat) {
+            setCurrentChat(m_newChat);
             return;
+        }
 
         // Create a new chat pointer and connect it to determine when it is populated
         m_newChat = new Chat(this);
@@ -114,20 +118,6 @@ public:
         emit countChanged();
     }
 
-    void setNewChat(Chat* chat)
-    {
-        // Don't add a new chat if we already have one
-        if (m_newChat)
-            return;
-
-        m_newChat = chat;
-        connect(m_newChat->chatModel(), &ChatModel::countChanged,
-            this, &ChatListModel::newChatCountChanged);
-        connect(m_newChat, &Chat::nameChanged,
-            this, &ChatListModel::nameChanged);
-        setCurrentChat(m_newChat);
-    }
-
     Q_INVOKABLE void removeChat(Chat* chat)
     {
         Q_ASSERT(chat != m_serverChat);
@@ -195,7 +185,11 @@ public:
     int count() const { return m_chats.size(); }
 
     // stop ChatLLM threads for clean shutdown
-    void destroyChats() { for (auto *chat: m_chats) { chat->destroy(); } }
+    void destroyChats()
+    {
+        for (auto *chat: m_chats) { chat->destroy(); }
+        ChatLLM::destroyStore();
+    }
 
     void removeChatFile(Chat *chat) const;
     Q_INVOKABLE void saveChats();

gpt4all-chat/chatllm.cpp

@@ -7,6 +7,18 @@
 #include "mysettings.h"
 #include "../gpt4all-backend/llmodel.h"
 
+#include <algorithm>
+#include <cctype>
+#include <cmath>
+#include <cstddef>
+#include <functional>
+#include <limits>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <QElapsedTimer>
+
 //#define DEBUG
 //#define DEBUG_MODEL_LOADING
 
@@ -18,16 +30,17 @@ public:
     static LLModelStore *globalInstance();
 
     LLModelInfo acquireModel(); // will block until llmodel is ready
-    void releaseModel(const LLModelInfo &info); // must be called when you are done
+    void releaseModel(LLModelInfo &&info); // must be called when you are done
+    void destroy();
 
 private:
     LLModelStore()
     {
         // seed with empty model
-        m_availableModels.append(LLModelInfo());
+        m_availableModel = LLModelInfo();
     }
     ~LLModelStore() {}
-    QVector<LLModelInfo> m_availableModels;
+    std::optional<LLModelInfo> m_availableModel;
     QMutex m_mutex;
     QWaitCondition m_condition;
     friend class MyLLModelStore;
@@ -43,19 +56,27 @@ LLModelStore *LLModelStore::globalInstance()
 LLModelInfo LLModelStore::acquireModel()
 {
     QMutexLocker locker(&m_mutex);
-    while (m_availableModels.isEmpty())
+    while (!m_availableModel)
         m_condition.wait(locker.mutex());
-    return m_availableModels.takeFirst();
+    auto first = std::move(*m_availableModel);
+    m_availableModel.reset();
+    return first;
 }
 
-void LLModelStore::releaseModel(const LLModelInfo &info)
+void LLModelStore::releaseModel(LLModelInfo &&info)
 {
     QMutexLocker locker(&m_mutex);
-    m_availableModels.append(info);
-    Q_ASSERT(m_availableModels.count() < 2);
+    Q_ASSERT(!m_availableModel);
+    m_availableModel = std::move(info);
     m_condition.wakeAll();
 }
 
+void LLModelStore::destroy()
+{
+    QMutexLocker locker(&m_mutex);
+    m_availableModel.reset();
+}
+
 ChatLLM::ChatLLM(Chat *parent, bool isServer)
     : QObject{nullptr}
     , m_promptResponseTokens(0)
@@ -64,7 +85,6 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_shouldBeLoaded(false)
     , m_forceUnloadModel(false)
     , m_markedForDeletion(false)
-    , m_shouldTrySwitchContext(false)
     , m_stopGenerating(false)
     , m_timer(nullptr)
     , m_isServer(isServer)
@@ -74,11 +94,9 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_restoreStateFromText(false)
 {
     moveToThread(&m_llmThread);
-    connect(this, &ChatLLM::sendStartup, Network::globalInstance(), &Network::sendStartup);
-    connect(this, &ChatLLM::sendModelLoaded, Network::globalInstance(), &Network::sendModelLoaded);
     connect(this, &ChatLLM::shouldBeLoadedChanged, this, &ChatLLM::handleShouldBeLoadedChanged,
         Qt::QueuedConnection); // explicitly queued
-    connect(this, &ChatLLM::shouldTrySwitchContextChanged, this, &ChatLLM::handleShouldTrySwitchContextChanged,
+    connect(this, &ChatLLM::trySwitchContextRequested, this, &ChatLLM::trySwitchContextOfLoadedModel,
         Qt::QueuedConnection); // explicitly queued
     connect(parent, &Chat::idChanged, this, &ChatLLM::handleChatIdChanged);
     connect(&m_llmThread, &QThread::started, this, &ChatLLM::handleThreadStarted);
@@ -98,7 +116,8 @@ ChatLLM::~ChatLLM()
     destroy();
 }
 
-void ChatLLM::destroy() {
+void ChatLLM::destroy()
+{
     m_stopGenerating = true;
     m_llmThread.quit();
     m_llmThread.wait();
@@ -106,11 +125,15 @@ void ChatLLM::destroy() {
     // The only time we should have a model loaded here is on shutdown
     // as we explicitly unload the model in all other circumstances
     if (isModelLoaded()) {
-        delete m_llModelInfo.model;
-        m_llModelInfo.model = nullptr;
+        m_llModelInfo.model.reset();
     }
 }
 
+void ChatLLM::destroyStore()
+{
+    LLModelStore::globalInstance()->destroy();
+}
+
 void ChatLLM::handleThreadStarted()
 {
     m_timer = new TokenTimer(this);
@@ -120,7 +143,7 @@ void ChatLLM::handleThreadStarted()
 
 void ChatLLM::handleForceMetalChanged(bool forceMetal)
 {
-#if defined(Q_OS_MAC) && defined(__arm__)
+#if defined(Q_OS_MAC) && defined(__aarch64__)
     m_forceMetal = forceMetal;
     if (isModelLoaded() && m_shouldBeLoaded) {
         m_reloadingToChangeVariant = true;
@@ -151,7 +174,7 @@ bool ChatLLM::loadDefaultModel()
     return loadModel(defaultModel);
 }
 
-bool ChatLLM::trySwitchContextOfLoadedModel(const ModelInfo &modelInfo)
+void ChatLLM::trySwitchContextOfLoadedModel(const ModelInfo &modelInfo)
 {
     // We're trying to see if the store already has the model fully loaded that we wish to use
     // and if so we just acquire it from the store and switch the context and return true. If the
@@ -159,10 +182,11 @@ bool ChatLLM::trySwitchContextOfLoadedModel(const ModelInfo &modelInfo)
 
     // If we're already loaded or a server or we're reloading to change the variant/device or the
     // modelInfo is empty, then this should fail
-    if (isModelLoaded() || m_isServer || m_reloadingToChangeVariant || modelInfo.name().isEmpty()) {
-        m_shouldTrySwitchContext = false;
-        emit trySwitchContextOfLoadedModelCompleted(false);
-        return false;
+    if (
+        isModelLoaded() || m_isServer || m_reloadingToChangeVariant || modelInfo.name().isEmpty() || !m_shouldBeLoaded
+    ) {
+        emit trySwitchContextOfLoadedModelCompleted(0);
+        return;
     }
 
     QString filePath = modelInfo.dirpath + modelInfo.filename();
@@ -170,33 +194,28 @@ bool ChatLLM::trySwitchContextOfLoadedModel(const ModelInfo &modelInfo)
 
     m_llModelInfo = LLModelStore::globalInstance()->acquireModel();
 #if defined(DEBUG_MODEL_LOADING)
-        qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model;
+        qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
 
     // The store gave us no already loaded model, the wrong type of model, then give it back to the
     // store and fail
-    if (!m_llModelInfo.model || m_llModelInfo.fileInfo != fileInfo) {
-        LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
-        m_llModelInfo = LLModelInfo();
-        m_shouldTrySwitchContext = false;
-        emit trySwitchContextOfLoadedModelCompleted(false);
-        return false;
+    if (!m_llModelInfo.model || m_llModelInfo.fileInfo != fileInfo || !m_shouldBeLoaded) {
+        LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+        emit trySwitchContextOfLoadedModelCompleted(0);
+        return;
     }
 
 #if defined(DEBUG_MODEL_LOADING)
-    qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model;
+    qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
 
-    // We should be loaded and now we are
-    m_shouldBeLoaded = true;
-    m_shouldTrySwitchContext = false;
+    emit trySwitchContextOfLoadedModelCompleted(2);
 
     // Restore, signal and process
     restoreState();
     emit modelLoadingPercentageChanged(1.0f);
-    emit trySwitchContextOfLoadedModelCompleted(true);
+    emit trySwitchContextOfLoadedModelCompleted(0);
     processSystemPrompt();
-    return true;
 }
 
 bool ChatLLM::loadModel(const ModelInfo &modelInfo)
@@ -213,6 +232,13 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     if (isModelLoaded() && this->modelInfo() == modelInfo)
         return true;
 
+    // reset status
+    emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
+    emit modelLoadingError("");
+    emit reportFallbackReason("");
+    emit reportDevice("");
+    m_pristineLoadedState = false;
+
     QString filePath = modelInfo.dirpath + modelInfo.filename();
     QFileInfo fileInfo(filePath);
 
@@ -221,28 +247,25 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     if (alreadyAcquired) {
         resetContext();
 #if defined(DEBUG_MODEL_LOADING)
-        qDebug() << "already acquired model deleted" << m_llmThread.objectName() << m_llModelInfo.model;
+        qDebug() << "already acquired model deleted" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
-        delete m_llModelInfo.model;
-        m_llModelInfo.model = nullptr;
-        emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
+        m_llModelInfo.model.reset();
     } else if (!m_isServer) {
         // This is a blocking call that tries to retrieve the model we need from the model store.
         // If it succeeds, then we just have to restore state. If the store has never had a model
         // returned to it, then the modelInfo.model pointer should be null which will happen on startup
         m_llModelInfo = LLModelStore::globalInstance()->acquireModel();
 #if defined(DEBUG_MODEL_LOADING)
-        qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model;
+        qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
         // At this point it is possible that while we were blocked waiting to acquire the model from the
         // store, that our state was changed to not be loaded. If this is the case, release the model
         // back into the store and quit loading
         if (!m_shouldBeLoaded) {
 #if defined(DEBUG_MODEL_LOADING)
-            qDebug() << "no longer need model" << m_llmThread.objectName() << m_llModelInfo.model;
+            qDebug() << "no longer need model" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
-            LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
-            m_llModelInfo = LLModelInfo();
+            LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
             emit modelLoadingPercentageChanged(0.0f);
             return false;
         }
@@ -250,7 +273,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         // Check if the store just gave us exactly the model we were looking for
         if (m_llModelInfo.model && m_llModelInfo.fileInfo == fileInfo && !m_reloadingToChangeVariant) {
 #if defined(DEBUG_MODEL_LOADING)
-            qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model;
+            qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
             restoreState();
             emit modelLoadingPercentageChanged(1.0f);
@@ -264,10 +287,9 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         } else {
             // Release the memory since we have to switch to a different model.
 #if defined(DEBUG_MODEL_LOADING)
-            qDebug() << "deleting model" << m_llmThread.objectName() << m_llModelInfo.model;
+            qDebug() << "deleting model" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
-            delete m_llModelInfo.model;
-            m_llModelInfo.model = nullptr;
+            m_llModelInfo.model.reset();
         }
     }
 
@@ -278,15 +300,16 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     m_llModelInfo.fileInfo = fileInfo;
 
     if (fileInfo.exists()) {
+        QVariantMap modelLoadProps;
         if (modelInfo.isOnline) {
             QString apiKey;
             QString modelName;
             {
                 QFile file(filePath);
-                file.open(QIODeviceBase::ReadOnly | QIODeviceBase::Text);
-                QTextStream stream(&file);
-                QString text = stream.readAll();
-                QJsonDocument doc = QJsonDocument::fromJson(text.toUtf8());
+                bool success = file.open(QIODeviceBase::ReadOnly);
+                (void)success;
+                Q_ASSERT(success);
+                QJsonDocument doc = QJsonDocument::fromJson(file.readAll());
                 QJsonObject obj = doc.object();
                 apiKey = obj["apiKey"].toString();
                 modelName = obj["modelName"].toString();
@@ -296,18 +319,46 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             model->setModelName(modelName);
             model->setRequestURL(modelInfo.url());
             model->setAPIKey(apiKey);
-            m_llModelInfo.model = model;
+            m_llModelInfo.model.reset(model);
         } else {
+            QElapsedTimer modelLoadTimer;
+            modelLoadTimer.start();
+
+            auto requestedDevice = MySettings::globalInstance()->device();
             auto n_ctx = MySettings::globalInstance()->modelContextLength(modelInfo);
             m_ctx.n_ctx = n_ctx;
             auto ngl = MySettings::globalInstance()->modelGpuLayers(modelInfo);
 
-            std::string buildVariant = "auto";
-#if defined(Q_OS_MAC) && defined(__arm__)
-            if (m_forceMetal)
-                buildVariant = "metal";
+            std::string backend = "auto";
+#ifdef Q_OS_MAC
+            if (requestedDevice == "CPU") {
+                backend = "cpu";
+            } else if (m_forceMetal) {
+#ifdef __aarch64__
+                backend = "metal";
 #endif
-            m_llModelInfo.model = LLModel::Implementation::construct(filePath.toStdString(), buildVariant, n_ctx);
+            }
+#else // !defined(Q_OS_MAC)
+            if (requestedDevice.startsWith("CUDA: "))
+                backend = "cuda";
+#endif
+
+            QString constructError;
+            m_llModelInfo.model.reset();
+            try {
+                auto *model = LLModel::Implementation::construct(filePath.toStdString(), backend, n_ctx);
+                m_llModelInfo.model.reset(model);
+            } catch (const LLModel::MissingImplementationError &e) {
+                modelLoadProps.insert("error", "missing_model_impl");
+                constructError = e.what();
+            } catch (const LLModel::UnsupportedModelError &e) {
+                modelLoadProps.insert("error", "unsupported_model_file");
+                constructError = e.what();
+            } catch (const LLModel::BadArchError &e) {
+                constructError = e.what();
+                modelLoadProps.insert("error", "unsupported_model_arch");
+                modelLoadProps.insert("model_arch", QString::fromStdString(e.arch()));
+            }
 
             if (m_llModelInfo.model) {
                 if (m_llModelInfo.model->isModelBlacklisted(filePath.toStdString())) {
@@ -322,92 +373,133 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                 }
 
                 m_llModelInfo.model->setProgressCallback([this](float progress) -> bool {
+                    progress = std::max(progress, std::numeric_limits<float>::min()); // keep progress above zero
                     emit modelLoadingPercentageChanged(progress);
                     return m_shouldBeLoaded;
                 });
 
-                // Pick the best match for the device
-                QString actualDevice = m_llModelInfo.model->implementation().buildVariant() == "metal" ? "Metal" : "CPU";
-                const QString requestedDevice = MySettings::globalInstance()->device();
-                if (requestedDevice == "CPU") {
-                    emit reportFallbackReason(""); // fallback not applicable
-                } else {
-                    const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString(), n_ctx, ngl);
-                    std::vector<LLModel::GPUDevice> availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
-                    LLModel::GPUDevice *device = nullptr;
+                auto approxDeviceMemGB = [](const LLModel::GPUDevice *dev) {
+                    float memGB = dev->heapSize / float(1024 * 1024 * 1024);
+                    return std::floor(memGB * 10.f) / 10.f; // truncate to 1 decimal place
+                };
 
-                    if (!availableDevices.empty() && requestedDevice == "Auto" && availableDevices.front().type == 2 /*a discrete gpu*/) {
-                        device = &availableDevices.front();
-                    } else {
-                        for (LLModel::GPUDevice &d : availableDevices) {
-                            if (QString::fromStdString(d.name) == requestedDevice) {
+                std::vector<LLModel::GPUDevice> availableDevices;
+                const LLModel::GPUDevice *defaultDevice = nullptr;
+                {
+                    const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString(), n_ctx, ngl);
+                    availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
+                    // Pick the best device
+                    // NB: relies on the fact that Kompute devices are listed first
+                    if (!availableDevices.empty() && availableDevices.front().type == 2 /*a discrete gpu*/) {
+                        defaultDevice = &availableDevices.front();
+                        float memGB = defaultDevice->heapSize / float(1024 * 1024 * 1024);
+                        memGB = std::floor(memGB * 10.f) / 10.f; // truncate to 1 decimal place
+                        modelLoadProps.insert("default_device", QString::fromStdString(defaultDevice->name));
+                        modelLoadProps.insert("default_device_mem", approxDeviceMemGB(defaultDevice));
+                    }
+                }
+
+                QString actualDevice("CPU");
+
+#if defined(Q_OS_MAC) && defined(__aarch64__)
+                if (m_llModelInfo.model->implementation().buildVariant() == "metal")
+                    actualDevice = "Metal";
+#else
+                if (requestedDevice != "CPU") {
+                    const auto *device = defaultDevice;
+                    if (requestedDevice != "Auto") {
+                        // Use the selected device
+                        for (const LLModel::GPUDevice &d : availableDevices) {
+                            if (QString::fromStdString(d.selectionName()) == requestedDevice) {
                                 device = &d;
                                 break;
                             }
                         }
                     }
 
-                    emit reportFallbackReason(""); // no fallback yet
                     std::string unavail_reason;
                     if (!device) {
                         // GPU not available
                     } else if (!m_llModelInfo.model->initializeGPUDevice(device->index, &unavail_reason)) {
                         emit reportFallbackReason(QString::fromStdString("<br>" + unavail_reason));
                     } else {
-                        actualDevice = QString::fromStdString(device->name);
+                        actualDevice = QString::fromStdString(device->reportedName());
+                        modelLoadProps.insert("requested_device_mem", approxDeviceMemGB(device));
                     }
                 }
+#endif
 
                 // Report which device we're actually using
                 emit reportDevice(actualDevice);
-
                 bool success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, ngl);
+
+                if (!m_shouldBeLoaded) {
+                    m_llModelInfo.model.reset();
+                    if (!m_isServer)
+                        LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+                    m_llModelInfo = LLModelInfo();
+                    emit modelLoadingPercentageChanged(0.0f);
+                    return false;
+                }
+
                 if (actualDevice == "CPU") {
                     // we asked llama.cpp to use the CPU
                 } else if (!success) {
                     // llama_init_from_file returned nullptr
                     emit reportDevice("CPU");
                     emit reportFallbackReason("<br>GPU loading failed (out of VRAM?)");
+                    modelLoadProps.insert("cpu_fallback_reason", "gpu_load_failed");
                     success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, 0);
+
+                    if (!m_shouldBeLoaded) {
+                        m_llModelInfo.model.reset();
+                        if (!m_isServer)
+                            LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+                        m_llModelInfo = LLModelInfo();
+                        emit modelLoadingPercentageChanged(0.0f);
+                        return false;
+                    }
                 } else if (!m_llModelInfo.model->usingGPUDevice()) {
                     // ggml_vk_init was not called in llama.cpp
                     // We might have had to fallback to CPU after load if the model is not possible to accelerate
                     // for instance if the quantization method is not supported on Vulkan yet
                     emit reportDevice("CPU");
                     emit reportFallbackReason("<br>model or quant has no GPU support");
+                    modelLoadProps.insert("cpu_fallback_reason", "gpu_unsupported_model");
                 }
 
                 if (!success) {
-                    delete m_llModelInfo.model;
-                    m_llModelInfo.model = nullptr;
+                    m_llModelInfo.model.reset();
                     if (!m_isServer)
-                        LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
+                        LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
                     m_llModelInfo = LLModelInfo();
                     emit modelLoadingError(QString("Could not load model due to invalid model file for %1").arg(modelInfo.filename()));
+                    modelLoadProps.insert("error", "loadmodel_failed");
                 } else {
                     switch (m_llModelInfo.model->implementation().modelType()[0]) {
                     case 'L': m_llModelType = LLModelType::LLAMA_; break;
                     case 'G': m_llModelType = LLModelType::GPTJ_; break;
                     default:
                         {
-                            delete m_llModelInfo.model;
-                            m_llModelInfo.model = nullptr;
+                            m_llModelInfo.model.reset();
                             if (!m_isServer)
-                                LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
+                                LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
                             m_llModelInfo = LLModelInfo();
                             emit modelLoadingError(QString("Could not determine model type for %1").arg(modelInfo.filename()));
                         }
                     }
+
+                    modelLoadProps.insert("$duration", modelLoadTimer.elapsed() / 1000.);
                 }
             } else {
                 if (!m_isServer)
-                    LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
+                    LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
                 m_llModelInfo = LLModelInfo();
-                emit modelLoadingError(QString("Could not load model due to invalid format for %1").arg(modelInfo.filename()));
+                emit modelLoadingError(QString("Error loading %1: %2").arg(modelInfo.filename()).arg(constructError));
             }
         }
 #if defined(DEBUG_MODEL_LOADING)
-        qDebug() << "new model" << m_llmThread.objectName() << m_llModelInfo.model;
+        qDebug() << "new model" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
         restoreState();
 #if defined(DEBUG)
@@ -416,15 +508,12 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 #endif
         emit modelLoadingPercentageChanged(isModelLoaded() ? 1.0f : 0.0f);
 
-        static bool isFirstLoad = true;
-        if (isFirstLoad) {
-            emit sendStartup();
-            isFirstLoad = false;
-        } else
-            emit sendModelLoaded();
+        modelLoadProps.insert("requestedDevice", MySettings::globalInstance()->device());
+        modelLoadProps.insert("model", modelInfo.filename());
+        Network::globalInstance()->trackChatEvent("model_load", modelLoadProps);
     } else {
         if (!m_isServer)
-            LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
+            LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo)); // release back into the store
         m_llModelInfo = LLModelInfo();
         emit modelLoadingError(QString("Could not find file for model %1").arg(modelInfo.filename()));
     }
@@ -433,7 +522,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         setModelInfo(modelInfo);
         processSystemPrompt();
     }
-    return m_llModelInfo.model;
+    return bool(m_llModelInfo.model);
 }
 
 bool ChatLLM::isModelLoaded() const
@@ -632,6 +721,8 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     printf("%s", qPrintable(prompt));
     fflush(stdout);
 #endif
+    QElapsedTimer totalTime;
+    totalTime.start();
     m_timer->start();
     if (!docsContext.isEmpty()) {
         auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode localdocs context without a response
@@ -644,28 +735,30 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     fflush(stdout);
 #endif
     m_timer->stop();
+    qint64 elapsed = totalTime.elapsed();
     std::string trimmed = trim_whitespace(m_response);
     if (trimmed != m_response) {
         m_response = trimmed;
         emit responseChanged(QString::fromStdString(m_response));
     }
-    emit responseStopped();
+    emit responseStopped(elapsed);
+    m_pristineLoadedState = false;
     return true;
 }
 
 void ChatLLM::setShouldBeLoaded(bool b)
 {
 #if defined(DEBUG_MODEL_LOADING)
-    qDebug() << "setShouldBeLoaded" << m_llmThread.objectName() << b << m_llModelInfo.model;
+    qDebug() << "setShouldBeLoaded" << m_llmThread.objectName() << b << m_llModelInfo.model.get();
 #endif
     m_shouldBeLoaded = b; // atomic
     emit shouldBeLoadedChanged();
 }
 
-void ChatLLM::setShouldTrySwitchContext(bool b)
+void ChatLLM::requestTrySwitchContext()
 {
-    m_shouldTrySwitchContext = b; // atomic
-    emit shouldTrySwitchContextChanged();
+    m_shouldBeLoaded = true; // atomic
+    emit trySwitchContextRequested(modelInfo());
 }
 
 void ChatLLM::handleShouldBeLoadedChanged()
@@ -676,12 +769,6 @@ void ChatLLM::handleShouldBeLoadedChanged()
         unloadModel();
 }
 
-void ChatLLM::handleShouldTrySwitchContextChanged()
-{
-    if (m_shouldTrySwitchContext)
-        trySwitchContextOfLoadedModel(modelInfo());
-}
-
 void ChatLLM::unloadModel()
 {
     if (!isModelLoaded() || m_isServer)
@@ -696,17 +783,16 @@ void ChatLLM::unloadModel()
         saveState();
 
 #if defined(DEBUG_MODEL_LOADING)
-    qDebug() << "unloadModel" << m_llmThread.objectName() << m_llModelInfo.model;
+    qDebug() << "unloadModel" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
 
     if (m_forceUnloadModel) {
-        delete m_llModelInfo.model;
-        m_llModelInfo.model = nullptr;
+        m_llModelInfo.model.reset();
         m_forceUnloadModel = false;
     }
 
-    LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
-    m_llModelInfo = LLModelInfo();
+    LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+    m_pristineLoadedState = false;
 }
 
 void ChatLLM::reloadModel()
@@ -718,7 +804,7 @@ void ChatLLM::reloadModel()
         return;
 
 #if defined(DEBUG_MODEL_LOADING)
-    qDebug() << "reloadModel" << m_llmThread.objectName() << m_llModelInfo.model;
+    qDebug() << "reloadModel" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
     const ModelInfo m = modelInfo();
     if (m.name().isEmpty())
@@ -733,28 +819,19 @@ void ChatLLM::generateName()
     if (!isModelLoaded())
         return;
 
-    QString instructPrompt("### Instruction:\n"
-                           "Describe response above in three words.\n"
-                           "### Response:\n");
+    auto promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     auto promptFunc = std::bind(&ChatLLM::handleNamePrompt, this, std::placeholders::_1);
-    auto responseFunc = std::bind(&ChatLLM::handleNameResponse, this, std::placeholders::_1,
-        std::placeholders::_2);
+    auto responseFunc = std::bind(&ChatLLM::handleNameResponse, this, std::placeholders::_1, std::placeholders::_2);
     auto recalcFunc = std::bind(&ChatLLM::handleNameRecalculate, this, std::placeholders::_1);
     LLModel::PromptContext ctx = m_ctx;
-#if defined(DEBUG)
-    printf("%s", qPrintable(instructPrompt));
-    fflush(stdout);
-#endif
-    m_llModelInfo.model->prompt(instructPrompt.toStdString(), "%1", promptFunc, responseFunc, recalcFunc, ctx);
-#if defined(DEBUG)
-    printf("\n");
-    fflush(stdout);
-#endif
+    m_llModelInfo.model->prompt("Describe the above conversation in three words or less.",
+                                promptTemplate.toStdString(), promptFunc, responseFunc, recalcFunc, ctx);
     std::string trimmed = trim_whitespace(m_nameResponse);
     if (trimmed != m_nameResponse) {
         m_nameResponse = trimmed;
         emit generatedNameChanged(QString::fromStdString(m_nameResponse));
     }
+    m_pristineLoadedState = false;
 }
 
 void ChatLLM::handleChatIdChanged(const QString &id)
@@ -894,7 +971,10 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
     // If we do not deserialize the KV or it is discarded, then we need to restore the state from the
     // text only. This will be a costly operation, but the chat has to be restored from the text archive
     // alone.
-    m_restoreStateFromText = !deserializeKV || discardKV;
+    if (!deserializeKV || discardKV) {
+        m_restoreStateFromText = true;
+        m_pristineLoadedState = true;
+    }
 
     if (!deserializeKV) {
 #if defined(DEBUG)
@@ -958,14 +1038,14 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
 
 void ChatLLM::saveState()
 {
-    if (!isModelLoaded())
+    if (!isModelLoaded() || m_pristineLoadedState)
         return;
 
     if (m_llModelType == LLModelType::API_) {
         m_state.clear();
         QDataStream stream(&m_state, QIODeviceBase::WriteOnly);
         stream.setVersion(QDataStream::Qt_6_4);
-        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model);
+        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model.get());
         stream << chatAPI->context();
         return;
     }
@@ -986,7 +1066,7 @@ void ChatLLM::restoreState()
     if (m_llModelType == LLModelType::API_) {
         QDataStream stream(&m_state, QIODeviceBase::ReadOnly);
         stream.setVersion(QDataStream::Qt_6_4);
-        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model);
+        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model.get());
         QList<QString> context;
         stream >> context;
         chatAPI->setContext(context);
@@ -1005,13 +1085,18 @@ void ChatLLM::restoreState()
     if (m_llModelInfo.model->stateSize() == m_state.size()) {
         m_llModelInfo.model->restoreState(static_cast<const uint8_t*>(reinterpret_cast<void*>(m_state.data())));
         m_processedSystemPrompt = true;
+        m_pristineLoadedState = true;
     } else {
         qWarning() << "restoring state from text because" << m_llModelInfo.model->stateSize() << "!=" << m_state.size();
         m_restoreStateFromText = true;
     }
 
-    m_state.clear();
-    m_state.squeeze();
+    // free local state copy unless unload is pending
+    if (m_shouldBeLoaded) {
+        m_state.clear();
+        m_state.squeeze();
+        m_pristineLoadedState = false;
+    }
 }
 
 void ChatLLM::processSystemPrompt()
@@ -1056,7 +1141,8 @@ void ChatLLM::processSystemPrompt()
     fflush(stdout);
 #endif
     auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode system prompt without a response
-    m_llModelInfo.model->prompt(systemPrompt, "%1", promptFunc, nullptr, recalcFunc, m_ctx, true);
+    // use "%1%2" and not "%1" to avoid implicit whitespace
+    m_llModelInfo.model->prompt(systemPrompt, "%1%2", promptFunc, nullptr, recalcFunc, m_ctx, true);
     m_ctx.n_predict = old_n_predict;
 #if defined(DEBUG)
     printf("\n");
@@ -1064,6 +1150,7 @@ void ChatLLM::processSystemPrompt()
 #endif
 
     m_processedSystemPrompt = m_stopGenerating == false;
+    m_pristineLoadedState = false;
 }
 
 void ChatLLM::processRestoreStateFromText()
@@ -1122,4 +1209,6 @@ void ChatLLM::processRestoreStateFromText()
 
     m_isRecalc = false;
     emit recalcChanged();
+
+    m_pristineLoadedState = false;
 }

gpt4all-chat/chatllm.h

@@ -5,6 +5,8 @@
 #include <QThread>
 #include <QFileInfo>
 
+#include <memory>
+
 #include "database.h"
 #include "modellist.h"
 #include "../gpt4all-backend/llmodel.h"
@@ -16,7 +18,7 @@ enum LLModelType {
 };
 
 struct LLModelInfo {
-    LLModel *model = nullptr;
+    std::unique_ptr<LLModel> model;
     QFileInfo fileInfo;
     // NOTE: This does not store the model type or name on purpose as this is left for ChatLLM which
     // must be able to serialize the information even if it is in the unloaded state
@@ -72,6 +74,7 @@ public:
     virtual ~ChatLLM();
 
     void destroy();
+    static void destroyStore();
     bool isModelLoaded() const;
     void regenerateResponse();
     void resetResponse();
@@ -81,7 +84,7 @@ public:
 
     bool shouldBeLoaded() const { return m_shouldBeLoaded; }
     void setShouldBeLoaded(bool b);
-    void setShouldTrySwitchContext(bool b);
+    void requestTrySwitchContext();
     void setForceUnloadModel(bool b) { m_forceUnloadModel = b; }
     void setMarkedForDeletion(bool b) { m_markedForDeletion = b; }
 
@@ -101,7 +104,7 @@ public:
 public Q_SLOTS:
     bool prompt(const QList<QString> &collectionList, const QString &prompt);
     bool loadDefaultModel();
-    bool trySwitchContextOfLoadedModel(const ModelInfo &modelInfo);
+    void trySwitchContextOfLoadedModel(const ModelInfo &modelInfo);
     bool loadModel(const ModelInfo &modelInfo);
     void modelChangeRequested(const ModelInfo &modelInfo);
     void unloadModel();
@@ -109,7 +112,6 @@ public Q_SLOTS:
     void generateName();
     void handleChatIdChanged(const QString &id);
     void handleShouldBeLoadedChanged();
-    void handleShouldTrySwitchContextChanged();
     void handleThreadStarted();
     void handleForceMetalChanged(bool forceMetal);
     void handleDeviceChanged();
@@ -123,15 +125,13 @@ Q_SIGNALS:
     void modelLoadingWarning(const QString &warning);
     void responseChanged(const QString &response);
     void promptProcessing();
-    void responseStopped();
-    void sendStartup();
-    void sendModelLoaded();
+    void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
     void stateChanged();
     void threadStarted();
     void shouldBeLoadedChanged();
-    void shouldTrySwitchContextChanged();
-    void trySwitchContextOfLoadedModelCompleted(bool);
+    void trySwitchContextRequested(const ModelInfo &modelInfo);
+    void trySwitchContextOfLoadedModelCompleted(int value);
     void requestRetrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
     void reportSpeed(const QString &speed);
     void reportDevice(const QString &device);
@@ -174,7 +174,6 @@ private:
     QThread m_llmThread;
     std::atomic<bool> m_stopGenerating;
     std::atomic<bool> m_shouldBeLoaded;
-    std::atomic<bool> m_shouldTrySwitchContext;
     std::atomic<bool> m_isRecalc;
     std::atomic<bool> m_forceUnloadModel;
     std::atomic<bool> m_markedForDeletion;
@@ -183,6 +182,10 @@ private:
     bool m_reloadingToChangeVariant;
     bool m_processedSystemPrompt;
     bool m_restoreStateFromText;
+    // m_pristineLoadedState is set if saveSate is unnecessary, either because:
+    // - an unload was queued during LLModel::restoreState()
+    // - the chat will be restored from text and hasn't been interacted with yet
+    bool m_pristineLoadedState = false;
     QVector<QPair<QString, QString>> m_stateFromText;
 };
 

gpt4all-chat/cmake/deploy-qt-linux.cmake.in

@@ -5,10 +5,7 @@ set(DATA_DIR ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN
 set(BIN_DIR ${DATA_DIR}/bin)
 set(Qt6_ROOT_DIR "@Qt6_ROOT_DIR@")
 set(ENV{LD_LIBRARY_PATH} "${BIN_DIR}:${Qt6_ROOT_DIR}/../lib/")
-execute_process(COMMAND ${LINUXDEPLOYQT} ${BIN_DIR}/chat -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -bundle-non-qt-libs -qmake=${Qt6_ROOT_DIR}/bin/qmake -verbose=2)
-file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/*llmodel.*)
-file(COPY ${MYLLMODELLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin)
+execute_process(COMMAND ${LINUXDEPLOYQT} ${BIN_DIR}/chat -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -bundle-non-qt-libs -qmake=${Qt6_ROOT_DIR}/bin/qmake -verbose=2 -exclude-libs=libcuda.so.1)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-32.png"
      DESTINATION ${DATA_DIR})
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-48.png"

gpt4all-chat/cmake/deploy-qt-mac.cmake.in

@@ -4,21 +4,16 @@ set(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@")
 execute_process(COMMAND ${MACDEPLOYQT} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -verbose=2)
 file(GLOB MYGPTJLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libgptj*)
 file(GLOB MYLLAMALIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllama*)
-file(GLOB MYBERTLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libbert*)
 file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllmodel.*)
 file(COPY ${MYGPTJLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLAMALIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYBERTLLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLMODELLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns"
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Resources)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-32.png"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-48.png"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)
-file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns"
+file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)

gpt4all-chat/cmake/deploy-qt-windows.cmake.in

@@ -2,12 +2,9 @@ set(WINDEPLOYQT "@WINDEPLOYQT@")
 set(COMPONENT_NAME_MAIN "@COMPONENT_NAME_MAIN@")
 set(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@")
 execute_process(COMMAND ${WINDEPLOYQT} --qmldir ${CMAKE_CURRENT_SOURCE_DIR} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin)
-file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/*llmodel.*)
-file(COPY ${MYLLMODELLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-32.png"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-48.png"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)
-file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.ico"
+file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.ico"
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data)

gpt4all-chat/cmake/installerscript.qs

@@ -19,7 +19,7 @@ Component.prototype.createOperations = function()
                     targetDirectory + "/bin/chat.exe",
                     "@UserProfile@/Desktop/GPT4All.lnk",
                     "workingDirectory=" + targetDirectory + "/bin",
-                    "iconPath=" + targetDirectory + "/favicon.ico",
+                    "iconPath=" + targetDirectory + "/gpt4all.ico",
                     "iconId=0", "description=Open GPT4All");
             } catch (e) {
                 print("ERROR: creating desktop shortcut" + e);
@@ -28,7 +28,7 @@ Component.prototype.createOperations = function()
                 targetDirectory + "/bin/chat.exe",
                 "@StartMenuDir@/GPT4All.lnk",
                 "workingDirectory=" + targetDirectory + "/bin",
-                "iconPath=" + targetDirectory + "/favicon.ico",
+                "iconPath=" + targetDirectory + "/gpt4all.ico",
                 "iconId=0", "description=Open GPT4All");
         } else if (systemInfo.productType === "osx") {
             var gpt4allAppPath = targetDirectory + "/bin/gpt4all.app";

gpt4all-chat/database.cpp

@@ -1,7 +1,9 @@
 #include "database.h"
-#include "mysettings.h"
-#include "embllm.h"
+
 #include "embeddings.h"
+#include "embllm.h"
+#include "mysettings.h"
+#include "network.h"
 
 #include <QTimer>
 #include <QPdfDocument>
@@ -410,8 +412,8 @@ bool updateDocument(QSqlQuery &q, int id, qint64 document_time)
 {
     if (!q.prepare(UPDATE_DOCUMENT_TIME_SQL))
         return false;
-    q.addBindValue(id);
     q.addBindValue(document_time);
+    q.addBindValue(id);
     return q.exec();
 }
 
@@ -490,7 +492,7 @@ QSqlError initDb()
     i.collection = collection_name;
     i.folder_path = folder_path;
     i.folder_id = folder_id;
-    emit addCollectionItem(i);
+    emit addCollectionItem(i, false);
 
     // Add a document
     int document_time = 123456789;
@@ -535,13 +537,13 @@ QSqlError initDb()
 
 Database::Database(int chunkSize)
     : QObject(nullptr)
-    , m_watcher(new QFileSystemWatcher(this))
     , m_chunkSize(chunkSize)
+    , m_scanTimer(new QTimer(this))
+    , m_watcher(new QFileSystemWatcher(this))
     , m_embLLM(new EmbeddingLLM)
     , m_embeddings(new Embeddings(this))
 {
     moveToThread(&m_dbThread);
-    connect(&m_dbThread, &QThread::started, this, &Database::start);
     m_dbThread.setObjectName("database");
     m_dbThread.start();
 }
@@ -550,17 +552,20 @@ Database::~Database()
 {
     m_dbThread.quit();
     m_dbThread.wait();
+    delete m_embLLM;
 }
 
 void Database::scheduleNext(int folder_id, size_t countForFolder)
 {
     emit updateCurrentDocsToIndex(folder_id, countForFolder);
     if (!countForFolder) {
-        emit updateIndexing(folder_id, false);
+        updateFolderStatus(folder_id, FolderStatus::Complete);
         emit updateInstalled(folder_id, true);
     }
-    if (!m_docsToScan.isEmpty())
-        QTimer::singleShot(0, this, &Database::scanQueue);
+    if (m_docsToScan.isEmpty()) {
+        m_scanTimer->stop();
+        updateIndexingStatus();
+    }
 }
 
 void Database::handleDocumentError(const QString &errorMessage,
@@ -721,7 +726,6 @@ void Database::removeFolderFromDocumentQueue(int folder_id)
         return;
     m_docsToScan.remove(folder_id);
     emit removeFolderById(folder_id);
-    emit docsToScanChanged();
 }
 
 void Database::enqueueDocumentInternal(const DocumentInfo &info, bool prepend)
@@ -745,13 +749,16 @@ void Database::enqueueDocuments(int folder_id, const QVector<DocumentInfo> &info
     const size_t bytes = countOfBytes(folder_id);
     emit updateCurrentBytesToIndex(folder_id, bytes);
     emit updateTotalBytesToIndex(folder_id, bytes);
-    emit docsToScanChanged();
+    m_scanTimer->start();
 }
 
 void Database::scanQueue()
 {
-    if (m_docsToScan.isEmpty())
+    if (m_docsToScan.isEmpty()) {
+        m_scanTimer->stop();
+        updateIndexingStatus();
         return;
+    }
 
     DocumentInfo info = dequeueDocument();
     const size_t countForFolder = countOfDocuments(info.folder);
@@ -818,6 +825,8 @@ void Database::scanQueue()
     QSqlDatabase::database().transaction();
     Q_ASSERT(document_id != -1);
     if (info.isPdf()) {
+        updateFolderStatus(folder_id, FolderStatus::Embedding, -1, info.currentPage == 0);
+
         QPdfDocument doc;
         if (QPdfDocument::Error::None != doc.load(info.doc.canonicalFilePath())) {
             handleDocumentError("ERROR: Could not load pdf",
@@ -850,6 +859,8 @@ void Database::scanQueue()
             emit subtractCurrentBytesToIndex(info.folder, bytes - (bytesPerPage * doc.pageCount()));
         }
     } else {
+        updateFolderStatus(folder_id, FolderStatus::Embedding, -1, info.currentPosition == 0);
+
         QFile file(document_path);
         if (!file.open(QIODevice::ReadOnly)) {
             handleDocumentError("ERROR: Cannot open file for scanning",
@@ -884,7 +895,7 @@ void Database::scanQueue()
     return scheduleNext(folder_id, countForFolder);
 }
 
-void Database::scanDocuments(int folder_id, const QString &folder_path)
+void Database::scanDocuments(int folder_id, const QString &folder_path, bool isNew)
 {
 #if defined(DEBUG)
     qDebug() << "scanning folder for documents" << folder_path;
@@ -915,7 +926,7 @@ void Database::scanDocuments(int folder_id, const QString &folder_path)
     }
 
     if (!infos.isEmpty()) {
-        emit updateIndexing(folder_id, true);
+        updateFolderStatus(folder_id, FolderStatus::Started, infos.count(), false, isNew);
         enqueueDocuments(folder_id, infos);
     }
 }
@@ -925,7 +936,7 @@ void Database::start()
     connect(m_watcher, &QFileSystemWatcher::directoryChanged, this, &Database::directoryChanged);
     connect(m_embLLM, &EmbeddingLLM::embeddingsGenerated, this, &Database::handleEmbeddingsGenerated);
     connect(m_embLLM, &EmbeddingLLM::errorGenerated, this, &Database::handleErrorGenerated);
-    connect(this, &Database::docsToScanChanged, this, &Database::scanQueue);
+    m_scanTimer->callOnTimeout(this, &Database::scanQueue);
     if (!QSqlDatabase::drivers().contains("QSQLITE")) {
         qWarning() << "ERROR: missing sqllite driver";
     } else {
@@ -937,10 +948,11 @@ void Database::start()
     if (m_embeddings->fileExists() && !m_embeddings->load())
         qWarning() << "ERROR: Could not load embeddings";
 
-    addCurrentFolders();
+    int nAdded = addCurrentFolders();
+    Network::globalInstance()->trackEvent("localdocs_startup", { {"doc_collections_total", nAdded} });
 }
 
-void Database::addCurrentFolders()
+int Database::addCurrentFolders()
 {
 #if defined(DEBUG)
     qDebug() << "addCurrentFolders";
@@ -950,21 +962,26 @@ void Database::addCurrentFolders()
     QList<CollectionItem> collections;
     if (!selectAllFromCollections(q, &collections)) {
         qWarning() << "ERROR: Cannot select collections" << q.lastError();
-        return;
+        return 0;
     }
 
     emit collectionListUpdated(collections);
 
+    int nAdded = 0;
     for (const auto &i : collections)
-        addFolder(i.collection, i.folder_path);
+        nAdded += addFolder(i.collection, i.folder_path, true);
+
+    updateIndexingStatus();
+
+    return collections.count() + nAdded;
 }
 
-void Database::addFolder(const QString &collection, const QString &path)
+bool Database::addFolder(const QString &collection, const QString &path, bool fromDb)
 {
     QFileInfo info(path);
     if (!info.exists() || !info.isReadable()) {
         qWarning() << "ERROR: Cannot add folder that doesn't exist or not readable" << path;
-        return;
+        return false;
     }
 
     QSqlQuery q;
@@ -973,13 +990,13 @@ void Database::addFolder(const QString &collection, const QString &path)
     // See if the folder exists in the db
     if (!selectFolder(q, path, &folder_id)) {
         qWarning() << "ERROR: Cannot select folder from path" << path << q.lastError();
-        return;
+        return false;
     }
 
     // Add the folder
     if (folder_id == -1 && !addFolderToDB(q, path, &folder_id)) {
         qWarning() << "ERROR: Cannot add folder to db with path" << path << q.lastError();
-        return;
+        return false;
     }
 
     Q_ASSERT(folder_id != -1);
@@ -988,24 +1005,32 @@ void Database::addFolder(const QString &collection, const QString &path)
     QList<int> folders;
     if (!selectFoldersFromCollection(q, collection, &folders)) {
         qWarning() << "ERROR: Cannot select folders from collections" << collection << q.lastError();
-        return;
+        return false;
     }
 
+    bool added = false;
     if (!folders.contains(folder_id)) {
         if (!addCollection(q, collection, folder_id)) {
             qWarning() << "ERROR: Cannot add folder to collection" << collection << path << q.lastError();
-            return;
+            return false;
         }
 
         CollectionItem i;
         i.collection = collection;
         i.folder_path = path;
         i.folder_id = folder_id;
-        emit addCollectionItem(i);
+        emit addCollectionItem(i, fromDb);
+        added = true;
     }
 
     addFolderToWatch(path);
-    scanDocuments(folder_id, path);
+    scanDocuments(folder_id, path, !fromDb);
+
+    if (!fromDb) {
+        updateIndexingStatus();
+    }
+
+    return added;
 }
 
 void Database::removeFolder(const QString &collection, const QString &path)
@@ -1285,5 +1310,69 @@ void Database::directoryChanged(const QString &path)
     cleanDB();
 
     // Rescan the documents associated with the folder
-    scanDocuments(folder_id, path);
+    scanDocuments(folder_id, path, false);
+    updateIndexingStatus();
+}
+
+void Database::updateIndexingStatus() {
+    Q_ASSERT(m_scanTimer->isActive() || m_docsToScan.isEmpty());
+    if (!m_indexingTimer.isValid() && m_scanTimer->isActive()) {
+        Network::globalInstance()->trackEvent("localdocs_indexing_start");
+        m_indexingTimer.start();
+    } else if (m_indexingTimer.isValid() && !m_scanTimer->isActive()) {
+        qint64 durationMs = m_indexingTimer.elapsed();
+        Network::globalInstance()->trackEvent("localdocs_indexing_complete", { {"$duration", durationMs / 1000.} });
+        m_indexingTimer.invalidate();
+    }
+}
+
+void Database::updateFolderStatus(int folder_id, Database::FolderStatus status, int numDocs, bool atStart, bool isNew) {
+    FolderStatusRecord *lastRecord = nullptr;
+    if (m_foldersBeingIndexed.contains(folder_id)) {
+        lastRecord = &m_foldersBeingIndexed[folder_id];
+    }
+    Q_ASSERT(lastRecord || status == FolderStatus::Started);
+
+    switch (status) {
+        case FolderStatus::Started:
+            if (lastRecord == nullptr) {
+                // record timestamp but don't send an event yet
+                m_foldersBeingIndexed.insert(folder_id, { QDateTime::currentMSecsSinceEpoch(), isNew, numDocs });
+                emit updateIndexing(folder_id, true);
+            }
+            break;
+        case FolderStatus::Embedding:
+            if (!lastRecord->docsChanged) {
+                Q_ASSERT(atStart);
+                // send start event with the original timestamp for folders that need updating
+                const auto *embeddingModels = ModelList::globalInstance()->installedEmbeddingModels();
+                Network::globalInstance()->trackEvent("localdocs_folder_indexing", {
+                    {"folder_id", folder_id},
+                    {"is_new_collection", lastRecord->isNew},
+                    {"document_count", lastRecord->numDocs},
+                    {"embedding_model", embeddingModels->defaultModelInfo().filename()},
+                    {"chunk_size", m_chunkSize},
+                    {"time", lastRecord->startTime},
+                });
+            }
+            lastRecord->docsChanged += atStart;
+            lastRecord->chunksRead++;
+            break;
+        case FolderStatus::Complete:
+            if (lastRecord->docsChanged) {
+                // send complete event for folders that were updated
+                qint64 durationMs = QDateTime::currentMSecsSinceEpoch() - lastRecord->startTime;
+                Network::globalInstance()->trackEvent("localdocs_folder_complete", {
+                    {"folder_id", folder_id},
+                    {"is_new_collection", lastRecord->isNew},
+                    {"documents_total", lastRecord->numDocs},
+                    {"documents_changed", lastRecord->docsChanged},
+                    {"chunks_read", lastRecord->chunksRead},
+                    {"$duration", durationMs / 1000.},
+                });
+            }
+            m_foldersBeingIndexed.remove(folder_id);
+            emit updateIndexing(folder_id, false);
+            break;
+    }
 }

gpt4all-chat/database.h

@@ -1,16 +1,19 @@
 #ifndef DATABASE_H
 #define DATABASE_H
 
-#include <QObject>
-#include <QtSql>
-#include <QQueue>
+#include <QElapsedTimer>
 #include <QFileInfo>
-#include <QThread>
 #include <QFileSystemWatcher>
+#include <QObject>
+#include <QQueue>
+#include <QThread>
+#include <QtSql>
 
 #include "embllm.h"
 
 class Embeddings;
+class QTimer;
+
 struct DocumentInfo
 {
     int folder;
@@ -58,9 +61,10 @@ public:
     virtual ~Database();
 
 public Q_SLOTS:
+    void start();
     void scanQueue();
-    void scanDocuments(int folder_id, const QString &folder_path);
-    void addFolder(const QString &collection, const QString &path);
+    void scanDocuments(int folder_id, const QString &folder_path, bool isNew);
+    bool addFolder(const QString &collection, const QString &path, bool fromDb);
     void removeFolder(const QString &collection, const QString &path);
     void retrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
     void cleanDB();
@@ -78,21 +82,22 @@ Q_SIGNALS:
     void updateTotalBytesToIndex(int folder_id, size_t totalBytesToIndex);
     void updateCurrentEmbeddingsToIndex(int folder_id, size_t currentBytesToIndex);
     void updateTotalEmbeddingsToIndex(int folder_id, size_t totalBytesToIndex);
-    void addCollectionItem(const CollectionItem &item);
+    void addCollectionItem(const CollectionItem &item, bool fromDb);
     void removeFolderById(int folder_id);
-    void removeCollectionItem(const QString &collectionName);
     void collectionListUpdated(const QList<CollectionItem> &collectionList);
 
 private Q_SLOTS:
-    void start();
     void directoryChanged(const QString &path);
     bool addFolderToWatch(const QString &path);
     bool removeFolderFromWatch(const QString &path);
-    void addCurrentFolders();
+    int addCurrentFolders();
     void handleEmbeddingsGenerated(const QVector<EmbeddingResult> &embeddings);
     void handleErrorGenerated(int folder_id, const QString &error);
 
 private:
+    enum class FolderStatus { Started, Embedding, Complete };
+    struct FolderStatusRecord { qint64 startTime; bool isNew; int numDocs, docsChanged, chunksRead; };
+
     void removeFolderInternal(const QString &collection, int folder_id, const QString &path);
     size_t chunkStream(QTextStream &stream, int folder_id, int document_id, const QString &file,
         const QString &title, const QString &author, const QString &subject, const QString &keywords, int page,
@@ -107,10 +112,15 @@ private:
     void removeFolderFromDocumentQueue(int folder_id);
     void enqueueDocumentInternal(const DocumentInfo &info, bool prepend = false);
     void enqueueDocuments(int folder_id, const QVector<DocumentInfo> &infos);
+    void updateIndexingStatus();
+    void updateFolderStatus(int folder_id, FolderStatus status, int numDocs = -1, bool atStart = false, bool isNew = false);
 
 private:
     int m_chunkSize;
+    QTimer *m_scanTimer;
     QMap<int, QQueue<DocumentInfo>> m_docsToScan;
+    QElapsedTimer m_indexingTimer;
+    QMap<int, FolderStatusRecord> m_foldersBeingIndexed;
     QList<ResultInfo> m_retrieve;
     QThread m_dbThread;
     QFileSystemWatcher *m_watcher;

gpt4all-chat/download.cpp

@@ -75,15 +75,25 @@ bool Download::hasNewerRelease() const
     return compareVersions(versions.first(), currentVersion);
 }
 
-bool Download::isFirstStart() const
+bool Download::isFirstStart(bool writeVersion) const
 {
+    auto *mySettings = MySettings::globalInstance();
+
     QSettings settings;
     settings.sync();
     QString lastVersionStarted = settings.value("download/lastVersionStarted").toString();
     bool first = lastVersionStarted != QCoreApplication::applicationVersion();
-    settings.setValue("download/lastVersionStarted", QCoreApplication::applicationVersion());
-    settings.sync();
-    return first;
+    if (first && writeVersion) {
+        settings.setValue("download/lastVersionStarted", QCoreApplication::applicationVersion());
+        // let the user select these again
+        settings.remove("network/usageStatsActive");
+        settings.remove("network/isActive");
+        settings.sync();
+        emit mySettings->networkUsageStatsActiveChanged();
+        emit mySettings->networkIsActiveChanged();
+    }
+
+    return first || !mySettings->isNetworkUsageStatsActiveSet() || !mySettings->isNetworkIsActiveSet();
 }
 
 void Download::updateReleaseNotes()
@@ -131,7 +141,7 @@ void Download::downloadModel(const QString &modelFile)
     ModelList::globalInstance()->updateDataByFilename(modelFile, {{ ModelList::DownloadingRole, true }});
     ModelInfo info = ModelList::globalInstance()->modelInfoByFilename(modelFile);
     QString url = !info.url().isEmpty() ? info.url() : "http://gpt4all.io/models/gguf/" + modelFile;
-    Network::globalInstance()->sendDownloadStarted(modelFile);
+    Network::globalInstance()->trackEvent("download_started", { {"model", modelFile} });
     QNetworkRequest request(url);
     request.setAttribute(QNetworkRequest::User, modelFile);
     request.setRawHeader("range", QString("bytes=%1-").arg(tempFile->pos()).toUtf8());
@@ -153,7 +163,7 @@ void Download::cancelDownload(const QString &modelFile)
         QNetworkReply *modelReply = m_activeDownloads.keys().at(i);
         QUrl url = modelReply->request().url();
         if (url.toString().endsWith(modelFile)) {
-            Network::globalInstance()->sendDownloadCanceled(modelFile);
+            Network::globalInstance()->trackEvent("download_canceled", { {"model", modelFile} });
 
             // Disconnect the signals
             disconnect(modelReply, &QNetworkReply::downloadProgress, this, &Download::handleDownloadProgress);
@@ -178,7 +188,8 @@ void Download::installModel(const QString &modelFile, const QString &apiKey)
     if (apiKey.isEmpty())
         return;
 
-    Network::globalInstance()->sendInstallModel(modelFile);
+    Network::globalInstance()->trackEvent("install_model", { {"model", modelFile} });
+
     QString filePath = MySettings::globalInstance()->modelPath() + modelFile;
     QFile file(filePath);
     if (file.open(QIODeviceBase::WriteOnly | QIODeviceBase::Text)) {
@@ -216,7 +227,7 @@ void Download::removeModel(const QString &modelFile)
         shouldRemoveInstalled = info.installed && !info.isClone() && (info.isDiscovered() || info.description() == "" /*indicates sideloaded*/);
         if (shouldRemoveInstalled)
             ModelList::globalInstance()->removeInstalled(info);
-        Network::globalInstance()->sendRemoveModel(modelFile);
+        Network::globalInstance()->trackEvent("remove_model", { {"model", modelFile} });
         file.remove();
     }
 
@@ -332,7 +343,11 @@ void Download::handleErrorOccurred(QNetworkReply::NetworkError code)
             .arg(modelReply->errorString());
     qWarning() << error;
     ModelList::globalInstance()->updateDataByFilename(modelFilename, {{ ModelList::DownloadErrorRole, error }});
-    Network::globalInstance()->sendDownloadError(modelFilename, (int)code, modelReply->errorString());
+    Network::globalInstance()->trackEvent("download_error", {
+        {"model", modelFilename},
+        {"code", (int)code},
+        {"error", modelReply->errorString()},
+    });
     cancelDownload(modelFilename);
 }
 
@@ -515,7 +530,7 @@ void Download::handleHashAndSaveFinished(bool success, const QString &error,
     // The hash and save should send back with tempfile closed
     Q_ASSERT(!tempFile->isOpen());
     QString modelFilename = modelReply->request().attribute(QNetworkRequest::User).toString();
-    Network::globalInstance()->sendDownloadFinished(modelFilename, success);
+    Network::globalInstance()->trackEvent("download_finished", { {"model", modelFilename}, {"success", success} });
 
     QVector<QPair<int, QVariant>> data {
         { ModelList::CalcHashRole, false },

gpt4all-chat/download.h

@@ -54,7 +54,7 @@ public:
     Q_INVOKABLE void cancelDownload(const QString &modelFile);
     Q_INVOKABLE void installModel(const QString &modelFile, const QString &apiKey);
     Q_INVOKABLE void removeModel(const QString &modelFile);
-    Q_INVOKABLE bool isFirstStart() const;
+    Q_INVOKABLE bool isFirstStart(bool writeVersion = false) const;
 
 public Q_SLOTS:
     void updateReleaseNotes();

gpt4all-chat/embllm.cpp

@@ -5,6 +5,7 @@ EmbeddingLLMWorker::EmbeddingLLMWorker()
     : QObject(nullptr)
     , m_networkManager(new QNetworkAccessManager(this))
     , m_model(nullptr)
+    , m_stopGenerating(false)
 {
     moveToThread(&m_workerThread);
     connect(this, &EmbeddingLLMWorker::finished, &m_workerThread, &QThread::quit, Qt::DirectConnection);
@@ -14,6 +15,10 @@ EmbeddingLLMWorker::EmbeddingLLMWorker()
 
 EmbeddingLLMWorker::~EmbeddingLLMWorker()
 {
+    m_stopGenerating = true;
+    m_workerThread.quit();
+    m_workerThread.wait();
+
     if (m_model) {
         delete m_model;
         m_model = nullptr;
@@ -42,17 +47,29 @@ bool EmbeddingLLMWorker::loadModel()
     }
 
     auto filename = fileInfo.fileName();
-    bool isNomic = filename.startsWith("nomic-") && filename.endsWith(".txt");
+    bool isNomic = filename.startsWith("gpt4all-nomic-") && filename.endsWith(".rmodel");
     if (isNomic) {
         QFile file(filePath);
-        file.open(QIODeviceBase::ReadOnly | QIODeviceBase::Text);
-        QTextStream stream(&file);
-        m_nomicAPIKey = stream.readAll();
+        if (!file.open(QIODeviceBase::ReadOnly)) {
+            qWarning() << "failed to open" << filePath << ":" << file.errorString();
+            m_model = nullptr;
+            return false;
+        }
+        QJsonDocument doc = QJsonDocument::fromJson(file.readAll());
+        QJsonObject obj = doc.object();
+        m_nomicAPIKey = obj["apiKey"].toString();
         file.close();
         return true;
     }
 
-    m_model = LLModel::Implementation::construct(filePath.toStdString());
+    try {
+        m_model = LLModel::Implementation::construct(filePath.toStdString());
+    } catch (const std::exception &e) {
+        qWarning() << "WARNING: Could not load embedding model:" << e.what();
+        m_model = nullptr;
+        return false;
+    }
+
     // NOTE: explicitly loads model on CPU to avoid GPU OOM
     // TODO(cebtenzzre): support GPU-accelerated embeddings
     bool success = m_model->loadModel(filePath.toStdString(), 2048, 0);
@@ -85,16 +102,7 @@ bool EmbeddingLLMWorker::isNomic() const
 // this function is always called for retrieval tasks
 std::vector<float> EmbeddingLLMWorker::generateSyncEmbedding(const QString &text)
 {
-    if (!hasModel() && !loadModel()) {
-        qWarning() << "WARNING: Could not load model for embeddings";
-        return {};
-    }
-
-    if (isNomic()) {
-        qWarning() << "WARNING: Request to generate sync embeddings for non-local model invalid";
-        return {};
-    }
-
+    Q_ASSERT(!isNomic());
     std::vector<float> embedding(m_model->embeddingSize());
     try {
         m_model->embed({text.toStdString()}, embedding.data(), true);
@@ -145,6 +153,9 @@ void EmbeddingLLMWorker::requestSyncEmbedding(const QString &text)
 // this function is always called for storage into the database
 void EmbeddingLLMWorker::requestAsyncEmbedding(const QVector<EmbeddingChunk> &chunks)
 {
+    if (m_stopGenerating)
+        return;
+
     if (!hasModel() && !loadModel()) {
         qWarning() << "WARNING: Could not load model for embeddings";
         return;
@@ -294,16 +305,21 @@ EmbeddingLLM::~EmbeddingLLM()
 
 std::vector<float> EmbeddingLLM::generateEmbeddings(const QString &text)
 {
+    if (!m_embeddingWorker->hasModel() && !m_embeddingWorker->loadModel()) {
+        qWarning() << "WARNING: Could not load model for embeddings";
+        return {};
+    }
+
     if (!m_embeddingWorker->isNomic()) {
         return m_embeddingWorker->generateSyncEmbedding(text);
-    } else {
-        EmbeddingLLMWorker worker;
-        connect(this, &EmbeddingLLM::requestSyncEmbedding, &worker,
-            &EmbeddingLLMWorker::requestSyncEmbedding, Qt::QueuedConnection);
-        emit requestSyncEmbedding(text);
-        worker.wait();
-        return worker.lastResponse();
     }
+
+    EmbeddingLLMWorker worker;
+    connect(this, &EmbeddingLLM::requestSyncEmbedding, &worker,
+        &EmbeddingLLMWorker::requestSyncEmbedding, Qt::QueuedConnection);
+    emit requestSyncEmbedding(text);
+    worker.wait();
+    return worker.lastResponse();
 }
 
 void EmbeddingLLM::generateAsyncEmbeddings(const QVector<EmbeddingChunk> &chunks)

gpt4all-chat/embllm.h

@@ -58,6 +58,7 @@ private:
     QNetworkAccessManager *m_networkManager;
     std::vector<float> m_lastResponse;
     LLModel *m_model = nullptr;
+    std::atomic<bool> m_stopGenerating;
     QThread m_workerThread;
 };
 

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.appdata.xml

@@ -2,9 +2,10 @@
 <component type="desktop">
     <id>io.gpt4all.gpt4all</id>
     <metadata_license>CC0-1.0</metadata_license>
-    <project_license>MIT License</project_license>
+    <project_license>MIT</project_license>
     <name>GPT4ALL</name>
-    <summary>Open-source assistant-style large language models that run locally on your CPU and GPU</summary>
+    <summary>Open-source assistant</summary>
+    <developer_name>Nomic-ai</developer_name>
     <description>
         <p>Cross platform Qt based GUI for GPT4All</p>
         <ul>
@@ -29,20 +30,12 @@
     <url type="bugtracker">https://github.com/nomic-ai/gpt4all/issues</url>
     <url type="vcs-browser">https://github.com/nomic-ai/gpt4all</url>
     <releases>
-      <release version="2.4.19" date="2023-09-16">
-        <description>
-          <p>
-            <ul>
-              <li>A bugfix for crashes on systems that have a corrupted Vulkan driver or a corrupted version of the Vulkan shared library</li>
-            </ul>
-          </p>
-        </description>
-        </release>              
+      <release version="2.7.5" date="2024-05-03"></release>              
     </releases>
     <launchable type="desktop-id">io.gpt4all.gpt4all.desktop</launchable>
-    <content_rating type="oars-1.0">
+    <content_rating type="oars-1.1">
         <content_attribute id="language-profanity">mild</content_attribute>
         <content_attribute id="language-humor">moderate</content_attribute>
         <content_attribute id="language-discrimination">mild</content_attribute>
     </content_rating>
-</component>
+</component>

gpt4all-chat/icons/chat.svg

@@ -1,3 +0,0 @@
-<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M26.1538 9.76H22.4615V5.92C22.4615 5.41078 22.267 4.92242 21.9208 4.56235C21.5746 4.20229 21.105 4 20.6154 4H5.84615C5.35652 4 4.88695 4.20229 4.54073 4.56235C4.1945 4.92242 4 5.41078 4 5.92V21.28C4.00054 21.4606 4.05006 21.6374 4.14287 21.79C4.23568 21.9427 4.36801 22.065 4.52467 22.143C4.68132 22.2209 4.85595 22.2513 5.02847 22.2307C5.201 22.2101 5.36444 22.1393 5.5 22.0264L9.53846 18.64V22.24C9.53846 22.7492 9.73297 23.2376 10.0792 23.5976C10.4254 23.9577 10.895 24.16 11.3846 24.16H22.1835L26.5 27.7864C26.6633 27.9238 26.8668 27.9991 27.0769 28C27.3217 28 27.5565 27.8989 27.7296 27.7188C27.9027 27.5388 28 27.2946 28 27.04V11.68C28 11.1708 27.8055 10.6824 27.4593 10.3224C27.1131 9.96229 26.6435 9.76 26.1538 9.76ZM8.90962 16.6936L5.84615 19.27V5.92H20.6154V16.48H9.49C9.27874 16.48 9.07387 16.5554 8.90962 16.6936ZM26.1538 25.03L23.0904 22.4536C22.9271 22.3162 22.7235 22.2408 22.5135 22.24H11.3846V18.4H20.6154C21.105 18.4 21.5746 18.1977 21.9208 17.8376C22.267 17.4776 22.4615 16.9892 22.4615 16.48V11.68H26.1538V25.03Z" fill="black"/>
-</svg>

gpt4all-chat/icons/db.svg

@@ -1,3 +1,5 @@
-<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M16 3C9.27125 3 4 6.075 4 10V22C4 25.925 9.27125 29 16 29C22.7288 29 28 25.925 28 22V10C28 6.075 22.7288 3 16 3ZM26 16C26 17.2025 25.015 18.4287 23.2987 19.365C21.3662 20.4187 18.7738 21 16 21C13.2262 21 10.6338 20.4187 8.70125 19.365C6.985 18.4287 6 17.2025 6 16V13.92C8.1325 15.795 11.7787 17 16 17C20.2213 17 23.8675 15.79 26 13.92V16ZM8.70125 6.635C10.6338 5.58125 13.2262 5 16 5C18.7738 5 21.3662 5.58125 23.2987 6.635C25.015 7.57125 26 8.7975 26 10C26 11.2025 25.015 12.4287 23.2987 13.365C21.3662 14.4187 18.7738 15 16 15C13.2262 15 10.6338 14.4187 8.70125 13.365C6.985 12.4287 6 11.2025 6 10C6 8.7975 6.985 7.57125 8.70125 6.635ZM23.2987 25.365C21.3662 26.4187 18.7738 27 16 27C13.2262 27 10.6338 26.4187 8.70125 25.365C6.985 24.4287 6 23.2025 6 22V19.92C8.1325 21.795 11.7787 23 16 23C20.2213 23 23.8675 21.79 26 19.92V22C26 23.2025 25.015 24.4287 23.2987 25.365Z" fill="black"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" fill="#7d7d8e" viewBox="0 0 448 512"><path d="M448 80v48c0 44.2-100.3 80-224 80S0 172.2 0 128V80C0 35.8 100.3 0 224 0S448 35.8 448 80zM393.2 214.7c20.8-7.4 39.9-16.9 54.8-28.6V288c0 44.2-100.3 80-224 80S0 332.2 0 288V186.1c14.9 11.8 34 21.2 54.8 28.6C99.7 230.7 159.5 240 224 240s124.3-9.3 169.2-25.3zM0 346.1c14.9 11.8 34 21.2 54.8 28.6C99.7 390.7 159.5 400 224 400s124.3-9.3 169.2-25.3c20.8-7.4 39.9-16.9 54.8-28.6V432c0 44.2-100.3 80-224 80S0 476.2 0 432V346.1z"/></svg>
+<!--
+Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
+License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+-->

gpt4all-chat/icons/edit.svg

@@ -1,3 +1,5 @@
-<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M28.4138 9.17125L22.8288 3.585C22.643 3.39924 22.4225 3.25188 22.1799 3.15134C21.9372 3.0508 21.6771 2.99905 21.4144 2.99905C21.1517 2.99905 20.8916 3.0508 20.6489 3.15134C20.4062 3.25188 20.1857 3.39924 20 3.585L4.58626 19C4.39973 19.185 4.25185 19.4053 4.15121 19.648C4.05057 19.8907 3.99917 20.151 4.00001 20.4138V26C4.00001 26.5304 4.21072 27.0391 4.5858 27.4142C4.96087 27.7893 5.46958 28 6.00001 28H11.5863C11.849 28.0008 12.1093 27.9494 12.352 27.8488C12.5947 27.7482 12.815 27.6003 13 27.4138L28.4138 12C28.5995 11.8143 28.7469 11.5938 28.8474 11.3511C28.948 11.1084 28.9997 10.8483 28.9997 10.5856C28.9997 10.3229 28.948 10.0628 28.8474 9.82015C28.7469 9.57747 28.5995 9.35698 28.4138 9.17125ZM6.41376 20L17 9.41375L19.0863 11.5L8.50001 22.085L6.41376 20ZM6.00001 22.4138L9.58626 26H6.00001V22.4138ZM12 25.5863L9.91376 23.5L20.5 12.9138L22.5863 15L12 25.5863ZM24 13.5863L18.4138 8L21.4138 5L27 10.585L24 13.5863Z" fill="black"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" fill="#7d7d8e" viewBox="0 0 576 512"><path d="M402.6 83.2l90.2 90.2c3.8 3.8 3.8 10 0 13.8L274.4 405.6l-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8L388.8 83.2c3.8-3.8 10-3.8 13.8 0zm162-22.9l-48.8-48.8c-15.2-15.2-39.9-15.2-55.2 0l-35.4 35.4c-3.8 3.8-3.8 10 0 13.8l90.2 90.2c3.8 3.8 10 3.8 13.8 0l35.4-35.4c15.2-15.3 15.2-40 0-55.2zM384 346.2V448H64V128h229.8c3.2 0 6.2-1.3 8.5-3.5l40-40c7.6-7.6 2.2-20.5-8.5-20.5H48C21.5 64 0 85.5 0 112v352c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V306.2c0-10.7-12.9-16-20.5-8.5l-40 40c-2.2 2.3-3.5 5.3-3.5 8.5z"/></svg>
+<!--
+Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
+License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+-->

gpt4all-chat/icons/info.svg

@@ -1,3 +0,0 @@
-<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M16 3C13.4288 3 10.9154 3.76244 8.77759 5.1909C6.63975 6.61935 4.97351 8.64968 3.98957 11.0251C3.00563 13.4006 2.74819 16.0144 3.2498 18.5362C3.75141 21.0579 4.98953 23.3743 6.80762 25.1924C8.6257 27.0105 10.9421 28.2486 13.4638 28.7502C15.9856 29.2518 18.5995 28.9944 20.9749 28.0104C23.3503 27.0265 25.3807 25.3603 26.8091 23.2224C28.2376 21.0846 29 18.5712 29 16C28.9964 12.5533 27.6256 9.24882 25.1884 6.81163C22.7512 4.37445 19.4467 3.00364 16 3ZM16 27C13.8244 27 11.6977 26.3549 9.88873 25.1462C8.07979 23.9375 6.66989 22.2195 5.83733 20.2095C5.00477 18.1995 4.78693 15.9878 5.21137 13.854C5.63581 11.7202 6.68345 9.7602 8.22183 8.22183C9.76021 6.68345 11.7202 5.6358 13.854 5.21136C15.9878 4.78692 18.1995 5.00476 20.2095 5.83733C22.2195 6.66989 23.9375 8.07979 25.1462 9.88873C26.3549 11.6977 27 13.8244 27 16C26.9967 18.9164 25.8367 21.7123 23.7745 23.7745C21.7123 25.8367 18.9164 26.9967 16 27ZM18 22C18 22.2652 17.8946 22.5196 17.7071 22.7071C17.5196 22.8946 17.2652 23 17 23C16.4696 23 15.9609 22.7893 15.5858 22.4142C15.2107 22.0391 15 21.5304 15 21V16C14.7348 16 14.4804 15.8946 14.2929 15.7071C14.1054 15.5196 14 15.2652 14 15C14 14.7348 14.1054 14.4804 14.2929 14.2929C14.4804 14.1054 14.7348 14 15 14C15.5304 14 16.0391 14.2107 16.4142 14.5858C16.7893 14.9609 17 15.4696 17 16V21C17.2652 21 17.5196 21.1054 17.7071 21.2929C17.8946 21.4804 18 21.7348 18 22ZM14 10.5C14 10.2033 14.088 9.91332 14.2528 9.66665C14.4176 9.41997 14.6519 9.22771 14.926 9.11418C15.2001 9.00065 15.5017 8.97094 15.7926 9.02882C16.0836 9.0867 16.3509 9.22956 16.5607 9.43934C16.7704 9.64912 16.9133 9.91639 16.9712 10.2074C17.0291 10.4983 16.9994 10.7999 16.8858 11.074C16.7723 11.3481 16.58 11.5824 16.3334 11.7472C16.0867 11.912 15.7967 12 15.5 12C15.1022 12 14.7206 11.842 14.4393 11.5607C14.158 11.2794 14 10.8978 14 10.5Z" fill="black"/>
-</svg>

gpt4all-chat/icons/local-docs.svg

@@ -1,3 +0,0 @@
-<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M26.7075 8.2925L21.7075 3.2925C21.6146 3.19967 21.5042 3.12605 21.3829 3.07586C21.2615 3.02568 21.1314 2.9999 21 3H11C10.4696 3 9.96086 3.21071 9.58579 3.58579C9.21071 3.96086 9 4.46957 9 5V7H7C6.46957 7 5.96086 7.21071 5.58579 7.58579C5.21071 7.96086 5 8.46957 5 9V27C5 27.5304 5.21071 28.0391 5.58579 28.4142C5.96086 28.7893 6.46957 29 7 29H21C21.5304 29 22.0391 28.7893 22.4142 28.4142C22.7893 28.0391 23 27.5304 23 27V25H25C25.5304 25 26.0391 24.7893 26.4142 24.4142C26.7893 24.0391 27 23.5304 27 23V9C27.0001 8.86864 26.9743 8.73855 26.9241 8.61715C26.8739 8.49576 26.8003 8.38544 26.7075 8.2925ZM21 27H7V9H16.5863L21 13.4137V23.98C21 23.9875 21 23.9937 21 24C21 24.0063 21 24.0125 21 24.02V27ZM25 23H23V13C23.0001 12.8686 22.9743 12.7385 22.9241 12.6172C22.8739 12.4958 22.8003 12.3854 22.7075 12.2925L17.7075 7.2925C17.6146 7.19967 17.5042 7.12605 17.3829 7.07586C17.2615 7.02568 17.1314 6.9999 17 7H11V5H20.5863L25 9.41375V23ZM18 19C18 19.2652 17.8946 19.5196 17.7071 19.7071C17.5196 19.8946 17.2652 20 17 20H11C10.7348 20 10.4804 19.8946 10.2929 19.7071C10.1054 19.5196 10 19.2652 10 19C10 18.7348 10.1054 18.4804 10.2929 18.2929C10.4804 18.1054 10.7348 18 11 18H17C17.2652 18 17.5196 18.1054 17.7071 18.2929C17.8946 18.4804 18 18.7348 18 19ZM18 23C18 23.2652 17.8946 23.5196 17.7071 23.7071C17.5196 23.8946 17.2652 24 17 24H11C10.7348 24 10.4804 23.8946 10.2929 23.7071C10.1054 23.5196 10 23.2652 10 23C10 22.7348 10.1054 22.4804 10.2929 22.2929C10.4804 22.1054 10.7348 22 11 22H17C17.2652 22 17.5196 22.1054 17.7071 22.2929C17.8946 22.4804 18 22.7348 18 23Z" fill="black"/>
-</svg>

gpt4all-chat/icons/models.svg

@@ -1,3 +0,0 @@
-<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M27.96 8.26876L16.96 2.25001C16.6661 2.08761 16.3358 2.00243 16 2.00243C15.6642 2.00243 15.3339 2.08761 15.04 2.25001L4.04 8.27126C3.72586 8.44314 3.46363 8.69621 3.28069 9.00404C3.09775 9.31187 3.00081 9.66317 3 10.0213V21.9763C3.00081 22.3343 3.09775 22.6856 3.28069 22.9935C3.46363 23.3013 3.72586 23.5544 4.04 23.7263L15.04 29.7475C15.3339 29.9099 15.6642 29.9951 16 29.9951C16.3358 29.9951 16.6661 29.9099 16.96 29.7475L27.96 23.7263C28.2741 23.5544 28.5364 23.3013 28.7193 22.9935C28.9023 22.6856 28.9992 22.3343 29 21.9763V10.0225C28.9999 9.66378 28.9032 9.3117 28.7203 9.00315C28.5373 8.6946 28.2747 8.44095 27.96 8.26876ZM16 4.00001L26.0425 9.50001L16 15L5.9575 9.50001L16 4.00001ZM5 11.25L15 16.7225V27.4463L5 21.9775V11.25ZM17 27.4463V16.7275L27 11.25V21.9725L17 27.4463Z" fill="black"/>
-</svg>

gpt4all-chat/icons/search.svg

@@ -1,6 +0,0 @@
-
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M505 442.7L405.3 343c-4.5-4.5-10.6-7-17-7H372c27.6-35.3 44-79.7 44-128C416 93.1 322.9 0 208 0S0 93.1 0 208s93.1 208 208 208c48.3 0 92.7-16.4 128-44v16.3c0 6.4 2.5 12.5 7 17l99.7 99.7c9.4 9.4 24.6 9.4 33.9 0l28.3-28.3c9.4-9.4 9.4-24.6.1-34zM208 336c-70.7 0-128-57.2-128-128 0-70.7 57.2-128 128-128 70.7 0 128 57.2 128 128 0 70.7-57.2 128-128 128z"/></svg>
-<!--
-Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
-License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
--->

gpt4all-chat/icons/trash.svg

@@ -1,3 +1,5 @@
-<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M27 6H5C4.73478 6 4.48043 6.10536 4.29289 6.29289C4.10536 6.48043 4 6.73478 4 7C4 7.26522 4.10536 7.51957 4.29289 7.70711C4.48043 7.89464 4.73478 8 5 8H6V26C6 26.5304 6.21071 27.0391 6.58579 27.4142C6.96086 27.7893 7.46957 28 8 28H24C24.5304 28 25.0391 27.7893 25.4142 27.4142C25.7893 27.0391 26 26.5304 26 26V8H27C27.2652 8 27.5196 7.89464 27.7071 7.70711C27.8946 7.51957 28 7.26522 28 7C28 6.73478 27.8946 6.48043 27.7071 6.29289C27.5196 6.10536 27.2652 6 27 6ZM24 26H8V8H24V26ZM10 3C10 2.73478 10.1054 2.48043 10.2929 2.29289C10.4804 2.10536 10.7348 2 11 2H21C21.2652 2 21.5196 2.10536 21.7071 2.29289C21.8946 2.48043 22 2.73478 22 3C22 3.26522 21.8946 3.51957 21.7071 3.70711C21.5196 3.89464 21.2652 4 21 4H11C10.7348 4 10.4804 3.89464 10.2929 3.70711C10.1054 3.51957 10 3.26522 10 3Z" fill="black"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" fill="#7d7d8e" viewBox="0 0 448 512"><path d="M0 84V56c0-13.3 10.7-24 24-24h112l9.4-18.7c4-8.2 12.3-13.3 21.4-13.3h114.3c9.1 0 17.4 5.1 21.5 13.3L312 32h112c13.3 0 24 10.7 24 24v28c0 6.6-5.4 12-12 12H12C5.4 96 0 90.6 0 84zm416 56v324c0 26.5-21.5 48-48 48H80c-26.5 0-48-21.5-48-48V140c0-6.6 5.4-12 12-12h360c6.6 0 12 5.4 12 12zm-272 68c0-8.8-7.2-16-16-16s-16 7.2-16 16v224c0 8.8 7.2 16 16 16s16-7.2 16-16V208zm96 0c0-8.8-7.2-16-16-16s-16 7.2-16 16v224c0 8.8 7.2 16 16 16s16-7.2 16-16V208zm96 0c0-8.8-7.2-16-16-16s-16 7.2-16 16v224c0 8.8 7.2 16 16 16s16-7.2 16-16V208z"/></svg>
+<!--
+Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
+License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+-->

gpt4all-chat/llm.cpp

@@ -49,7 +49,7 @@ bool LLM::checkForUpdates() const
     #pragma message "offline installer build will not check for updates!"
     return QDesktopServices::openUrl(QUrl("https://gpt4all.io/"));
     #else
-    Network::globalInstance()->sendCheckForUpdates();
+    Network::globalInstance()->trackEvent("check_for_updates");
 
 #if defined(Q_OS_LINUX)
     QString tool("maintenancetool");

gpt4all-chat/localdocs.cpp

@@ -18,6 +18,8 @@ LocalDocs::LocalDocs()
     // Create the DB with the chunk size from settings
     m_database = new Database(MySettings::globalInstance()->localDocsChunkSize());
 
+    connect(this, &LocalDocs::requestStart, m_database,
+        &Database::start, Qt::QueuedConnection);
     connect(this, &LocalDocs::requestAddFolder, m_database,
         &Database::addFolder, Qt::QueuedConnection);
     connect(this, &LocalDocs::requestRemoveFolder, m_database,
@@ -50,8 +52,6 @@ LocalDocs::LocalDocs()
         m_localDocsModel, &LocalDocsModel::addCollectionItem, Qt::QueuedConnection);
     connect(m_database, &Database::removeFolderById,
         m_localDocsModel, &LocalDocsModel::removeFolderById, Qt::QueuedConnection);
-    connect(m_database, &Database::removeCollectionItem,
-        m_localDocsModel, &LocalDocsModel::removeCollectionItem, Qt::QueuedConnection);
     connect(m_database, &Database::collectionListUpdated,
         m_localDocsModel, &LocalDocsModel::collectionListUpdated, Qt::QueuedConnection);
 
@@ -68,7 +68,7 @@ void LocalDocs::addFolder(const QString &collection, const QString &path)
 {
     const QUrl url(path);
     const QString localPath = url.isLocalFile() ? url.toLocalFile() : path;
-    emit requestAddFolder(collection, localPath);
+    emit requestAddFolder(collection, localPath, false);
 }
 
 void LocalDocs::removeFolder(const QString &collection, const QString &path)

gpt4all-chat/localdocs.h

@@ -26,7 +26,8 @@ public Q_SLOTS:
     void aboutToQuit();
 
 Q_SIGNALS:
-    void requestAddFolder(const QString &collection, const QString &path);
+    void requestStart();
+    void requestAddFolder(const QString &collection, const QString &path, bool fromDb);
     void requestRemoveFolder(const QString &collection, const QString &path);
     void requestChunkSizeChange(int chunkSize);
     void localDocsModelChanged();

gpt4all-chat/localdocsdb.h

@@ -1,105 +0,0 @@
-#ifndef LOCALDOCS_H
-#define LOCALDOCS_H
-
-#include "localdocsmodel.h"
-
-#include <QObject>
-#include <QtSql>
-#include <QQueue>
-#include <QFileInfo>
-#include <QThread>
-#include <QFileSystemWatcher>
-
-struct DocumentInfo
-{
-    int folder;
-    QFileInfo doc;
-};
-
-struct CollectionItem {
-    QString collection;
-    QString folder_path;
-    int folder_id = -1;
-};
-Q_DECLARE_METATYPE(CollectionItem)
-
-class Database : public QObject
-{
-    Q_OBJECT
-public:
-    Database();
-
-public Q_SLOTS:
-    void scanQueue();
-    void scanDocuments(int folder_id, const QString &folder_path);
-    void addFolder(const QString &collection, const QString &path);
-    void removeFolder(const QString &collection, const QString &path);
-    void retrieveFromDB(const QList<QString> &collections, const QString &text);
-    void cleanDB();
-
-Q_SIGNALS:
-    void docsToScanChanged();
-    void retrieveResult(const QList<QString> &result);
-    void collectionListUpdated(const QList<CollectionItem> &collectionList);
-
-private Q_SLOTS:
-    void start();
-    void directoryChanged(const QString &path);
-    bool addFolderToWatch(const QString &path);
-    bool removeFolderFromWatch(const QString &path);
-    void addCurrentFolders();
-    void updateCollectionList();
-
-private:
-    void removeFolderInternal(const QString &collection, int folder_id, const QString &path);
-    void chunkStream(QTextStream &stream, int document_id);
-    void handleDocumentErrorAndScheduleNext(const QString &errorMessage,
-        int document_id, const QString &document_path, const QSqlError &error);
-
-private:
-    QQueue<DocumentInfo> m_docsToScan;
-    QList<QString> m_retrieve;
-    QThread m_dbThread;
-    QFileSystemWatcher *m_watcher;
-};
-
-class LocalDocs : public QObject
-{
-    Q_OBJECT
-    Q_PROPERTY(LocalDocsModel *localDocsModel READ localDocsModel NOTIFY localDocsModelChanged)
-
-public:
-    static LocalDocs *globalInstance();
-
-    LocalDocsModel *localDocsModel() const { return m_localDocsModel; }
-
-    void addFolder(const QString &collection, const QString &path);
-    void removeFolder(const QString &collection, const QString &path);
-
-    QList<QString> result() const { return m_retrieveResult; }
-    void requestRetrieve(const QList<QString> &collections, const QString &text);
-
-Q_SIGNALS:
-    void requestAddFolder(const QString &collection, const QString &path);
-    void requestRemoveFolder(const QString &collection, const QString &path);
-    void requestRetrieveFromDB(const QList<QString> &collections, const QString &text);
-    void receivedResult();
-    void localDocsModelChanged();
-
-private Q_SLOTS:
-    void handleRetrieveResult(const QList<QString> &result);
-    void handleCollectionListUpdated(const QList<CollectionItem> &collectionList);
-
-private:
-    LocalDocsModel *m_localDocsModel;
-    Database *m_database;
-    QList<QString> m_retrieveResult;
-    QList<CollectionItem> m_collectionList;
-
-private:
-    explicit LocalDocs();
-    ~LocalDocs() {}
-    friend class MyLocalDocs;
-};
-
-#endif // LOCALDOCS_H

gpt4all-chat/localdocsmodel.cpp

@@ -1,6 +1,7 @@
 #include "localdocsmodel.h"
 
 #include "localdocs.h"
+#include "network.h"
 
 LocalDocsCollectionsModel::LocalDocsCollectionsModel(QObject *parent)
     : QSortFilterProxyModel(parent)
@@ -158,50 +159,43 @@ void LocalDocsModel::updateTotalEmbeddingsToIndex(int folder_id, size_t totalEmb
         [](CollectionItem& item, size_t val) { item.totalEmbeddingsToIndex += val; }, {TotalEmbeddingsToIndexRole});
 }
 
-void LocalDocsModel::addCollectionItem(const CollectionItem &item)
+void LocalDocsModel::addCollectionItem(const CollectionItem &item, bool fromDb)
 {
     beginInsertRows(QModelIndex(), m_collectionList.size(), m_collectionList.size());
     m_collectionList.append(item);
     endInsertRows();
+
+    if (!fromDb) {
+        Network::globalInstance()->trackEvent("doc_collection_add", {
+            {"collection_count", m_collectionList.count()},
+        });
+    }
+}
+
+void LocalDocsModel::removeCollectionIf(std::function<bool(CollectionItem)> const &predicate) {
+    for (int i = 0; i < m_collectionList.size();) {
+        if (predicate(m_collectionList.at(i))) {
+            beginRemoveRows(QModelIndex(), i, i);
+            m_collectionList.removeAt(i);
+            endRemoveRows();
+
+            Network::globalInstance()->trackEvent("doc_collection_remove", {
+                {"collection_count", m_collectionList.count()},
+            });
+        } else {
+            ++i;
+        }
+    }
 }
 
 void LocalDocsModel::removeFolderById(int folder_id)
 {
-    for (int i = 0; i < m_collectionList.size();) {
-        if (m_collectionList.at(i).folder_id == folder_id) {
-            beginRemoveRows(QModelIndex(), i, i);
-            m_collectionList.removeAt(i);
-            endRemoveRows();
-        } else {
-            ++i;
-        }
-    }
+    removeCollectionIf([folder_id](const auto &c) { return c.folder_id == folder_id; });
 }
 
 void LocalDocsModel::removeCollectionPath(const QString &name, const QString &path)
 {
-    for (int i = 0; i < m_collectionList.size();) {
-        if (m_collectionList.at(i).collection == name && m_collectionList.at(i).folder_path == path) {
-            beginRemoveRows(QModelIndex(), i, i);
-            m_collectionList.removeAt(i);
-            endRemoveRows();
-        } else {
-            ++i;
-        }
-    }
-}
-
-void LocalDocsModel::removeCollectionItem(const QString &collectionName)
-{
-    for (int i = 0; i < m_collectionList.size();) {
-        if (m_collectionList.at(i).collection == collectionName) {
-            beginRemoveRows(QModelIndex(), i, i);
-            m_collectionList.removeAt(i);
-            endRemoveRows();
-        } else {
-            ++i;
-        }
-    }
+    removeCollectionIf([&name, &path](const auto &c) { return c.collection == name && c.folder_path == path; });
 }
 
 void LocalDocsModel::collectionListUpdated(const QList<CollectionItem> &collectionList)

gpt4all-chat/localdocsmodel.h

@@ -55,10 +55,9 @@ public Q_SLOTS:
     void updateTotalBytesToIndex(int folder_id, size_t totalBytesToIndex);
     void updateCurrentEmbeddingsToIndex(int folder_id, size_t currentBytesToIndex);
     void updateTotalEmbeddingsToIndex(int folder_id, size_t totalBytesToIndex);
-    void addCollectionItem(const CollectionItem &item);
+    void addCollectionItem(const CollectionItem &item, bool fromDb);
     void removeFolderById(int folder_id);
     void removeCollectionPath(const QString &name, const QString &path);
-    void removeCollectionItem(const QString &collectionName);
     void collectionListUpdated(const QList<CollectionItem> &collectionList);
 
 private:
@@ -66,6 +65,7 @@ private:
     void updateField(int folder_id, T value,
         const std::function<void(CollectionItem&, T)>& updater,
         const QVector<int>& roles);
+    void removeCollectionIf(std::function<bool(CollectionItem)> const &predicate);
 
 private:
     QList<CollectionItem> m_collectionList;

gpt4all-chat/main.qml

@@ -43,25 +43,6 @@ Window {
         font.pixelSize: theme.fontSizeLarge
     }
 
-    NetworkDialog {
-        id: networkDialog
-        anchors.centerIn: parent
-        width: Math.min(1024, window.width - (window.width * .2))
-        height: Math.min(600, window.height - (window.height * .2))
-        Item {
-            Accessible.role: Accessible.Dialog
-            Accessible.name: qsTr("Network dialog")
-            Accessible.description: qsTr("opt-in to share feedback/conversations")
-        }
-    }
-
-    AboutDialog {
-        id: aboutDialog
-        anchors.centerIn: parent
-        width: Math.min(1024, window.width - (window.width * .2))
-        height: Math.min(600, window.height - (window.height * .2))
-    }
-
     onClosing: function(close) {
         if (window.hasSaved)
             return;
@@ -82,173 +63,7 @@ Window {
 
     color: theme.black
 
-    Rectangle {
-        id: viewBar
-        anchors.top: parent.top
-        anchors.bottom: parent.bottom
-        anchors.left: parent.left
-        width: 80
-        color: theme.viewBarBackground
-
-        ColumnLayout {
-            id: viewsLayout
-            anchors.top: parent.top
-            anchors.topMargin: 30
-            anchors.horizontalCenter: parent.horizontalCenter
-            Layout.margins: 0
-            spacing: 25
-
-            MyToolButton {
-                id: chatButton
-                backgroundColor: toggled ? theme.iconBackgroundViewBarToggled : theme.iconBackgroundViewBar
-                backgroundColorHovered: toggled ? backgroundColor : theme.iconBackgroundViewBarHovered
-                Layout.preferredWidth: 40
-                Layout.preferredHeight: 40
-                Layout.alignment: Qt.AlignCenter
-                toggledWidth: 0
-                toggled: stackLayout.currentIndex === 0
-                toggledColor: theme.iconBackgroundViewBarToggled
-                scale: 1.5
-                source: "qrc:/gpt4all/icons/chat.svg"
-                Accessible.name: qsTr("Chat view")
-                Accessible.description: qsTr("Chat view to interact with models")
-                onClicked: {
-                    stackLayout.currentIndex = 0
-                }
-            }
-
-            MyToolButton {
-                id: searchButton
-                backgroundColor: toggled ? theme.iconBackgroundViewBarToggled : theme.iconBackgroundViewBar
-                backgroundColorHovered: toggled ? backgroundColor : theme.iconBackgroundViewBarHovered
-                Layout.preferredWidth: 40
-                Layout.preferredHeight: 40
-                toggledWidth: 0
-                toggled: stackLayout.currentIndex === 1
-                toggledColor: theme.iconBackgroundViewBarToggled
-                scale: 1.5
-                source: "qrc:/gpt4all/icons/models.svg"
-                Accessible.name: qsTr("Search")
-                Accessible.description: qsTr("Launch a dialog to download new models")
-                onClicked: {
-                    stackLayout.currentIndex = 1
-                }
-            }
-
-            MyToolButton {
-                id: settingsButton
-                backgroundColor: toggled ? theme.iconBackgroundViewBarToggled : theme.iconBackgroundViewBar
-                backgroundColorHovered: toggled ? backgroundColor : theme.iconBackgroundViewBarHovered
-                Layout.preferredWidth: 40
-                Layout.preferredHeight: 40
-                toggledWidth: 0
-                toggledColor: theme.iconBackgroundViewBarToggled
-                toggled: stackLayout.currentIndex === 2
-                scale: 1.5
-                source: "qrc:/gpt4all/icons/settings.svg"
-                Accessible.name: qsTr("Settings")
-                Accessible.description: qsTr("Reveals a dialogue with settings")
-
-                onClicked: {
-                    stackLayout.currentIndex = 2
-                }
-            }
-        }
-
-        ColumnLayout {
-            id: buttonsLayout
-            anchors.bottom: parent.bottom
-            anchors.margins: 0
-            anchors.bottomMargin: 25
-            anchors.horizontalCenter: parent.horizontalCenter
-            Layout.margins: 0
-            spacing: 25
-
-            MyToolButton {
-                id: networkButton
-                backgroundColor: toggled ? theme.iconBackgroundViewBarToggled : theme.iconBackgroundViewBar
-                backgroundColorHovered: toggled ? backgroundColor : theme.iconBackgroundViewBarHovered
-                toggledColor: theme.iconBackgroundViewBar
-                Layout.preferredWidth: 40
-                Layout.preferredHeight: 40
-                scale: 1.2
-                toggled: MySettings.networkIsActive
-                source: "qrc:/gpt4all/icons/network.svg"
-                Accessible.name: qsTr("Network")
-                Accessible.description: qsTr("Reveals a dialogue where you can opt-in for sharing data over network")
-
-                onClicked: {
-                    if (MySettings.networkIsActive) {
-                        MySettings.networkIsActive = false
-                        Network.sendNetworkToggled(false);
-                    } else
-                        networkDialog.open()
-                }
-            }
-
-            MyToolButton {
-                id: infoButton
-                backgroundColor: theme.iconBackgroundViewBar
-                backgroundColorHovered: theme.iconBackgroundViewBarHovered
-                Layout.preferredWidth: 40
-                Layout.preferredHeight: 40
-                scale: 1.2
-                source: "qrc:/gpt4all/icons/info.svg"
-                Accessible.name: qsTr("About")
-                Accessible.description: qsTr("Reveals an about dialog")
-                onClicked: {
-                    aboutDialog.open()
-                }
-            }
-        }
-    }
-
-    StackLayout {
-        id: stackLayout
-        anchors.top: parent.top
-        anchors.bottom: parent.bottom
-        anchors.left: viewBar.right
-        anchors.right: parent.right
-
-        ChatView {
-            id: chatView
-            Layout.fillWidth: true
-            Layout.fillHeight: true
-
-            Connections {
-                target: chatView
-                function onDownloadViewRequested(showEmbeddingModels) {
-                    console.log("onDownloadViewRequested")
-                    stackLayout.currentIndex = 1;
-                    if (showEmbeddingModels)
-                        downloadView.showEmbeddingModels();
-                }
-                function onSettingsViewRequested(page) {
-                    settingsDialog.pageToDisplay = page;
-                    stackLayout.currentIndex = 2;
-                }
-            }
-        }
-
-        ModelDownloaderView {
-            id: downloadView
-            Layout.fillWidth: true
-            Layout.fillHeight: true
-            Item {
-                Accessible.role: Accessible.Dialog
-                Accessible.name: qsTr("Download new models")
-                Accessible.description: qsTr("View for downloading new models")
-            }
-        }
-
-        SettingsView {
-            id: settingsDialog
-            Layout.fillWidth: true
-            Layout.fillHeight: true
-            onDownloadClicked: {
-                stackLayout.currentIndex = 1
-                downloadView.showEmbeddingModels()
-            }
-        }
+    ChatView {
+        anchors.fill: parent
     }
 }

gpt4all-chat/metadata/models2.json

@@ -13,7 +13,7 @@
     "description": "<strong>Best overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
     "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.gguf2.Q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n",
-    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI. For multi-step problems, write out your reasoning for each step.\n<|im_end|>"
+    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI.\n<|im_end|>"
   },
   {
     "order": "b",

gpt4all-chat/metadata/models3.json

@@ -1,6 +1,22 @@
 [
   {
     "order": "a",
+    "md5sum": "c87ad09e1e4c8f9c35a5fcef52b6f1c9",
+    "name": "Llama 3 Instruct",
+    "filename": "Meta-Llama-3-8B-Instruct.Q4_0.gguf",
+    "filesize": "4661724384",
+    "requires": "2.7.1",
+    "ramrequired": "8",
+    "parameters": "8 billion",
+    "quant": "q4_0",
+    "type": "LLaMA3",
+    "description": "<ul><li>Fast responses</li><li>Chat based model</li><li>Accepts system prompts in Llama 3 format</li><li>Trained by Meta</li><li>License: <a href=\"https://llama.meta.com/llama3/license/\">Meta Llama 3 Community License</a></li></ul>",
+    "url": "https://gpt4all.io/models/gguf/Meta-Llama-3-8B-Instruct.Q4_0.gguf",
+    "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2<|eot_id|>",
+    "systemPrompt": ""
+  },
+  {
+    "order": "b",
     "md5sum": "a5f6b4eabd3992da4d7fb7f020f921eb",
     "name": "Nous Hermes 2 Mistral DPO",
     "filename": "Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf",
@@ -15,22 +31,6 @@
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
     "systemPrompt": ""
   },
-  {
-    "order": "b",
-    "md5sum": "f692417a22405d80573ac10cb0cd6c6a",
-    "name": "Mistral OpenOrca",
-    "filename": "mistral-7b-openorca.gguf2.Q4_0.gguf",
-    "filesize": "4108928128",
-    "requires": "2.5.0",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "Mistral",
-    "description": "<strong>Strong overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
-    "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.gguf2.Q4_0.gguf",
-    "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
-    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI. For multi-step problems, write out your reasoning for each step.\n<|im_end|>\n"
-  },
   {
     "order": "c",
     "md5sum": "97463be739b50525df56d33b26b00852",
@@ -49,6 +49,22 @@
   },
   {
     "order": "d",
+    "md5sum": "f692417a22405d80573ac10cb0cd6c6a",
+    "name": "Mistral OpenOrca",
+    "filename": "mistral-7b-openorca.gguf2.Q4_0.gguf",
+    "filesize": "4108928128",
+    "requires": "2.7.1",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "description": "<strong>Strong overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
+    "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.gguf2.Q4_0.gguf",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
+    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI.\n<|im_end|>\n"
+  },
+  {
+    "order": "e",
     "md5sum": "c4c78adf744d6a20f05c8751e3961b84",
     "name": "GPT4All Falcon",
     "filename": "gpt4all-falcon-newbpe-q4_0.gguf",
@@ -64,7 +80,7 @@
     "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
   },
   {
-    "order": "e",
+    "order": "f",
     "md5sum": "00c8593ba57f5240f59662367b3ed4a5",
     "name": "Orca 2 (Medium)",
     "filename": "orca-2-7b.Q4_0.gguf",
@@ -79,7 +95,7 @@
     "url": "https://gpt4all.io/models/gguf/orca-2-7b.Q4_0.gguf"
   },
   {
-    "order": "f",
+    "order": "g",
     "md5sum": "3c0d63c4689b9af7baa82469a6f51a19",
     "name": "Orca 2 (Full)",
     "filename": "orca-2-13b.Q4_0.gguf",
@@ -94,7 +110,7 @@
     "url": "https://gpt4all.io/models/gguf/orca-2-13b.Q4_0.gguf"
   },
   {
-    "order": "g",
+    "order": "h",
     "md5sum": "5aff90007499bce5c64b1c0760c0b186",
     "name": "Wizard v1.2",
     "filename": "wizardlm-13b-v1.2.Q4_0.gguf",
@@ -109,7 +125,23 @@
     "url": "https://gpt4all.io/models/gguf/wizardlm-13b-v1.2.Q4_0.gguf"
   },
   {
-    "order": "h",
+    "order": "i",
+    "md5sum": "31b47b4e8c1816b62684ac3ca373f9e1",
+    "name": "Ghost 7B v0.9.1",
+    "filename": "ghost-7b-v0.9.1-Q4_0.gguf",
+    "filesize": "4108916960",
+    "requires": "2.7.1",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "description": "<strong>Ghost 7B v0.9.1</strong> fast, powerful and smooth for Vietnamese and English languages.",
+    "url": "https://huggingface.co/lamhieu/ghost-7b-v0.9.1-gguf/resolve/main/ghost-7b-v0.9.1-Q4_0.gguf",
+    "promptTemplate": "<|user|>\n%1</s>\n<|assistant|>\n%2</s>\n",
+    "systemPrompt": "<|system|>\nYou are Ghost created by Lam Hieu. You are a helpful and knowledgeable assistant. You like to help and always give honest information, in its original language. In communication, you are always respectful, equal and promote positive behavior.\n</s>"
+  },
+  {
+    "order": "j",
     "md5sum": "3d12810391d04d1153b692626c0c6e16",
     "name": "Hermes",
     "filename": "nous-hermes-llama2-13b.Q4_0.gguf",
@@ -125,7 +157,7 @@
     "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
   },
   {
-    "order": "i",
+    "order": "k",
     "md5sum": "40388eb2f8d16bb5d08c96fdfaac6b2c",
     "name": "Snoozy",
     "filename": "gpt4all-13b-snoozy-q4_0.gguf",
@@ -140,12 +172,12 @@
     "url": "https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf"
   },
   {
-    "order": "j",
+    "order": "l",
     "md5sum": "15dcb4d7ea6de322756449c11a0b7545",
     "name": "MPT Chat",
     "filename": "mpt-7b-chat-newbpe-q4_0.gguf",
     "filesize": "3912373472",
-    "requires": "2.6.0",
+    "requires": "2.7.1",
     "removedIn": "2.7.3",
     "ramrequired": "8",
     "parameters": "7 billion",
@@ -157,7 +189,7 @@
     "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
   },
   {
-    "order": "j",
+    "order": "l",
     "md5sum": "ab5d8e8a2f79365ea803c1f1d0aa749d",
     "name": "MPT Chat",
     "filename": "mpt-7b-chat.gguf4.Q4_0.gguf",
@@ -173,7 +205,23 @@
     "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
   },
   {
-    "order": "k",
+    "order": "m",
+    "md5sum": "f8347badde9bfc2efbe89124d78ddaf5",
+    "name": "Phi-3 Mini Instruct",
+    "filename": "Phi-3-mini-4k-instruct.Q4_0.gguf",
+    "filesize": "2176181568",
+    "requires": "2.7.1",
+    "ramrequired": "4",
+    "parameters": "4 billion",
+    "quant": "q4_0",
+    "type": "Phi-3",
+    "description": "<ul><li>Very fast responses</li><li>Chat based model</li><li>Accepts system prompts in Phi-3 format</li><li>Trained by Microsoft</li><li>License: <a href=\"https://opensource.org/license/mit\">MIT</a></li><li>No restrictions on commercial use</li></ul>",
+    "url": "https://gpt4all.io/models/gguf/Phi-3-mini-4k-instruct.Q4_0.gguf",
+    "promptTemplate": "<|user|>\n%1<|end|>\n<|assistant|>\n%2<|end|>\n",
+    "systemPrompt": ""
+  },
+  {
+    "order": "n",
     "md5sum": "0e769317b90ac30d6e09486d61fefa26",
     "name": "Mini Orca (Small)",
     "filename": "orca-mini-3b-gguf2-q4_0.gguf",
@@ -183,13 +231,13 @@
     "parameters": "3 billion",
     "quant": "q4_0",
     "type": "OpenLLaMa",
-    "description": "<strong>Small version of new model with novel dataset</strong><br><ul><li>Instruction based<li>Explain tuned datasets<li>Orca Research Paper dataset construction approaches<li>Cannot be used commercially</ul>",
+    "description": "<strong>Small version of new model with novel dataset</strong><br><ul><li>Very fast responses</li><li>Instruction based</li><li>Explain tuned datasets</li><li>Orca Research Paper dataset construction approaches</li><li>Cannot be used commercially</li></ul>",
     "url": "https://gpt4all.io/models/gguf/orca-mini-3b-gguf2-q4_0.gguf",
     "promptTemplate": "### User:\n%1\n\n### Response:\n",
     "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
   },
   {
-    "order": "l",
+    "order": "o",
     "md5sum": "c232f17e09bca4b7ee0b5b1f4107c01e",
     "disableGUI": "true",
     "name": "Replit",
@@ -206,7 +254,7 @@
     "url": "https://gpt4all.io/models/gguf/replit-code-v1_5-3b-newbpe-q4_0.gguf"
   },
   {
-    "order": "m",
+    "order": "p",
     "md5sum": "70841751ccd95526d3dcfa829e11cd4c",
     "disableGUI": "true",
     "name": "Starcoder",
@@ -223,7 +271,7 @@
     "url": "https://gpt4all.io/models/gguf/starcoder-newbpe-q4_0.gguf"
   },
   {
-    "order": "n",
+    "order": "q",
     "md5sum": "e973dd26f0ffa6e46783feaea8f08c83",
     "disableGUI": "true",
     "name": "Rift coder",
@@ -240,7 +288,7 @@
     "url": "https://gpt4all.io/models/gguf/rift-coder-v0-7b-q4_0.gguf"
   },
   {
-    "order": "o",
+    "order": "r",
     "md5sum": "e479e6f38b59afc51a470d1953a6bfc7",
     "disableGUI": "true",
     "name": "SBert",
@@ -258,7 +306,7 @@
     "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf"
   },
   {
-    "order": "o",
+    "order": "r",
     "md5sum": "dd90e2cb7f8e9316ac3796cece9883b5",
     "name": "SBert",
     "filename": "all-MiniLM-L6-v2.gguf2.f16.gguf",
@@ -273,7 +321,7 @@
     "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2.gguf2.f16.gguf"
   },
   {
-    "order": "p",
+    "order": "s",
     "md5sum": "919de4dd6f25351bcb0223790db1932d",
     "name": "EM German Mistral",
     "filename": "em_german_mistral_v01.Q4_0.gguf",
@@ -289,7 +337,7 @@
     "systemPrompt": "Du bist ein hilfreicher Assistent. "
   },
   {
-    "order": "q",
+    "order": "t",
     "md5sum": "60ea031126f82db8ddbbfecc668315d2",
     "disableGUI": "true",
     "name": "Nomic Embed Text v1",
@@ -306,7 +354,7 @@
     "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.f16.gguf"
   },
   {
-    "order": "r",
+    "order": "u",
     "md5sum": "a5401e7f7e46ed9fcaed5b60a281d547",
     "disableGUI": "true",
     "name": "Nomic Embed Text v1.5",
@@ -321,21 +369,5 @@
     "systemPrompt": "",
     "description": "nomic-embed-text-v1.5",
     "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.5.f16.gguf"
-  },
-  {
-    "order": "g",
-    "md5sum": "31b47b4e8c1816b62684ac3ca373f9e1",
-    "name": "Ghost 7B v0.9.1",
-    "filename": "ghost-7b-v0.9.1-Q4_0.gguf",
-    "filesize": "4108916960",
-    "requires": "2.5.0",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "Mistral",
-    "description": "<strong>Ghost 7B v0.9.1</strong> fast, powerful and smooth for Vietnamese and English languages.",
-    "url": "https://huggingface.co/lamhieu/ghost-7b-v0.9.1-gguf/resolve/main/ghost-7b-v0.9.1-Q4_0.gguf",
-    "promptTemplate": "<|user|>\n%1</s>\n<|assistant|>\n%2</s>\n",
-    "systemPrompt": "<|system|>\nYou are Ghost created by Lam Hieu. You are a helpful and knowledgeable assistant. You like to help and always give honest information, in its original language. In communication, you are always respectful, equal and promote positive behavior.\n</s>"
   }
 ]

gpt4all-chat/metadata/release.json

@@ -752,6 +752,64 @@
 * Jared Van Bortel (Nomic AI)
 * Adam Treat (Nomic AI)
 * Community (beta testers, bug reporters, bindings authors)
+"
+  },
+  {
+    "version": "2.7.4",
+    "notes":
+"
+<b>&mdash; What's New &mdash;</b>
+* Add a right-click menu to the chat (by @kryotek777 in PR #2108)
+* Change the left sidebar to stay open (PR #2117)
+* Limit the width of text in the chat (PR #2118)
+* Move to llama.cpp's SBert implementation (PR #2086)
+* Support models provided by the Mistral AI API (by @Olyxz16 in PR #2053)
+* Models List: Add Ghost 7B v0.9.1 (by @lh0x00 in PR #2127)
+* Add Documentation and FAQ links to the New Chat page (by @3Simplex in PR #2183)
+* Models List: Simplify Mistral OpenOrca system prompt (PR #2220)
+* Models List: Add Llama 3 Instruct (PR #2242)
+* Models List: Add Phi-3 Mini Instruct (PR #2252)
+* Improve accuracy of anonymous usage statistics (PR #2238)
+
+<b>&mdash; Fixes &mdash;</b>
+* Detect unsupported CPUs correctly on Windows (PR #2141)
+* Fix the colors used by the server chat (PR #2150)
+* Fix startup issues when encountering non-Latin characters in paths (PR #2162)
+* Fix issues causing LocalDocs context links to not work sometimes (PR #2218)
+* Fix incorrect display of certain code block syntax in the chat (PR #2232)
+* Fix an issue causing unnecessary indexing of document collections on startup (PR #2236)
+",
+    "contributors":
+"
+* Jared Van Bortel (Nomic AI)
+* Adam Treat (Nomic AI)
+* Lam Hieu (`@lh0x00`)
+* 3Simplex (`@3Simplex`)
+* Kryotek (`@kryotek777`)
+* Olyxz16 (`@Olyxz16`)
+* Robin Verduijn (`@robinverduijn`)
+* Tim453 (`@Tim453`)
+* Xu Zhen (`@xuzhen`)
+* Community (beta testers, bug reporters, bindings authors)
+"
+  },
+  {
+    "version": "2.7.5",
+    "notes":
+"
+<b>&mdash; What's New &mdash;</b>
+* Improve accuracy of anonymous usage statistics (PR #2297, PR #2299)
+
+<b>&mdash; Fixes &mdash;</b>
+* Fix some issues with anonymous usage statistics (PR #2270, PR #2296)
+* Default to GPU with most VRAM on Windows and Linux, not least (PR #2297)
+* Fix initial failure to generate embeddings with Nomic Embed (PR #2284)
+",
+    "contributors":
+"
+* Jared Van Bortel (Nomic AI)
+* Adam Treat (Nomic AI)
+* Community (beta testers, bug reporters, bindings authors)
 "
   }
 ]

gpt4all-chat/modellist.cpp

@@ -12,7 +12,7 @@
 
 const char * const KNOWN_EMBEDDING_MODELS[] {
     "all-MiniLM-L6-v2.gguf2.f16.gguf",
-    "nomic-embed-text-v1.txt",
+    "gpt4all-nomic-embed-text-v1.rmodel",
 };
 
 QString ModelInfo::id() const
@@ -1178,7 +1178,7 @@ void ModelList::updateModelsFromDirectory()
             it.next();
             if (!it.fileInfo().isDir()) {
                 QString filename = it.fileName();
-                if (filename.startsWith("chatgpt-") && filename.endsWith(".txt")) {
+                if (filename.endsWith(".txt") && (filename.startsWith("chatgpt-") || filename.startsWith("nomic-"))) {
                     QString apikey;
                     QString modelname(filename);
                     modelname.chop(4); // strip ".txt" extension
@@ -1648,7 +1648,7 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             "<li>You can apply for an API key <a href=\"https://atlas.nomic.ai/\">with Nomic Atlas.</a></li>");
         const QString modelName = "Nomic Embed";
         const QString id = modelName;
-        const QString modelFilename = "nomic-embed-text-v1.txt"; // FIXME: This should be made to use '.rmodel' as well
+        const QString modelFilename = "gpt4all-nomic-embed-text-v1.rmodel";
         if (contains(modelFilename))
             changeId(modelFilename, id);
         if (!contains(id))

gpt4all-chat/mysettings.cpp

@@ -65,10 +65,14 @@ MySettings::MySettings()
 {
     QSettings::setDefaultFormat(QSettings::IniFormat);
 
-    std::vector<LLModel::GPUDevice> devices = LLModel::Implementation::availableGPUDevices();
     QVector<QString> deviceList{ "Auto" };
+#if defined(Q_OS_MAC) && defined(__aarch64__)
+    deviceList << "Metal";
+#else
+    std::vector<LLModel::GPUDevice> devices = LLModel::Implementation::availableGPUDevices();
     for (LLModel::GPUDevice &d : devices)
-        deviceList << QString::fromStdString(d.name);
+        deviceList << QString::fromStdString(d.selectionName());
+#endif
     deviceList << "CPU";
     setDeviceList(deviceList);
 }
@@ -786,7 +790,23 @@ QString MySettings::device() const
 {
     QSettings setting;
     setting.sync();
-    return setting.value("device", default_device).toString();
+    auto value = setting.value("device");
+    if (!value.isValid())
+        return default_device;
+
+    auto device = value.toString();
+    if (!device.isEmpty()) {
+        auto deviceStr = device.toStdString();
+        auto newNameStr = LLModel::GPUDevice::updateSelectionName(deviceStr);
+        if (newNameStr != deviceStr) {
+            auto newName = QString::fromStdString(newNameStr);
+            qWarning() << "updating device name:" << device << "->" << newName;
+            device = newName;
+            setting.setValue("device", device);
+            setting.sync();
+        }
+    }
+    return device;
 }
 
 void MySettings::setDevice(const QString &u)
@@ -910,15 +930,23 @@ bool MySettings::networkIsActive() const
     return setting.value("network/isActive", default_networkIsActive).toBool();
 }
 
+bool MySettings::isNetworkIsActiveSet() const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value("network/isActive").isValid();
+}
+
 void MySettings::setNetworkIsActive(bool b)
 {
-    if (networkIsActive() == b)
-        return;
-
     QSettings setting;
-    setting.setValue("network/isActive", b);
     setting.sync();
-    emit networkIsActiveChanged();
+    auto cur = setting.value("network/isActive");
+    if (!cur.isValid() || cur.toBool() != b) {
+        setting.setValue("network/isActive", b);
+        setting.sync();
+        emit networkIsActiveChanged();
+    }
 }
 
 bool MySettings::networkUsageStatsActive() const
@@ -928,13 +956,21 @@ bool MySettings::networkUsageStatsActive() const
     return setting.value("network/usageStatsActive", default_networkUsageStatsActive).toBool();
 }
 
+bool MySettings::isNetworkUsageStatsActiveSet() const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value("network/usageStatsActive").isValid();
+}
+
 void MySettings::setNetworkUsageStatsActive(bool b)
 {
-    if (networkUsageStatsActive() == b)
-        return;
-
     QSettings setting;
-    setting.setValue("network/usageStatsActive", b);
     setting.sync();
-    emit networkUsageStatsActiveChanged();
+    auto cur = setting.value("network/usageStatsActive");
+    if (!cur.isValid() || cur.toBool() != b) {
+        setting.setValue("network/usageStatsActive", b);
+        setting.sync();
+        emit networkUsageStatsActiveChanged();
+    }
 }

gpt4all-chat/mysettings.h

@@ -129,8 +129,10 @@ public:
     QString networkAttribution() const;
     void setNetworkAttribution(const QString &a);
     bool networkIsActive() const;
+    Q_INVOKABLE bool isNetworkIsActiveSet() const;
     void setNetworkIsActive(bool b);
     bool networkUsageStatsActive() const;
+    Q_INVOKABLE bool isNetworkUsageStatsActiveSet() const;
     void setNetworkUsageStatsActive(bool b);
     int networkPort() const;
     void setNetworkPort(int c);

gpt4all-chat/network.cpp

@@ -1,8 +1,13 @@
 #include "network.h"
-#include "llm.h"
+
 #include "chatlistmodel.h"
+#include "download.h"
+#include "llm.h"
+#include "localdocs.h"
 #include "mysettings.h"
 
+#include <cmath>
+
 #include <QCoreApplication>
 #include <QGuiApplication>
 #include <QUuid>
@@ -14,16 +19,55 @@
 
 //#define DEBUG
 
+static const char MIXPANEL_TOKEN[] = "ce362e568ddaee16ed243eaffb5860a2";
+
 #if defined(Q_OS_MAC)
+
 #include <sys/sysctl.h>
-std::string getCPUModel() {
+static QString getCPUModel() {
     char buffer[256];
     size_t bufferlen = sizeof(buffer);
     sysctlbyname("machdep.cpu.brand_string", &buffer, &bufferlen, NULL, 0);
-    return std::string(buffer);
+    return buffer;
 }
+
+#elif defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)
+
+#ifndef _MSC_VER
+static void get_cpuid(int level, int *regs) {
+    asm volatile("cpuid" : "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), "=d" (regs[3]) : "0" (level) : "memory");
+}
+#else
+#define get_cpuid(level, regs) __cpuid(regs, level)
 #endif
 
+static QString getCPUModel() {
+    int regs[12];
+
+    // EAX=800000000h: Get Highest Extended Function Implemented
+    get_cpuid(0x80000000, regs);
+    if (regs[0] < 0x80000004)
+        return "(unknown)";
+
+    // EAX=800000002h-800000004h: Processor Brand String
+    get_cpuid(0x80000002, regs);
+    get_cpuid(0x80000003, regs + 4);
+    get_cpuid(0x80000004, regs + 8);
+
+    char str[sizeof(regs) + 1];
+    memcpy(str, regs, sizeof(regs));
+    str[sizeof(regs)] = 0;
+
+    return QString(str).trimmed();
+}
+
+#else
+
+static QString getCPUModel() { return "(non-x86)"; }
+
+#endif
+
+
 class MyNetwork: public Network { };
 Q_GLOBAL_STATIC(MyNetwork, networkInstance)
 Network *Network::globalInstance()
@@ -33,40 +77,43 @@ Network *Network::globalInstance()
 
 Network::Network()
     : QObject{nullptr}
-    , m_shouldSendStartup(false)
 {
     QSettings settings;
     settings.sync();
     m_uniqueId = settings.value("uniqueId", generateUniqueId()).toString();
     settings.setValue("uniqueId", m_uniqueId);
     settings.sync();
-    connect(MySettings::globalInstance(), &MySettings::networkIsActiveChanged, this, &Network::handleIsActiveChanged);
-    connect(MySettings::globalInstance(), &MySettings::networkUsageStatsActiveChanged, this, &Network::handleUsageStatsActiveChanged);
-    if (MySettings::globalInstance()->networkIsActive())
+    m_sessionId = generateUniqueId();
+
+    // allow sendMixpanel to be called from any thread
+    connect(this, &Network::requestMixpanel, this, &Network::sendMixpanel, Qt::QueuedConnection);
+
+    const auto *mySettings = MySettings::globalInstance();
+    connect(mySettings, &MySettings::networkIsActiveChanged, this, &Network::handleIsActiveChanged);
+    connect(mySettings, &MySettings::networkUsageStatsActiveChanged, this, &Network::handleUsageStatsActiveChanged);
+
+    m_hasSentOptIn  = !Download::globalInstance()->isFirstStart() &&  mySettings->networkUsageStatsActive();
+    m_hasSentOptOut = !Download::globalInstance()->isFirstStart() && !mySettings->networkUsageStatsActive();
+
+    if (mySettings->networkIsActive())
         sendHealth();
-    if (MySettings::globalInstance()->networkUsageStatsActive())
-        sendIpify();
     connect(&m_networkManager, &QNetworkAccessManager::sslErrors, this,
         &Network::handleSslErrors);
 }
 
+// NOTE: this won't be useful until we make it possible to change this via the settings page
+void Network::handleUsageStatsActiveChanged()
+{
+    if (!MySettings::globalInstance()->networkUsageStatsActive())
+        m_sendUsageStats = false;
+}
+
 void Network::handleIsActiveChanged()
 {
     if (MySettings::globalInstance()->networkUsageStatsActive())
         sendHealth();
 }
 
-void Network::handleUsageStatsActiveChanged()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        sendOptOut();
-    else {
-        // model might be loaded already when user opt-in for first time
-        sendStartup();
-        sendIpify();
-    }
-}
-
 QString Network::generateUniqueId() const
 {
     return QUuid::createUuid().toString(QUuid::WithoutBraces);
@@ -167,8 +214,8 @@ void Network::handleSslErrors(QNetworkReply *reply, const QList<QSslError> &erro
 void Network::sendOptOut()
 {
     QJsonObject properties;
-    properties.insert("token", "ce362e568ddaee16ed243eaffb5860a2");
-    properties.insert("time", QDateTime::currentSecsSinceEpoch());
+    properties.insert("token", MIXPANEL_TOKEN);
+    properties.insert("time", QDateTime::currentMSecsSinceEpoch());
     properties.insert("distinct_id", m_uniqueId);
     properties.insert("$insert_id", generateUniqueId());
 
@@ -181,7 +228,7 @@ void Network::sendOptOut()
 
     QJsonDocument doc;
     doc.setArray(array);
-    sendMixpanel(doc.toJson(QJsonDocument::Compact), true /*isOptOut*/);
+    emit requestMixpanel(doc.toJson(QJsonDocument::Compact));
 
 #if defined(DEBUG)
     printf("%s %s\n", qPrintable("opt_out"), qPrintable(doc.toJson(QJsonDocument::Indented)));
@@ -189,215 +236,77 @@ void Network::sendOptOut()
 #endif
 }
 
-void Network::sendModelLoaded()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("model_load");
-}
-
-void Network::sendResetContext(int conversationLength)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-
-    KeyValue kv;
-    kv.key = QString("length");
-    kv.value = QJsonValue(conversationLength);
-    sendMixpanelEvent("reset_context", QVector<KeyValue>{kv});
-}
-
 void Network::sendStartup()
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
+    const auto *mySettings = MySettings::globalInstance();
+    Q_ASSERT(mySettings->isNetworkUsageStatsActiveSet());
+    if (!mySettings->networkUsageStatsActive()) {
+        // send a single opt-out per session after the user has made their selections,
+        // unless this is a normal start (same version) and the user was already opted out
+        if (!m_hasSentOptOut) {
+            sendOptOut();
+            m_hasSentOptOut = true;
+        }
         return;
-    m_shouldSendStartup = true;
-    if (m_ipify.isEmpty())
-        return; // when it completes it will send
-    sendMixpanelEvent("startup");
+    }
+
+    // only chance to enable usage stats is at the start of a new session
+    m_sendUsageStats = true;
+
+    const auto *display = QGuiApplication::primaryScreen();
+    trackEvent("startup", {
+        {"$screen_dpi", std::round(display->physicalDotsPerInch())},
+        {"display", QString("%1x%2").arg(display->size().width()).arg(display->size().height())},
+        {"ram", LLM::globalInstance()->systemTotalRAMInGB()},
+        {"cpu", getCPUModel()},
+        {"cpu_supports_avx2", LLModel::Implementation::cpuSupportsAVX2()},
+        {"datalake_active", mySettings->networkIsActive()},
+    });
+    sendIpify();
+
+    // mirror opt-out logic so the ratio can be used to infer totals
+    if (!m_hasSentOptIn) {
+        trackEvent("opt_in");
+        m_hasSentOptIn = true;
+    }
 }
 
-void Network::sendCheckForUpdates()
+void Network::trackChatEvent(const QString &ev, QVariantMap props)
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("check_for_updates");
+    const auto &curChat = ChatListModel::globalInstance()->currentChat();
+    if (!props.contains("model"))
+        props.insert("model", curChat->modelInfo().filename());
+    props.insert("actualDevice", curChat->device());
+    props.insert("doc_collections_enabled", curChat->collectionList().count());
+    props.insert("doc_collections_total", LocalDocs::globalInstance()->localDocsModel()->rowCount());
+    props.insert("datalake_active", MySettings::globalInstance()->networkIsActive());
+    props.insert("using_server", ChatListModel::globalInstance()->currentChat()->isServer());
+    trackEvent(ev, props);
 }
 
-void Network::sendModelDownloaderDialog()
+void Network::trackEvent(const QString &ev, const QVariantMap &props)
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("download_dialog");
-}
-
-void Network::sendInstallModel(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("install_model", QVector<KeyValue>{kv});
-}
-
-void Network::sendRemoveModel(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("remove_model", QVector<KeyValue>{kv});
-}
-
-void Network::sendDownloadStarted(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("download_started", QVector<KeyValue>{kv});
-}
-
-void Network::sendDownloadCanceled(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("download_canceled", QVector<KeyValue>{kv});
-}
-
-void Network::sendDownloadError(const QString &model, int code, const QString &errorString)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    KeyValue kvCode;
-    kvCode.key = QString("code");
-    kvCode.value = QJsonValue(code);
-    KeyValue kvError;
-    kvError.key = QString("error");
-    kvError.value = QJsonValue(errorString);
-    sendMixpanelEvent("download_error", QVector<KeyValue>{kv, kvCode, kvError});
-}
-
-void Network::sendDownloadFinished(const QString &model, bool success)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    KeyValue kvSuccess;
-    kvSuccess.key = QString("success");
-    kvSuccess.value = QJsonValue(success);
-    sendMixpanelEvent("download_finished", QVector<KeyValue>{kv, kvSuccess});
-}
-
-void Network::sendSettingsDialog()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("settings_dialog");
-}
-
-void Network::sendNetworkToggled(bool isActive)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("isActive");
-    kv.value = QJsonValue(isActive);
-    sendMixpanelEvent("network_toggled", QVector<KeyValue>{kv});
-}
-
-void Network::sendNewChat(int count)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("number_of_chats");
-    kv.value = QJsonValue(count);
-    sendMixpanelEvent("new_chat", QVector<KeyValue>{kv});
-}
-
-void Network::sendRemoveChat()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("remove_chat");
-}
-
-void Network::sendRenameChat()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("rename_chat");
-}
-
-void Network::sendChatStarted()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("chat_started");
-}
-
-void Network::sendRecalculatingContext(int conversationLength)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-
-    KeyValue kv;
-    kv.key = QString("length");
-    kv.value = QJsonValue(conversationLength);
-    sendMixpanelEvent("recalc_context", QVector<KeyValue>{kv});
-}
-
-void Network::sendNonCompatHardware()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("noncompat_hardware");
-}
-
-void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &values)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
+    if (!m_sendUsageStats)
         return;
 
     Q_ASSERT(ChatListModel::globalInstance()->currentChat());
     QJsonObject properties;
-    properties.insert("token", "ce362e568ddaee16ed243eaffb5860a2");
-    properties.insert("time", QDateTime::currentSecsSinceEpoch());
-    properties.insert("distinct_id", m_uniqueId);
+
+    properties.insert("token", MIXPANEL_TOKEN);
+    if (!props.contains("time"))
+        properties.insert("time", QDateTime::currentMSecsSinceEpoch());
+    properties.insert("distinct_id", m_uniqueId); // effectively a device ID
     properties.insert("$insert_id", generateUniqueId());
-    properties.insert("$os", QSysInfo::prettyProductName());
+
     if (!m_ipify.isEmpty())
         properties.insert("ip", m_ipify);
-    properties.insert("name", QCoreApplication::applicationName() + " v"
-        + QCoreApplication::applicationVersion());
-    properties.insert("model", ChatListModel::globalInstance()->currentChat()->modelInfo().filename());
-    properties.insert("requestedDevice", MySettings::globalInstance()->device());
-    properties.insert("actualDevice", ChatListModel::globalInstance()->currentChat()->device());
 
-    // Some additional startup information
-    if (ev == "startup") {
-        const QSize display = QGuiApplication::primaryScreen()->size();
-        properties.insert("display", QString("%1x%2").arg(display.width()).arg(display.height()));
-        properties.insert("ram", LLM::globalInstance()->systemTotalRAMInGB());
-#if defined(Q_OS_MAC)
-        properties.insert("cpu", QString::fromStdString(getCPUModel()));
-#endif
-    }
+    properties.insert("$os", QSysInfo::prettyProductName());
+    properties.insert("session_id", m_sessionId);
+    properties.insert("name", QCoreApplication::applicationName() + " v" + QCoreApplication::applicationVersion());
 
-    for (const auto& p : values)
-        properties.insert(p.key, p.value);
+    for (const auto &[key, value]: props.asKeyValueRange())
+        properties.insert(key, QJsonValue::fromVariant(value));
 
     QJsonObject event;
     event.insert("event", ev);
@@ -408,7 +317,7 @@ void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &valu
 
     QJsonDocument doc;
     doc.setArray(array);
-    sendMixpanel(doc.toJson(QJsonDocument::Compact));
+    emit requestMixpanel(doc.toJson(QJsonDocument::Compact));
 
 #if defined(DEBUG)
     printf("%s %s\n", qPrintable(ev), qPrintable(doc.toJson(QJsonDocument::Indented)));
@@ -418,7 +327,7 @@ void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &valu
 
 void Network::sendIpify()
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive() || !m_ipify.isEmpty())
+    if (!m_sendUsageStats || !m_ipify.isEmpty())
         return;
 
     QUrl ipifyUrl("https://api.ipify.org");
@@ -431,11 +340,8 @@ void Network::sendIpify()
     connect(reply, &QNetworkReply::finished, this, &Network::handleIpifyFinished);
 }
 
-void Network::sendMixpanel(const QByteArray &json, bool isOptOut)
+void Network::sendMixpanel(const QByteArray &json)
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive() && !isOptOut)
-        return;
-
     QUrl trackUrl("https://api.mixpanel.com/track");
     QNetworkRequest request(trackUrl);
     QSslConfiguration conf = request.sslConfiguration();
@@ -449,7 +355,6 @@ void Network::sendMixpanel(const QByteArray &json, bool isOptOut)
 
 void Network::handleIpifyFinished()
 {
-    Q_ASSERT(MySettings::globalInstance()->networkUsageStatsActive());
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply)
         return;
@@ -469,8 +374,7 @@ void Network::handleIpifyFinished()
 #endif
     reply->deleteLater();
 
-    if (m_shouldSendStartup)
-        sendStartup();
+    trackEvent("ipify_complete");
 }
 
 void Network::handleMixpanelFinished()

gpt4all-chat/network.h

@@ -19,31 +19,15 @@ public:
 
     Q_INVOKABLE QString generateUniqueId() const;
     Q_INVOKABLE bool sendConversation(const QString &ingestId, const QString &conversation);
+    Q_INVOKABLE void trackChatEvent(const QString &event, QVariantMap props = QVariantMap());
+    Q_INVOKABLE void trackEvent(const QString &event, const QVariantMap &props = QVariantMap());
 
 Q_SIGNALS:
     void healthCheckFailed(int code);
+    void requestMixpanel(const QByteArray &json, bool isOptOut = false);
 
 public Q_SLOTS:
-    void sendOptOut();
-    void sendModelLoaded();
     void sendStartup();
-    void sendCheckForUpdates();
-    Q_INVOKABLE void sendModelDownloaderDialog();
-    Q_INVOKABLE void sendResetContext(int conversationLength);
-    void sendInstallModel(const QString &model);
-    void sendRemoveModel(const QString &model);
-    void sendDownloadStarted(const QString &model);
-    void sendDownloadCanceled(const QString &model);
-    void sendDownloadError(const QString &model, int code, const QString &errorString);
-    void sendDownloadFinished(const QString &model, bool success);
-    Q_INVOKABLE void sendSettingsDialog();
-    Q_INVOKABLE void sendNetworkToggled(bool active);
-    Q_INVOKABLE void sendNewChat(int count);
-    Q_INVOKABLE void sendRemoveChat();
-    Q_INVOKABLE void sendRenameChat();
-    Q_INVOKABLE void sendNonCompatHardware();
-    void sendChatStarted();
-    void sendRecalculatingContext(int conversationLength);
 
 private Q_SLOTS:
     void handleIpifyFinished();
@@ -53,18 +37,21 @@ private Q_SLOTS:
     void handleMixpanelFinished();
     void handleIsActiveChanged();
     void handleUsageStatsActiveChanged();
+    void sendMixpanel(const QByteArray &json);
 
 private:
+    void sendOptOut();
     void sendHealth();
     void sendIpify();
-    void sendMixpanelEvent(const QString &event, const QVector<KeyValue> &values = QVector<KeyValue>());
-    void sendMixpanel(const QByteArray &json, bool isOptOut = false);
     bool packageAndSendJson(const QString &ingestId, const QString &json);
 
 private:
-    bool m_shouldSendStartup;
+    bool m_sendUsageStats = false;
+    bool m_hasSentOptIn;
+    bool m_hasSentOptOut;
     QString m_ipify;
     QString m_uniqueId;
+    QString m_sessionId;
     QNetworkAccessManager m_networkManager;
     QVector<QNetworkReply*> m_activeUploads;
 

gpt4all-chat/qml/AboutDialog.qml

@@ -98,17 +98,4 @@ MyDialog {
             Accessible.description: qsTr("Contains embedded link to https://home.nomic.ai")
         }
     }
-
-    MyButton {
-        id: checkForUpdatesButton
-        anchors.right: parent.right
-        anchors.bottom: parent.bottom
-        text: qsTr("Check for updates...")
-        font.pixelSize: theme.fontSizeLarge
-        Accessible.description: qsTr("Launch an external application that will check for updates to the installer")
-        onClicked: {
-            if (!LLM.checkForUpdates())
-                checkForUpdatesError.open()
-        }
-    }
 }

gpt4all-chat/qml/ChatDrawer.qml

@@ -16,22 +16,13 @@ Rectangle {
         id: theme
     }
 
+    signal downloadClicked
+    signal aboutClicked
+
     color: theme.containerBackground
 
-    Rectangle {
-        id: borderRight
-        anchors.top: parent.top
-        anchors.bottom: parent.bottom
-        anchors.right: parent.right
-        width: 2
-        color: theme.containerForeground
-    }
-
     Item {
-        anchors.top: parent.top
-        anchors.bottom: parent.bottom
-        anchors.left: parent.left
-        anchors.right: borderRight.left
+        anchors.fill: parent
         anchors.margins: 10
 
         Accessible.role: Accessible.Pane
@@ -48,8 +39,9 @@ Rectangle {
             text: qsTr("\uFF0B New chat")
             Accessible.description: qsTr("Create a new chat")
             onClicked: {
-                ChatListModel.addChat();
-                Network.sendNewChat(ChatListModel.count)
+                ChatListModel.addChat()
+                conversationList.positionViewAtIndex(0, ListView.Beginning)
+                Network.trackEvent("new_chat", {"number_of_chats": ChatListModel.count})
             }
         }
 
@@ -59,7 +51,7 @@ Rectangle {
             anchors.rightMargin: -10
             anchors.topMargin: 10
             anchors.top: newChat.bottom
-            anchors.bottom: parent.bottom
+            anchors.bottom: checkForUpdatesButton.top
             anchors.bottomMargin: 10
             ScrollBar.vertical.policy: ScrollBar.AlwaysOff
             clip: true
@@ -69,6 +61,9 @@ Rectangle {
                 anchors.fill: parent
                 anchors.rightMargin: 10
                 model: ChatListModel
+
+                Component.onCompleted: ChatListModel.loadChats()
+
                 ScrollBar.vertical: ScrollBar {
                     parent: conversationList.parent
                     anchors.top: conversationList.top
@@ -119,8 +114,8 @@ Rectangle {
                             // having focus
                             if (chatName.readOnly)
                                 return;
+                            Network.trackChatEvent("rename_chat")
                             changeName();
-                            Network.sendRenameChat()
                         }
                         function changeName() {
                             ChatListModel.get(index).name = chatName.text
@@ -203,8 +198,8 @@ Rectangle {
                                     color: "transparent"
                                 }
                                 onClicked: {
+                                    Network.trackChatEvent("remove_chat")
                                     ChatListModel.removeChat(ChatListModel.get(index))
-                                    Network.sendRemoveChat()
                                 }
                                 Accessible.role: Accessible.Button
                                 Accessible.name: qsTr("Confirm chat deletion")
@@ -245,5 +240,45 @@ Rectangle {
                 Accessible.description: qsTr("List of chats in the drawer dialog")
             }
         }
+
+        MyButton {
+            id: checkForUpdatesButton
+            anchors.left: parent.left
+            anchors.right: parent.right
+            anchors.bottom: downloadButton.top
+            anchors.bottomMargin: 10
+            text: qsTr("Updates")
+            font.pixelSize: theme.fontSizeLarge
+            Accessible.description: qsTr("Launch an external application that will check for updates to the installer")
+            onClicked: {
+                if (!LLM.checkForUpdates())
+                    checkForUpdatesError.open()
+            }
+        }
+
+        MyButton {
+            id: downloadButton
+            anchors.left: parent.left
+            anchors.right: parent.right
+            anchors.bottom: aboutButton.top
+            anchors.bottomMargin: 10
+            text: qsTr("Downloads")
+            Accessible.description: qsTr("Launch a dialog to download new models")
+            onClicked: {
+                downloadClicked()
+            }
+        }
+
+        MyButton {
+            id: aboutButton
+            anchors.left: parent.left
+            anchors.right: parent.right
+            anchors.bottom: parent.bottom
+            text: qsTr("About")
+            Accessible.description: qsTr("Launch a dialog to show the about page")
+            onClicked: {
+                aboutClicked()
+            }
+        }
     }
 }

gpt4all-chat/qml/ChatView.qml

@@ -1,16 +1,18 @@
+import Qt5Compat.GraphicalEffects
 import QtCore
 import QtQuick
 import QtQuick.Controls
 import QtQuick.Controls.Basic
 import QtQuick.Layouts
-import Qt5Compat.GraphicalEffects
-import llm
+
 import chatlistmodel
 import download
-import modellist
-import network
 import gpt4all
+import llm
+import localdocs
+import modellist
 import mysettings
+import network
 
 Rectangle {
     id: window
@@ -21,8 +23,6 @@ Rectangle {
 
     property var currentChat: ChatListModel.currentChat
     property var chatModel: currentChat.chatModel
-    signal settingsViewRequested(int page)
-    signal downloadViewRequested(bool showEmbeddingModels)
 
     color: theme.black
 
@@ -31,6 +31,10 @@ Rectangle {
         startupDialogs();
     }
 
+    Component.onDestruction: {
+        Network.trackEvent("session_end")
+    }
+
     Connections {
         target: firstStartDialog
         function onClosed() {
@@ -38,6 +42,13 @@ Rectangle {
         }
     }
 
+    Connections {
+        target: downloadNewModels
+        function onClosed() {
+            startupDialogs();
+        }
+    }
+
     Connections {
         target: Download
         function onHasNewerReleaseChanged() {
@@ -61,12 +72,12 @@ Rectangle {
     }
 
     property bool hasShownModelDownload: false
-    property bool hasShownFirstStart: false
+    property bool hasCheckedFirstStart: false
     property bool hasShownSettingsAccess: false
 
     function startupDialogs() {
         if (!LLM.compatHardware()) {
-            Network.sendNonCompatHardware();
+            Network.trackEvent("noncompat_hardware")
             errorCompatHardware.open();
             return;
         }
@@ -79,21 +90,29 @@ Rectangle {
         }
 
         // check for first time start of this version
-        if (!hasShownFirstStart && Download.isFirstStart()) {
-            firstStartDialog.open();
-            hasShownFirstStart = true;
-            return;
+        if (!hasCheckedFirstStart) {
+            if (Download.isFirstStart(/*writeVersion*/ true)) {
+                firstStartDialog.open();
+                return;
+            }
+
+            // send startup or opt-out now that the user has made their choice
+            Network.sendStartup()
+            // start localdocs
+            LocalDocs.requestStart()
+
+            hasCheckedFirstStart = true
         }
 
-        // check for any current models and if not, open download view once
+        // check for any current models and if not, open download dialog once
         if (!hasShownModelDownload && ModelList.installedModels.count === 0 && !firstStartDialog.opened) {
-            downloadViewRequested();
+            downloadNewModels.open();
             hasShownModelDownload = true;
             return;
         }
 
         // check for new version
-        if (Download.hasNewerRelease && !firstStartDialog.opened) {
+        if (Download.hasNewerRelease && !firstStartDialog.opened && !downloadNewModels.opened) {
             newVersionDialog.open();
             return;
         }
@@ -103,10 +122,6 @@ Rectangle {
         return ModelList.modelInfo(currentChat.modelInfo.id).name;
     }
 
-    property bool isCurrentlyLoading: false
-    property real modelLoadingPercentage: 0.0
-    property bool trySwitchContextInProgress: false
-
     PopupDialog {
         id: errorCompatHardware
         anchors.centerIn: parent
@@ -148,6 +163,13 @@ Rectangle {
         anchors.centerIn: parent
     }
 
+    AboutDialog {
+        id: aboutDialog
+        anchors.centerIn: parent
+        width: Math.min(1024, window.width - (window.width * .2))
+        height: Math.min(600, window.height - (window.height * .2))
+    }
+
     Item {
         Accessible.role: Accessible.Window
         Accessible.name: title
@@ -282,265 +304,250 @@ Rectangle {
         anchors.top: parent.top
         height: 100
         color: theme.mainHeader
+        Item {
+            anchors.centerIn: parent
+            height: childrenRect.height
+            visible: true
 
-        RowLayout {
-            id: comboLayout
-            height: 80
-            anchors.left: parent.left
-            anchors.right: parent.right
-            anchors.verticalCenter: parent.verticalCenter
-            spacing: 20
-
-            Rectangle {
-                Layout.alignment: Qt.AlignLeft
-                Layout.leftMargin: 30
-                Layout.fillWidth: true
-                Layout.preferredWidth: 100
-                Layout.topMargin: 20
-                color: "transparent"
-                Layout.preferredHeight: childrenRect.height
-                MyToolButton {
-                    id: drawerButton
-                    anchors.left: parent.left
-                    backgroundColor: theme.iconBackgroundLight
-                    width: 40
-                    height: 40
-                    scale: 1.5
-                    padding: 15
-                    source: conversation.state === "expanded" ? "qrc:/gpt4all/icons/left_panel_open.svg" : "qrc:/gpt4all/icons/left_panel_closed.svg"
-                    Accessible.role: Accessible.ButtonMenu
-                    Accessible.name: qsTr("Chat panel")
-                    Accessible.description: qsTr("Chat panel with options")
-                    onClicked: {
-                        conversation.toggleLeftPanel()
-                    }
+            Label {
+                id: modelLabel
+                color: theme.textColor
+                padding: 20
+                font.pixelSize: theme.fontSizeLarger
+                text: ""
+                background: Rectangle {
+                    color: theme.mainHeader
                 }
+                horizontalAlignment: TextInput.AlignRight
             }
 
-            MyComboBox {
-                id: comboBox
-                Layout.alignment: Qt.AlignHCenter
-                Layout.fillHeight: true
-                Layout.fillWidth: true
-                Layout.preferredWidth: 100
-                Layout.maximumWidth: 675
-                enabled: !currentChat.isServer
-                    && !window.trySwitchContextInProgress
-                    && !window.isCurrentlyLoading
-                model: ModelList.installedModels
-                valueRole: "id"
-                textRole: "name"
+            RowLayout {
+                id: comboLayout
+                anchors.top: modelLabel.top
+                anchors.bottom: modelLabel.bottom
+                anchors.horizontalCenter: parent.horizontalCenter
+                anchors.horizontalCenterOffset: window.width >= 950 ? 0 : Math.max(-((950 - window.width) / 2), -99.5)
+                spacing: 20
 
-                function changeModel(index) {
-                    window.modelLoadingPercentage = 0.0;
-                    window.isCurrentlyLoading = true;
-                    currentChat.stopGenerating()
-                    currentChat.reset();
-                    currentChat.modelInfo = ModelList.modelInfo(comboBox.valueAt(index))
-                }
+                MyComboBox {
+                    id: comboBox
+                    Layout.fillWidth: true
+                    Layout.fillHeight: true
+                    implicitWidth: 575
+                    width: window.width >= 750 ? implicitWidth : implicitWidth - (750 - window.width)
+                    enabled: !currentChat.isServer
+                        && !currentChat.trySwitchContextInProgress
+                        && !currentChat.isCurrentlyLoading
+                    model: ModelList.installedModels
+                    valueRole: "id"
+                    textRole: "name"
 
-                Connections {
-                    target: currentChat
-                    function onModelLoadingPercentageChanged() {
-                        window.modelLoadingPercentage = currentChat.modelLoadingPercentage;
-                        window.isCurrentlyLoading = currentChat.modelLoadingPercentage !== 0.0
-                            && currentChat.modelLoadingPercentage !== 1.0;
+                    function changeModel(index) {
+                        currentChat.stopGenerating()
+                        currentChat.reset();
+                        currentChat.modelInfo = ModelList.modelInfo(comboBox.valueAt(index))
                     }
-                    function onTrySwitchContextOfLoadedModelAttempted() {
-                        window.trySwitchContextInProgress = true;
-                    }
-                    function onTrySwitchContextOfLoadedModelCompleted() {
-                        window.trySwitchContextInProgress = false;
-                    }
-                }
-                Connections {
-                    target: switchModelDialog
-                    function onAccepted() {
-                        comboBox.changeModel(switchModelDialog.index)
-                    }
-                }
 
-                background: ProgressBar {
-                    id: modelProgress
-                    value: window.modelLoadingPercentage
-                    background: Rectangle {
-                        color: theme.mainComboBackground
-                        radius: 10
-                    }
-                    contentItem: Item {
-                        Rectangle {
-                            visible: window.isCurrentlyLoading
-                            anchors.bottom: parent.bottom
-                            width: modelProgress.visualPosition * parent.width
-                            height: 10
-                            radius: 2
-                            color: theme.progressForeground
+                    Connections {
+                        target: switchModelDialog
+                        function onAccepted() {
+                            comboBox.changeModel(switchModelDialog.index)
                         }
                     }
-                }
-                contentItem: Text {
-                    anchors.horizontalCenter: parent.horizontalCenter
-                    leftPadding: 10
-                    rightPadding: {
-                        if (ejectButton.visible && reloadButton)
-                            return 105;
-                        if (reloadButton.visible)
-                            return 65
-                        return 25
-                    }
-                    text: {
-                        if (currentChat.modelLoadingError !== "")
-                            return qsTr("Model loading error...")
-                        if (window.trySwitchContextInProgress)
-                            return qsTr("Switching context...")
-                        if (currentModelName() === "")
-                            return qsTr("Choose a model...")
-                        if (currentChat.modelLoadingPercentage === 0.0)
-                            return qsTr("Reload \u00B7 ") + currentModelName()
-                        if (window.isCurrentlyLoading)
-                            return qsTr("Loading \u00B7 ") + currentModelName()
-                        return currentModelName()
-                    }
-                    font.pixelSize: theme.fontSizeLarger
-                    color: theme.white
-                    verticalAlignment: Text.AlignVCenter
-                    horizontalAlignment: Text.AlignHCenter
-                    elide: Text.ElideRight
-                }
-                delegate: ItemDelegate {
-                    id: comboItemDelegate
-                    width: comboBox.width
-                    contentItem: Text {
-                        text: name
-                        color: theme.textColor
-                        font: comboBox.font
-                        elide: Text.ElideRight
-                        verticalAlignment: Text.AlignVCenter
-                    }
-                    background: Rectangle {
-                        color: (index % 2 === 0 ? theme.darkContrast : theme.lightContrast)
-                        border.width: highlighted
-                        border.color: theme.accentColor
-                    }
-                    highlighted: comboBox.highlightedIndex === index
-                }
-                Accessible.role: Accessible.ComboBox
-                Accessible.name: currentModelName()
-                Accessible.description: qsTr("The top item is the current model")
-                onActivated: function (index) {
-                    var newInfo = ModelList.modelInfo(comboBox.valueAt(index));
-                    if (newInfo === currentChat.modelInfo) {
-                        currentChat.reloadModel();
-                    } else if (currentModelName() !== "" && chatModel.count !== 0) {
-                        switchModelDialog.index = index;
-                        switchModelDialog.open();
-                    } else {
-                        comboBox.changeModel(index);
-                    }
-                }
 
-                MyMiniButton {
-                    id: ejectButton
-                    visible: currentChat.isModelLoaded && !window.isCurrentlyLoading
-                    z: 500
-                    anchors.right: parent.right
-                    anchors.rightMargin: 50
-                    anchors.verticalCenter: parent.verticalCenter
-                    source: "qrc:/gpt4all/icons/eject.svg"
-                    backgroundColor: theme.gray300
-                    backgroundColorHovered: theme.iconBackgroundLight
-                    onClicked: {
-                        currentChat.forceUnloadModel();
-                    }
-                    ToolTip.text: qsTr("Eject the currently loaded model")
-                    ToolTip.visible: hovered
-                }
-
-                MyMiniButton {
-                    id: reloadButton
-                    visible: currentChat.modelLoadingError === ""
-                        && !window.trySwitchContextInProgress
-                        && !window.isCurrentlyLoading
-                        && (currentChat.isModelLoaded || currentModelName() !== "")
-                    z: 500
-                    anchors.right: ejectButton.visible ? ejectButton.left : parent.right
-                    anchors.rightMargin: ejectButton.visible ? 10 : 50
-                    anchors.verticalCenter: parent.verticalCenter
-                    source: "qrc:/gpt4all/icons/regenerate.svg"
-                    backgroundColor: theme.gray300
-                    backgroundColorHovered: theme.iconBackgroundLight
-                    onClicked: {
-                        if (currentChat.isModelLoaded)
-                            currentChat.forceReloadModel();
-                        else
-                            currentChat.reloadModel();
-                    }
-                    ToolTip.text: qsTr("Reload the currently loaded model")
-                    ToolTip.visible: hovered
-                }
-            }
-
-            Rectangle {
-                color: "transparent"
-                Layout.alignment: Qt.AlignRight
-                Layout.rightMargin: 30
-                Layout.fillWidth: true
-                Layout.preferredWidth: 100
-                Layout.preferredHeight: childrenRect.height
-                Layout.topMargin: 20
-
-                RowLayout {
-                    spacing: 20
-                    anchors.right: parent.right
-                    MyButton {
-                        id: collectionsButton
-                        Image {
-                            id: collectionsImage
-                            anchors.verticalCenter: parent.verticalCenter
-                            anchors.left: parent.left
-                            anchors.leftMargin: 15
-                            width: 24
-                            height: 24
-                            mipmap: true
-                            source: "qrc:/gpt4all/icons/db.svg"
-                        }
-
-                        ColorOverlay {
-                            anchors.fill: collectionsImage
-                            source: collectionsImage
-                            color: collectionsButton.hovered || collectionsImage.toggled ? theme.iconBackgroundHovered : theme.iconBackgroundLight
-                        }
-
-                        leftPadding: 50
-                        borderWidth: 0
-                        backgroundColor: theme.mainComboBackground
-                        backgroundColorHovered: theme.conversationButtonBackgroundHovered
-                        backgroundRadius: 5
-                        padding: 15
-                        topPadding: 8
-                        bottomPadding: 8
-                        textColor: hovered || toggled ? theme.iconBackgroundHovered : theme.iconBackgroundLight
-                        text: qsTr("LocalDocs")
-                        fontPixelSize: theme.fontSizeSmall
-
-                        property bool toggled: currentChat.collectionList.length
+                    background: ProgressBar {
+                        id: modelProgress
+                        value: currentChat.modelLoadingPercentage
                         background: Rectangle {
-                            radius: collectionsButton.backgroundRadius
-                            color: collectionsButton.toggled ? collectionsButton.backgroundColorHovered : collectionsButton.backgroundColor
+                            color: theme.mainComboBackground
+                            radius: 10
                         }
+                        contentItem: Item {
+                            Rectangle {
+                                visible: currentChat.isCurrentlyLoading
+                                anchors.bottom: parent.bottom
+                                width: modelProgress.visualPosition * parent.width
+                                height: 10
+                                radius: 2
+                                color: theme.progressForeground
+                            }
+                        }
+                    }
+                    contentItem: Text {
+                        anchors.horizontalCenter: parent.horizontalCenter
+                        leftPadding: 10
+                        rightPadding: {
+                            if (ejectButton.visible && reloadButton)
+                                return 105;
+                            if (reloadButton.visible)
+                                return 65
+                            return 25
+                        }
+                        text: {
+                            if (currentChat.modelLoadingError !== "")
+                                return qsTr("Model loading error...")
+                            if (currentChat.trySwitchContextInProgress == 1)
+                                return qsTr("Waiting for model...")
+                            if (currentChat.trySwitchContextInProgress == 2)
+                                return qsTr("Switching context...")
+                            if (currentModelName() === "")
+                                return qsTr("Choose a model...")
+                            if (currentChat.modelLoadingPercentage === 0.0)
+                                return qsTr("Reload \u00B7 ") + currentModelName()
+                            if (currentChat.isCurrentlyLoading)
+                                return qsTr("Loading \u00B7 ") + currentModelName()
+                            return currentModelName()
+                        }
+                        font.pixelSize: theme.fontSizeLarger
+                        color: theme.white
+                        verticalAlignment: Text.AlignVCenter
+                        horizontalAlignment: Text.AlignHCenter
+                        elide: Text.ElideRight
+                    }
+                    delegate: ItemDelegate {
+                        id: comboItemDelegate
+                        width: comboBox.width
+                        contentItem: Text {
+                            text: name
+                            color: theme.textColor
+                            font: comboBox.font
+                            elide: Text.ElideRight
+                            verticalAlignment: Text.AlignVCenter
+                        }
+                        background: Rectangle {
+                            color: (index % 2 === 0 ? theme.darkContrast : theme.lightContrast)
+                            border.width: highlighted
+                            border.color: theme.accentColor
+                        }
+                        highlighted: comboBox.highlightedIndex === index
+                    }
+                    Accessible.role: Accessible.ComboBox
+                    Accessible.name: currentModelName()
+                    Accessible.description: qsTr("The top item is the current model")
+                    onActivated: function (index) {
+                        var newInfo = ModelList.modelInfo(comboBox.valueAt(index));
+                        if (newInfo === currentChat.modelInfo) {
+                            currentChat.reloadModel();
+                        } else if (currentModelName() !== "" && chatModel.count !== 0) {
+                            switchModelDialog.index = index;
+                            switchModelDialog.open();
+                        } else {
+                            comboBox.changeModel(index);
+                        }
+                    }
 
-                        Accessible.name: qsTr("Add documents")
-                        Accessible.description: qsTr("add collections of documents to the chat")
-
+                    MyMiniButton {
+                        id: ejectButton
+                        visible: currentChat.isModelLoaded && !currentChat.isCurrentlyLoading
+                        z: 500
+                        anchors.right: parent.right
+                        anchors.rightMargin: 50
+                        anchors.verticalCenter: parent.verticalCenter
+                        source: "qrc:/gpt4all/icons/eject.svg"
+                        backgroundColor: theme.gray300
+                        backgroundColorHovered: theme.iconBackgroundLight
                         onClicked: {
-                            collectionsDialog.open()
+                            currentChat.forceUnloadModel();
                         }
+                        ToolTip.text: qsTr("Eject the currently loaded model")
+                        ToolTip.visible: hovered
+                    }
+
+                    MyMiniButton {
+                        id: reloadButton
+                        visible: currentChat.modelLoadingError === ""
+                            && !currentChat.trySwitchContextInProgress
+                            && !currentChat.isCurrentlyLoading
+                            && (currentChat.isModelLoaded || currentModelName() !== "")
+                        z: 500
+                        anchors.right: ejectButton.visible ? ejectButton.left : parent.right
+                        anchors.rightMargin: ejectButton.visible ? 10 : 50
+                        anchors.verticalCenter: parent.verticalCenter
+                        source: "qrc:/gpt4all/icons/regenerate.svg"
+                        backgroundColor: theme.gray300
+                        backgroundColorHovered: theme.iconBackgroundLight
+                        onClicked: {
+                            if (currentChat.isModelLoaded)
+                                currentChat.forceReloadModel();
+                            else
+                                currentChat.reloadModel();
+                        }
+                        ToolTip.text: qsTr("Reload the currently loaded model")
+                        ToolTip.visible: hovered
                     }
                 }
             }
         }
     }
 
+    SettingsDialog {
+        id: settingsDialog
+        anchors.centerIn: parent
+        width: Math.min(1920, window.width - (window.width * .1))
+        height: window.height - (window.height * .1)
+        onDownloadClicked: {
+            downloadNewModels.showEmbeddingModels = true
+            downloadNewModels.open()
+        }
+    }
+
+    MyToolButton {
+        id: drawerButton
+        backgroundColor: theme.iconBackgroundLight
+        anchors.left: parent.left
+        anchors.top: parent.top
+        anchors.topMargin: 42.5
+        anchors.leftMargin: 30
+        width: 40
+        height: 40
+        scale: 1.5
+        z: 200
+        padding: 15
+        source: conversation.state === "expanded" ? "qrc:/gpt4all/icons/left_panel_open.svg" : "qrc:/gpt4all/icons/left_panel_closed.svg"
+        Accessible.role: Accessible.ButtonMenu
+        Accessible.name: qsTr("Chat panel")
+        Accessible.description: qsTr("Chat panel with options")
+        onClicked: {
+            conversation.toggleLeftPanel()
+        }
+    }
+
+    NetworkDialog {
+        id: networkDialog
+        anchors.centerIn: parent
+        width: Math.min(1024, window.width - (window.width * .2))
+        height: Math.min(600, window.height - (window.height * .2))
+        Item {
+            Accessible.role: Accessible.Dialog
+            Accessible.name: qsTr("Network dialog")
+            Accessible.description: qsTr("opt-in to share feedback/conversations")
+        }
+    }
+
+    MyToolButton {
+        id: networkButton
+        backgroundColor: theme.iconBackgroundLight
+        anchors.right: parent.right
+        anchors.top: parent.top
+        anchors.topMargin: 42.5
+        anchors.rightMargin: 30
+        width: 40
+        height: 40
+        z: 200
+        padding: 15
+        toggled: MySettings.networkIsActive
+        source: "qrc:/gpt4all/icons/network.svg"
+        Accessible.name: qsTr("Network")
+        Accessible.description: qsTr("Reveals a dialogue where you can opt-in for sharing data over network")
+
+        onClicked: {
+            if (MySettings.networkIsActive) {
+                MySettings.networkIsActive = false
+            } else
+                networkDialog.open()
+        }
+    }
+
     Connections {
         target: Network
         function onHealthCheckFailed(code) {
@@ -552,7 +559,49 @@ Rectangle {
         id: collectionsDialog
         anchors.centerIn: parent
         onAddRemoveClicked: {
-            settingsViewRequested(2 /*page 2*/)
+            settingsDialog.pageToDisplay = 2;
+            settingsDialog.open();
+        }
+    }
+
+    MyToolButton {
+        id: collectionsButton
+        backgroundColor: theme.iconBackgroundLight
+        anchors.right: networkButton.left
+        anchors.top: parent.top
+        anchors.topMargin: 42.5
+        anchors.rightMargin: 10
+        width: 40
+        height: 42.5
+        z: 200
+        padding: 15
+        toggled: currentChat.collectionList.length
+        source: "qrc:/gpt4all/icons/db.svg"
+        Accessible.name: qsTr("Add documents")
+        Accessible.description: qsTr("add collections of documents to the chat")
+
+        onClicked: {
+            collectionsDialog.open()
+        }
+    }
+
+    MyToolButton {
+        id: settingsButton
+        backgroundColor: theme.iconBackgroundLight
+        anchors.right: collectionsButton.left
+        anchors.top: parent.top
+        anchors.topMargin: 42.5
+        anchors.rightMargin: 10
+        width: 40
+        height: 40
+        z: 200
+        padding: 15
+        source: "qrc:/gpt4all/icons/settings.svg"
+        Accessible.name: qsTr("Settings")
+        Accessible.description: qsTr("Reveals a dialogue with settings")
+
+        onClicked: {
+            settingsDialog.open()
         }
     }
 
@@ -596,6 +645,35 @@ Rectangle {
         }
     }
 
+    MyToolButton {
+        id: copyButton
+        backgroundColor: theme.iconBackgroundLight
+        anchors.right: settingsButton.left
+        anchors.top: parent.top
+        anchors.topMargin: 42.5
+        anchors.rightMargin: 10
+        width: 40
+        height: 40
+        z: 200
+        padding: 15
+        source: "qrc:/gpt4all/icons/copy.svg"
+        Accessible.name: qsTr("Copy")
+        Accessible.description: qsTr("Copy the conversation to the clipboard")
+
+        TextEdit{
+            id: copyEdit
+            visible: false
+        }
+
+        onClicked: {
+            var conversation = getConversation()
+            copyEdit.text = conversation
+            copyEdit.selectAll()
+            copyEdit.copy()
+            copyMessage.open()
+        }
+    }
+
     function getConversation() {
         var conversation = "";
         for (var i = 0; i < chatModel.count; i++) {
@@ -633,6 +711,29 @@ Rectangle {
         return str + "]}"
     }
 
+    MyToolButton {
+        id: resetContextButton
+        backgroundColor: theme.iconBackgroundLight
+        anchors.right: copyButton.left
+        anchors.top: parent.top
+        anchors.topMargin: 42.5
+        anchors.rightMargin: 10
+        width: 40
+        height: 40
+        z: 200
+        padding: 15
+        source: "qrc:/gpt4all/icons/regenerate.svg"
+
+        Accessible.name: text
+        Accessible.description: qsTr("Reset the context and erase current conversation")
+
+        onClicked: {
+            Network.trackChatEvent("reset_context", { "length": chatModel.count })
+            currentChat.reset();
+            currentChat.processSystemPrompt();
+        }
+    }
+
     Dialog {
         id: checkForUpdatesError
         anchors.centerIn: parent
@@ -662,12 +763,31 @@ Rectangle {
         }
     }
 
+    ModelDownloaderDialog {
+        id: downloadNewModels
+        anchors.centerIn: parent
+        width: Math.min(1920, window.width - (window.width * .1))
+        height: window.height - (window.height * .1)
+        Item {
+            Accessible.role: Accessible.Dialog
+            Accessible.name: qsTr("Download new models")
+            Accessible.description: qsTr("Dialog for downloading new models")
+        }
+    }
+
     ChatDrawer {
         id: drawer
         anchors.left: parent.left
         anchors.top: accentRibbon.bottom
         anchors.bottom: parent.bottom
-        width: Math.max(180, Math.min(600, 0.2 * window.width))
+        width: Math.min(600, 0.2 * window.width)
+        onDownloadClicked: {
+            downloadNewModels.showEmbeddingModels = false
+            downloadNewModels.open()
+        }
+        onAboutClicked: {
+            aboutDialog.open()
+        }
     }
 
     PopupDialog {
@@ -847,7 +967,7 @@ Rectangle {
                             padding: 18
                             leftPadding: 50
                             Image {
-                                id: downloadImage
+                                id: image
                                 anchors.verticalCenter: parent.verticalCenter
                                 anchors.left: parent.left
                                 anchors.leftMargin: 15
@@ -857,13 +977,12 @@ Rectangle {
                                 source: "qrc:/gpt4all/icons/download.svg"
                             }
                             ColorOverlay {
-                                anchors.fill: downloadImage
-                                source: downloadImage
+                                anchors.fill: image
+                                source: image
                                 color: theme.accentColor
                             }
                             onClicked: {
-                                console.log("download button")
-                                downloadViewRequested(false /*showEmbeddingModels*/);
+                                downloadNewModels.open();
                             }
                         }
                     }
@@ -872,7 +991,10 @@ Rectangle {
                 ListView {
                     id: listView
                     visible: ModelList.installedModels.count !== 0 && chatModel.count !== 0
-                    anchors.fill: parent
+                    anchors.top: parent.top
+                    anchors.bottom: parent.bottom
+                    anchors.horizontalCenter: parent.horizontalCenter
+                    width: Math.min(1280, parent.width)
                     model: chatModel
 
                     ScrollBar.vertical: ScrollBar {
@@ -885,9 +1007,8 @@ Rectangle {
 
                     delegate: TextArea {
                         id: myTextArea
-                        text: value + references
-                        anchors.horizontalCenter: listView.contentItem.horizontalCenter
-                        width: Math.min(1280, listView.contentItem.width)
+                        text: value + (MySettings.localDocsShowReferences ? references : "")
+                        width: listView.width
                         color: {
                             if (!currentChat.isServer)
                                 return theme.textColor
@@ -930,7 +1051,7 @@ Rectangle {
                             anchors.fill: parent
                             acceptedButtons: Qt.RightButton
 
-                            onClicked: {
+                            onClicked: (mouse) => {
                                 if (mouse.button === Qt.RightButton) {
                                     conversationContextMenu.x = conversationMouseArea.mouseX
                                     conversationContextMenu.y = conversationMouseArea.mouseY
@@ -943,11 +1064,19 @@ Rectangle {
                             id: conversationContextMenu
                             MenuItem {
                                 text: qsTr("Copy")
+                                enabled: myTextArea.selectedText !== ""
+                                height: enabled ? implicitHeight : 0
                                 onTriggered: myTextArea.copy()
                             }
                             MenuItem {
-                                text: qsTr("Select All")
-                                onTriggered: myTextArea.selectAll()
+                                text: qsTr("Copy Message")
+                                enabled: myTextArea.selectedText === ""
+                                height: enabled ? implicitHeight : 0
+                                onTriggered: {
+                                    myTextArea.selectAll()
+                                    myTextArea.copy()
+                                    myTextArea.deselect()
+                                }
                             }
                         }
 
@@ -1162,9 +1291,11 @@ Rectangle {
                     var listElement = chatModel.get(index);
 
                     if (currentChat.responseInProgress) {
+                        Network.trackChatEvent("stop_generating_clicked")
                         listElement.stopped = true
                         currentChat.stopGenerating()
                     } else {
+                        Network.trackChatEvent("regenerate_clicked")
                         currentChat.regenerateResponse()
                         if (chatModel.count) {
                             if (listElement.name === qsTr("Response: ")) {
@@ -1195,8 +1326,9 @@ Rectangle {
                 textColor: theme.textColor
                 visible: !currentChat.isServer
                     && !currentChat.isModelLoaded
-                    && !window.trySwitchContextInProgress
-                    && !window.isCurrentlyLoading
+                    && currentChat.modelLoadingError === ""
+                    && !currentChat.trySwitchContextInProgress
+                    && !currentChat.isCurrentlyLoading
                     && currentModelName() !== ""
 
                 Image {
@@ -1267,11 +1399,11 @@ Rectangle {
                 Accessible.role: Accessible.EditableText
                 Accessible.name: placeholderText
                 Accessible.description: qsTr("Send messages/prompts to the model")
-                Keys.onReturnPressed: (event)=> {
-                    if (event.modifiers & Qt.ControlModifier || event.modifiers & Qt.ShiftModifier)
-                        event.accepted = false;
-                    else {
-                        editingFinished();
+                Keys.onReturnPressed: (event) => {
+                    if (event.modifiers & Qt.ControlModifier || event.modifiers & Qt.ShiftModifier) {
+                        event.accepted = false
+                    } else if (!currentChat.responseInProgress) {
+                        editingFinished()
                         sendMessage()
                     }
                 }
@@ -1279,6 +1411,7 @@ Rectangle {
                     if (textInput.text === "")
                         return
 
+                    Network.trackChatEvent("send_message")
                     currentChat.stopGenerating()
                     currentChat.newPromptResponsePair(textInput.text);
                     currentChat.prompt(textInput.text,
@@ -1299,7 +1432,7 @@ Rectangle {
                     anchors.fill: parent
                     acceptedButtons: Qt.RightButton
 
-                    onClicked: {
+                    onClicked: (mouse) => {
                         if (mouse.button === Qt.RightButton) {
                             textInputContextMenu.x = textInputMouseArea.mouseX
                             textInputContextMenu.y = textInputMouseArea.mouseY
@@ -1312,10 +1445,14 @@ Rectangle {
                     id: textInputContextMenu
                     MenuItem {
                         text: qsTr("Cut")
+                        enabled: textInput.selectedText !== ""
+                        height: enabled ? implicitHeight : 0
                         onTriggered: textInput.cut()
                     }
                     MenuItem {
                         text: qsTr("Copy")
+                        enabled: textInput.selectedText !== ""
+                        height: enabled ? implicitHeight : 0
                         onTriggered: textInput.copy()
                     }
                     MenuItem {
@@ -1340,6 +1477,7 @@ Rectangle {
             width: 30
             height: 30
             visible: !currentChat.isServer
+            enabled: !currentChat.responseInProgress
             source: "qrc:/gpt4all/icons/send_message.svg"
             Accessible.name: qsTr("Send message")
             Accessible.description: qsTr("Sends the message/prompt contained in textfield to the model")

gpt4all-chat/qml/ModelDownloaderDialog.qml

@@ -11,15 +11,21 @@ import modellist
 import network
 import mysettings
 
-Rectangle {
+MyDialog {
     id: modelDownloaderDialog
-    color: theme.containerBackground
+    modal: true
+    closePolicy: Popup.CloseOnEscape | Popup.CloseOnPressOutside
+    padding: 10
+    property bool showEmbeddingModels: false
 
-    function showEmbeddingModels() {
-        Network.sendModelDownloaderDialog();
-        ModelList.downloadableModels.expanded = true
-        var targetModelIndex = ModelList.defaultEmbeddingModelIndex
-        modelListView.positionViewAtIndex(targetModelIndex, ListView.Beginning)
+    onOpened: {
+        Network.trackEvent("download_dialog")
+
+        if (showEmbeddingModels) {
+            ModelList.downloadableModels.expanded = true
+            var targetModelIndex = ModelList.defaultEmbeddingModelIndex
+            modelListView.positionViewAtIndex(targetModelIndex, ListView.Beginning)
+        }
     }
 
     PopupDialog {
@@ -30,7 +36,7 @@ Rectangle {
 
     ColumnLayout {
         anchors.fill: parent
-        anchors.margins: 20
+        anchors.margins: 10
         spacing: 30
 
         Label {

gpt4all-chat/qml/MyToolButton.qml

@@ -9,8 +9,6 @@ Button {
     padding: 10
     property color backgroundColor: theme.iconBackgroundDark
     property color backgroundColorHovered: theme.iconBackgroundHovered
-    property color toggledColor: theme.accentColor
-    property real toggledWidth: 1
     property bool toggled: false
     property alias source: image.source
     property alias fillMode: image.fillMode
@@ -27,12 +25,11 @@ Button {
         anchors.fill: parent
         Rectangle {
             anchors.fill: parent
-            color: myButton.toggledColor
+            color: "transparent"
             visible: myButton.toggled
-            border.color: myButton.toggledColor
-            border.width: myButton.toggledWidth
-            radius: 6
-            opacity: .2
+            border.color: theme.accentColor
+            border.width: 1
+            radius: 10
         }
         Image {
             id: image

gpt4all-chat/qml/NetworkDialog.qml

@@ -100,16 +100,10 @@ NOTE: By turning on this feature, you will be sending your data to the GPT4All O
     }
 
     onAccepted: {
-        if (MySettings.networkIsActive)
-            return
-        MySettings.networkIsActive = true;
-        Network.sendNetworkToggled(true);
+        MySettings.networkIsActive = true
     }
 
     onRejected: {
-        if (!MySettings.networkIsActive)
-            return
-        MySettings.networkIsActive = false;
-        Network.sendNetworkToggled(false);
+        MySettings.networkIsActive = false
     }
 }

gpt4all-chat/qml/SettingsDialog.qml

@@ -0,0 +1,132 @@
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import QtQuick.Dialogs
+import QtQuick.Layouts
+import Qt.labs.folderlistmodel
+import download
+import modellist
+import network
+import llm
+import mysettings
+
+MyDialog {
+    id: settingsDialog
+    modal: true
+    padding: 20
+    onOpened: {
+        Network.trackEvent("settings_dialog")
+    }
+
+    signal downloadClicked
+    property alias pageToDisplay: listView.currentIndex
+
+    Item {
+        Accessible.role: Accessible.Dialog
+        Accessible.name: qsTr("Settings")
+        Accessible.description: qsTr("Contains various application settings")
+    }
+
+    ListModel {
+        id: stacksModel
+        ListElement {
+            title: qsTr("Models")
+        }
+        ListElement {
+            title: qsTr("Application")
+        }
+        ListElement {
+            title: qsTr("LocalDocs")
+        }
+    }
+
+    Rectangle {
+        id: stackList
+        anchors.top: parent.top
+        anchors.bottom: parent.bottom
+        anchors.left: parent.left
+        width: 220
+        color: theme.controlBackground
+        radius: 10
+
+        ScrollView {
+            anchors.top: parent.top
+            anchors.bottom: parent.bottom
+            anchors.left: parent.left
+            anchors.right: parent.right
+            anchors.topMargin: 10
+            ScrollBar.vertical.policy: ScrollBar.AsNeeded
+            clip: true
+
+            ListView {
+                id: listView
+                anchors.fill: parent
+                model: stacksModel
+
+                delegate: Rectangle {
+                    id: item
+                    width: listView.width
+                    height: titleLabel.height + 10
+                    color: "transparent"
+
+                    MyButton {
+                        id: titleLabel
+                        backgroundColor: index === listView.currentIndex ? theme.buttonBackground : theme.controlBackground
+                        backgroundColorHovered: index === listView.currentIndex ? backgroundColor : theme.containerBackground
+                        borderColor: index === listView.currentIndex ? theme.accentColor : "transparent"
+                        borderWidth: index === listView.currentIndex ? 1 : 0
+                        textColor: index === listView.currentIndex ? theme.oppositeTextColor : theme.titleTextColor
+                        anchors.verticalCenter: parent.verticalCenter
+                        anchors.left: parent.left
+                        anchors.right: parent.right
+                        anchors.margins: 10
+                        font.bold: index === listView.currentIndex
+                        text: title
+                        font.pixelSize: theme.fontSizeLarge
+                        onClicked: {
+                            listView.currentIndex = index
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    StackLayout {
+        id: stackLayout
+        anchors.top: parent.top
+        anchors.bottom: parent.bottom
+        anchors.left: stackList.right
+        anchors.right: parent.right
+        currentIndex: listView.currentIndex
+
+        MySettingsStack {
+            title: qsTr("Model/Character Settings")
+            tabs: [
+                Component { ModelSettings { } }
+            ]
+        }
+
+        MySettingsStack {
+            title: qsTr("Application General Settings")
+            tabs: [
+                Component { ApplicationSettings { } }
+            ]
+        }
+
+        MySettingsStack {
+            title: qsTr("Local Document Collections")
+            tabs: [
+                Component {
+                    LocalDocsSettings {
+                        id: localDocsSettings
+                        Component.onCompleted: {
+                             localDocsSettings.downloadClicked.connect(settingsDialog.downloadClicked);
+                        }
+                    }
+                }
+            ]
+        }
+    }
+}

gpt4all-chat/qml/SettingsView.qml

@@ -1,132 +0,0 @@
-import QtCore
-import QtQuick
-import QtQuick.Controls
-import QtQuick.Controls.Basic
-import QtQuick.Dialogs
-import QtQuick.Layouts
-import Qt.labs.folderlistmodel
-import download
-import modellist
-import network
-import llm
-import mysettings
-
-Rectangle {
-    id: settingsDialog
-    color: theme.containerBackground
-
-    signal downloadClicked
-    property alias pageToDisplay: listView.currentIndex
-
-    Item {
-        Accessible.role: Accessible.Dialog
-        Accessible.name: qsTr("Settings")
-        Accessible.description: qsTr("Contains various application settings")
-    }
-
-    ListModel {
-        id: stacksModel
-        ListElement {
-            title: qsTr("Models")
-        }
-        ListElement {
-            title: qsTr("Application")
-        }
-        ListElement {
-            title: qsTr("LocalDocs")
-        }
-    }
-
-    Item {
-        anchors.fill: parent
-        anchors.margins: 20
-        Rectangle {
-            id: stackList
-            anchors.top: parent.top
-            anchors.bottom: parent.bottom
-            anchors.left: parent.left
-            width: 220
-            color: theme.controlBackground
-            radius: 10
-
-            ScrollView {
-                anchors.top: parent.top
-                anchors.bottom: parent.bottom
-                anchors.left: parent.left
-                anchors.right: parent.right
-                anchors.topMargin: 10
-                ScrollBar.vertical.policy: ScrollBar.AsNeeded
-                clip: true
-
-                ListView {
-                    id: listView
-                    anchors.fill: parent
-                    model: stacksModel
-
-                    delegate: Rectangle {
-                        id: item
-                        width: listView.width
-                        height: titleLabel.height + 10
-                        color: "transparent"
-
-                        MyButton {
-                            id: titleLabel
-                            backgroundColor: index === listView.currentIndex ? theme.buttonBackground : theme.controlBackground
-                            backgroundColorHovered: index === listView.currentIndex ? backgroundColor : theme.containerBackground
-                            borderColor: index === listView.currentIndex ? theme.accentColor : "transparent"
-                            borderWidth: index === listView.currentIndex ? 1 : 0
-                            textColor: index === listView.currentIndex ? theme.oppositeTextColor : theme.titleTextColor
-                            anchors.verticalCenter: parent.verticalCenter
-                            anchors.left: parent.left
-                            anchors.right: parent.right
-                            anchors.margins: 10
-                            font.bold: index === listView.currentIndex
-                            text: title
-                            font.pixelSize: theme.fontSizeLarge
-                            onClicked: {
-                                listView.currentIndex = index
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        StackLayout {
-            id: stackLayout
-            anchors.top: parent.top
-            anchors.bottom: parent.bottom
-            anchors.left: stackList.right
-            anchors.right: parent.right
-            currentIndex: listView.currentIndex
-
-            MySettingsStack {
-                title: qsTr("Model/Character Settings")
-                tabs: [
-                    Component { ModelSettings { } }
-                ]
-            }
-
-            MySettingsStack {
-                title: qsTr("Application General Settings")
-                tabs: [
-                    Component { ApplicationSettings { } }
-                ]
-            }
-
-            MySettingsStack {
-                title: qsTr("Local Document Collections")
-                tabs: [
-                    Component {
-                        LocalDocsSettings {
-                            id: localDocsSettings
-                            Component.onCompleted: {
-                                 localDocsSettings.downloadClicked.connect(settingsDialog.downloadClicked);
-                            }
-                        }
-                    }
-                ]
-            }
-        }
-    }
-}

gpt4all-chat/qml/StartupDialog.qml

@@ -123,8 +123,6 @@ model release that uses your data!")
                 buttons: optInStatisticsRadio.children
                 onClicked: {
                     MySettings.networkUsageStatsActive = optInStatisticsRadio.checked
-                    if (!optInStatisticsRadio.checked)
-                        Network.sendOptOut();
                     if (optInNetworkRadio.choiceMade && optInStatisticsRadio.choiceMade)
                         startupDialog.close();
                 }
@@ -140,7 +138,7 @@ model release that uses your data!")
 
                 RadioButton {
                     id: optInStatisticsRadioYes
-                    checked: false
+                    checked: MySettings.networkUsageStatsActive
                     text: qsTr("Yes")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton
@@ -182,6 +180,7 @@ model release that uses your data!")
                 }
                 RadioButton {
                     id: optInStatisticsRadioNo
+                    checked: MySettings.isNetworkUsageStatsActiveSet() && !MySettings.networkUsageStatsActive
                     text: qsTr("No")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton
@@ -254,7 +253,7 @@ model release that uses your data!")
 
                 RadioButton {
                     id: optInNetworkRadioYes
-                    checked: false
+                    checked: MySettings.networkIsActive
                     text: qsTr("Yes")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton
@@ -296,6 +295,7 @@ model release that uses your data!")
                 }
                 RadioButton {
                     id: optInNetworkRadioNo
+                    checked: MySettings.isNetworkIsActiveSet() && !MySettings.networkIsActive
                     text: qsTr("No")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton