qml: fix missing localdocs and prefill progress (#3330)

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2.1
 setup: true
 orbs:
-  path-filtering: circleci/path-filtering@0.0.1
+  path-filtering: circleci/path-filtering@1.1.0
 
 workflows:
   version: 2.1

.circleci/continue_config.yml

@@ -19,6 +19,13 @@ parameters:
     default: false
 
 jobs:
+  # work around CircleCI-Public/path-filtering-orb#20
+  noop:
+    docker:
+      - image: cimg/base:current
+    steps:
+      - run: "true"
+
   build-offline-chat-installer-macos:
     macos:
       xcode: 15.4.0
@@ -37,11 +44,11 @@ jobs:
           command: softwareupdate --install-rosetta --agree-to-license  # needed for QtIFW
       - run:
           name: Install dependencies
-          command: brew install ccache
+          command: brew install ccache wget
       - run:
           name: Installing Qt
           command: |
-            curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
+            wget "https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg"
             hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
             /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.48 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
             hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
@@ -175,11 +182,11 @@ jobs:
           command: softwareupdate --install-rosetta --agree-to-license  # needed for QtIFW
       - run:
           name: Install dependencies
-          command: brew install ccache
+          command: brew install ccache wget
       - run:
           name: Installing Qt
           command: |
-            curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
+            wget "https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg"
             hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
             /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.48 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
             hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
@@ -312,9 +319,11 @@ jobs:
       - run:
           name: Setup Linux and Dependencies
           command: |
-            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-1.3.290-jammy.list https://packages.lunarg.com/vulkan/1.3.290/lunarg-vulkan-1.3.290-jammy.list
-            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            # Prevent apt-get from interactively prompting for service restart
+            echo "\$nrconf{restart} = 'l'" | sudo tee /etc/needrestart/conf.d/90-autorestart.conf >/dev/null
+            wget -qO- "https://packages.lunarg.com/lunarg-signing-key-pub.asc" | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            wget -qO- "https://packages.lunarg.com/vulkan/1.3.290/lunarg-vulkan-1.3.290-jammy.list" | sudo tee /etc/apt/sources.list.d/lunarg-vulkan-1.3.290-jammy.list
+            wget "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb"
             sudo dpkg -i cuda-keyring_1.1-1_all.deb
             packages=(
               bison build-essential ccache cuda-compiler-11-8 flex g++-12 gperf libcublas-dev-11-8 libfontconfig1
@@ -329,7 +338,7 @@ jobs:
       - run:
           name: Installing Qt
           command: |
-            wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
+            wget "https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run"
             chmod +x qt-unified-linux-x64-4.6.0-online.run
             ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.48 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
       - run:
@@ -388,9 +397,11 @@ jobs:
       - run:
           name: Setup Linux and Dependencies
           command: |
-            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-1.3.290-jammy.list https://packages.lunarg.com/vulkan/1.3.290/lunarg-vulkan-1.3.290-jammy.list
-            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            # Prevent apt-get from interactively prompting for service restart
+            echo "\$nrconf{restart} = 'l'" | sudo tee /etc/needrestart/conf.d/90-autorestart.conf >/dev/null
+            wget -qO- "https://packages.lunarg.com/lunarg-signing-key-pub.asc" | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            wget -qO- "https://packages.lunarg.com/vulkan/1.3.290/lunarg-vulkan-1.3.290-jammy.list" | sudo tee /etc/apt/sources.list.d/lunarg-vulkan-1.3.290-jammy.list
+            wget "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb"
             sudo dpkg -i cuda-keyring_1.1-1_all.deb
             packages=(
               bison build-essential ccache cuda-compiler-11-8 flex g++-12 gperf libcublas-dev-11-8 libfontconfig1
@@ -405,7 +416,7 @@ jobs:
       - run:
           name: Installing Qt
           command: |
-            wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
+            wget "https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run"
             chmod +x qt-unified-linux-x64-4.6.0-online.run
             ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.48 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
       - run:
@@ -451,7 +462,8 @@ jobs:
 
   build-offline-chat-installer-windows:
     machine:
-      image: windows-server-2022-gui:current
+      # we use 2024.04.01 because nvcc complains about the MSVC ver if we use anything newer
+      image: windows-server-2022-gui:2024.04.1
       resource_class: windows.large
       shell: powershell.exe -ExecutionPolicy Bypass
     steps:
@@ -466,21 +478,21 @@ jobs:
             - ccache-gpt4all-win-amd64-
       - run:
           name: Install dependencies
-          command: choco install -y ccache
+          command: choco install -y ccache wget
       - run:
           name: Installing Qt
           command: |
-            Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
+            wget.exe "https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe"
             & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.48 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
       - run:
           name: Install VulkanSDK
           command: |
-            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            wget.exe "https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe"
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
       - run:
           name: Install CUDA Toolkit
           command: |
-            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe -OutFile cuda_11.8.0_windows_network.exe
+            wget.exe "https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe"
             .\cuda_11.8.0_windows_network.exe -s cudart_11.8 nvcc_11.8 cublas_11.8 cublas_dev_11.8
       - run:
           name: "Install Dotnet 8"
@@ -488,7 +500,7 @@ jobs:
             mkdir dotnet
             cd dotnet
             $dotnet_url="https://download.visualstudio.microsoft.com/download/pr/5af098e1-e433-4fda-84af-3f54fd27c108/6bd1c6e48e64e64871957289023ca590/dotnet-sdk-8.0.302-win-x64.zip"
-            Invoke-WebRequest -Uri $dotnet_url -Outfile dotnet-sdk-8.0.302-win-x64.zip
+            wget.exe "$dotnet_url"
             Expand-Archive -LiteralPath .\dotnet-sdk-8.0.302-win-x64.zip
             $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet-sdk-8.0.302-win-x64"
             $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
@@ -542,20 +554,23 @@ jobs:
 
   sign-offline-chat-installer-windows:
     machine:
-      image: windows-server-2022-gui:current
+      image: windows-server-2022-gui:2024.04.1
       resource_class: windows.large
       shell: powershell.exe -ExecutionPolicy Bypass
     steps:
       - checkout
       - attach_workspace:
           at: build
+      - run:
+          name: Install dependencies
+          command: choco install -y wget
       - run:
           name: "Install Dotnet 8 && Azure Sign Tool"
           command: |
             mkdir dotnet
             cd dotnet
             $dotnet_url="https://download.visualstudio.microsoft.com/download/pr/5af098e1-e433-4fda-84af-3f54fd27c108/6bd1c6e48e64e64871957289023ca590/dotnet-sdk-8.0.302-win-x64.zip"
-            Invoke-WebRequest -Uri $dotnet_url -Outfile dotnet-sdk-8.0.302-win-x64.zip
+            wget.exe "$dotnet_url"
             Expand-Archive -LiteralPath .\dotnet-sdk-8.0.302-win-x64.zip
             $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet-sdk-8.0.302-win-x64"
             $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
@@ -572,7 +587,7 @@ jobs:
 
   build-online-chat-installer-windows:
     machine:
-      image: windows-server-2022-gui:current
+      image: windows-server-2022-gui:2024.04.1
       resource_class: windows.large
       shell: powershell.exe -ExecutionPolicy Bypass
     steps:
@@ -587,21 +602,21 @@ jobs:
             - ccache-gpt4all-win-amd64-
       - run:
           name: Install dependencies
-          command: choco install -y ccache
+          command: choco install -y ccache wget
       - run:
           name: Installing Qt
           command: |
-            Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
+            wget.exe "https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe"
             & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.48 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
       - run:
           name: Install VulkanSDK
           command: |
-            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            wget.exe "https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe"
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
       - run:
           name: Install CUDA Toolkit
           command: |
-            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe -OutFile cuda_11.8.0_windows_network.exe
+            wget.exe "https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe"
             .\cuda_11.8.0_windows_network.exe -s cudart_11.8 nvcc_11.8 cublas_11.8 cublas_dev_11.8
       - run:
           name: "Install Dotnet 8"
@@ -609,7 +624,7 @@ jobs:
             mkdir dotnet
             cd dotnet
             $dotnet_url="https://download.visualstudio.microsoft.com/download/pr/5af098e1-e433-4fda-84af-3f54fd27c108/6bd1c6e48e64e64871957289023ca590/dotnet-sdk-8.0.302-win-x64.zip"
-            Invoke-WebRequest -Uri $dotnet_url -Outfile dotnet-sdk-8.0.302-win-x64.zip
+            wget.exe "$dotnet_url"
             Expand-Archive -LiteralPath .\dotnet-sdk-8.0.302-win-x64.zip
             $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet-sdk-8.0.302-win-x64"
             $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
@@ -670,20 +685,23 @@ jobs:
 
   sign-online-chat-installer-windows:
     machine:
-      image: windows-server-2022-gui:current
+      image: windows-server-2022-gui:2024.04.1
       resource_class: windows.large
       shell: powershell.exe -ExecutionPolicy Bypass
     steps:
       - checkout
       - attach_workspace:
           at: build
+      - run:
+          name: Install dependencies
+          command: choco install -y wget
       - run:
           name: "Install Dotnet 8"
           command: |
             mkdir dotnet
             cd dotnet
             $dotnet_url="https://download.visualstudio.microsoft.com/download/pr/5af098e1-e433-4fda-84af-3f54fd27c108/6bd1c6e48e64e64871957289023ca590/dotnet-sdk-8.0.302-win-x64.zip"
-            Invoke-WebRequest -Uri $dotnet_url -Outfile dotnet-sdk-8.0.302-win-x64.zip
+            wget.exe "$dotnet_url"
             Expand-Archive -LiteralPath .\dotnet-sdk-8.0.302-win-x64.zip
             $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet-sdk-8.0.302-win-x64"
             $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
@@ -719,9 +737,11 @@ jobs:
       - run:
           name: Setup Linux and Dependencies
           command: |
-            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-1.3.290-jammy.list https://packages.lunarg.com/vulkan/1.3.290/lunarg-vulkan-1.3.290-jammy.list
-            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            # Prevent apt-get from interactively prompting for service restart
+            echo "\$nrconf{restart} = 'l'" | sudo tee /etc/needrestart/conf.d/90-autorestart.conf >/dev/null
+            wget -qO- "https://packages.lunarg.com/lunarg-signing-key-pub.asc" | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            wget -qO- "https://packages.lunarg.com/vulkan/1.3.290/lunarg-vulkan-1.3.290-jammy.list" | sudo tee /etc/apt/sources.list.d/lunarg-vulkan-1.3.290-jammy.list
+            wget "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb"
             sudo dpkg -i cuda-keyring_1.1-1_all.deb
             packages=(
               bison build-essential ccache cuda-compiler-11-8 flex g++-12 gperf libcublas-dev-11-8 libfontconfig1
@@ -736,7 +756,7 @@ jobs:
       - run:
           name: Installing Qt
           command: |
-            wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
+            wget "https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run"
             chmod +x qt-unified-linux-x64-4.6.0-online.run
             ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.48 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
       - run:
@@ -765,7 +785,7 @@ jobs:
 
   build-gpt4all-chat-windows:
     machine:
-      image: windows-server-2022-gui:current
+      image: windows-server-2022-gui:2024.04.1
       resource_class: windows.large
       shell: powershell.exe -ExecutionPolicy Bypass
     steps:
@@ -780,21 +800,21 @@ jobs:
             - ccache-gpt4all-win-amd64-
       - run:
           name: Install dependencies
-          command: choco install -y ccache
+          command: choco install -y ccache wget
       - run:
           name: Installing Qt
           command: |
-            Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
+            wget.exe "https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe"
             & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.48 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
       - run:
           name: Install VulkanSDK
           command: |
-            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            wget.exe "https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe"
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
       - run:
           name: Install CUDA Toolkit
           command: |
-            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe -OutFile cuda_11.8.0_windows_network.exe
+            wget.exe "https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe"
             .\cuda_11.8.0_windows_network.exe -s cudart_11.8 nvcc_11.8 cublas_11.8 cublas_dev_11.8
       - run:
           name: Build
@@ -842,11 +862,11 @@ jobs:
           command: softwareupdate --install-rosetta --agree-to-license  # needed for QtIFW
       - run:
           name: Install dependencies
-          command: brew install ccache
+          command: brew install ccache wget
       - run:
           name: Installing Qt
           command: |
-            curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
+            wget "https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg"
             hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
             /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.48 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
             hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
@@ -932,9 +952,9 @@ jobs:
       - run:
           name: Install dependencies
           command: |
-            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-1.3.290-jammy.list https://packages.lunarg.com/vulkan/1.3.290/lunarg-vulkan-1.3.290-jammy.list
-            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            wget -qO- "https://packages.lunarg.com/lunarg-signing-key-pub.asc" | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            wget -qO- "https://packages.lunarg.com/vulkan/1.3.290/lunarg-vulkan-1.3.290-jammy.list" | sudo tee /etc/apt/sources.list.d/lunarg-vulkan-1.3.290-jammy.list
+            wget "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb"
             sudo dpkg -i cuda-keyring_1.1-1_all.deb
             packages=(
               build-essential ccache cmake cuda-compiler-11-8 g++-12 libcublas-dev-11-8 libnvidia-compute-550-server
@@ -1028,7 +1048,7 @@ jobs:
 
   build-py-windows:
     machine:
-      image: windows-server-2022-gui:current
+      image: windows-server-2022-gui:2024.04.1
       resource_class: windows.large
       shell: powershell.exe -ExecutionPolicy Bypass
     steps:
@@ -1041,20 +1061,20 @@ jobs:
       - restore_cache:
           keys:
             - ccache-gpt4all-win-amd64-
+      - run:
+          name: Install dependencies
+          command:
+            choco install -y ccache cmake ninja wget --installargs 'ADD_CMAKE_TO_PATH=System'
       - run:
           name: Install VulkanSDK
           command: |
-            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            wget.exe "https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe"
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
       - run:
           name: Install CUDA Toolkit
           command: |
-            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe -OutFile cuda_11.8.0_windows_network.exe
+            wget.exe "https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe"
             .\cuda_11.8.0_windows_network.exe -s cudart_11.8 nvcc_11.8 cublas_11.8 cublas_dev_11.8
-      - run:
-          name: Install dependencies
-          command:
-            choco install -y ccache cmake ninja --installargs 'ADD_CMAKE_TO_PATH=System'
       - run:
           name: Install Python dependencies
           command: pip install setuptools wheel cmake
@@ -1096,7 +1116,7 @@ jobs:
           paths:
             - "*.whl"
 
-  store-and-upload-wheels:
+  publish-wheels:
     docker:
       - image: circleci/python:3.8
     steps:
@@ -1132,9 +1152,9 @@ jobs:
       - run:
           name: Install dependencies
           command: |
-            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-1.3.290-jammy.list https://packages.lunarg.com/vulkan/1.3.290/lunarg-vulkan-1.3.290-jammy.list
-            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            wget -qO- "https://packages.lunarg.com/lunarg-signing-key-pub.asc" | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            wget -qO- "https://packages.lunarg.com/vulkan/1.3.290/lunarg-vulkan-1.3.290-jammy.list" | sudo tee /etc/apt/sources.list.d/lunarg-vulkan-1.3.290-jammy.list
+            wget "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb"
             sudo dpkg -i cuda-keyring_1.1-1_all.deb
             packages=(
               build-essential ccache cmake cuda-compiler-11-8 g++-12 libcublas-dev-11-8 libnvidia-compute-550-server
@@ -1225,7 +1245,7 @@ jobs:
 
   build-bindings-backend-windows:
     machine:
-      image: windows-server-2022-gui:current
+      image: windows-server-2022-gui:2024.04.1
       resource_class: windows.large
       shell: powershell.exe -ExecutionPolicy Bypass
     steps:
@@ -1238,20 +1258,20 @@ jobs:
       - restore_cache:
           keys:
             - ccache-gpt4all-win-amd64-
+      - run:
+          name: Install dependencies
+          command: |
+            choco install -y ccache cmake ninja wget --installargs 'ADD_CMAKE_TO_PATH=System'
       - run:
           name: Install VulkanSDK
           command: |
-            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            wget.exe "https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe"
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
       - run:
           name: Install CUDA Toolkit
           command: |
-            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe -OutFile cuda_11.8.0_windows_network.exe
+            wget.exe "https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe"
             .\cuda_11.8.0_windows_network.exe -s cudart_11.8 nvcc_11.8 cublas_11.8 cublas_dev_11.8
-      - run:
-          name: Install dependencies
-          command: |
-            choco install -y ccache cmake ninja --installargs 'ADD_CMAKE_TO_PATH=System'
       - run:
           name: Build Libraries
           no_output_timeout: 30m
@@ -1361,14 +1381,14 @@ jobs:
           at: /tmp/gpt4all-backend
       - run: choco install wget -y
       - run:
-         command: wget https://nodejs.org/dist/v18.16.0/node-v18.16.0-x86.msi -P C:\Users\circleci\Downloads\
-         shell: cmd.exe
-      - run: MsiExec.exe /i C:\Users\circleci\Downloads\node-v18.16.0-x86.msi /qn
+          command: |
+            wget.exe "https://nodejs.org/dist/v18.16.0/node-v18.16.0-x86.msi" -P C:\Users\circleci\Downloads\
+            MsiExec.exe /i C:\Users\circleci\Downloads\node-v18.16.0-x86.msi /qn
       - run:
           command: |
-              Start-Process powershell -verb runAs -Args "-start GeneralProfile"
-              nvm install 18.16.0
-              nvm use 18.16.0
+            Start-Process powershell -verb runAs -Args "-start GeneralProfile"
+            nvm install 18.16.0
+            nvm use 18.16.0
       - run: node --version
       - run: corepack enable
       - run:
@@ -1449,8 +1469,23 @@ jobs:
             npm set //registry.npmjs.org/:_authToken=$NPM_TOKEN
             npm publish
 
+main_only: &main_only
+  filters:
+    branches:
+      only: main
+
 workflows:
   version: 2
+  noop:
+    when:
+      not:
+        or:
+          - << pipeline.parameters.run-all-workflows >>
+          - << pipeline.parameters.run-python-workflow >>
+          - << pipeline.parameters.run-ts-workflow >>
+          - << pipeline.parameters.run-chat-workflow >>
+    jobs:
+      - noop
   build-chat-offline-installers:
     when:
       or:
@@ -1484,23 +1519,30 @@ workflows:
         - << pipeline.parameters.run-chat-workflow >>
     jobs:
       - hold:
+          <<: *main_only
           type: approval
       - build-online-chat-installer-macos:
+          <<: *main_only
           requires:
             - hold
       - sign-online-chat-installer-macos:
+          <<: *main_only
           requires:
             - build-online-chat-installer-macos
       - notarize-online-chat-installer-macos:
+          <<: *main_only
           requires:
             - sign-online-chat-installer-macos
       - build-online-chat-installer-windows:
+          <<: *main_only
           requires:
             - hold
       - sign-online-chat-installer-windows:
+          <<: *main_only
           requires:
             - build-online-chat-installer-windows
       - build-online-chat-installer-linux:
+          <<: *main_only
           requires:
             - hold
   build-and-test-gpt4all-chat:
@@ -1527,47 +1569,31 @@ workflows:
         - << pipeline.parameters.run-python-workflow >>
     jobs:
       - build-ts-docs:
-          filters:
-            branches:
-              only:
-                - main
+          <<: *main_only
       - build-py-docs:
-          filters:
-            branches:
-              only:
-                - main
-  build-py-deploy:
+          <<: *main_only
+  build-python:
     when:
       or:
         - << pipeline.parameters.run-all-workflows >>
         - << pipeline.parameters.run-python-workflow >>
     jobs:
       - pypi-hold:
+          <<: *main_only
           type: approval
       - hold:
           type: approval
       - build-py-linux:
-          filters:
-            branches:
-              only:
           requires:
             - hold
       - build-py-macos:
-          filters:
-            branches:
-              only:
           requires:
             - hold
       - build-py-windows:
-          filters:
-            branches:
-              only:
           requires:
             - hold
-      - store-and-upload-wheels:
-          filters:
-            branches:
-              only:
+      - publish-wheels:
+          <<: *main_only
           requires:
             - pypi-hold
             - build-py-windows
@@ -1582,59 +1608,38 @@ workflows:
     jobs:
       - hold:
           type: approval
-      - nuget-hold:
-          type: approval
       - nodejs-hold:
           type: approval
       - npm-hold:
+          <<: *main_only
           type: approval
       - build-bindings-backend-linux:
-          filters:
-            branches:
-              only:
           requires:
             - hold
       - build-bindings-backend-macos:
-          filters:
-            branches:
-              only:
           requires:
             - hold
       - build-bindings-backend-windows:
-          filters:
-            branches:
-              only:
           requires:
             - hold
 
       # NodeJs Jobs
       - prepare-npm-pkg:
-          filters:
-            branches:
-              only:
+          <<: *main_only
           requires:
             - npm-hold
             - build-nodejs-linux
             - build-nodejs-windows
             - build-nodejs-macos
       - build-nodejs-linux:
-          filters:
-            branches:
-              only:
           requires:
             - nodejs-hold
             - build-bindings-backend-linux
       - build-nodejs-windows:
-          filters:
-            branches:
-              only:
           requires:
             - nodejs-hold
             - build-bindings-backend-windows
       - build-nodejs-macos:
-          filters:
-            branches:
-              only:
           requires:
             - nodejs-hold
             - build-bindings-backend-macos

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
-ignore-words-list = blong, afterall, som, assistent, crasher
-skip = .git,*.pdf,*.svg,*.lock,*.ts
+ignore-words-list = blong, afterall, assistent, crasher, requestor
+skip = ./.git,./gpt4all-chat/translations,*.pdf,*.svg,*.lock

.gitignore

@@ -182,6 +182,7 @@ gpt4all-chat/models/*
 build_*
 build-*
 cmake-build-*
+/gpt4all-chat/tests/python/config.py
 
 # IntelliJ
 .idea/

.gitmodules

@@ -17,3 +17,9 @@
 [submodule "gpt4all-chat/deps/QXlsx"]
 	path = gpt4all-chat/deps/QXlsx
 	url = https://github.com/nomic-ai/QXlsx.git
+[submodule "gpt4all-chat/deps/Jinja2Cpp"]
+	path = gpt4all-chat/deps/Jinja2Cpp
+	url = https://github.com/nomic-ai/jinja2cpp.git
+[submodule "gpt4all-chat/deps/rapidjson"]
+	path = gpt4all-chat/deps/rapidjson
+	url = https://github.com/nomic-ai/rapidjson.git

MAINTAINERS.md

@@ -51,11 +51,6 @@ Thiago Ramos ([@thiagojramos](https://github.com/thiagojramos))<br/>
 E-mail: thiagojramos@outlook.com<br/>
 - pt\_BR translation
 
-Victor Emanuel ([@SINAPSA-IC](https://github.com/SINAPSA-IC))<br/>
-E-mail: contact@sinapsaro.ro<br/>
-Discord: `@sinapsa_ic_56124_99632`
-- ro\_RO translation
-
 不知火 Shiranui ([@supersonictw](https://github.com/supersonictw))<br/>
 E-mail: supersonic@livemail.tw<br/>
 Discord: `@supersonictw`

common/common.cmake

@@ -11,8 +11,7 @@ function(gpt4all_add_warning_options target)
         -Wextra-semi
         -Wformat=2
         -Wmissing-include-dirs
-        -Wnull-dereference
-        -Wstrict-overflow=2
+        -Wsuggest-override
         -Wvla
         # errors
         -Werror=format-security
@@ -22,8 +21,6 @@ function(gpt4all_add_warning_options target)
         # disabled warnings
         -Wno-sign-compare
         -Wno-unused-parameter
-        -Wno-unused-function
-        -Wno-unused-variable
     )
     if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
         target_compile_options("${target}" PRIVATE

gpt4all-backend/include/gpt4all-backend/llmodel.h

@@ -5,6 +5,7 @@
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
+#include <expected>
 #include <functional>
 #include <optional>
 #include <span>
@@ -24,6 +25,10 @@ using namespace std::string_literals;
 class LLModel {
 public:
     using Token = int32_t;
+    using PromptCallback      = std::function<bool(std::span<const Token> batch, bool cached)>;
+    using ResponseCallback    = std::function<bool(Token token, std::string_view piece)>;
+    using EmbedCancelCallback = bool(unsigned *batchSizes, unsigned nBatch, const char *backend);
+    using ProgressCallback    = std::function<bool(float progress)>;
 
     class BadArchError: public std::runtime_error {
     public:
@@ -101,6 +106,7 @@ public:
         static int32_t maxContextLength(const std::string &modelPath);
         static int32_t layerCount(const std::string &modelPath);
         static bool isEmbeddingModel(const std::string &modelPath);
+        static auto chatTemplate(const char *modelPath) -> std::expected<std::string, std::string>;
         static void setImplementationsSearchPath(const std::string &path);
         static const std::string &implementationsSearchPath();
         static bool hasSupportedCPU();
@@ -124,9 +130,6 @@ public:
     };
 
     struct PromptContext {
-        std::vector<int32_t> tokens;    // current tokens in the context window
-        int32_t n_past = 0;             // number of tokens in past conversation
-        int32_t n_ctx = 0;              // number of tokens possible in context window
         int32_t n_predict = 200;
         int32_t top_k = 40;
         float   top_p = 0.9f;
@@ -138,8 +141,6 @@ public:
         float   contextErase = 0.5f;    // percent of context to erase if we exceed the context window
     };
 
-    using ProgressCallback = std::function<bool(float progress)>;
-
     explicit LLModel() {}
     virtual ~LLModel() {}
 
@@ -151,21 +152,17 @@ public:
     virtual bool isModelLoaded() const = 0;
     virtual size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) = 0;
     virtual size_t stateSize() const = 0;
-    virtual size_t saveState(std::span<uint8_t> dest) const = 0;
-    virtual size_t restoreState(std::span<const uint8_t> src) = 0;
+    virtual size_t saveState(std::span<uint8_t> stateOut, std::vector<Token> &inputTokensOut) const = 0;
+    virtual size_t restoreState(std::span<const uint8_t> state, std::span<const Token> inputTokens) = 0;
 
     // This method requires the model to return true from supportsCompletion otherwise it will throw
     // an error
-    virtual void prompt(const std::string &prompt,
-                        const std::string &promptTemplate,
-                        std::function<bool(int32_t)> promptCallback,
-                        std::function<bool(int32_t, const std::string&)> responseCallback,
-                        bool allowContextShift,
-                        PromptContext &ctx,
-                        bool special = false,
-                        std::optional<std::string_view> fakeReply = {});
+    virtual void prompt(std::string_view        prompt,
+                        const PromptCallback   &promptCallback,
+                        const ResponseCallback &responseCallback,
+                        const PromptContext    &ctx);
 
-    using EmbedCancelCallback = bool(unsigned *batchSizes, unsigned nBatch, const char *backend);
+    virtual int32_t countPromptTokens(std::string_view prompt) const;
 
     virtual size_t embeddingSize() const {
         throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
@@ -210,17 +207,24 @@ public:
 
     void setProgressCallback(ProgressCallback callback) { m_progressCallback = callback; }
 
+    virtual int32_t contextLength() const = 0;
+    virtual auto specialTokens() -> std::unordered_map<std::string, std::string> const = 0;
+
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
-    virtual std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special = false) = 0;
+    virtual std::vector<Token> tokenize(std::string_view str) const = 0;
     virtual bool isSpecialToken(Token id) const = 0;
     virtual std::string tokenToString(Token id) const = 0;
-    virtual void initSampler(PromptContext &ctx) = 0;
+    virtual void initSampler(const PromptContext &ctx) = 0;
     virtual Token sampleToken() const = 0;
-    virtual bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const = 0;
-    virtual void shiftContext(PromptContext &promptCtx) = 0;
-    virtual int32_t contextLength() const = 0;
+    virtual bool evalTokens(int32_t nPast, std::span<const Token> tokens) const = 0;
+    virtual void shiftContext(const PromptContext &promptCtx, int32_t *nPast) = 0;
+    virtual int32_t inputLength() const = 0;
+    virtual int32_t computeModelInputPosition(std::span<const Token> input) const = 0;
+    virtual void setModelInputPosition(int32_t pos) = 0;
+    virtual void appendInputToken(Token tok) = 0;
+    virtual std::span<const Token> inputTokens() const = 0;
     virtual const std::vector<Token> &endTokens() const = 0;
     virtual bool shouldAddBOS() const = 0;
 
@@ -236,6 +240,12 @@ protected:
         return -1;
     }
 
+    virtual auto chatTemplate(const char *modelPath) const -> std::expected<std::string, std::string>
+    {
+        (void)modelPath;
+        return std::unexpected("not implemented");
+    }
+
     const Implementation *m_implementation = nullptr;
 
     ProgressCallback m_progressCallback;
@@ -247,17 +257,15 @@ protected:
         return true;
     }
 
-    bool decodePrompt(std::function<bool(int32_t)> promptCallback,
-                      std::function<bool(int32_t, const std::string&)> responseCallback,
-                      bool allowContextShift,
-                      PromptContext &promptCtx,
-                      std::vector<Token> embd_inp,
-                      bool isResponse = false);
-    void generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
-                          bool allowContextShift,
-                          PromptContext &promptCtx);
-
-    Token m_tokenize_last_token = -1; // not serialized
+    // prefill context with prompt
+    auto decodePrompt(const PromptCallback &promptCallback,
+                      const PromptContext  &promptCtx,
+                      std::vector<Token>    embd_inp)
+        -> std::optional<int32_t>;
+    // generate a response
+    void generateResponse(const ResponseCallback &responseCallback,
+                          const PromptContext    &promptCtx,
+                          int32_t                 nPast);
 
     friend class LLMImplementation;
 };

gpt4all-backend/include/gpt4all-backend/llmodel_c.h

@@ -23,6 +23,11 @@ extern "C" {
  */
 typedef void *llmodel_model;
 
+/**
+ * A token.
+ */
+typedef int32_t token_t;
+
 /**
  * llmodel_prompt_context structure for holding the prompt context.
  * NOTE: The implementation takes care of all the memory handling of the raw logits pointer and the
@@ -30,19 +35,15 @@ typedef void *llmodel_model;
  * behavior.
  */
 struct llmodel_prompt_context {
-    int32_t *tokens;        // current tokens in the context window
-    size_t tokens_size;     // the size of the raw tokens vector
-    int32_t n_past;         // number of tokens in past conversation
-    int32_t n_ctx;          // number of tokens possible in context window
     int32_t n_predict;      // number of tokens to predict
     int32_t top_k;          // top k logits to sample from
-    float top_p;            // nucleus sampling probability threshold
-    float min_p;            // Min P sampling
-    float temp;             // temperature to adjust model's output distribution
+    float   top_p;          // nucleus sampling probability threshold
+    float   min_p;          // Min P sampling
+    float   temp;           // temperature to adjust model's output distribution
     int32_t n_batch;        // number of predictions to generate in parallel
-    float repeat_penalty;   // penalty factor for repeated tokens
+    float   repeat_penalty; // penalty factor for repeated tokens
     int32_t repeat_last_n;  // last n tokens to penalize
-    float context_erase;    // percent of context to erase if we exceed the context window
+    float   context_erase;  // percent of context to erase if we exceed the context window
 };
 
 struct llmodel_gpu_device {
@@ -61,10 +62,12 @@ typedef struct llmodel_gpu_device llmodel_gpu_device;
 
 /**
  * Callback type for prompt processing.
- * @param token_id The token id of the prompt.
+ * @param token_ids An array of token ids of the prompt.
+ * @param n_token_ids The number of tokens in the array.
+ * @param cached Whether the tokens were already in cache.
  * @return a bool indicating whether the model should keep processing.
  */
-typedef bool (*llmodel_prompt_callback)(int32_t token_id);
+typedef bool (*llmodel_prompt_callback)(const token_t *token_ids, size_t n_token_ids, bool cached);
 
 /**
  * Callback type for response.
@@ -72,7 +75,7 @@ typedef bool (*llmodel_prompt_callback)(int32_t token_id);
  * @param response The response string. NOTE: a token_id of -1 indicates the string is an error string.
  * @return a bool indicating whether the model should keep generating.
  */
-typedef bool (*llmodel_response_callback)(int32_t token_id, const char *response);
+typedef bool (*llmodel_response_callback)(token_t token_id, const char *response);
 
 /**
  * Embedding cancellation callback for use with llmodel_embed.
@@ -83,6 +86,8 @@ typedef bool (*llmodel_response_callback)(int32_t token_id, const char *response
  */
 typedef bool (*llmodel_emb_cancel_callback)(unsigned *batch_sizes, unsigned n_batch, const char *backend);
 
+typedef void (*llmodel_special_token_callback)(const char *name, const char *token);
+
 /**
  * Create a llmodel instance.
  * Recognises correct model type from file at model_path
@@ -141,48 +146,57 @@ bool llmodel_isModelLoaded(llmodel_model model);
  * @param model A pointer to the llmodel_model instance.
  * @return the size in bytes of the internal state of the model
  */
-uint64_t llmodel_get_state_size(llmodel_model model);
+uint64_t llmodel_state_get_size(llmodel_model model);
 
 /**
- * Saves the internal state of the model to the specified destination address.
+ * Saves the internal state of the model.
  * NOTE: This state data is specific to the type of model you have created.
  * @param model A pointer to the llmodel_model instance.
- * @param dest A pointer to the destination.
- * @param size The size of the destination buffer.
- * @return the number of bytes copied, or zero on error.
+ * @param state Where to store the state. This must be a buffer of at least llmodel_state_get_size() bytes.
+ * @param state_size The size of the destination for the state.
+ * @param input_tokens_out Where to store the address of the token cache state. This is dynamically allocated and must
+ * be freed with llmodel_state_free_input_tokens.
+ * @param n_input_tokens Where to store the size of the token cache state.
+ * @return The number of bytes copied. On error, zero is returned, the token cache is set to NULL, and the token cache
+ * size is set to zero.
  */
-uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest, uint64_t size);
+uint64_t llmodel_state_get_data(llmodel_model model, uint8_t *state_out, uint64_t state_size,
+                                token_t **input_tokens_out, uint64_t *n_input_tokens);
+
+/**
+ * Frees the temporary token cache buffer created by a call to llmodel_state_get_data().
+ * @param input_tokens The token cache buffer.
+ */
+void llmodel_state_free_input_tokens(token_t *input_tokens);
 
 /**
  * Restores the internal state of the model using data from the specified address.
  * NOTE: This state data is specific to the type of model you have created.
  * @param model A pointer to the llmodel_model instance.
- * @param src A pointer to the state data.
- * @param size The size of the source data.
+ * @param state A pointer to the state data.
+ * @param state_size The size of the state data.
+ * @param input_tokens The token cache associated with the saved state.
+ * @param n_input_tokens The number of tokens in input_tokens.
  * @return The number of bytes read, or zero on error.
  */
-uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src, size_t size);
+uint64_t llmodel_state_set_data(llmodel_model model, const uint8_t *state, uint64_t state_size,
+                                const token_t *input_tokens, uint64_t n_input_tokens);
 
 /**
  * Generate a response using the model.
  * @param model A pointer to the llmodel_model instance.
  * @param prompt A string representing the input prompt.
- * @param prompt_template A string representing the input prompt template.
  * @param prompt_callback A callback function for handling the processing of prompt.
  * @param response_callback A callback function for handling the generated response.
- * @param allow_context_shift Whether to allow shifting of context to make room for more input.
- * @param special True if special tokens in the prompt should be processed, false otherwise.
- * @param fake_reply A string to insert into context as the model's reply, or NULL to generate one.
  * @param ctx A pointer to the llmodel_prompt_context structure.
+ * @param error A pointer to a string; will only be set on error.
  */
-void llmodel_prompt(llmodel_model model, const char *prompt,
-                    const char *prompt_template,
-                    llmodel_prompt_callback prompt_callback,
-                    llmodel_response_callback response_callback,
-                    bool allow_context_shift,
-                    llmodel_prompt_context *ctx,
-                    bool special,
-                    const char *fake_reply);
+bool llmodel_prompt(llmodel_model               model,
+                    const char                 *prompt,
+                    llmodel_prompt_callback     prompt_callback,
+                    llmodel_response_callback   response_callback,
+                    llmodel_prompt_context     *ctx,
+                    const char                **error);
 
 /**
  * Generate an embedding using the model.
@@ -294,6 +308,10 @@ const char *llmodel_model_backend_name(llmodel_model model);
  */
 const char *llmodel_model_gpu_device_name(llmodel_model model);
 
+int32_t llmodel_count_prompt_tokens(llmodel_model model, const char *prompt, const char **error);
+
+void llmodel_model_foreach_special_token(llmodel_model model, llmodel_special_token_callback callback);
+
 #ifdef __cplusplus
 }
 #endif

gpt4all-backend/src/llamamodel.cpp

@@ -202,7 +202,7 @@ static int32_t get_arch_key_u32(std::string const &modelPath, std::string const
         if (keyidx != -1) {
             value = gguf_get_val_u32(ctx, keyidx);
         } else {
-            std::cerr << __func__ << ": " << key << "not found in " << modelPath << "\n";
+            std::cerr << __func__ << ": " << key << " not found in " << modelPath << "\n";
         }
     }
 
@@ -218,6 +218,7 @@ struct LLamaPrivate {
     int64_t                      n_threads    = 0;
     std::vector<LLModel::Token>  end_tokens;
     const char                  *backend_name = nullptr;
+    std::vector<LLModel::Token>  inputTokens;
 
     llama_model          *model        = nullptr;
     llama_context        *ctx          = nullptr;
@@ -501,28 +502,29 @@ size_t LLamaModel::stateSize() const
     return llama_state_get_size(d_ptr->ctx);
 }
 
-size_t LLamaModel::saveState(std::span<uint8_t> dest) const
+size_t LLamaModel::saveState(std::span<uint8_t> stateOut, std::vector<Token> &inputTokensOut) const
 {
-    return llama_state_get_data(d_ptr->ctx, dest.data(), dest.size());
+    size_t bytesWritten = llama_state_get_data(d_ptr->ctx, stateOut.data(), stateOut.size());
+    if (bytesWritten)
+        inputTokensOut.assign(d_ptr->inputTokens.begin(), d_ptr->inputTokens.end());
+    return bytesWritten;
 }
 
-size_t LLamaModel::restoreState(std::span<const uint8_t> src)
+size_t LLamaModel::restoreState(std::span<const uint8_t> state, std::span<const Token> inputTokens)
 {
-    return llama_state_set_data(d_ptr->ctx, src.data(), src.size());
+    size_t bytesRead = llama_state_set_data(d_ptr->ctx, state.data(), state.size());
+    if (bytesRead)
+        d_ptr->inputTokens.assign(inputTokens.begin(), inputTokens.end());
+    return bytesRead;
 }
 
-std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, std::string_view str, bool special)
+std::vector<LLModel::Token> LLamaModel::tokenize(std::string_view str) const
 {
-    bool atStart = m_tokenize_last_token == -1;
-    bool insertSpace = atStart || isSpecialToken(m_tokenize_last_token);
     std::vector<LLModel::Token> fres(str.length() + 4);
-    int32_t fres_len = llama_tokenize_gpt4all(
-        d_ptr->model, str.data(), str.length(), fres.data(), fres.size(), /*add_special*/ atStart,
-        /*parse_special*/ special, /*insert_space*/ insertSpace
+    int32_t fres_len = llama_tokenize(
+        d_ptr->model, str.data(), str.length(), fres.data(), fres.size(), /*add_special*/ true, /*parse_special*/ true
     );
     fres.resize(fres_len);
-    if (fres_len)
-        m_tokenize_last_token = fres.back();
     return fres;
 }
 
@@ -548,7 +550,7 @@ std::string LLamaModel::tokenToString(Token id) const
     return std::string(result.data(), result.size());
 }
 
-void LLamaModel::initSampler(PromptContext &promptCtx)
+void LLamaModel::initSampler(const PromptContext &promptCtx)
 {
     auto *model = d_ptr->model;
     auto *chain = d_ptr->sampler_chain;
@@ -582,7 +584,8 @@ void LLamaModel::initSampler(PromptContext &promptCtx)
             llama_sampler_init_top_p(promptCtx.top_p, 1),
             llama_sampler_init_min_p(promptCtx.min_p, 1),
             llama_sampler_init_temp(promptCtx.temp),
-            llama_sampler_init_dist(LLAMA_DEFAULT_SEED)
+            llama_sampler_init_softmax(),
+            llama_sampler_init_dist(LLAMA_DEFAULT_SEED),
         };
         for (auto *smpl : samplers)
             llama_sampler_chain_add(chain, smpl);
@@ -594,9 +597,11 @@ LLModel::Token LLamaModel::sampleToken() const
     return llama_sampler_sample(d_ptr->sampler_chain, d_ptr->ctx, -1);
 }
 
-bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
+bool LLamaModel::evalTokens(int32_t nPast, std::span<const Token> tokens) const
 {
-    llama_kv_cache_seq_rm(d_ptr->ctx, 0, ctx.n_past, -1);
+    assert(!tokens.empty());
+
+    llama_kv_cache_seq_rm(d_ptr->ctx, 0, nPast, -1);
 
     llama_batch batch = llama_batch_init(tokens.size(), 0, 1);
 
@@ -604,7 +609,7 @@ bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &toke
 
     for (int32_t i = 0; i < batch.n_tokens; i++) {
         batch.token   [i] = tokens[i];
-        batch.pos     [i] = ctx.n_past + i;
+        batch.pos     [i] = nPast + i;
         batch.n_seq_id[i] = 1;
         batch.seq_id  [i][0] = 0;
         batch.logits  [i] = false;
@@ -618,14 +623,14 @@ bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &toke
     return res == 0;
 }
 
-void LLamaModel::shiftContext(PromptContext &promptCtx)
+void LLamaModel::shiftContext(const PromptContext &promptCtx, int32_t *nPast)
 {
     // infinite text generation via context shifting
 
     // erase up to n_ctx*contextErase tokens
     int n_keep = shouldAddBOS();
-    int n_past = promptCtx.n_past;
-    int n_discard = std::min(n_past - n_keep, int(promptCtx.n_ctx * promptCtx.contextErase));
+    int n_past = *nPast;
+    int n_discard = std::min(n_past - n_keep, int(contextLength() * promptCtx.contextErase));
 
     assert(n_discard > 0);
     if (n_discard <= 0)
@@ -638,8 +643,9 @@ void LLamaModel::shiftContext(PromptContext &promptCtx)
     llama_kv_cache_seq_rm (d_ptr->ctx, 0, n_keep,             n_keep + n_discard);
     llama_kv_cache_seq_add(d_ptr->ctx, 0, n_keep + n_discard, n_past,             -n_discard);
 
-    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
-    promptCtx.n_past = promptCtx.tokens.size();
+    auto &inp = d_ptr->inputTokens;
+    inp.erase(inp.begin() + n_keep, inp.begin() + n_keep + n_discard);
+    *nPast = inp.size();
 }
 
 int32_t LLamaModel::contextLength() const
@@ -647,6 +653,56 @@ int32_t LLamaModel::contextLength() const
     return llama_n_ctx(d_ptr->ctx);
 }
 
+auto LLamaModel::specialTokens() -> std::unordered_map<std::string, std::string> const
+{
+    if (!d_ptr->model)
+        throw std::logic_error("model not loaded");
+
+    std::unordered_map<std::string, std::string> tokens;
+    if (auto id = llama_token_bos(d_ptr->model); id != LLAMA_TOKEN_NULL)
+        tokens.emplace("bos_token", tokenToString(id));
+    if (auto id = llama_token_eos(d_ptr->model); id != LLAMA_TOKEN_NULL)
+        tokens.emplace("eos_token", tokenToString(id));
+    return tokens;
+}
+
+int32_t LLamaModel::inputLength() const
+{
+    return d_ptr->inputTokens.size();
+}
+
+int32_t LLamaModel::computeModelInputPosition(std::span<const Token> input) const
+{
+    // find common prefix
+    auto cacheIt = d_ptr->inputTokens.begin();
+    auto inputIt = input.begin();
+    while (cacheIt < d_ptr->inputTokens.end() && inputIt < input.end() && *cacheIt == *inputIt) {
+        ++cacheIt; ++inputIt;
+    }
+    // tell the caller to ignore the tokens between [begin, inputIt)
+    return inputIt - input.begin();
+}
+
+void LLamaModel::setModelInputPosition(int32_t pos)
+{
+    auto &inp = d_ptr->inputTokens;
+    assert(pos >= 0);
+    assert(pos <= inp.size());
+    // truncate token cache to end at the new n_past
+    if (pos < inp.size())
+        inp.resize(pos);
+}
+
+void LLamaModel::appendInputToken(Token tok)
+{
+    d_ptr->inputTokens.push_back(tok);
+}
+
+auto LLamaModel::inputTokens() const -> std::span<const Token>
+{
+    return d_ptr->inputTokens;
+}
+
 const std::vector<LLModel::Token> &LLamaModel::endTokens() const
 {
     return d_ptr->end_tokens;
@@ -667,6 +723,37 @@ int32_t LLamaModel::layerCount(std::string const &modelPath) const
     return get_arch_key_u32(modelPath, "block_count");
 }
 
+// TODO(jared): reduce redundant code and operations by combining all metadata getters for unloaded
+//              models into a class that keeps the model file open
+auto LLamaModel::chatTemplate(const char *modelPath) const -> std::expected<std::string, std::string>
+{
+    auto *ctx = load_gguf(modelPath);
+    if (!ctx)
+        return std::unexpected("failed to open model file");
+
+    std::expected<std::string, std::string> result;
+    enum gguf_type ktype;
+    const int kid = gguf_find_key(ctx, "tokenizer.chat_template");
+    if (kid == -1) {
+        result = std::unexpected("key not found");
+        goto cleanup;
+    }
+
+    ktype = gguf_get_kv_type(ctx, kid);
+    if (ktype != GGUF_TYPE_STRING) {
+        result = std::unexpected(
+            "expected key type STRING (" + std::to_string(GGUF_TYPE_STRING) + "), got " + std::to_string(ktype)
+        );
+        goto cleanup;
+    }
+
+    result = gguf_get_val_str(ctx, kid);
+
+cleanup:
+    gguf_free(ctx);
+    return result;
+}
+
 #ifdef GGML_USE_VULKAN
 static const char *getVulkanVendorName(uint32_t vendorID)
 {

gpt4all-backend/src/llamamodel_impl.h

@@ -11,6 +11,7 @@
 #include <string>
 #include <string_view>
 #include <vector>
+#include <unordered_map>
 
 struct LLamaPrivate;
 struct EmbModelSpec;
@@ -28,8 +29,8 @@ public:
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
-    size_t saveState(std::span<uint8_t> dest) const override;
-    size_t restoreState(std::span<const uint8_t> src) override;
+    size_t saveState(std::span<uint8_t> stateOut, std::vector<Token> &inputTokensOut) const override;
+    size_t restoreState(std::span<const uint8_t> state, std::span<const Token> inputTokens) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
     std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0) const override;
@@ -48,28 +49,36 @@ public:
     void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality = -1,
                size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
 
-private:
-    std::unique_ptr<LLamaPrivate> d_ptr;
-    bool m_supportsEmbedding = false;
-    bool m_supportsCompletion = false;
+    int32_t contextLength() const override;
+    auto specialTokens() -> std::unordered_map<std::string, std::string> const override;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special) override;
+    std::vector<Token> tokenize(std::string_view str) const override;
     bool isSpecialToken(Token id) const override;
     std::string tokenToString(Token id) const override;
-    void initSampler(PromptContext &ctx) override;
+    void initSampler(const PromptContext &ctx) override;
     Token sampleToken() const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    void shiftContext(PromptContext &promptCtx) override;
-    int32_t contextLength() const override;
+    bool evalTokens(int32_t nPast, std::span<const Token> tokens) const override;
+    void shiftContext(const PromptContext &promptCtx, int32_t *nPast) override;
+    int32_t inputLength() const override;
+    int32_t computeModelInputPosition(std::span<const Token> input) const override;
+    void setModelInputPosition(int32_t pos) override;
+    void appendInputToken(Token tok) override;
+    std::span<const Token> inputTokens() const override;
     const std::vector<Token> &endTokens() const override;
     bool shouldAddBOS() const override;
     int32_t maxContextLength(std::string const &modelPath) const override;
     int32_t layerCount(std::string const &modelPath) const override;
+    auto chatTemplate(const char *modelPath) const -> std::expected<std::string, std::string> override;
 
     void embedInternal(const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
                        size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb,
                        const EmbModelSpec *spec);
+
+private:
+    std::unique_ptr<LLamaPrivate> d_ptr;
+    bool m_supportsEmbedding = false;
+    bool m_supportsCompletion = false;
 };
 
 #endif // LLAMAMODEL_H

gpt4all-backend/src/llmodel.cpp

@@ -326,6 +326,12 @@ bool LLModel::Implementation::isEmbeddingModel(const std::string &modelPath)
     return llama && llama->isEmbeddingModel(modelPath);
 }
 
+auto LLModel::Implementation::chatTemplate(const char *modelPath) -> std::expected<std::string, std::string>
+{
+    auto *llama = constructGlobalLlama();
+    return llama ? llama->chatTemplate(modelPath) : std::unexpected("backend not available");
+}
+
 void LLModel::Implementation::setImplementationsSearchPath(const std::string& path)
 {
     s_implementations_search_path = path;

gpt4all-backend/src/llmodel_c.cpp

@@ -7,17 +7,20 @@
 #include <cstdlib>
 #include <cstring>
 #include <exception>
-#include <functional>
 #include <iostream>
 #include <memory>
 #include <optional>
 #include <string>
 #include <string_view>
 #include <vector>
+#include <span>
+
+namespace ranges = std::ranges;
+
+static_assert(sizeof(token_t) == sizeof(LLModel::Token));
 
 struct LLModelWrapper {
     LLModel *llModel = nullptr;
-    LLModel::PromptContext promptContext;
     ~LLModelWrapper() { delete llModel; }
 };
 
@@ -85,74 +88,80 @@ bool llmodel_isModelLoaded(llmodel_model model)
     return wrapper->llModel->isModelLoaded();
 }
 
-uint64_t llmodel_get_state_size(llmodel_model model)
+uint64_t llmodel_state_get_size(llmodel_model model)
 {
     auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->stateSize();
 }
 
-uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest, uint64_t size)
+uint64_t llmodel_state_get_data(llmodel_model model, uint8_t *state_out, uint64_t state_size,
+                                token_t **input_tokens_out, uint64_t *n_input_tokens)
 {
     auto *wrapper = static_cast<LLModelWrapper *>(model);
-    return wrapper->llModel->saveState({dest, size_t(size)});
+    std::vector<LLModel::Token> inputTokens;
+    auto bytesWritten = wrapper->llModel->saveState({state_out, size_t(state_size)}, inputTokens);
+    if (bytesWritten) {
+        auto *buf = new LLModel::Token[inputTokens.size()];
+        ranges::copy(inputTokens, buf);
+        *input_tokens_out = buf;
+        *n_input_tokens = uint64_t(inputTokens.size());
+    } else {
+        *input_tokens_out = nullptr;
+        *n_input_tokens = 0;
+    }
+    return bytesWritten;
 }
 
-uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src, uint64_t size)
+void llmodel_state_free_input_tokens(LLModel::Token *input_tokens)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
-    return wrapper->llModel->restoreState({src, size_t(size)});
+    delete[] input_tokens;
 }
 
-void llmodel_prompt(llmodel_model model, const char *prompt,
-                    const char *prompt_template,
-                    llmodel_prompt_callback prompt_callback,
-                    llmodel_response_callback response_callback,
-                    bool allow_context_shift,
-                    llmodel_prompt_context *ctx,
-                    bool special,
-                    const char *fake_reply)
+uint64_t llmodel_state_set_data(llmodel_model model, const uint8_t *state, uint64_t state_size,
+                                const token_t *input_tokens, uint64_t n_input_tokens)
 {
     auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->restoreState({state, size_t(state_size)}, {input_tokens, size_t(n_input_tokens)});
+}
 
-    auto response_func = [response_callback](int32_t token_id, const std::string &response) {
-        return response_callback(token_id, response.c_str());
-    };
+bool llmodel_prompt(llmodel_model               model,
+                    const char                 *prompt,
+                    llmodel_prompt_callback     prompt_callback,
+                    llmodel_response_callback   response_callback,
+                    llmodel_prompt_context     *ctx,
+                    const char                **error)
+{
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
 
     // Copy the C prompt context
-    wrapper->promptContext.n_past = ctx->n_past;
-    wrapper->promptContext.n_ctx = ctx->n_ctx;
-    wrapper->promptContext.n_predict = ctx->n_predict;
-    wrapper->promptContext.top_k = ctx->top_k;
-    wrapper->promptContext.top_p = ctx->top_p;
-    wrapper->promptContext.min_p = ctx->min_p;
-    wrapper->promptContext.temp = ctx->temp;
-    wrapper->promptContext.n_batch = ctx->n_batch;
-    wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
-    wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
-    wrapper->promptContext.contextErase = ctx->context_erase;
+    LLModel::PromptContext promptContext {
+        .n_predict      = ctx->n_predict,
+        .top_k          = ctx->top_k,
+        .top_p          = ctx->top_p,
+        .min_p          = ctx->min_p,
+        .temp           = ctx->temp,
+        .n_batch        = ctx->n_batch,
+        .repeat_penalty = ctx->repeat_penalty,
+        .repeat_last_n  = ctx->repeat_last_n,
+        .contextErase   = ctx->context_erase,
+    };
+
+    auto prompt_func = [prompt_callback](std::span<const LLModel::Token> token_ids, bool cached) {
+        return prompt_callback(token_ids.data(), token_ids.size(), cached);
+    };
+    auto response_func = [response_callback](LLModel::Token token_id, std::string_view piece) {
+        return response_callback(token_id, piece.data());
+    };
 
     // Call the C++ prompt method
-    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, allow_context_shift,
-                             wrapper->promptContext, special,
-                             fake_reply ? std::make_optional<std::string_view>(fake_reply) : std::nullopt);
+    try {
+        wrapper->llModel->prompt(prompt, prompt_func, response_func, promptContext);
+    } catch (std::exception const &e) {
+        llmodel_set_error(error, e.what());
+        return false;
+    }
 
-    // Update the C context by giving access to the wrappers raw pointers to std::vector data
-    // which involves no copies
-    ctx->tokens = wrapper->promptContext.tokens.data();
-    ctx->tokens_size = wrapper->promptContext.tokens.size();
-
-    // Update the rest of the C prompt context
-    ctx->n_past = wrapper->promptContext.n_past;
-    ctx->n_ctx = wrapper->promptContext.n_ctx;
-    ctx->n_predict = wrapper->promptContext.n_predict;
-    ctx->top_k = wrapper->promptContext.top_k;
-    ctx->top_p = wrapper->promptContext.top_p;
-    ctx->min_p = wrapper->promptContext.min_p;
-    ctx->temp = wrapper->promptContext.temp;
-    ctx->n_batch = wrapper->promptContext.n_batch;
-    ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
-    ctx->repeat_last_n = wrapper->promptContext.repeat_last_n;
-    ctx->context_erase = wrapper->promptContext.contextErase;
+    return true;
 }
 
 float *llmodel_embed(
@@ -291,3 +300,21 @@ const char *llmodel_model_gpu_device_name(llmodel_model model)
     const auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->gpuDeviceName();
 }
+
+int32_t llmodel_count_prompt_tokens(llmodel_model model, const char *prompt, const char **error)
+{
+    auto *wrapper = static_cast<const LLModelWrapper *>(model);
+    try {
+        return wrapper->llModel->countPromptTokens(prompt);
+    } catch (const std::exception& e) {
+        llmodel_set_error(error, e.what());
+        return -1;
+    }
+}
+
+void llmodel_model_foreach_special_token(llmodel_model model, llmodel_special_token_callback callback)
+{
+    auto *wrapper = static_cast<const LLModelWrapper *>(model);
+    for (auto &[name, token] : wrapper->llModel->specialTokens())
+        callback(name.c_str(), token.c_str());
+}

gpt4all-backend/src/llmodel_shared.cpp

@@ -4,211 +4,120 @@
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
-#include <functional>
 #include <iostream>
+#include <iterator>
 #include <optional>
-#include <regex>
-#include <sstream>
+#include <ranges>
 #include <stdexcept>
 #include <string>
 #include <string_view>
 #include <vector>
 
 namespace ranges = std::ranges;
+namespace views  = std::ranges::views;
 
-static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err)
-{
-    static const std::regex placeholderRegex(R"(%[1-2](?![0-9]))");
+void LLModel::prompt(
+    std::string_view        prompt,
+    const PromptCallback   &promptCallback,
+    const ResponseCallback &responseCallback,
+    const PromptContext    &promptCtx
+) {
+    if (!isModelLoaded())
+        throw std::invalid_argument("Attempted to prompt an unloaded model.");
+    if (!supportsCompletion())
+        throw std::invalid_argument("Not a text completion model.");
+    if (!promptCtx.n_batch)
+        throw std::invalid_argument("Batch size cannot be zero.");
+    if (!promptCtx.n_predict)
+        return; // nothing requested
 
-    auto it = std::sregex_iterator(tmpl.begin(), tmpl.end(), placeholderRegex);
-    placeholders.clear();
-    placeholders.insert(placeholders.end(), it, std::sregex_iterator());
+    auto embd_inp = tokenize(prompt);
+    if (embd_inp.empty())
+        throw std::invalid_argument("Prompt tokenized to zero tokens.");
 
-    if (placeholders.size() > 2) {
-        err = "ERROR: expected at most two placeholders, got " + std::to_string(placeholders.size());
-        return false;
-    }
-    if (placeholders.size() >= 1 && placeholders[0].str() != "%1") {
-        err = "ERROR: first placeholder must be %1, got " + placeholders[0].str();
-        return false;
-    }
-    if (placeholders.size() >= 2 && placeholders[1].str() != "%2") {
-        err = "ERROR: second placeholder must be %2, got " + placeholders[1].str();
-        return false;
-    }
-    return true;
+    if (auto res = decodePrompt(promptCallback, promptCtx, std::move(embd_inp)))
+        generateResponse(responseCallback, promptCtx, /*n_past*/ *res);
 }
 
-void LLModel::prompt(const std::string &prompt,
-                     const std::string &promptTemplate,
-                     std::function<bool(int32_t)> promptCallback,
-                     std::function<bool(int32_t, const std::string&)> responseCallback,
-                     bool allowContextShift,
-                     PromptContext &promptCtx,
-                     bool special,
-                     std::optional<std::string_view> fakeReply)
+int32_t LLModel::countPromptTokens(std::string_view prompt) const
 {
-    if (!isModelLoaded()) {
-        std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
-        return;
-    }
-
-    if (!supportsCompletion()) {
-        std::string errorMessage = "ERROR: this model does not support text completion or chat!";
-        responseCallback(-1, errorMessage);
-        std::cerr << implementation().modelType() << " " << errorMessage << "\n";
-        return;
-    }
-
-    // sanity checks
-    if (promptCtx.n_past > contextLength()) {
-        std::ostringstream ss;
-        ss << "n_past=" << promptCtx.n_past << " is past end of context length=" << contextLength();
-        throw std::out_of_range(ss.str());
-    }
-    if (promptCtx.n_past > promptCtx.tokens.size()) {
-        std::ostringstream ss;
-        ss << "n_past=" << promptCtx.n_past << " is past end of token cache length=" << promptCtx.tokens.size();
-        throw std::out_of_range(ss.str());
-    }
-
-    promptCtx.n_ctx = contextLength();
-    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
-
-    if (promptCtx.n_past < promptCtx.tokens.size())
-        promptCtx.tokens.resize(promptCtx.n_past);
-    m_tokenize_last_token = promptCtx.tokens.empty() ? -1 : promptCtx.tokens.back(); // not serialized
-
-    // parse the prompt template
-    std::vector<std::smatch> placeholders;
-    {
-        std::string err;
-        if (!parsePromptTemplate(promptTemplate, placeholders, err)) {
-            responseCallback(-1, err);
-            std::cerr << err << "\n";
-            return;
-        }
-    }
-
-    auto old_n_past = promptCtx.n_past; // prepare to fake n_past for tokenize
-
-    // tokenize the user prompt
-    std::vector<Token> embd_inp;
-    if (placeholders.empty()) {
-        // this is unusual, but well-defined
-        std::cerr << __func__ << ": prompt template has no placeholder\n";
-        embd_inp = tokenize(promptCtx, promptTemplate, true);
-    } else {
-        // template: beginning of user prompt
-        const auto &phUser = placeholders[0];
-        std::string userPrefix(phUser.prefix());
-        if (!userPrefix.empty()) {
-            embd_inp = tokenize(promptCtx, userPrefix, true);
-            promptCtx.n_past += embd_inp.size();
-        }
-
-        // user input (shouldn't have special token processing)
-        auto tokens = tokenize(promptCtx, prompt, special);
-        embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
-        promptCtx.n_past += tokens.size();
-
-        // template: end of user prompt + start of assistant prompt
-        size_t start = phUser.position() + phUser.length();
-        size_t end = placeholders.size() >= 2 ? placeholders[1].position() : promptTemplate.length();
-        auto userToAsst = promptTemplate.substr(start, end - start);
-        if (!userToAsst.empty()) {
-            tokens = tokenize(promptCtx, userToAsst, true);
-            embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
-            promptCtx.n_past += tokens.size();
-        }
-    }
-
-    promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
-
-    // decode the user prompt
-    if (!decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp))
-        return; // error
-
-    // decode the assistant's reply, either generated or spoofed
-    if (!fakeReply) {
-        generateResponse(responseCallback, allowContextShift, promptCtx);
-    } else {
-        embd_inp = tokenize(promptCtx, *fakeReply, false);
-        if (!decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp, true))
-            return; // error
-    }
-
-    // decode the rest of the prompt template
-    // template: end of assistant prompt
-    std::string asstSuffix;
-    if (placeholders.size() >= 2) {
-        size_t start = placeholders[1].position() + placeholders[1].length();
-        asstSuffix = promptTemplate.substr(start);
-    } else {
-        asstSuffix = "\n\n"; // default to a blank link, good for e.g. Alpaca
-    }
-    if (!asstSuffix.empty()) {
-        embd_inp = tokenize(promptCtx, asstSuffix, true);
-        decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp);
-    }
+    if (!isModelLoaded())
+        throw std::invalid_argument("Attempted to tokenize with an unloaded model.");
+    return int32_t(tokenize(prompt).size());
 }
 
-// returns false on error
-bool LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
-                           std::function<bool(int32_t, const std::string&)> responseCallback,
-                           bool allowContextShift,
-                           PromptContext &promptCtx,
-                           std::vector<Token> embd_inp,
-                           bool isResponse) {
-    if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
-        // FIXME: (Adam) We should find a way to bubble these strings to the UI level to allow for
-        // translation
-        responseCallback(-1, "Your message was too long and could not be processed. Please try again with something shorter.");
-        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
-            " tokens and the context window is " << promptCtx.n_ctx << "!\n";
-        return false;
+auto LLModel::decodePrompt(
+    const PromptCallback &promptCallback,
+    const PromptContext  &promptCtx,
+    std::vector<Token>    embd_inp
+) -> std::optional<int32_t>
+{
+    assert(!embd_inp.empty());
+
+    int32_t nCtx = contextLength();
+    int32_t n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
+
+    // Find the greatest n_past where the beginning of embd_inp matches the end of the token cache, starting at the
+    // requested n_past.
+    // This is used to skip unnecessary work when the prompt shares a common prefix with the previous result.
+    int32_t nPast = computeModelInputPosition(embd_inp);
+
+    // always decode up to a full batch before generating, even if cached
+    nPast -= std::min(n_batch, nPast);
+
+    // TODO(jared): generalize this to find the smallest new_embd_inp.size() - nPast given the cache
+    if (!nPast && int32_t(embd_inp.size()) > nCtx) {
+        // no cache hit -> shift the input before even processing
+
+        int32_t nKeep     = shouldAddBOS();
+        auto    newLength = int32_t(nCtx * (1.f - promptCtx.contextErase));
+        int32_t nDiscard  = int32_t(embd_inp.size()) - std::max(1, std::min(nCtx, newLength));
+
+        // execute the callback even for skipped tokens. this misrepresents the position of BOS but we don't care
+        auto discardedTokens = embd_inp | views::drop(nKeep) | views::take(nDiscard);
+        if (!promptCallback(discardedTokens, true))
+            return std::nullopt;
+
+        // erase nDiscard tokens
+        embd_inp.erase(discardedTokens.begin(), discardedTokens.end());
+        assert(int32_t(embd_inp.size()) <= nCtx);
+
+        // check the cache again, just in case
+        nPast = computeModelInputPosition(embd_inp);
+        nPast -= std::min(n_batch, nPast);
     }
 
-    // FIXME(jared): There are mitigations for this situation, such as making room before
-    // copying the prompt context, or restoring the KV cache when we restore the prompt
-    // context.
-    if (!allowContextShift && promptCtx.n_past + embd_inp.size() > promptCtx.n_ctx) {
-        std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_eval=" << embd_inp.size()
-                  << ", n_ctx=" << promptCtx.n_ctx << "\n";
-        return false;
-    }
+    setModelInputPosition(nPast);
+
+    // execute the callback even for skipped tokens
+    if (!promptCallback(embd_inp | views::take(nPast), true))
+        return std::nullopt;
 
     // process the prompt in batches
-    size_t i = 0;
-    while (i < embd_inp.size()) {
-        size_t batch_end = std::min(i + promptCtx.n_batch, embd_inp.size());
-        std::vector<Token> batch(embd_inp.begin() + i, embd_inp.begin() + batch_end);
+    for (int32_t i = nPast; i < embd_inp.size();) {
+        auto batch_end = std::min(i + n_batch, int32_t(embd_inp.size()));
+        std::span batch(embd_inp.begin() + i, embd_inp.begin() + batch_end);
 
         // Check if the context has run out...
-        if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
-            assert(allowContextShift);
-            shiftContext(promptCtx);
-            assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
+        if (nPast + int32_t(batch.size()) > nCtx) {
+            shiftContext(promptCtx, &nPast);
+            assert(nPast + int32_t(batch.size()) <= nCtx);
         }
 
-        if (!evalTokens(promptCtx, batch)) {
-            std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
-            return false;
-        }
+        // FIXME(Adam): We should find a way to bubble these strings to the UI level to allow for translation
+        if (!evalTokens(nPast, batch))
+            throw std::runtime_error("An internal error was encountered during prompt processing.");
 
-        size_t tokens = batch_end - i;
-        for (size_t t = 0; t < tokens; ++t) {
-            promptCtx.tokens.push_back(batch.at(t));
-            promptCtx.n_past += 1;
-            Token tok = batch.at(t);
-            bool res = isResponse ? responseCallback(tok, tokenToString(tok)) : promptCallback(tok);
-            if (!res)
-                return false;
+        for (auto &tok : batch) {
+            appendInputToken(tok);
+            nPast++;
+            if (!promptCallback({ &tok, 1 }, false))
+                return std::nullopt;
         }
         i = batch_end;
     }
 
-    return true;
+    return nPast;
 }
 
 /*
@@ -230,22 +139,16 @@ static std::string::size_type stringsOverlap(const std::string &s, const std::st
     return std::string::npos;
 }
 
-void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
-                               bool allowContextShift,
-                               PromptContext &promptCtx) {
+void LLModel::generateResponse(
+    const ResponseCallback &responseCallback,
+    const PromptContext    &promptCtx,
+    int32_t                 nPast
+) {
     static const char *stopSequences[] {
-        "### Instruction", "### Prompt", "### Response", "### Human", "### Assistant", "### Context",
+        "### System", "### Instruction", "### Human", "### User", "### Response", "### Assistant", "### Context",
+        "<|im_start|>", "<|im_end|>", "<|endoftext|>",
     };
 
-    // Don't even start if there is no room
-    if (!promptCtx.n_predict)
-        return;
-    if (!allowContextShift && promptCtx.n_past >= promptCtx.n_ctx) {
-        std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_ctx=" << promptCtx.n_ctx
-                  << "\n";
-        return;
-    }
-
     initSampler(promptCtx);
 
     std::string cachedResponse;
@@ -260,26 +163,20 @@ void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)>
         cachedTokens.push_back(new_tok.value());
         cachedResponse += new_piece;
 
-        auto accept = [this, &promptCtx, &new_tok, allowContextShift]() -> bool {
+        auto accept = [this, &promptCtx, &new_tok, &nPast] {
             // Shift context if out of space
-            if (promptCtx.n_past >= promptCtx.n_ctx) {
-                (void)allowContextShift;
-                assert(allowContextShift);
-                shiftContext(promptCtx);
-                assert(promptCtx.n_past < promptCtx.n_ctx);
+            if (nPast >= contextLength()) {
+                shiftContext(promptCtx, &nPast);
+                assert(nPast < contextLength());
             }
 
             // Accept the token
             Token tok = std::exchange(new_tok, std::nullopt).value();
-            if (!evalTokens(promptCtx, { tok })) {
-                // TODO(jared): raise an exception
-                std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
-                return false;
-            }
+            if (!evalTokens(nPast, { &tok, 1 }))
+                throw std::runtime_error("An internal error was encountered during response generation.");
 
-            promptCtx.tokens.push_back(tok);
-            promptCtx.n_past += 1;
-            return true;
+            appendInputToken(tok);
+            nPast++;
         };
 
         // Check for EOS
@@ -316,13 +213,6 @@ void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)>
             lengthLimit = cachedResponse.size() - new_piece.size();
         }
 
-        // Optionally stop if the context will run out
-        if (!allowContextShift && promptCtx.n_past + cachedTokens.size() >= promptCtx.n_ctx) {
-            std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_ctx="
-                      << promptCtx.n_ctx << "\n";
-            stop = true;
-        }
-
         // Empty the cache, up to the length limit
         std::string::size_type responseLength = 0;
         while (!cachedTokens.empty()) {
@@ -339,8 +229,8 @@ void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)>
             cachedResponse.erase(cachedResponse.begin(), cachedResponse.begin() + piece.size());
 
             // Accept the token, if needed (not cached)
-            if (cachedTokens.empty() && new_tok && !accept())
-                return;
+            if (cachedTokens.empty() && new_tok)
+                accept();
 
             // Send the token
             if (!responseCallback(tok, piece) || ++n_predicted >= promptCtx.n_predict) {
@@ -359,24 +249,23 @@ void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)>
             assert(!cachedTokens.empty() && cachedTokens.back() == new_tok);
             if (stop) {
                 cachedTokens.pop_back();
-            } else if (!accept()) {
-                return;
+            } else {
+                accept();
             }
         }
     }
 
-    auto &tokens = promptCtx.tokens;
-    if (tokens.size() < cachedTokens.size()) {
+    if (inputLength() < cachedTokens.size()) {
         /* This is theoretically possible if the longest stop sequence is greater than
          * n_ctx * contextErase tokens. */
         throw std::runtime_error("shifted too much context, can't go back");
     }
 
-    auto discard_start = tokens.end() - cachedTokens.size();
-    assert(std::equal(discard_start, tokens.end(), cachedTokens.begin()));
-    tokens.erase(discard_start, tokens.end());
-
-    promptCtx.n_past -= cachedTokens.size();
+#ifndef NDEBUG
+    auto inp = inputTokens();
+    auto discard_start = inp.end() - cachedTokens.size();
+    assert(std::equal(discard_start, inp.end(), cachedTokens.begin()));
+#endif
 }
 
 void LLModel::embed(

gpt4all-bindings/cli/app.py

@@ -113,10 +113,7 @@ def _old_loop(gpt4all_instance):
         full_response = gpt4all_instance.chat_completion(
             MESSAGES,
             # preferential kwargs for chat ux
-            logits_size=0,
-            tokens_size=0,
             n_past=0,
-            n_ctx=0,
             n_predict=200,
             top_k=40,
             top_p=0.9,

gpt4all-bindings/python/CHANGELOG.md

@@ -8,11 +8,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ### Added
 - Warn on Windows if the Microsoft Visual C++ runtime libraries are not found ([#2920](https://github.com/nomic-ai/gpt4all/pull/2920))
+- Basic cache for faster prefill when the input shares a prefix with previous context ([#3073](https://github.com/nomic-ai/gpt4all/pull/3073))
+- Add ability to modify or replace the history of an active chat session ([#3147](https://github.com/nomic-ai/gpt4all/pull/3147))
 
 ### Changed
 - Rebase llama.cpp on latest upstream as of September 26th ([#2998](https://github.com/nomic-ai/gpt4all/pull/2998))
 - Change the error message when a message is too long ([#3004](https://github.com/nomic-ai/gpt4all/pull/3004))
 - Fix CalledProcessError on Intel Macs since v2.8.0 ([#3045](https://github.com/nomic-ai/gpt4all/pull/3045))
+- Use Jinja for chat templates instead of per-message QString.arg-style templates ([#3147](https://github.com/nomic-ai/gpt4all/pull/3147))
 
 ## [2.8.2] - 2024-08-14
 

gpt4all-bindings/python/docs/gpt4all_desktop/chat_templates.md

@@ -0,0 +1,206 @@
+## What are chat templates?
+Natively, large language models only know how to complete plain text and do not know the difference between their input and their output. In order to support a chat with a person, LLMs are designed to use a template to convert the conversation to plain text using a specific format.
+
+For a given model, it is important to use an appropriate chat template, as each model is designed to work best with a specific format. The chat templates included with the built-in models should be sufficient for most purposes.
+
+There are two reasons you would want to alter the chat template:
+
+- You are sideloading a model and there is no chat template available,
+- You would like to have greater control over the input to the LLM than a system message provides.
+
+
+## What is a system message?
+A system message is a message that controls the responses from the LLM in a way that affects the entire conversation. System messages can be short, such as "Speak like a pirate.", or they can be long and contain a lot of context for the LLM to keep in mind.
+
+Not all models are designed to use a system message, so they work with some models better than others.
+
+
+## How do I customize the chat template or system message?
+To customize the chat template or system message, go to Settings > Model. Make sure to select the correct model at the top. If you clone a model, you can use a different chat template or system message from the base model, enabling you to use different settings for each conversation.
+
+These settings take effect immediately. After changing them, you can click "Redo last response" in the chat view, and the response will take the new settings into account.
+
+
+## Do I need to write a chat template?
+You typically do not need to write your own chat template. The exception is models that are not in the official model list and do not come with a chat template built-in. These will show a "Clear" option above the chat template field in the Model Settings page instead of a "Reset" option. See the section on [finding] or [creating] a chat template.
+
+[finding]: #how-do-i-find-a-chat-template
+[creating]: #advanced-how-do-chat-templates-work
+
+
+## What changed in GPT4All v3.5?
+GPT4All v3.5 overhauled the chat template system. There are three crucial differences:
+
+- The chat template now formats an entire conversation instead of a single pair of messages,
+- The chat template now uses Jinja syntax instead of `%1` and `%2` placeholders,
+- And the system message should no longer contain control tokens or trailing whitespace.
+
+If you are using any chat templates or system messages that had been added or altered from the default before upgrading to GPT4All v3.5 or newer, these will no longer work. See below for how to solve common errors you may see after upgrading.
+
+
+## Error/Warning: System message is not plain text.
+This is easy to fix. Go to the model's settings and look at the system prompt. There are three things to look for:
+
+- Control tokens such as `<|im_start|>`, `<|start_header_id|>`, or `<|system|>`
+- A prefix such as `### System` or `SYSTEM:`
+- Trailing whitespace, such as a space character or blank line.
+
+If you see any of these things, remove them. For example, this legacy system prompt:
+```
+<|start_header_id|>system<|end_header_id|>
+You are a helpful assistant.<|eot_id|>
+```
+
+Should become this:
+```
+You are a helpful assistant.
+```
+
+If you do not see anything that needs to be changed, you can dismiss the error by making a minor modification to the message and then changing it back.
+
+If you see a warning, your system message does not appear to be plain text. If you believe this warning is incorrect, it can be safely ignored. If in doubt, ask on the [Discord].
+
+[Discord]: https://discord.gg/mGZE39AS3e
+
+
+## Error: Legacy system prompt needs to be updated in Settings.
+This is the same as [above][above-1], but appears on the chat page.
+
+[above-1]: #errorwarning-system-message-is-not-plain-text
+
+
+## Error/Warning: Chat template is not in Jinja format.
+This is the result of attempting to use an old-style template (possibly from a previous version) in GPT4All 3.5+.
+
+Go to the Model Settings page and select the affected model. If you see a "Reset" button, and you have not intentionally modified the prompt template, you can click "Reset". Otherwise, this is what you can do:
+
+1. Back up your chat template by copying it safely to a text file and saving it. In the next step, it will be removed from GPT4All.
+2. Click "Reset" or "Clear".
+3. If you clicked "Clear", the chat template is now gone. Follow the steps to [find][finding] or [create][creating] a basic chat template for your model.
+4. Customize the chat template to suit your needs. For help, read the section about [creating] a chat template.
+
+
+## Error: Legacy prompt template needs to be updated in Settings.
+This is the same as [above][above-2], but appears on the chat page.
+
+[above-2]: #errorwarning-chat-template-is-not-in-jinja-format
+
+
+## The chat template has a syntax error.
+If there is a syntax error while editing the chat template, the details will be displayed in an error message above the input box. This could be because the chat template is not actually in Jinja format (see [above][above-2]).
+
+Otherwise, you have either typed something correctly, or the model comes with a template that is incompatible with GPT4All. See [the below section][creating] on creating chat templates and make sure that everything is correct. When in doubt, ask on the [Discord].
+
+
+## Error: No chat template configured.
+This may appear for models that are not from the official model list and do not include a chat template. Older versions of GPT4All picked a poor default in this case. You will get much better results if you follow the steps to [find][finding] or [create][creating] a chat template for your model.
+
+
+## Error: The chat template cannot be blank.
+If the button above the chat template on the Model Settings page says "Clear", see [above][above-3]. If you see "Reset", click that button to restore a reasonable default. Also see the section on [syntax errors][chat-syntax-error].
+
+[above-3]: #error-no-chat-template-configured
+[chat-syntax-error]: #the-chat-template-has-a-syntax-error
+
+
+## How do I find a chat template?
+When in doubt, you can always ask the [Discord] community for help. Below are the instructions to find one on your own.
+
+The authoritative source for a model's chat template is the HuggingFace repo that the original (non-GGUF) model came from. First, you should find this page. If you just have a model file, you can try a google search for the model's name. If you know the page you downloaded the GGUF model from, its README usually links to the original non-GGUF model.
+
+Once you have located the original model, there are two methods you can use to extract its chat template. Pick whichever one you are most comfortable with.
+
+### Using the CLI (all models)
+1. Install `jq` using your preferred package manager - e.g. Chocolatey (Windows), Homebrew (macOS), or apt (Ubuntu).
+2. Download `tokenizer_config.json` from the model's "Files and versions" tab.
+3. Open a command prompt in the directory which you have downloaded the model file.
+4. Run `jq -r ".chat_template" tokenizer_config.json`. This shows the chat template in a human-readable form. You can copy this and paste it into the settings page.
+5. (Optional) You can save the output to a text file like this: `jq -r ".chat_template" tokenizer_config.json >chat_template.txt`
+
+If the output is "null", the model does not provide a chat template. See the [below instructions][creating] on creating a chat template.
+
+### Python (open models)
+1. Install `transformers` using your preferred python package manager, e.g. `pip install transformers`. Make sure it is at least version v4.43.0.
+2. Copy the ID of the HuggingFace model, using the clipboard icon next to the name. For example, if the URL is `https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-8B`, the ID is `NousResearch/Hermes-2-Pro-Llama-3-8B`.
+3. Open a python interpreter (`python`) and run the following commands. Change the model ID in the example to the one you copied.
+```
+>>> from transformers import AutoTokenizer
+>>> tokenizer = AutoTokenizer.from_pretrained('NousResearch/Hermes-2-Pro-Llama-3-8B')
+>>> print(tokenizer.get_chat_template())
+```
+You can copy the output and paste it into the settings page.
+4. (Optional) You can save the output to a text file like this:
+```
+>>> open('chat_template.txt', 'w').write(tokenizer.get_chat_template())
+```
+
+If you get a ValueError exception, this model does not provide a chat template. See the [below instructions][creating] on creating a chat template.
+
+
+### Python (gated models)
+Some models, such as Llama and Mistral, do not allow public access to their chat template. You must either use the CLI method above, or follow the following instructions to use Python:
+
+1. For these steps, you must have git and git-lfs installed.
+2. You must have a HuggingFace account and be logged in.
+3. You must already have access to the gated model. Otherwise, request access.
+4. You must have an SSH key configured for git access to HuggingFace.
+5. `git clone` the model's HuggingFace repo using the SSH clone URL. There is no need to download the entire model, which is very large. A good way to do this on Linux is:
+```console
+$ GIT_LFS_SKIP_SMUDGE=1 git clone hf.co:meta-llama/Llama-3.1-8B-Instruct.git
+$ cd Llama-3.1-8B-Instruct
+$ git lfs pull -I "tokenizer.*"
+```
+6. Follow the above instructions for open models, but replace the model ID with the path to the directory containing `tokenizer\_config.json`:
+```
+>>> tokenizer = AutoTokenizer.from_pretrained('.')
+```
+
+
+## Advanced: How do chat templates work?
+The chat template is applied to the entire conversation you see in the chat window. The template loops over the list of messages, each containing `role` and `content` fields. `role` is either `user`, `assistant`, or `system`.
+
+GPT4All also supports the special variables `bos_token`, `eos_token`, and `add_generation_prompt`. See the [HuggingFace docs] for what those do.
+
+[HuggingFace docs]: https://huggingface.co/docs/transformers/v4.46.3/en/chat_templating#special-variables
+
+
+## Advanced: How do I make a chat template?
+The best way to create a chat template is to start by using an existing one as a reference. Then, modify it to use the format documented for the given model. Its README page may explicitly give an example of its template. Or, it may mention the name of a well-known standard template, such as ChatML, Alpaca, Vicuna. GPT4All does not yet include presets for these templates, so they will have to be found in other models or taken from the community.
+
+For more information, see the very helpful [HuggingFace guide]. Some of this is not applicable, such as the information about tool calling and RAG - GPT4All implements those features differently.
+
+Some models use a prompt template that does not intuitively map to a multi-turn chat, because it is more intended for single instructions. The [FastChat] implementation of these templates is a useful reference for the correct way to extend them to multiple messages.
+
+[HuggingFace guide]: https://huggingface.co/docs/transformers/v4.46.3/en/chat_templating#advanced-template-writing-tips
+[FastChat]: https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py
+
+
+# Advanced: What are GPT4All v1 templates?
+GPT4All supports its own template syntax, which is nonstandard but provides complete control over the way LocalDocs sources and file attachments are inserted into the conversation. These templates begin with `{# gpt4all v1 #}` and look similar to the example below.
+
+For standard templates, GPT4All combines the user message, sources, and attachments into the `content` field. For GPT4All v1 templates, this is not done, so they must be used directly in the template for those features to work correctly.
+
+```jinja
+{# gpt4all v1 #}
+{%- for message in messages %}
+    {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}
+    {%- if message['role'] == 'user' %}
+        {%- for source in message['sources'] %}
+            {%- if loop.first %}
+                {{- '### Context:\n' }}
+            {%- endif %}
+            {{- 'Collection: ' + source['collection'] + '\n'   +
+                'Path: '       + source['path']       + '\n'   +
+                'Excerpt: '    + source['text']       + '\n\n' }}
+        {%- endfor %}
+    {%- endif %}
+    {%- for attachment in message['prompt_attachments'] %}
+        {{- attachment['processed_content'] + '\n\n' }}
+    {%- endfor %}
+    {{- message['content'] | trim }}
+    {{- '<|eot_id|>' }}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
+```

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -9,7 +9,7 @@ import textwrap
 import threading
 from enum import Enum
 from queue import Queue
-from typing import TYPE_CHECKING, Any, Callable, Generic, Iterable, Literal, NoReturn, TypeVar, overload
+from typing import TYPE_CHECKING, Any, Callable, Generic, Iterable, Iterator, Literal, NoReturn, TypeVar, overload
 
 if sys.version_info >= (3, 9):
     import importlib.resources as importlib_resources
@@ -23,7 +23,9 @@ else:
     from typing import TypedDict
 
 if TYPE_CHECKING:
-    from typing_extensions import TypeAlias
+    from typing_extensions import ParamSpec, TypeAlias
+    T = TypeVar("T")
+    P = ParamSpec("P")
 
 EmbeddingsType = TypeVar('EmbeddingsType', bound='list[Any]')
 
@@ -31,7 +33,7 @@ cuda_found: bool = False
 
 
 # TODO(jared): use operator.call after we drop python 3.10 support
-def _operator_call(obj, /, *args, **kwargs):
+def _operator_call(obj: Callable[P, T], /, *args: P.args, **kwargs: P.kwargs) -> T:
     return obj(*args, **kwargs)
 
 
@@ -116,19 +118,15 @@ llmodel = load_llmodel_library()
 
 class LLModelPromptContext(ctypes.Structure):
     _fields_ = [
-        ("tokens", ctypes.POINTER(ctypes.c_int32)),
-        ("tokens_size", ctypes.c_size_t),
-        ("n_past", ctypes.c_int32),
-        ("n_ctx", ctypes.c_int32),
-        ("n_predict", ctypes.c_int32),
-        ("top_k", ctypes.c_int32),
-        ("top_p", ctypes.c_float),
-        ("min_p", ctypes.c_float),
-        ("temp", ctypes.c_float),
-        ("n_batch", ctypes.c_int32),
+        ("n_predict",      ctypes.c_int32),
+        ("top_k",          ctypes.c_int32),
+        ("top_p",          ctypes.c_float),
+        ("min_p",          ctypes.c_float),
+        ("temp",           ctypes.c_float),
+        ("n_batch",        ctypes.c_int32),
         ("repeat_penalty", ctypes.c_float),
-        ("repeat_last_n", ctypes.c_int32),
-        ("context_erase", ctypes.c_float),
+        ("repeat_last_n",  ctypes.c_int32),
+        ("context_erase",  ctypes.c_float),
     ]
 
 
@@ -160,23 +158,21 @@ llmodel.llmodel_required_mem.restype = ctypes.c_size_t
 llmodel.llmodel_isModelLoaded.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_isModelLoaded.restype = ctypes.c_bool
 
-PromptCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32)
-ResponseCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32, ctypes.c_char_p)
-EmbCancelCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.POINTER(ctypes.c_uint), ctypes.c_uint, ctypes.c_char_p)
+PromptCallback       = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.POINTER(ctypes.c_int32), ctypes.c_size_t, ctypes.c_bool)
+ResponseCallback     = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32, ctypes.c_char_p)
+EmbCancelCallback    = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.POINTER(ctypes.c_uint), ctypes.c_uint, ctypes.c_char_p)
+SpecialTokenCallback = ctypes.CFUNCTYPE(None, ctypes.c_char_p, ctypes.c_char_p)
 
 llmodel.llmodel_prompt.argtypes = [
     ctypes.c_void_p,
     ctypes.c_char_p,
-    ctypes.c_char_p,
     PromptCallback,
     ResponseCallback,
-    ctypes.c_bool,
     ctypes.POINTER(LLModelPromptContext),
-    ctypes.c_bool,
-    ctypes.c_char_p,
+    ctypes.POINTER(ctypes.c_char_p),
 ]
 
-llmodel.llmodel_prompt.restype = None
+llmodel.llmodel_prompt.restype = ctypes.c_bool
 
 llmodel.llmodel_embed.argtypes = [
     ctypes.c_void_p,
@@ -225,6 +221,12 @@ llmodel.llmodel_model_backend_name.restype = ctypes.c_char_p
 llmodel.llmodel_model_gpu_device_name.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_model_gpu_device_name.restype = ctypes.c_char_p
 
+llmodel.llmodel_count_prompt_tokens.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_char_p)]
+llmodel.llmodel_count_prompt_tokens.restype = ctypes.c_int32
+
+llmodel.llmodel_model_foreach_special_token.argtypes = [ctypes.c_void_p, SpecialTokenCallback]
+llmodel.llmodel_model_foreach_special_token.restype = None
+
 ResponseCallbackType = Callable[[int, str], bool]
 RawResponseCallbackType = Callable[[int, bytes], bool]
 EmbCancelCallbackType: TypeAlias = 'Callable[[list[int], str], bool]'
@@ -269,7 +271,6 @@ class LLModel:
         self.model_path = model_path.encode()
         self.n_ctx = n_ctx
         self.ngl = ngl
-        self.context: LLModelPromptContext | None = None
         self.buffer = bytearray()
         self.buff_expecting_cont_bytes: int = 0
 
@@ -289,6 +290,10 @@ class LLModel:
 
             raise RuntimeError(f"Unable to instantiate model: {errmsg}")
         self.model: ctypes.c_void_p | None = model
+        self.special_tokens_map: dict[str, str] = {}
+        llmodel.llmodel_model_foreach_special_token(
+            self.model, lambda n, t: self.special_tokens_map.__setitem__(n.decode(), t.decode()),
+        )
 
     def __del__(self, llmodel=llmodel):
         if hasattr(self, 'model'):
@@ -315,6 +320,19 @@ class LLModel:
         dev = llmodel.llmodel_model_gpu_device_name(self.model)
         return None if dev is None else dev.decode()
 
+    def count_prompt_tokens(self, prompt: str) -> int:
+        if self.model is None:
+            self._raise_closed()
+        err = ctypes.c_char_p()
+        n_tok = llmodel.llmodel_count_prompt_tokens(self.model, prompt, ctypes.byref(err))
+        if n_tok < 0:
+            s = err.value
+            errmsg = 'null' if s is None else s.decode()
+            raise RuntimeError(f'Unable to count prompt tokens: {errmsg}')
+        return n_tok
+
+    llmodel.llmodel_count_prompt_tokens.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
+
     @staticmethod
     def list_gpus(mem_required: int = 0) -> list[str]:
         """
@@ -378,50 +396,6 @@ class LLModel:
             raise Exception("Model not loaded")
         return llmodel.llmodel_threadCount(self.model)
 
-    def _set_context(
-        self,
-        n_predict: int = 4096,
-        top_k: int = 40,
-        top_p: float = 0.9,
-        min_p: float = 0.0,
-        temp: float = 0.1,
-        n_batch: int = 8,
-        repeat_penalty: float = 1.2,
-        repeat_last_n: int = 10,
-        context_erase: float = 0.75,
-        reset_context: bool = False,
-    ):
-        if self.context is None:
-            context = LLModelPromptContext(
-                tokens_size=0,
-                n_past=0,
-                n_ctx=0,
-                n_predict=n_predict,
-                top_k=top_k,
-                top_p=top_p,
-                min_p=min_p,
-                temp=temp,
-                n_batch=n_batch,
-                repeat_penalty=repeat_penalty,
-                repeat_last_n=repeat_last_n,
-                context_erase=context_erase,
-            )
-            self.context = context
-        else:
-            context = self.context
-            if reset_context:
-                self.context.n_past = 0
-
-        self.context.n_predict = n_predict
-        self.context.top_k = top_k
-        self.context.top_p = top_p
-        self.context.min_p = min_p
-        self.context.temp = temp
-        self.context.n_batch = n_batch
-        self.context.repeat_penalty = repeat_penalty
-        self.context.repeat_last_n = repeat_last_n
-        self.context.context_erase = context_erase
-
     @overload
     def generate_embeddings(
         self, text: str, prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
@@ -491,20 +465,18 @@ class LLModel:
 
     def prompt_model(
         self,
-        prompt: str,
-        prompt_template: str,
-        callback: ResponseCallbackType,
-        n_predict: int = 4096,
-        top_k: int = 40,
-        top_p: float = 0.9,
-        min_p: float = 0.0,
-        temp: float = 0.1,
-        n_batch: int = 8,
-        repeat_penalty: float = 1.2,
-        repeat_last_n: int = 10,
-        context_erase: float = 0.75,
-        reset_context: bool = False,
-        special: bool = False,
+        prompt          : str,
+        callback        : ResponseCallbackType,
+        n_predict       : int                  = 4096,
+        top_k           : int                  = 40,
+        top_p           : float                = 0.9,
+        min_p           : float                = 0.0,
+        temp            : float                = 0.1,
+        n_batch         : int                  = 8,
+        repeat_penalty  : float                = 1.2,
+        repeat_last_n   : int                  = 10,
+        context_erase   : float                = 0.75,
+        reset_context   : bool                 = False,
     ):
         """
         Generate response from model from a prompt.
@@ -527,34 +499,38 @@ class LLModel:
         self.buffer.clear()
         self.buff_expecting_cont_bytes = 0
 
-        self._set_context(
-            n_predict=n_predict,
-            top_k=top_k,
-            top_p=top_p,
-            min_p=min_p,
-            temp=temp,
-            n_batch=n_batch,
-            repeat_penalty=repeat_penalty,
-            repeat_last_n=repeat_last_n,
-            context_erase=context_erase,
-            reset_context=reset_context,
+        context = LLModelPromptContext(
+            n_predict      = n_predict,
+            top_k          = top_k,
+            top_p          = top_p,
+            min_p          = min_p,
+            temp           = temp,
+            n_batch        = n_batch,
+            repeat_penalty = repeat_penalty,
+            repeat_last_n  = repeat_last_n,
+            context_erase  = context_erase,
         )
 
-        llmodel.llmodel_prompt(
+        error_msg: bytes | None = None
+        def error_callback(msg: bytes) -> None:
+            nonlocal error_msg
+            error_msg = msg
+
+        err = ctypes.c_char_p()
+        if not llmodel.llmodel_prompt(
             self.model,
             ctypes.c_char_p(prompt.encode()),
-            ctypes.c_char_p(prompt_template.encode()),
             PromptCallback(self._prompt_callback),
             ResponseCallback(self._callback_decoder(callback)),
-            True,
-            self.context,
-            special,
-            ctypes.c_char_p(),
-        )
+            context,
+            ctypes.byref(err),
+        ):
+            s = err.value
+            raise RuntimeError(f"prompt error: {'null' if s is None else s.decode()}")
 
     def prompt_model_streaming(
-        self, prompt: str, prompt_template: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
-    ) -> Iterable[str]:
+        self, prompt: str, callback: ResponseCallbackType = empty_response_callback, **kwargs: Any,
+    ) -> Iterator[str]:
         if self.model is None:
             self._raise_closed()
 
@@ -573,15 +549,15 @@ class LLModel:
 
             return _generator_callback
 
-        def run_llmodel_prompt(prompt: str, prompt_template: str, callback: ResponseCallbackType, **kwargs):
-            self.prompt_model(prompt, prompt_template, callback, **kwargs)
+        def run_llmodel_prompt(prompt: str, callback: ResponseCallbackType, **kwargs):
+            self.prompt_model(prompt, callback, **kwargs)
             output_queue.put(Sentinel.TERMINATING_SYMBOL)
 
         # Kick off llmodel_prompt in separate thread so we can return generator
         # immediately
         thread = threading.Thread(
             target=run_llmodel_prompt,
-            args=(prompt, prompt_template, _generator_callback_wrapper(callback)),
+            args=(prompt, _generator_callback_wrapper(callback)),
             kwargs=kwargs,
         )
         thread.start()
@@ -636,5 +612,5 @@ class LLModel:
 
     # Empty prompt callback
     @staticmethod
-    def _prompt_callback(token_id: int) -> bool:
+    def _prompt_callback(token_ids: ctypes._Pointer[ctypes.c_int32], n_token_ids: int, cached: bool) -> bool:
         return True

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -4,37 +4,66 @@ Python only API for running all GPT4All models.
 from __future__ import annotations
 
 import hashlib
+import json
 import os
 import platform
 import re
 import sys
 import warnings
 from contextlib import contextmanager
+from datetime import datetime
 from pathlib import Path
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, Iterable, Literal, Protocol, overload
+from typing import TYPE_CHECKING, Any, Iterable, Iterator, Literal, NamedTuple, NoReturn, Protocol, TypedDict, overload
 
+import jinja2
 import requests
+from jinja2.sandbox import ImmutableSandboxedEnvironment
 from requests.exceptions import ChunkedEncodingError
 from tqdm import tqdm
 from urllib3.exceptions import IncompleteRead, ProtocolError
 
 from ._pyllmodel import (CancellationError as CancellationError, EmbCancelCallbackType, EmbedResult as EmbedResult,
-                         LLModel, ResponseCallbackType, empty_response_callback)
+                         LLModel, ResponseCallbackType, _operator_call, empty_response_callback)
 
 if TYPE_CHECKING:
     from typing_extensions import Self, TypeAlias
 
-if sys.platform == 'darwin':
+if sys.platform == "darwin":
     import fcntl
 
 # TODO: move to config
 DEFAULT_MODEL_DIRECTORY = Path.home() / ".cache" / "gpt4all"
 
-DEFAULT_PROMPT_TEMPLATE = "### Human:\n{0}\n\n### Assistant:\n"
+ConfigType: TypeAlias = "dict[str, Any]"
 
-ConfigType: TypeAlias = 'dict[str, Any]'
-MessageType: TypeAlias = 'dict[str, str]'
+# Environment setup adapted from HF transformers
+@_operator_call
+def _jinja_env() -> ImmutableSandboxedEnvironment:
+    def raise_exception(message: str) -> NoReturn:
+        raise jinja2.exceptions.TemplateError(message)
+
+    def tojson(obj: Any, indent: int | None = None) -> str:
+        return json.dumps(obj, ensure_ascii=False, indent=indent)
+
+    def strftime_now(fmt: str) -> str:
+        return datetime.now().strftime(fmt)
+
+    env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True)
+    env.filters["tojson"         ] = tojson
+    env.globals["raise_exception"] = raise_exception
+    env.globals["strftime_now"   ] = strftime_now
+    return env
+
+
+class MessageType(TypedDict):
+    role: str
+    content: str
+
+
+class ChatSession(NamedTuple):
+    template: jinja2.Template
+    history: list[MessageType]
 
 
 class Embed4All:
@@ -54,7 +83,7 @@ class Embed4All:
             kwargs: Remaining keyword arguments are passed to the `GPT4All` constructor.
         """
         if model_name is None:
-            model_name = 'all-MiniLM-L6-v2.gguf2.f16.gguf'
+            model_name = "all-MiniLM-L6-v2.gguf2.f16.gguf"
         self.gpt4all = GPT4All(model_name, n_threads=n_threads, device=device, **kwargs)
 
     def __enter__(self) -> Self:
@@ -145,18 +174,18 @@ class Embed4All:
             dimensionality = -1
         else:
             if dimensionality <= 0:
-                raise ValueError(f'Dimensionality must be None or a positive integer, got {dimensionality}')
+                raise ValueError(f"Dimensionality must be None or a positive integer, got {dimensionality}")
             if dimensionality < self.MIN_DIMENSIONALITY:
                 warnings.warn(
-                    f'Dimensionality {dimensionality} is less than the suggested minimum of {self.MIN_DIMENSIONALITY}.'
-                    ' Performance may be degraded.'
+                    f"Dimensionality {dimensionality} is less than the suggested minimum of {self.MIN_DIMENSIONALITY}."
+                    " Performance may be degraded."
                 )
         try:
             do_mean = {"mean": True, "truncate": False}[long_text_mode]
         except KeyError:
             raise ValueError(f"Long text mode must be one of 'mean' or 'truncate', got {long_text_mode!r}")
         result = self.gpt4all.model.generate_embeddings(text, prefix, dimensionality, do_mean, atlas, cancel_cb)
-        return result if return_dict else result['embeddings']
+        return result if return_dict else result["embeddings"]
 
 
 class GPT4All:
@@ -204,8 +233,7 @@ class GPT4All:
         """
 
         self.model_type = model_type
-        self._history: list[MessageType] | None = None
-        self._current_prompt_template: str = "{0}"
+        self._chat_session: ChatSession | None = None
 
         device_init = None
         if sys.platform == "darwin":
@@ -264,7 +292,13 @@ class GPT4All:
 
     @property
     def current_chat_session(self) -> list[MessageType] | None:
-        return None if self._history is None else list(self._history)
+        return None if self._chat_session is None else self._chat_session.history
+
+    @current_chat_session.setter
+    def current_chat_session(self, history: list[MessageType]) -> None:
+        if self._chat_session is None:
+            raise ValueError("current_chat_session may only be set when there is an active chat session")
+        self._chat_session.history[:] = history
 
     @staticmethod
     def list_models() -> list[ConfigType]:
@@ -276,7 +310,7 @@ class GPT4All:
         """
         resp = requests.get("https://gpt4all.io/models/models3.json")
         if resp.status_code != 200:
-            raise ValueError(f'Request failed: HTTP {resp.status_code} {resp.reason}')
+            raise ValueError(f"Request failed: HTTP {resp.status_code} {resp.reason}")
         return resp.json()
 
     @classmethod
@@ -306,15 +340,9 @@ class GPT4All:
         # get the config for the model
         config: ConfigType = {}
         if allow_download:
-            available_models = cls.list_models()
-
-            for m in available_models:
-                if model_filename == m["filename"]:
-                    tmpl = m.get("promptTemplate", DEFAULT_PROMPT_TEMPLATE)
-                    # change to Python-style formatting
-                    m["promptTemplate"] = tmpl.replace("%1", "{0}", 1).replace("%2", "{1}", 1)
-                    config.update(m)
-                    break
+            models = cls.list_models()
+            if (model := next((m for m in models if m["filename"] == model_filename), None)) is not None:
+                config.update(model)
 
         # Validate download directory
         if model_path is None:
@@ -378,13 +406,13 @@ class GPT4All:
             headers = {}
             if offset:
                 print(f"\nDownload interrupted, resuming from byte position {offset}", file=sys.stderr)
-                headers['Range'] = f'bytes={offset}-'  # resume incomplete response
+                headers["Range"] = f"bytes={offset}-"  # resume incomplete response
                 headers["Accept-Encoding"] = "identity"  # Content-Encoding changes meaning of ranges
             response = requests.get(url, stream=True, headers=headers)
             if response.status_code not in (200, 206):
-                raise ValueError(f'Request failed: HTTP {response.status_code} {response.reason}')
-            if offset and (response.status_code != 206 or str(offset) not in response.headers.get('Content-Range', '')):
-                raise ValueError('Connection was interrupted and server does not support range requests')
+                raise ValueError(f"Request failed: HTTP {response.status_code} {response.reason}")
+            if offset and (response.status_code != 206 or str(offset) not in response.headers.get("Content-Range", "")):
+                raise ValueError("Connection was interrupted and server does not support range requests")
             if (enc := response.headers.get("Content-Encoding")) is not None:
                 raise ValueError(f"Expected identity Content-Encoding, got {enc}")
             return response
@@ -483,19 +511,19 @@ class GPT4All:
 
     def generate(
         self,
-        prompt: str,
+        prompt         : str,
         *,
-        max_tokens: int = 200,
-        temp: float = 0.7,
-        top_k: int = 40,
-        top_p: float = 0.4,
-        min_p: float = 0.0,
-        repeat_penalty: float = 1.18,
-        repeat_last_n: int = 64,
-        n_batch: int = 8,
-        n_predict: int | None = None,
-        streaming: bool = False,
-        callback: ResponseCallbackType = empty_response_callback,
+        max_tokens     : int                  = 200,
+        temp           : float                = 0.7,
+        top_k          : int                  = 40,
+        top_p          : float                = 0.4,
+        min_p          : float                = 0.0,
+        repeat_penalty : float                = 1.18,
+        repeat_last_n  : int                  = 64,
+        n_batch        : int                  = 8,
+        n_predict      : int | None           = None,
+        streaming      : bool                 = False,
+        callback       : ResponseCallbackType = empty_response_callback,
     ) -> Any:
         """
         Generate outputs from any GPT4All model.
@@ -520,122 +548,94 @@ class GPT4All:
 
         # Preparing the model request
         generate_kwargs: dict[str, Any] = dict(
-            temp=temp,
-            top_k=top_k,
-            top_p=top_p,
-            min_p=min_p,
-            repeat_penalty=repeat_penalty,
-            repeat_last_n=repeat_last_n,
-            n_batch=n_batch,
-            n_predict=n_predict if n_predict is not None else max_tokens,
+            temp           = temp,
+            top_k          = top_k,
+            top_p          = top_p,
+            min_p          = min_p,
+            repeat_penalty = repeat_penalty,
+            repeat_last_n  = repeat_last_n,
+            n_batch        = n_batch,
+            n_predict      = n_predict if n_predict is not None else max_tokens,
         )
 
-        if self._history is not None:
-            # check if there is only one message, i.e. system prompt:
-            reset = len(self._history) == 1
-            self._history.append({"role": "user", "content": prompt})
-
-            fct_func = self._format_chat_prompt_template.__func__  # type: ignore[attr-defined]
-            if fct_func is GPT4All._format_chat_prompt_template:
-                if reset:
-                    # ingest system prompt
-                    # use "%1%2" and not "%1" to avoid implicit whitespace
-                    self.model.prompt_model(self._history[0]["content"], "%1%2",
-                                            empty_response_callback,
-                                            n_batch=n_batch, n_predict=0, reset_context=True, special=True)
-                prompt_template = self._current_prompt_template.format("%1", "%2")
-            else:
-                warnings.warn(
-                    "_format_chat_prompt_template is deprecated. Please use a chat session with a prompt template.",
-                    DeprecationWarning,
-                )
-                # special tokens won't be processed
-                prompt = self._format_chat_prompt_template(
-                    self._history[-1:],
-                    self._history[0]["content"] if reset else "",
-                )
-                prompt_template = "%1"
-                generate_kwargs["reset_context"] = reset
-        else:
-            prompt_template = "%1"
-            generate_kwargs["reset_context"] = True
-
         # Prepare the callback, process the model response
-        output_collector: list[MessageType]
-        output_collector = [
-            {"content": ""}
-        ]  # placeholder for the self._history if chat session is not activated
+        full_response = ""
 
-        if self._history is not None:
-            self._history.append({"role": "assistant", "content": ""})
-            output_collector = self._history
+        def _callback_wrapper(token_id: int, response: str) -> bool:
+            nonlocal full_response
+            full_response += response
+            return callback(token_id, response)
 
-        def _callback_wrapper(
-            callback: ResponseCallbackType,
-            output_collector: list[MessageType],
-        ) -> ResponseCallbackType:
-            def _callback(token_id: int, response: str) -> bool:
-                nonlocal callback, output_collector
+        last_msg_rendered = prompt
+        if self._chat_session is not None:
+            session = self._chat_session
+            def render(messages: list[MessageType]) -> str:
+                return session.template.render(
+                    messages=messages,
+                    add_generation_prompt=True,
+                    **self.model.special_tokens_map,
+                )
+            session.history.append(MessageType(role="user", content=prompt))
+            prompt = render(session.history)
+            if len(session.history) > 1:
+                last_msg_rendered = render(session.history[-1:])
 
-                output_collector[-1]["content"] += response
-
-                return callback(token_id, response)
-
-            return _callback
+        # Check request length
+        last_msg_len = self.model.count_prompt_tokens(last_msg_rendered)
+        if last_msg_len > (limit := self.model.n_ctx - 4):
+            raise ValueError(f"Your message was too long and could not be processed ({last_msg_len} > {limit}).")
 
         # Send the request to the model
         if streaming:
-            return self.model.prompt_model_streaming(
-                prompt,
-                prompt_template,
-                _callback_wrapper(callback, output_collector),
-                **generate_kwargs,
-            )
+            def stream() -> Iterator[str]:
+                yield from self.model.prompt_model_streaming(prompt, _callback_wrapper, **generate_kwargs)
+                if self._chat_session is not None:
+                    self._chat_session.history.append(MessageType(role="assistant", content=full_response))
+            return stream()
 
-        self.model.prompt_model(
-            prompt,
-            prompt_template,
-            _callback_wrapper(callback, output_collector),
-            **generate_kwargs,
-        )
-
-        return output_collector[-1]["content"]
+        self.model.prompt_model(prompt, _callback_wrapper, **generate_kwargs)
+        if self._chat_session is not None:
+            self._chat_session.history.append(MessageType(role="assistant", content=full_response))
+        return full_response
 
     @contextmanager
     def chat_session(
         self,
-        system_prompt: str | None = None,
-        prompt_template: str | None = None,
+        system_message: str | Literal[False] | None = None,
+        chat_template: str | None = None,
     ):
         """
         Context manager to hold an inference optimized chat session with a GPT4All model.
 
         Args:
-            system_prompt: An initial instruction for the model.
-            prompt_template: Template for the prompts with {0} being replaced by the user message.
+            system_message: An initial instruction for the model, None to use the model default, or False to disable. Defaults to None.
+            chat_template: Jinja template for the conversation, or None to use the model default. Defaults to None.
         """
 
-        if system_prompt is None:
-            system_prompt = self.config.get("systemPrompt", "")
+        if system_message is None:
+            system_message = self.config.get("systemMessage", False)
 
-        if prompt_template is None:
-            if (tmpl := self.config.get("promptTemplate")) is None:
-                warnings.warn("Use of a sideloaded model or allow_download=False without specifying a prompt template "
-                              "is deprecated. Defaulting to Alpaca.", DeprecationWarning)
-                tmpl = DEFAULT_PROMPT_TEMPLATE
-            prompt_template = tmpl
+        if chat_template is None:
+            if "name" not in self.config:
+                raise ValueError("For sideloaded models or with allow_download=False, you must specify a chat template.")
+            if "chatTemplate" not in self.config:
+                raise NotImplementedError("This model appears to have a built-in chat template, but loading it is not "
+                                          "currently implemented. Please pass a template to chat_session() directly.")
+            if (tmpl := self.config["chatTemplate"]) is None:
+                raise ValueError(f"The model {self.config['name']!r} does not support chat.")
+            chat_template = tmpl
 
-        if re.search(r"%1(?![0-9])", prompt_template):
-            raise ValueError("Prompt template containing a literal '%1' is not supported. For a prompt "
-                             "placeholder, please use '{0}' instead.")
-
-        self._history = [{"role": "system", "content": system_prompt}]
-        self._current_prompt_template = prompt_template
+        history = []
+        if system_message is not False:
+            history.append(MessageType(role="system", content=system_message))
+        self._chat_session = ChatSession(
+            template=_jinja_env.from_string(chat_template),
+            history=history,
+        )
         try:
             yield self
         finally:
-            self._history = None
-            self._current_prompt_template = "{0}"
+            self._chat_session = None
 
     @staticmethod
     def list_gpus() -> list[str]:
@@ -647,43 +647,6 @@ class GPT4All:
         """
         return LLModel.list_gpus()
 
-    def _format_chat_prompt_template(
-        self,
-        messages: list[MessageType],
-        default_prompt_header: str = "",
-        default_prompt_footer: str = "",
-    ) -> str:
-        """
-        Helper method for building a prompt from list of messages using the self._current_prompt_template as a template for each message.
-
-        Warning:
-            This function was deprecated in version 2.3.0, and will be removed in a future release.
-
-        Args:
-            messages:  List of dictionaries. Each dictionary should have a "role" key
-                with value of "system", "assistant", or "user" and a "content" key with a
-                string value. Messages are organized such that "system" messages are at top of prompt,
-                and "user" and "assistant" messages are displayed in order. Assistant messages get formatted as
-                "Response: {content}".
-
-        Returns:
-            Formatted prompt.
-        """
-
-        full_prompt = default_prompt_header + "\n\n" if default_prompt_header != "" else ""
-
-        for message in messages:
-            if message["role"] == "user":
-                user_message = self._current_prompt_template.format(message["content"])
-                full_prompt += user_message
-            if message["role"] == "assistant":
-                assistant_message = message["content"] + "\n"
-                full_prompt += assistant_message
-
-        full_prompt += "\n\n" + default_prompt_footer if default_prompt_footer != "" else ""
-
-        return full_prompt
-
 
 def append_extension_if_missing(model_name):
     if not model_name.endswith((".bin", ".gguf")):
@@ -696,7 +659,7 @@ class _HasFileno(Protocol):
 
 
 def _fsync(fd: int | _HasFileno) -> None:
-    if sys.platform == 'darwin':
+    if sys.platform == "darwin":
         # Apple's fsync does not flush the drive write cache
         try:
             fcntl.fcntl(fd, fcntl.F_FULLFSYNC)

gpt4all-bindings/python/mkdocs.yml

@@ -14,6 +14,7 @@ nav:
     - 'Models' : 'gpt4all_desktop/models.md'
     - 'LocalDocs' : 'gpt4all_desktop/localdocs.md'
     - 'Settings' : 'gpt4all_desktop/settings.md'
+    - 'Chat Templates' : 'gpt4all_desktop/chat_templates.md'
     - 'Cookbook':
       - 'Local AI Chat with Microsoft Excel': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-microsoft-excel.md'
       - 'Local AI Chat with your Google Drive': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-google-drive.md'

gpt4all-bindings/python/setup.py

@@ -88,9 +88,10 @@ setup(
     python_requires='>=3.8',
     packages=find_packages(),
     install_requires=[
+        'importlib_resources; python_version < "3.9"',
+        'jinja2~=3.1',
         'requests',
         'tqdm',
-        'importlib_resources; python_version < "3.9"',
         'typing-extensions>=4.3.0; python_version >= "3.9" and python_version < "3.11"',
     ],
     extras_require={

gpt4all-chat/.flake8

@@ -0,0 +1,5 @@
+# vim: set syntax=dosini:
+[flake8]
+exclude = .*,__pycache__
+max-line-length = 120
+extend-ignore = B001,C408,D,DAR,E221,E303,E722,E741,E800,N801,N806,P101,S101,S324,S404,S406,S410,S603,WPS100,WPS110,WPS111,WPS113,WPS114,WPS115,WPS120,WPS2,WPS300,WPS301,WPS304,WPS305,WPS306,WPS309,WPS316,WPS317,WPS318,WPS319,WPS322,WPS323,WPS326,WPS329,WPS330,WPS332,WPS336,WPS337,WPS347,WPS360,WPS361,WPS407,WPS414,WPS420,WPS421,WPS429,WPS430,WPS431,WPS432,WPS433,WPS437,WPS440,WPS440,WPS441,WPS442,WPS457,WPS458,WPS460,WPS462,WPS463,WPS473,WPS501,WPS504,WPS505,WPS508,WPS509,WPS510,WPS515,WPS516,WPS519,WPS520,WPS529,WPS531,WPS602,WPS604,WPS605,WPS608,WPS609,WPS613,WPS615

gpt4all-chat/CHANGELOG.md

@@ -4,6 +4,90 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## [3.6.0] - 2024-12-19
+
+### Added
+- Automatically substitute chat templates that are not compatible with Jinja2Cpp in GGUFs ([#3327](https://github.com/nomic-ai/gpt4all/pull/3327))
+- Built-in javascript code interpreter tool plus model ([#3173](https://github.com/nomic-ai/gpt4all/pull/3173))
+
+### Fixed
+- Fix remote model template to allow for XML in messages ([#3318](https://github.com/nomic-ai/gpt4all/pull/3318))
+- Fix Jinja2Cpp bug that broke system message detection in chat templates ([#3325](https://github.com/nomic-ai/gpt4all/pull/3325))
+- Fix LocalDocs sources displaying in unconsolidated form after v3.5.0 ([#3328](https://github.com/nomic-ai/gpt4all/pull/3328))
+
+## [3.5.3] - 2024-12-16
+
+### Fixed
+- Fix LocalDocs not using information from sources in v3.5.2 ([#3302](https://github.com/nomic-ai/gpt4all/pull/3302))
+
+## [3.5.2] - 2024-12-13
+
+### Added
+- Create separate download pages for built-in and HuggingFace models ([#3269](https://github.com/nomic-ai/gpt4all/pull/3269))
+
+### Fixed
+- Fix API server ignoring assistant messages in history after v3.5.0 ([#3256](https://github.com/nomic-ai/gpt4all/pull/3256))
+- Fix API server replying with incorrect token counts and stop reason after v3.5.0 ([#3256](https://github.com/nomic-ai/gpt4all/pull/3256))
+- Fix API server remembering previous, unrelated conversations after v3.5.0 ([#3256](https://github.com/nomic-ai/gpt4all/pull/3256))
+- Fix mishandling of default chat template and system message of cloned models in v3.5.0 ([#3262](https://github.com/nomic-ai/gpt4all/pull/3262))
+- Fix untranslated text on the startup dialog ([#3293](https://github.com/nomic-ai/gpt4all/pull/3293))
+
+## [3.5.1] - 2024-12-10
+
+### Fixed
+- Fix an incorrect value for currentResponse ([#3245](https://github.com/nomic-ai/gpt4all/pull/3245))
+- Fix the default model button so it works again after 3.5.0 ([#3246](https://github.com/nomic-ai/gpt4all/pull/3246))
+- Fix chat templates for Nous Hermes 2 Mistral, Mistral OpenOrca, Qwen 2, and remote models ([#3250](https://github.com/nomic-ai/gpt4all/pull/3250))
+- Fix chat templates for Llama 3.2 models ([#3251](https://github.com/nomic-ai/gpt4all/pull/3251))
+
+## [3.5.0] - 2024-12-09
+
+### Changed
+- Update Italian translation (by [@Harvester62](https://github.com/Harvester62) in [#3236](https://github.com/nomic-ai/gpt4all/pull/3236))
+- Update Romanian translation (by [@SINAPSA-IC](https://github.com/SINAPSA-IC) in [#3232](https://github.com/nomic-ai/gpt4all/pull/3232))
+
+### Fixed
+- Fix a few more problems with the Jinja changes ([#3239](https://github.com/nomic-ai/gpt4all/pull/3239))
+
+## [3.5.0-rc2] - 2024-12-06
+
+### Changed
+- Fade messages out with an animation when they are removed from the chat view ([#3227](https://github.com/nomic-ai/gpt4all/pull/3227))
+- Tweak wording of edit/redo confirmation dialogs ([#3228](https://github.com/nomic-ai/gpt4all/pull/3228))
+- Make edit/redo buttons disabled instead of invisible when they are temporarily unavailable ([#3228](https://github.com/nomic-ai/gpt4all/pull/3228))
+
+## [3.5.0-rc1] - 2024-12-04
+
+### Added
+- Add ability to attach text, markdown, and rst files to chat ([#3135](https://github.com/nomic-ai/gpt4all/pull/3135))
+- Add feature to minimize to system tray (by [@bgallois](https://github.com/bgallois) in [#3109](https://github.com/nomic-ai/gpt4all/pull/3109))
+- Basic cache for faster prefill when the input shares a prefix with previous context ([#3073](https://github.com/nomic-ai/gpt4all/pull/3073))
+- Add ability to edit prompts and regenerate any response ([#3147](https://github.com/nomic-ai/gpt4all/pull/3147))
+
+### Changed
+- Implement Qt 6.8 compatibility ([#3121](https://github.com/nomic-ai/gpt4all/pull/3121))
+- Use Jinja for chat templates instead of per-message QString.arg-style templates ([#3147](https://github.com/nomic-ai/gpt4all/pull/3147))
+- API server: Use system message(s) from client instead of settings ([#3147](https://github.com/nomic-ai/gpt4all/pull/3147))
+- API server: Accept messages in any order supported by the model instead of requiring user/assistant pairs ([#3147](https://github.com/nomic-ai/gpt4all/pull/3147))
+- Remote models: Pass system message with "system" role instead of joining with user message ([#3147](https://github.com/nomic-ai/gpt4all/pull/3147))
+
+### Removed
+- Remove option to save binary model state to disk ([#3147](https://github.com/nomic-ai/gpt4all/pull/3147))
+
+### Fixed
+- Fix bug in GUI when localdocs encounters binary data ([#3137](https://github.com/nomic-ai/gpt4all/pull/3137))
+- Fix LocalDocs bugs that prevented some docx files from fully chunking ([#3140](https://github.com/nomic-ai/gpt4all/pull/3140))
+- Fix missing softmax that was causing crashes and effectively infinite temperature since 3.4.0 ([#3202](https://github.com/nomic-ai/gpt4all/pull/3202))
+
+## [3.4.2] - 2024-10-16
+
+### Fixed
+- Limit bm25 retrieval to only specified collections ([#3083](https://github.com/nomic-ai/gpt4all/pull/3083))
+- Fix bug removing documents because of a wrong case sensitive file suffix check ([#3083](https://github.com/nomic-ai/gpt4all/pull/3083))
+- Fix bug with hybrid localdocs search where database would get out of sync ([#3083](https://github.com/nomic-ai/gpt4all/pull/3083))
+- Fix GUI bug where the localdocs embedding device appears blank ([#3083](https://github.com/nomic-ai/gpt4all/pull/3083))
+- Prevent LocalDocs from not making progress in certain cases ([#3094](https://github.com/nomic-ai/gpt4all/pull/3094))
+
 ## [3.4.1] - 2024-10-11
 
 ### Fixed
@@ -155,6 +239,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Fix several Vulkan resource management issues ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
 - Fix crash/hang when some models stop generating, by showing special tokens ([#2701](https://github.com/nomic-ai/gpt4all/pull/2701))
 
+[3.6.0]: https://github.com/nomic-ai/gpt4all/compare/v3.5.3...v3.6.0
+[3.5.3]: https://github.com/nomic-ai/gpt4all/compare/v3.5.2...v3.5.3
+[3.5.2]: https://github.com/nomic-ai/gpt4all/compare/v3.5.1...v3.5.2
+[3.5.1]: https://github.com/nomic-ai/gpt4all/compare/v3.5.0...v3.5.1
+[3.5.0]: https://github.com/nomic-ai/gpt4all/compare/v3.5.0-rc2...v3.5.0
+[3.5.0-rc2]: https://github.com/nomic-ai/gpt4all/compare/v3.5.0-rc1...v3.5.0-rc2
+[3.5.0-rc1]: https://github.com/nomic-ai/gpt4all/compare/v3.4.2...v3.5.0-rc1
+[3.4.2]: https://github.com/nomic-ai/gpt4all/compare/v3.4.1...v3.4.2
 [3.4.1]: https://github.com/nomic-ai/gpt4all/compare/v3.4.0...v3.4.1
 [3.4.0]: https://github.com/nomic-ai/gpt4all/compare/v3.3.0...v3.4.0
 [3.3.1]: https://github.com/nomic-ai/gpt4all/compare/v3.3.0...v3.3.1

gpt4all-chat/CMakeLists.txt

@@ -3,8 +3,8 @@ cmake_minimum_required(VERSION 3.25)  # for try_compile SOURCE_FROM_VAR
 include(../common/common.cmake)
 
 set(APP_VERSION_MAJOR 3)
-set(APP_VERSION_MINOR 4)
-set(APP_VERSION_PATCH 1)
+set(APP_VERSION_MINOR 6)
+set(APP_VERSION_PATCH 0)
 set(APP_VERSION_BASE "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 set(APP_VERSION "${APP_VERSION_BASE}")
 
@@ -22,6 +22,9 @@ if(APPLE)
   endif()
 endif()
 
+find_package(Python3 3.12 QUIET COMPONENTS Interpreter)
+
+option(GPT4ALL_TEST "Build the tests" ${Python3_FOUND})
 option(GPT4ALL_LOCALHOST "Build installer for localhost repo" OFF)
 option(GPT4ALL_OFFLINE_INSTALLER "Build an offline installer" OFF)
 option(GPT4ALL_SIGN_INSTALL "Sign installed binaries and installers (requires signing identities)" OFF)
@@ -30,6 +33,10 @@ option(GPT4ALL_SIGN_INSTALL "Sign installed binaries and installers (requires si
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
+if (MSVC)
+    # Enable accurate __cplusplus macro to fix errors in Jinja2Cpp
+    add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/Zc:__cplusplus>)
+endif()
 
 
 # conftests
@@ -72,7 +79,12 @@ configure_file(
   "${CMAKE_CURRENT_BINARY_DIR}/config.h"
 )
 
-find_package(Qt6 6.4 COMPONENTS Core HttpServer LinguistTools Pdf Quick QuickDialogs2 Sql Svg REQUIRED)
+set(CMAKE_FIND_PACKAGE_TARGETS_GLOBAL ON)
+find_package(Qt6 6.5 COMPONENTS Core HttpServer LinguistTools Pdf Quick QuickDialogs2 Sql Svg REQUIRED)
+
+if (QT_KNOWN_POLICY_QTP0004)
+    qt_policy(SET QTP0004 NEW)  # generate extra qmldir files on Qt 6.8+
+endif()
 
 # Get the Qt6Core target properties
 get_target_property(Qt6Core_INCLUDE_DIRS Qt6::Core INTERFACE_INCLUDE_DIRECTORIES)
@@ -90,9 +102,41 @@ message(STATUS "Qt 6 root directory: ${Qt6_ROOT_DIR}")
 
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
+if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/install CACHE PATH "..." FORCE)
+endif()
+
 add_subdirectory(deps)
 add_subdirectory(../gpt4all-backend llmodel)
 
+if (GPT4ALL_TEST)
+    enable_testing()
+
+    # Llama-3.2-1B model
+    set(TEST_MODEL "Llama-3.2-1B-Instruct-Q4_0.gguf")
+    set(TEST_MODEL_MD5 "48ff0243978606fdba19d899b77802fc")
+    set(TEST_MODEL_PATH "${CMAKE_BINARY_DIR}/resources/${TEST_MODEL}")
+    set(TEST_MODEL_URL "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/${TEST_MODEL}")
+
+    # Create a custom command to download the file if it does not exist or if the checksum does not match
+    add_custom_command(
+        OUTPUT "${TEST_MODEL_PATH}"
+        COMMAND ${CMAKE_COMMAND} -E echo "Downloading test model from ${TEST_MODEL_URL} ..."
+        COMMAND ${CMAKE_COMMAND} -DURL="${TEST_MODEL_URL}" -DOUTPUT_PATH="${TEST_MODEL_PATH}" -DEXPECTED_MD5="${TEST_MODEL_MD5}" -P "${CMAKE_SOURCE_DIR}/cmake/download_model.cmake"
+        DEPENDS "${CMAKE_SOURCE_DIR}/cmake/download_model.cmake"
+    )
+
+    # Define a custom target that depends on the downloaded model
+    add_custom_target(download_test_model
+        DEPENDS "${TEST_MODEL_PATH}"
+    )
+
+    add_subdirectory(tests)
+
+    # The 'check' target makes sure the tests and their dependencies are up-to-date before running them
+    add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure DEPENDS download_test_model chat gpt4all_tests)
+endif()
+
 set(CHAT_EXE_RESOURCES)
 
 # Metal shader library
@@ -137,17 +181,26 @@ if (APPLE)
     set_source_files_properties(${CHAT_EXE_RESOURCES} PROPERTIES MACOSX_PACKAGE_LOCATION Resources)
 endif()
 
+set(MACOS_SOURCES)
+if (APPLE)
+    find_library(COCOA_LIBRARY Cocoa)
+    list(APPEND MACOS_SOURCES src/macosdock.mm src/macosdock.h)
+endif()
+
 qt_add_executable(chat
     src/main.cpp
     src/chat.cpp                  src/chat.h
     src/chatapi.cpp               src/chatapi.h
     src/chatlistmodel.cpp         src/chatlistmodel.h
     src/chatllm.cpp               src/chatllm.h
-    src/chatmodel.h
+    src/chatmodel.h               src/chatmodel.cpp
     src/chatviewtextprocessor.cpp src/chatviewtextprocessor.h
+    src/codeinterpreter.cpp       src/codeinterpreter.h
     src/database.cpp              src/database.h
     src/download.cpp              src/download.h
     src/embllm.cpp                src/embllm.h
+    src/jinja_helpers.cpp         src/jinja_helpers.h
+    src/jinja_replacements.cpp    src/jinja_replacements.h
     src/llm.cpp                   src/llm.h
     src/localdocs.cpp             src/localdocs.h
     src/localdocsmodel.cpp        src/localdocsmodel.h
@@ -156,8 +209,12 @@ qt_add_executable(chat
     src/mysettings.cpp            src/mysettings.h
     src/network.cpp               src/network.h
     src/server.cpp                src/server.h
+    src/tool.cpp                  src/tool.h
+    src/toolcallparser.cpp        src/toolcallparser.h
+    src/toolmodel.cpp             src/toolmodel.h
     src/xlsxtomd.cpp              src/xlsxtomd.h
     ${CHAT_EXE_RESOURCES}
+    ${MACOS_SOURCES}
 )
 gpt4all_add_warning_options(chat)
 
@@ -169,8 +226,14 @@ qt_add_qml_module(chat
       main.qml
       qml/AddCollectionView.qml
       qml/AddModelView.qml
+      qml/AddGPT4AllModelView.qml
+      qml/AddHFModelView.qml
       qml/ApplicationSettings.qml
       qml/ChatDrawer.qml
+      qml/ChatCollapsibleItem.qml
+      qml/ChatItemView.qml
+      qml/ChatMessageButton.qml
+      qml/ChatTextItem.qml
       qml/ChatView.qml
       qml/CollectionsDrawer.qml
       qml/HomeView.qml
@@ -183,18 +246,20 @@ qt_add_qml_module(chat
       qml/PopupDialog.qml
       qml/SettingsView.qml
       qml/StartupDialog.qml
-      qml/SwitchModelDialog.qml
+      qml/ConfirmationDialog.qml
       qml/Theme.qml
       qml/ThumbsDownDialog.qml
       qml/Toast.qml
       qml/ToastManager.qml
       qml/MyBusyIndicator.qml
       qml/MyButton.qml
+      qml/MyTabButton.qml
       qml/MyCheckBox.qml
       qml/MyComboBox.qml
       qml/MyDialog.qml
       qml/MyDirectoryField.qml
       qml/MyFileDialog.qml
+      qml/MyFileIcon.qml
       qml/MyFolderDialog.qml
       qml/MyFancyLink.qml
       qml/MyMenu.qml
@@ -229,6 +294,7 @@ qt_add_qml_module(chat
       icons/eject.svg
       icons/email.svg
       icons/file-doc.svg
+      icons/file-docx.svg
       icons/file-md.svg
       icons/file-pdf.svg
       icons/file-txt.svg
@@ -340,17 +406,16 @@ target_include_directories(chat PRIVATE deps/usearch/include
 target_link_libraries(chat
     PRIVATE Qt6::Core Qt6::HttpServer Qt6::Pdf Qt6::Quick Qt6::Sql Qt6::Svg)
 target_link_libraries(chat
-    PRIVATE llmodel SingleApplication fmt::fmt duckx::duckx QXlsx)
+    PRIVATE llmodel SingleApplication fmt::fmt duckx::duckx QXlsx jinja2cpp)
 
+if (APPLE)
+    target_link_libraries(chat PRIVATE ${COCOA_LIBRARY})
+endif()
 
 # -- install --
 
 set(COMPONENT_NAME_MAIN ${PROJECT_NAME})
 
-if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-  set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/install CACHE PATH "..." FORCE)
-endif()
-
 install(TARGETS chat DESTINATION bin COMPONENT ${COMPONENT_NAME_MAIN})
 
 install(
@@ -463,6 +528,7 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
     set(CPACK_BUNDLE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns")
 endif()
 
+set(CPACK_COMPONENTS_ALL gpt4all)  # exclude development components
 set(CPACK_PACKAGE_INSTALL_DIRECTORY ${COMPONENT_NAME_MAIN})
 set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR})
 set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR})

gpt4all-chat/cmake/download_model.cmake

@@ -0,0 +1,12 @@
+if(NOT DEFINED URL OR NOT DEFINED OUTPUT_PATH OR NOT DEFINED EXPECTED_MD5)
+    message(FATAL_ERROR "Usage: cmake -DURL=<url> -DOUTPUT_PATH=<path> -DEXPECTED_MD5=<md5> -P download_model.cmake")
+endif()
+
+message(STATUS "Downloading model from ${URL} to ${OUTPUT_PATH} ...")
+
+file(DOWNLOAD "${URL}" "${OUTPUT_PATH}" EXPECTED_MD5 "${EXPECTED_MD5}" STATUS status)
+
+list(GET status 0 status_code)
+if(NOT status_code EQUAL 0)
+    message(FATAL_ERROR "Failed to download model: ${status}")
+endif()

gpt4all-chat/deps/CMakeLists.txt

@@ -3,7 +3,7 @@ set(BUILD_SHARED_LIBS OFF)
 set(FMT_INSTALL OFF)
 add_subdirectory(fmt)
 
-set(QAPPLICATION_CLASS QGuiApplication)
+set(QAPPLICATION_CLASS QApplication)
 add_subdirectory(SingleApplication)
 
 set(DUCKX_INSTALL OFF)
@@ -11,3 +11,12 @@ add_subdirectory(DuckX)
 
 set(QT_VERSION_MAJOR 6)
 add_subdirectory(QXlsx/QXlsx)
+
+# forked dependency of Jinja2Cpp
+set(RAPIDJSON_BUILD_DOC                  OFF)
+set(RAPIDJSON_BUILD_EXAMPLES             OFF)
+set(RAPIDJSON_BUILD_TESTS                OFF)
+set(RAPIDJSON_ENABLE_INSTRUMENTATION_OPT OFF)
+add_subdirectory(rapidjson)
+
+add_subdirectory(Jinja2Cpp)

gpt4all-chat/dev-requirements.txt

@@ -0,0 +1,11 @@
+-r test-requirements.txt
+
+# dev tools
+flake8~=7.1
+mypy~=1.12
+pytype>=2024.10.11
+wemake-python-styleguide~=0.19.2
+
+# type stubs and other optional modules
+types-requests~=2.32
+urllib3[socks]

gpt4all-chat/icons/edit.svg

@@ -1,3 +1 @@
-<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M28.4138 9.17125L22.8288 3.585C22.643 3.39924 22.4225 3.25188 22.1799 3.15134C21.9372 3.0508 21.6771 2.99905 21.4144 2.99905C21.1517 2.99905 20.8916 3.0508 20.6489 3.15134C20.4062 3.25188 20.1857 3.39924 20 3.585L4.58626 19C4.39973 19.185 4.25185 19.4053 4.15121 19.648C4.05057 19.8907 3.99917 20.151 4.00001 20.4138V26C4.00001 26.5304 4.21072 27.0391 4.5858 27.4142C4.96087 27.7893 5.46958 28 6.00001 28H11.5863C11.849 28.0008 12.1093 27.9494 12.352 27.8488C12.5947 27.7482 12.815 27.6003 13 27.4138L28.4138 12C28.5995 11.8143 28.7469 11.5938 28.8474 11.3511C28.948 11.1084 28.9997 10.8483 28.9997 10.5856C28.9997 10.3229 28.948 10.0628 28.8474 9.82015C28.7469 9.57747 28.5995 9.35698 28.4138 9.17125ZM6.41376 20L17 9.41375L19.0863 11.5L8.50001 22.085L6.41376 20ZM6.00001 22.4138L9.58626 26H6.00001V22.4138ZM12 25.5863L9.91376 23.5L20.5 12.9138L22.5863 15L12 25.5863ZM24 13.5863L18.4138 8L21.4138 5L27 10.585L24 13.5863Z" fill="black"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="#000000" viewBox="0 0 256 256"><path d="M227.31,73.37,182.63,28.68a16,16,0,0,0-22.63,0L36.69,152A15.86,15.86,0,0,0,32,163.31V208a16,16,0,0,0,16,16H92.69A15.86,15.86,0,0,0,104,219.31L227.31,96a16,16,0,0,0,0-22.63ZM92.69,208H48V163.31l88-88L180.69,120ZM192,108.68,147.31,64l24-24L216,84.68Z"></path></svg>

gpt4all-chat/icons/file-docx.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256"><rect width="256" height="256" fill="none"/><line x1="152" y1="96" x2="208" y2="96" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><line x1="152" y1="160" x2="208" y2="160" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><path d="M64,72V40a8,8,0,0,1,8-8H200a8,8,0,0,1,8,8V216a8,8,0,0,1-8,8H72a8,8,0,0,1-8-8V184" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><polyline points="64 104 76 152 92 120 108 152 120 104" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><rect x="32" y="72" width="120" height="112" rx="8" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/></svg>

gpt4all-chat/main.qml

@@ -12,6 +12,7 @@ import network
 import gpt4all
 import localdocs
 import mysettings
+import Qt.labs.platform
 
 Window {
     id: window
@@ -22,6 +23,43 @@ Window {
     visible: true
     title: qsTr("GPT4All v%1").arg(Qt.application.version)
 
+    SystemTrayIcon {
+        id: systemTrayIcon
+        property bool shouldClose: false
+        visible: MySettings.systemTray && !shouldClose
+        icon.source: "qrc:/gpt4all/icons/gpt4all.svg"
+
+        function restore() {
+            LLM.showDockIcon();
+            window.show();
+            window.raise();
+            window.requestActivate();
+        }
+        onActivated: function(reason) {
+            if (reason === SystemTrayIcon.Context && Qt.platform.os !== "osx")
+                menu.open();
+            else if (reason === SystemTrayIcon.Trigger)
+                restore();
+        }
+
+        menu: Menu {
+            MenuItem {
+                text: qsTr("Restore")
+                onTriggered: systemTrayIcon.restore()
+            }
+            MenuItem {
+                text: qsTr("Quit")
+                onTriggered: {
+                    systemTrayIcon.restore();
+                    systemTrayIcon.shouldClose = true;
+                    window.shouldClose = true;
+                    savingPopup.open();
+                    ChatListModel.saveChats();
+                }
+            }
+        }
+    }
+
     Settings {
         property alias x: window.x
         property alias y: window.y
@@ -156,7 +194,7 @@ Window {
         font.pixelSize: theme.fontSizeLarge
     }
 
-    property bool hasSaved: false
+    property bool shouldClose: false
 
     PopupDialog {
         id: savingPopup
@@ -180,9 +218,18 @@ Window {
     }
 
     onClosing: function(close) {
-        if (window.hasSaved)
+        if (systemTrayIcon.visible) {
+            LLM.hideDockIcon();
+            window.visible = false;
+            ChatListModel.saveChats();
+            close.accepted = false;
+            return;
+        }
+
+        if (window.shouldClose)
             return;
 
+        window.shouldClose = true;
         savingPopup.open();
         ChatListModel.saveChats();
         close.accepted = false
@@ -191,9 +238,9 @@ Window {
     Connections {
         target: ChatListModel
         function onSaveChatsFinished() {
-            window.hasSaved = true;
             savingPopup.close();
-            window.close()
+            if (window.shouldClose)
+                window.close()
         }
     }
 
@@ -627,9 +674,6 @@ Window {
 
             function show() {
                 stackLayout.currentIndex = 2;
-                // FIXME This expanded code should be removed and we should be changing the names of
-                // the classes here in ModelList for the proxy/filter models
-                ModelList.downloadableModels.expanded = true
             }
 
             function isShown() {

gpt4all-chat/metadata/latestnews.md

@@ -1,18 +1,22 @@
 ## Latest News
 
-GPT4All v3.4.1 was released on October 11th, and fixes several issues with LocalDocs from the previous release.
+GPT4All v3.5.3 was released on December 16th. It fixes a serious issue with LocalDocs in v3.5.2 that resulted in sources not being used.
 
-* **IMPORTANT NOTE:** If you are coming from v3.4.0, be sure to "Rebuild" your collections at least once after updating!
+GPT4All v3.5.2 was released on December 13th. It changes the "Explore Models" page and fixes issues with the API server and cloned models.
+
+GPT4All v3.5.1 was released on December 10th and fixes several issues with the new chat templates. Additionally, it fixes a bug with the default model button as well an issue with remote models.
 
 ---
 
-GPT4All v3.4.0 was released on October 8th. Changes include:
+GPT4All v3.5.0 was released on December 9th. Changes include:
 
-* **Attached Files:** You can now attach a small Microsoft Excel spreadsheet (.xlsx) to a chat message and ask the model about it.
-* **LocalDocs Accuracy:** The LocalDocs algorithm has been enhanced to find more accurate references for some queries.
-* **Word Document Support:** LocalDocs now supports Microsoft Word (.docx) documents natively.
-  * **IMPORTANT NOTE:** If .docx files are not found, make sure Settings > LocalDocs > Allowed File Extensions includes "docx".
-* **Forgetful Model Fixes:** Issues with the "Redo last chat response" button, and with continuing chats from previous sessions, have been fixed.
-* **Chat Saving Improvements:** On exit, GPT4All will no longer save chats that are not new or modified. As a bonus, downgrading without losing access to all chats will be possible in the future, should the need arise.
-* **UI Fixes:** The model list no longer scrolls to the top when you start downloading a model.
-* **New Models:** LLama 3.2 Instruct 3B and 1B models now available in model list.
+* **Message Editing:**
+  * You can now edit any message you've sent by clicking the pencil icon below it.
+  * You can now redo earlier responses in the conversation.
+* **Templates:** Chat templates have been completely overhauled! They now use Jinja-style syntax. You may notice warnings or errors in the UI. Read the linked docs, and if you have any questions, please ask on the Discord.
+* **File Attachments:** Markdown and plain text files are now supported as file attachments.
+* **System Tray:** There is now an option in Application Settings to allow GPT4All to minimize to the system tray instead of closing.
+* **Local API Server:**
+  * The API server now supports system messages from the client and no longer uses the system message in settings.
+  * You can now send messages to the API server in any order supported by the model instead of just user/assistant pairs.
+* **Translations:** The Italian and Romanian translations have been improved.

gpt4all-chat/metadata/models3.json

@@ -1,6 +1,22 @@
 [
   {
     "order": "a",
+    "md5sum": "a54c08a7b90e4029a8c2ab5b5dc936aa",
+    "name": "Reasoner v1",
+    "filename": "qwen2.5-coder-7b-instruct-q4_0.gguf",
+    "filesize": "4431390720",
+    "requires": "3.6.0",
+    "ramrequired": "8",
+    "parameters": "8 billion",
+    "quant": "q4_0",
+    "type": "qwen2",
+    "description": "<ul><li>Based on <a href=\"https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct\">Qwen2.5-Coder 7B</a></li><li>Uses built-in javascript code interpreter</li><li>Use for complex reasoning tasks that can be aided by computation analysis</li><li>License: <a href=\"https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct/blob/main/LICENSE\">Apache License Version 2.0</a></li><li>#reasoning</li></ul>",
+    "url": "https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF/resolve/main/qwen2.5-coder-7b-instruct-q4_0.gguf",
+    "chatTemplate": "{{- '<|im_start|>system\\n' }}\n{% if toolList|length > 0 %}You have access to the following functions:\n{% for tool in toolList %}\nUse the function '{{tool.function}}' to: '{{tool.description}}'\n{% if tool.parameters|length > 0 %}\nparameters:\n{% for info in tool.parameters %}\n  {{info.name}}:\n    type: {{info.type}}\n    description: {{info.description}}\n    required: {{info.required}}\n{% endfor %}\n{% endif %}\n# Tool Instructions\nIf you CHOOSE to call this function ONLY reply with the following format:\n'{{tool.symbolicFormat}}'\nHere is an example. If the user says, '{{tool.examplePrompt}}', then you reply\n'{{tool.exampleCall}}'\nAfter the result you might reply with, '{{tool.exampleReply}}'\n{% endfor %}\nYou MUST include both the start and end tags when you use a function.\n\nYou are a helpful AI assistant who uses the functions to break down, analyze, perform, and verify complex reasoning tasks. You SHOULD try to verify your answers using the functions where possible.\n{% endif %}\n{{- '<|im_end|>\\n' }}\n{% for message in messages %}\n{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}\n{% endfor %}\n{% if add_generation_prompt %}\n{{ '<|im_start|>assistant\\n' }}\n{% endif %}\n",
+    "systemPrompt": ""
+  },
+  {
+    "order": "aa",
     "md5sum": "c87ad09e1e4c8f9c35a5fcef52b6f1c9",
     "name": "Llama 3 8B Instruct",
     "filename": "Meta-Llama-3-8B-Instruct.Q4_0.gguf",
@@ -13,7 +29,8 @@
     "description": "<ul><li>Fast responses</li><li>Chat based model</li><li>Accepts system prompts in Llama 3 format</li><li>Trained by Meta</li><li>License: <a href=\"https://llama.meta.com/llama3/license/\">Meta Llama 3 Community License</a></li></ul>",
     "url": "https://gpt4all.io/models/gguf/Meta-Llama-3-8B-Instruct.Q4_0.gguf",
     "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2<|eot_id|>",
-    "systemPrompt": ""
+    "systemPrompt": "",
+    "chatTemplate": "{%- set loop_messages = messages %}\n{%- for message in loop_messages %}\n    {%- set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' %}\n    {{- content }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}"
   },
   {
     "order": "b",
@@ -29,7 +46,8 @@
     "description": "<ul><li>Fast responses</li><li>Instruct model</li><li>Multilingual dialogue use</li><li>Agentic system capable</li><li>Trained by Meta</li><li>License: <a href=\"https://llama.meta.com/llama3_2/license/\">Meta Llama 3.2 Community License</a></li></ul>",
     "url": "https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF/resolve/main/Llama-3.2-3B-Instruct-Q4_0.gguf",
     "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2",
-    "systemPrompt": "<|start_header_id|>system<|end_header_id|>\nCutting Knowledge Date: December 2023\n\nYou are a helpful assistant.<|eot_id|>"
+    "systemPrompt": "<|start_header_id|>system<|end_header_id|>\nCutting Knowledge Date: December 2023\n\nYou are a helpful assistant.<|eot_id|>",
+    "chatTemplate": "{{- bos_token }}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now('%d %b %Y') %}\n    {%- else %}\n        {%- set date_string = '26 Jul 2024' %}\n    {%- endif %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] | trim %}\n    {%- set loop_start = 1 %}\n{%- else %}\n    {%- set system_message = '' %}\n    {%- set loop_start = 0 %}\n{%- endif %}\n\n{#- System message #}\n{{- '<|start_header_id|>system<|end_header_id|>\\n\\n' }}\n{{- 'Cutting Knowledge Date: December 2023\\n' }}\n{{- 'Today Date: ' + date_string + '\\n\\n' }}\n{{- system_message }}\n{{- '<|eot_id|>' }}\n\n{%- for message in messages %}\n    {%- if loop.index0 >= loop_start %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' + message['content'] | trim + '<|eot_id|>' }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}"
   },
   {
     "order": "c",
@@ -45,7 +63,8 @@
     "description": "<ul><li>Fast responses</li><li>Instruct model</li><li>Multilingual dialogue use</li><li>Agentic system capable</li><li>Trained by Meta</li><li>License: <a href=\"https://llama.meta.com/llama3_2/license/\">Meta Llama 3.2 Community License</a></li></ul>",
     "url": "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf",
     "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2",
-    "systemPrompt": "<|start_header_id|>system<|end_header_id|>\nCutting Knowledge Date: December 2023\n\nYou are a helpful assistant.<|eot_id|>"
+    "systemPrompt": "<|start_header_id|>system<|end_header_id|>\nCutting Knowledge Date: December 2023\n\nYou are a helpful assistant.<|eot_id|>",
+    "chatTemplate": "{{- bos_token }}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now('%d %b %Y') %}\n    {%- else %}\n        {%- set date_string = '26 Jul 2024' %}\n    {%- endif %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] | trim %}\n    {%- set loop_start = 1 %}\n{%- else %}\n    {%- set system_message = '' %}\n    {%- set loop_start = 0 %}\n{%- endif %}\n\n{#- System message #}\n{{- '<|start_header_id|>system<|end_header_id|>\\n\\n' }}\n{{- 'Cutting Knowledge Date: December 2023\\n' }}\n{{- 'Today Date: ' + date_string + '\\n\\n' }}\n{{- system_message }}\n{{- '<|eot_id|>' }}\n\n{%- for message in messages %}\n    {%- if loop.index0 >= loop_start %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' + message['content'] | trim + '<|eot_id|>' }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}"
   },
   {
     "order": "d",
@@ -61,7 +80,8 @@
     "description": "<strong>Good overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Accepts system prompts in ChatML format</li><li>Trained by Mistral AI<li>Finetuned by Nous Research on the OpenHermes-2.5 dataset<li>Licensed for commercial use</ul>",
     "url": "https://huggingface.co/NousResearch/Nous-Hermes-2-Mistral-7B-DPO-GGUF/resolve/main/Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
-    "systemPrompt": ""
+    "systemPrompt": "",
+    "chatTemplate": "{%- for message in messages %}\n    {{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}"
   },
   {
     "order": "e",
@@ -77,7 +97,8 @@
     "systemPrompt": "",
     "description": "<strong>Strong overall fast instruction following model</strong><br><ul><li>Fast responses</li><li>Trained by Mistral AI<li>Uncensored</li><li>Licensed for commercial use</li></ul>",
     "url": "https://gpt4all.io/models/gguf/mistral-7b-instruct-v0.1.Q4_0.gguf",
-    "promptTemplate": "[INST] %1 [/INST]"
+    "promptTemplate": "[INST] %1 [/INST]",
+    "chatTemplate": "{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] %}\n    {%- set loop_start = 1 %}\n{%- else %}\n    {%- set loop_start = 0 %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if loop.index0 >= loop_start %}\n        {%- if (message['role'] == 'user') != ((loop.index0 - loop_start) % 2 == 0) %}\n            {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n        {%- endif %}\n        {%- if message['role'] == 'user' %}\n            {%- if loop.first and system_message is defined %}\n                {{- ' [INST] ' + system_message + '\\n\\n' + message['content'] + ' [/INST]' }}\n            {%- else %}\n                {{- ' [INST] ' + message['content'] + ' [/INST]' }}\n            {%- endif %}\n        {%- elif message['role'] == 'assistant' %}\n            {{- ' ' + message['content'] + eos_token }}\n        {%- else %}\n            {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}"
   },
   {
     "order": "f",
@@ -93,7 +114,8 @@
     "description": "<ul><li><strong>For advanced users only. Not recommended for use on Windows or Linux without selecting CUDA due to speed issues.</strong></li><li>Fast responses</li><li>Chat based model</li><li>Large context size of 128k</li><li>Accepts agentic system prompts in Llama 3.1 format</li><li>Trained by Meta</li><li>License: <a href=\"https://llama.meta.com/llama3_1/license/\">Meta Llama 3.1 Community License</a></li></ul>",
     "url": "https://huggingface.co/GPT4All-Community/Meta-Llama-3.1-8B-Instruct-128k/resolve/main/Meta-Llama-3.1-8B-Instruct-128k-Q4_0.gguf",
     "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2",
-    "systemPrompt": "<|start_header_id|>system<|end_header_id|>\nCutting Knowledge Date: December 2023\n\nYou are a helpful assistant.<|eot_id|>"
+    "systemPrompt": "<|start_header_id|>system<|end_header_id|>\nCutting Knowledge Date: December 2023\n\nYou are a helpful assistant.<|eot_id|>",
+    "chatTemplate": "{%- set loop_messages = messages %}\n{%- for message in loop_messages %}\n    {%- set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' %}\n    {%- if loop.index0 == 0 %}\n        {%- set content = bos_token + content %}\n    {%- endif %}\n    {{- content }}\n{%- endfor %}\n{{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}"
   },
   {
     "order": "g",
@@ -109,7 +131,8 @@
     "description": "<strong>Strong overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
     "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.gguf2.Q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
-    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI.\n<|im_end|>\n"
+    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI.\n<|im_end|>\n",
+    "chatTemplate": "{%- if not add_generation_prompt is defined %}\n    {%- set add_generation_prompt = false %}\n{%- endif %}\n{%- for message in messages %}\n    {{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}"
   },
   {
     "order": "h",
@@ -125,7 +148,8 @@
     "systemPrompt": "",
     "description": "<strong>Very fast model with good quality</strong><br><ul><li>Fastest responses</li><li>Instruction based</li><li>Trained by TII<li>Finetuned by Nomic AI<li>Licensed for commercial use</ul>",
     "url": "https://gpt4all.io/models/gguf/gpt4all-falcon-newbpe-q4_0.gguf",
-    "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
+    "promptTemplate": "### Instruction:\n%1\n\n### Response:\n",
+    "chatTemplate": "{%- if messages[0]['role'] == 'system' %}\n    {%- set loop_start = 1 %}\n    {{- messages[0]['content'] + '\\n\\n' }}\n{%- else %}\n    {%- set loop_start = 0 %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if loop.index0 >= loop_start %}\n        {%- if message['role'] == 'user' %}\n            {{- '### User: ' + message['content'] + '\\n\\n' }}\n        {%- elif message['role'] == 'assistant' %}\n            {{- '### Assistant: ' + message['content'] + '\\n\\n' }}\n        {%- else %}\n            {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '### Assistant:' }}\n{%- endif %}"
   },
   {
     "order": "i",
@@ -140,7 +164,8 @@
     "type": "LLaMA2",
     "systemPrompt": "",
     "description": "<ul><li>Instruction based<li>Trained by Microsoft<li>Cannot be used commercially</ul>",
-    "url": "https://gpt4all.io/models/gguf/orca-2-7b.Q4_0.gguf"
+    "url": "https://gpt4all.io/models/gguf/orca-2-7b.Q4_0.gguf",
+    "chatTemplate": "{%- for message in messages %}\n    {{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}"
   },
   {
     "order": "j",
@@ -155,7 +180,8 @@
     "type": "LLaMA2",
     "systemPrompt": "",
     "description": "<ul><li>Instruction based<li>Trained by Microsoft<li>Cannot be used commercially</ul>",
-    "url": "https://gpt4all.io/models/gguf/orca-2-13b.Q4_0.gguf"
+    "url": "https://gpt4all.io/models/gguf/orca-2-13b.Q4_0.gguf",
+    "chatTemplate": "{%- for message in messages %}\n    {{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}"
   },
   {
     "order": "k",
@@ -170,7 +196,9 @@
     "type": "LLaMA2",
     "systemPrompt": "",
     "description": "<strong>Strong overall larger model</strong><br><ul><li>Instruction based<li>Gives very long responses<li>Finetuned with only 1k of high-quality data<li>Trained by Microsoft and Peking University<li>Cannot be used commercially</ul>",
-    "url": "https://gpt4all.io/models/gguf/wizardlm-13b-v1.2.Q4_0.gguf"
+    "url": "https://gpt4all.io/models/gguf/wizardlm-13b-v1.2.Q4_0.gguf",
+    "chatTemplate": "{%- if messages[0]['role'] == 'system' %}\n    {%- set loop_start = 1 %}\n    {{- messages[0]['content'] + ' ' }}\n{%- else %}\n    {%- set loop_start = 0 %}\n{%- endif %}\n{%- for message in loop_messages %}\n    {%- if loop.index0 >= loop_start %}\n        {%- if message['role'] == 'user' %}\n            {{- 'USER: ' + message['content'] }}\n        {%- elif message['role'] == 'assistant' %}\n            {{- 'ASSISTANT: ' + message['content'] }}\n        {%- else %}\n            {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n        {%- endif %}\n        {%- if (loop.index0 - loop_start) % 2 == 0 %}\n            {{- ' ' }}\n        {%- else %}\n            {{- eos_token }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- 'ASSISTANT:' }}\n{%- endif %}",
+    "systemMessage": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions."
   },
   {
     "order": "l",
@@ -186,7 +214,9 @@
     "description": "<strong>Ghost 7B v0.9.1</strong> fast, powerful and smooth for Vietnamese and English languages.",
     "url": "https://huggingface.co/lamhieu/ghost-7b-v0.9.1-gguf/resolve/main/ghost-7b-v0.9.1-Q4_0.gguf",
     "promptTemplate": "<|user|>\n%1</s>\n<|assistant|>\n%2</s>\n",
-    "systemPrompt": "<|system|>\nYou are Ghost created by Lam Hieu. You are a helpful and knowledgeable assistant. You like to help and always give honest information, in its original language. In communication, you are always respectful, equal and promote positive behavior.\n</s>"
+    "systemPrompt": "<|system|>\nYou are Ghost created by Lam Hieu. You are a helpful and knowledgeable assistant. You like to help and always give honest information, in its original language. In communication, you are always respectful, equal and promote positive behavior.\n</s>",
+    "chatTemplate": "{%- for message in messages %}\n    {%- if message['role'] == 'user' %}\n        {{- '<|user|>\\n' + message['content'] + eos_token }}\n    {%- elif message['role'] == 'system' %}\n        {{- '<|system|>\\n' + message['content'] + eos_token }}\n    {%- elif message['role'] == 'assistant' %}\n        {{- '<|assistant|>\\n'  + message['content'] + eos_token }}\n    {%- endif %}\n    {%- if loop.last and add_generation_prompt %}\n        {{- '<|assistant|>' }}\n    {%- endif %}\n{%- endfor %}",
+    "systemMessage": "You are Ghost created by Lam Hieu. You are a helpful and knowledgeable assistant. You like to help and always give honest information, in its original language. In communication, you are always respectful, equal and promote positive behavior."
   },
   {
     "order": "m",
@@ -202,7 +232,8 @@
     "systemPrompt": "",
     "description": "<strong>Extremely good model</strong><br><ul><li>Instruction based<li>Gives long responses<li>Curated with 300,000 uncensored instructions<li>Trained by Nous Research<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/nous-hermes-llama2-13b.Q4_0.gguf",
-    "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
+    "promptTemplate": "### Instruction:\n%1\n\n### Response:\n",
+    "chatTemplate": "{%- if messages[0]['role'] == 'system' %}\n    {%- set loop_start = 1 %}\n    {{- messages[0]['content'] + '\\n\\n' }}\n{%- else %}\n    {%- set loop_start = 0 %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if loop.index0 >= loop_start %}\n        {%- if message['role'] == 'user' %}\n            {{- '### Instruction:\\n' + message['content'] + '\\n\\n' }}\n        {%- elif message['role'] == 'assistant' %}\n            {{- '### Response:\\n' + message['content'] + '\\n\\n' }}\n        {%- else %}\n            {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '### Instruction:\\n' }}\n{%- endif %}"
   },
   {
     "order": "n",
@@ -217,7 +248,9 @@
     "type": "LLaMA",
     "systemPrompt": "",
     "description": "<strong>Very good overall model</strong><br><ul><li>Instruction based<li>Based on the same dataset as Groovy<li>Slower than Groovy, with higher quality responses<li>Trained by Nomic AI<li>Cannot be used commercially</ul>",
-    "url": "https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf"
+    "url": "https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf",
+    "chatTemplate": "{%- if messages[0]['role'] == 'system' %}\n    {%- set loop_start = 1 %}\n    {{- messages[0]['content'] + '\\n\\n' }}\n{%- else %}\n    {%- set loop_start = 0 %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if loop.index0 >= loop_start %}\n        {%- if message['role'] == 'user' %}\n            {{- '### Instruction:\\n' + message['content'] + '\\n\\n' }}\n        {%- elif message['role'] == 'assistant' %}\n            {{- '### Response:\\n' + message['content'] + '\\n\\n' }}\n        {%- else %}\n            {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '### Response:\\n' }}\n{%- endif %}",
+    "systemMessage": "Below is an instruction that describes a task. Write a response that appropriately completes the request."
   },
   {
     "order": "o",
@@ -234,7 +267,8 @@
     "description": "<strong>Good model with novel architecture</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/mpt-7b-chat-newbpe-q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
-    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
+    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n",
+    "chatTemplate": "{%- for message in messages %}\n    {{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}"
   },
   {
     "order": "p",
@@ -250,7 +284,8 @@
     "description": "<strong>Good model with novel architecture</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/mpt-7b-chat.gguf4.Q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
-    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
+    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n",
+    "chatTemplate": "{%- for message in messages %}\n    {{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}"
   },
   {
     "order": "q",
@@ -266,7 +301,8 @@
     "description": "<ul><li>Very fast responses</li><li>Chat based model</li><li>Accepts system prompts in Phi-3 format</li><li>Trained by Microsoft</li><li>License: <a href=\"https://opensource.org/license/mit\">MIT</a></li><li>No restrictions on commercial use</li></ul>",
     "url": "https://gpt4all.io/models/gguf/Phi-3-mini-4k-instruct.Q4_0.gguf",
     "promptTemplate": "<|user|>\n%1<|end|>\n<|assistant|>\n%2<|end|>\n",
-    "systemPrompt": ""
+    "systemPrompt": "",
+    "chatTemplate": "{{- bos_token }}\n{%- for message in messages %}\n    {{- '<|' + message['role'] + '|>\\n' + message['content'] + '<|end|>\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|assistant|>\\n' }}\n{%- else %}\n    {{- eos_token }}\n{%- endif %}"
   },
   {
     "order": "r",
@@ -282,7 +318,8 @@
     "description": "<strong>Small version of new model with novel dataset</strong><br><ul><li>Very fast responses</li><li>Instruction based</li><li>Explain tuned datasets</li><li>Orca Research Paper dataset construction approaches</li><li>Cannot be used commercially</li></ul>",
     "url": "https://gpt4all.io/models/gguf/orca-mini-3b-gguf2-q4_0.gguf",
     "promptTemplate": "### User:\n%1\n\n### Response:\n",
-    "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
+    "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n",
+    "chatTemplate": "{%- if messages[0]['role'] == 'system' %}\n    {%- set loop_start = 1 %}\n    {{- '### System:\\n' + messages[0]['content'] + '\\n\\n' }}\n{%- else %}\n    {%- set loop_start = 0 %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if loop.index0 >= loop_start %}\n        {%- if message['role'] == 'user' %}\n            {{- '### User:\\n' + message['content'] + '\\n\\n' }}\n        {%- elif message['role'] == 'assistant' %}\n            {{- '### Response:\\n' + message['content'] + '\\n\\n' }}\n        {%- else %}\n            {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '### Response:\\n' }}\n{%- endif %}"
   },
   {
     "order": "s",
@@ -299,7 +336,8 @@
     "systemPrompt": "",
     "promptTemplate": "%1",
     "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>Licensed for commercial use<li>WARNING: Not available for chat GUI</ul>",
-    "url": "https://gpt4all.io/models/gguf/replit-code-v1_5-3b-newbpe-q4_0.gguf"
+    "url": "https://gpt4all.io/models/gguf/replit-code-v1_5-3b-newbpe-q4_0.gguf",
+    "chatTemplate": null
   },
   {
     "order": "t",
@@ -316,7 +354,8 @@
     "systemPrompt": "",
     "promptTemplate": "%1",
     "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>WARNING: Not available for chat GUI</ul>",
-    "url": "https://gpt4all.io/models/gguf/starcoder-newbpe-q4_0.gguf"
+    "url": "https://gpt4all.io/models/gguf/starcoder-newbpe-q4_0.gguf",
+    "chatTemplate": null
   },
   {
     "order": "u",
@@ -333,7 +372,8 @@
     "systemPrompt": "",
     "promptTemplate": "%1",
     "description": "<strong>Trained on collection of Python and TypeScript</strong><br><ul><li>Code completion based<li>WARNING: Not available for chat GUI</li>",
-    "url": "https://gpt4all.io/models/gguf/rift-coder-v0-7b-q4_0.gguf"
+    "url": "https://gpt4all.io/models/gguf/rift-coder-v0-7b-q4_0.gguf",
+    "chatTemplate": null
   },
   {
     "order": "v",
@@ -351,7 +391,8 @@
     "embeddingModel": true,
     "systemPrompt": "",
     "description": "<strong>LocalDocs text embeddings model</strong><br><ul><li>For use with LocalDocs feature<li>Used for retrieval augmented generation (RAG)",
-    "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf"
+    "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf",
+    "chatTemplate": null
   },
   {
     "order": "w",
@@ -367,7 +408,8 @@
     "type": "Bert",
     "embeddingModel": true,
     "description": "<strong>LocalDocs text embeddings model</strong><br><ul><li>For use with LocalDocs feature<li>Used for retrieval augmented generation (RAG)",
-    "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2.gguf2.f16.gguf"
+    "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2.gguf2.f16.gguf",
+    "chatTemplate": null
   },
   {
     "order": "x",
@@ -383,7 +425,9 @@
     "description": "<strong>Mistral-based model for German-language applications</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by ellamind<li>Finetuned on German instruction and chat data</a><li>Licensed for commercial use</ul>",
     "url": "https://huggingface.co/TheBloke/em_german_mistral_v01-GGUF/resolve/main/em_german_mistral_v01.Q4_0.gguf",
     "promptTemplate": "USER: %1 ASSISTANT: ",
-    "systemPrompt": "Du bist ein hilfreicher Assistent. "
+    "systemPrompt": "Du bist ein hilfreicher Assistent. ",
+    "chatTemplate": "{%- set system_message = false %}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set loop_start = 1 %}\n    {%- set system_message = true %}\n    {{- messages[0]['content'] }}\n{%- else %}\n    {%- set loop_start = 0 %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if loop.index0 >= loop_start %}\n        {%- if (not loop.first) or (system_message is not none)  %}\n            {{- ' ' }}\n        {%- endif %}\n        {%- if message['role'] == 'user' %}\n            {{- 'USER: ' + message['content'] }}\n        {%- elif message['role'] == 'assistant' %}\n            {{- 'ASSISTANT: ' + message['content'] }}\n        {%- else %}\n            {{- raise_exception('After the optional system message, conversation roles must be either user or assistant.') }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {%- if messages %}\n        {{- ' ' }}\n    {%- endif %}\n    {{- 'ASSISTANT:' }}\n{%- endif %}",
+    "systemMessage": "Du bist ein hilfreicher Assistent."
   },
   {
     "order": "y",
@@ -400,7 +444,8 @@
     "embeddingModel": true,
     "systemPrompt": "",
     "description": "nomic-embed-text-v1",
-    "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.f16.gguf"
+    "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.f16.gguf",
+    "chatTemplate": null
   },
   {
     "order": "z",
@@ -417,7 +462,8 @@
     "embeddingModel": true,
     "systemPrompt": "",
     "description": "nomic-embed-text-v1.5",
-    "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.5.f16.gguf"
+    "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.5.f16.gguf",
+    "chatTemplate": null
   },
   {
     "order": "zzz",
@@ -433,6 +479,7 @@
     "description": "<ul><li>Very fast responses</li><li>Instruction based model</li><li>Usage of LocalDocs (RAG): Highly recommended</li><li>Supports context length of up to 32768</li><li>Trained and finetuned by Qwen (Alibaba Cloud)</li><li>License: <a href=\"https://www.apache.org/licenses/LICENSE-2.0.html/\">Apache 2.0</a></li></ul>",
     "url": "https://huggingface.co/Qwen/Qwen2-1.5B-Instruct-GGUF/resolve/main/qwen2-1_5b-instruct-q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>",
-    "systemPrompt": "<|im_start|>system\nBelow is an instruction that describes a task. Write a response that appropriately completes the request.<|im_end|>\n"
+    "systemPrompt": "<|im_start|>system\nBelow is an instruction that describes a task. Write a response that appropriately completes the request.<|im_end|>\n",
+    "chatTemplate": "{%- for message in messages %}\n    {%- if loop.first and messages[0]['role'] != 'system' %}\n        {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n    {{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}"
   }
 ]

gpt4all-chat/metadata/release.json

@@ -223,5 +223,30 @@
     "version": "3.4.1",
     "notes": "* **LocalDocs Fixes:** Several issues with LocalDocs in v3.4.0 have been fixed, including missing words and very slow indexing.\n* **Syntax Highlighting:** Go code is now highlighted with the correct colors.\n* **Cache Fixes:** The model list cache is now stored with a version number, and in a more appropriate directory.\n* **Translation Updates:** The Italian translation has been improved.\n",
     "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* John Parent (Kitware)\n* Riccardo Giovanetti (`@Harvester62`)"
+  },
+  {
+    "version": "3.4.2",
+    "notes": "* **LocalDocs Fixes:** Several issues with LocalDocs, some of which were introduced in v3.4.0, have been fixed.\n  * Fixed the possible use of references from unselected collections.\n  * Fixed unnecessary reindexing of files with uppercase extensions.\n  * Fixed hybrid search failure due to inconsistent database state.\n  * Fully fixed the blank Embeddings Device selection in LocalDocs settings.\n  * Fixed LocalDocs indexing of large PDFs making very slow progress or even stalling.\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Jared Van Bortel (Nomic AI)"
+  },
+  {
+    "version": "3.5.0",
+    "notes": "* **Message Editing:**\n  * You can now edit any message you've sent by clicking the pencil icon below it.\n  * You can now redo earlier responses in the conversation.\n* **Templates:** Chat templates have been completely overhauled! They now use Jinja-style syntax. You may notice warnings or errors in the UI. Read the linked docs, and if you have any questions, please ask on the Discord.\n* **File Attachments:** Markdown and plain text files are now supported as file attachments.\n* **System Tray:** There is now an option in Application Settings to allow GPT4All to minimize to the system tray instead of closing.\n* **Local API Server:**\n  * The API server now supports system messages from the client and no longer uses the system message in settings.\n  * You can now send messages to the API server in any order supported by the model instead of just user/assistant pairs.\n* **Translations:** The Italian and Romanian translations have been improved.\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Benjamin Gallois (`@bgallois`)\n* Riccardo Giovanetti (`@Harvester62`)\n* Victor Emanuel (`@SINAPSA-IC`)"
+  },
+  {
+    "version": "3.5.1",
+    "notes": "* **Chat template fixes:** Llama 3.2 models, Nous Hermes 2 Mistral, Mistral OpenOrca, Qwen 2 and remote models\n* **Bugfix:** Fix the default model button so it works again after 3.5.0\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)"
+  },
+  {
+    "version": "3.5.2",
+    "notes": "* **Model Search:** There are now separate tabs for official and third-party models.\n* **Local Server Fixes:** Several mistakes in v3.5's changes to the API server have been corrected.\n* **Cloned Model Fixes:** The chat template and system message of cloned models now manage their defaults correctly.\n* **Translation Improvements:** The Romanian and Italian translations have been updated.\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Riccardo Giovanetti (`@Harvester62`)\n* Victor Emanuel (`@SINAPSA-IC`)"
+  },
+  {
+    "version": "3.5.3",
+    "notes": "* **LocalDocs Fix:** A serious issue causing LocalDocs to not work properly in v3.5.2 has been fixed.\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)"
   }
 ]

gpt4all-chat/pyproject.toml

@@ -0,0 +1,29 @@
+[tool.pytest.ini_options]
+addopts = ['--import-mode=importlib']
+
+[tool.mypy]
+files = 'tests/python'
+pretty = true
+strict = true
+warn_unused_ignores = false
+
+[tool.pytype]
+inputs = ['tests/python']
+jobs = 'auto'
+bind_decorated_methods = true
+none_is_not_bool = true
+overriding_renamed_parameter_count_checks = true
+strict_none_binding = true
+precise_return = true
+# protocols:
+# - https://github.com/google/pytype/issues/1423
+# - https://github.com/google/pytype/issues/1424
+strict_import = true
+strict_parameter_checks = true
+strict_primitive_comparisons = true
+# strict_undefined_checks: too many false positives
+
+[tool.isort]
+src_paths = ['tests/python']
+line_length = 120
+combine_as_imports = true

gpt4all-chat/qml/AddGPT4AllModelView.qml

@@ -0,0 +1,592 @@
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import QtQuick.Layouts
+import QtQuick.Dialogs
+import Qt.labs.folderlistmodel
+import Qt5Compat.GraphicalEffects
+
+import llm
+import chatlistmodel
+import download
+import modellist
+import network
+import gpt4all
+import mysettings
+import localdocs
+
+ColumnLayout {
+    Layout.fillWidth: true
+    Layout.alignment: Qt.AlignTop
+    spacing: 5
+
+    Label {
+        Layout.topMargin: 0
+        Layout.bottomMargin: 25
+        Layout.rightMargin: 150 * theme.fontScale
+        Layout.alignment: Qt.AlignTop
+        Layout.fillWidth: true
+        verticalAlignment: Text.AlignTop
+        text: qsTr("These models have been specifically configured for use in GPT4All. The first few models on the " +
+                   "list are known to work the best, but you should only attempt to use models that will fit in your " +
+                   "available memory.")
+        font.pixelSize: theme.fontSizeLarger
+        color: theme.textColor
+        wrapMode: Text.WordWrap
+    }
+
+    Label {
+        visible: !ModelList.gpt4AllDownloadableModels.count && !ModelList.asyncModelRequestOngoing
+        Layout.fillWidth: true
+        Layout.fillHeight: true
+        horizontalAlignment: Qt.AlignHCenter
+        verticalAlignment: Qt.AlignVCenter
+        text: qsTr("Network error: could not retrieve %1").arg("http://gpt4all.io/models/models3.json")
+        font.pixelSize: theme.fontSizeLarge
+        color: theme.mutedTextColor
+    }
+
+    MyBusyIndicator {
+        visible: !ModelList.gpt4AllDownloadableModels.count && ModelList.asyncModelRequestOngoing
+        running: ModelList.asyncModelRequestOngoing
+        Accessible.role: Accessible.Animation
+        Layout.alignment: Qt.AlignCenter
+        Accessible.name: qsTr("Busy indicator")
+        Accessible.description: qsTr("Displayed when the models request is ongoing")
+    }
+
+    RowLayout {
+        ButtonGroup {
+            id: buttonGroup
+            exclusive: true
+        }
+        MyButton {
+            text: qsTr("All")
+            checked: true
+            borderWidth: 0
+            backgroundColor: checked ? theme.lightButtonBackground : "transparent"
+            backgroundColorHovered: theme.lighterButtonBackgroundHovered
+            backgroundRadius: 5
+            padding: 15
+            topPadding: 8
+            bottomPadding: 8
+            textColor: theme.lighterButtonForeground
+            fontPixelSize: theme.fontSizeLarge
+            fontPixelBold: true
+            checkable: true
+            ButtonGroup.group: buttonGroup
+            onClicked: {
+                ModelList.gpt4AllDownloadableModels.filter("");
+            }
+
+        }
+        MyButton {
+            text: qsTr("Reasoning")
+            borderWidth: 0
+            backgroundColor: checked ? theme.lightButtonBackground : "transparent"
+            backgroundColorHovered: theme.lighterButtonBackgroundHovered
+            backgroundRadius: 5
+            padding: 15
+            topPadding: 8
+            bottomPadding: 8
+            textColor: theme.lighterButtonForeground
+            fontPixelSize: theme.fontSizeLarge
+            fontPixelBold: true
+            checkable: true
+            ButtonGroup.group: buttonGroup
+            onClicked: {
+                ModelList.gpt4AllDownloadableModels.filter("#reasoning");
+            }
+        }
+        Layout.bottomMargin: 10
+    }
+
+    ScrollView {
+        id: scrollView
+        ScrollBar.vertical.policy: ScrollBar.AsNeeded
+        Layout.fillWidth: true
+        Layout.fillHeight: true
+        clip: true
+
+        ListView {
+            id: modelListView
+            model: ModelList.gpt4AllDownloadableModels
+            boundsBehavior: Flickable.StopAtBounds
+            spacing: 30
+
+            delegate: Rectangle {
+                id: delegateItem
+                width: modelListView.width
+                height: childrenRect.height + 60
+                color: theme.conversationBackground
+                radius: 10
+                border.width: 1
+                border.color: theme.controlBorder
+
+                ColumnLayout {
+                    anchors.top: parent.top
+                    anchors.left: parent.left
+                    anchors.right: parent.right
+                    anchors.margins: 30
+
+                    Text {
+                        Layout.fillWidth: true
+                        Layout.alignment: Qt.AlignLeft
+                        text: name
+                        elide: Text.ElideRight
+                        color: theme.titleTextColor
+                        font.pixelSize: theme.fontSizeLargest
+                        font.bold: true
+                        Accessible.role: Accessible.Paragraph
+                        Accessible.name: qsTr("Model file")
+                        Accessible.description: qsTr("Model file to be downloaded")
+                    }
+
+
+                    Rectangle {
+                        Layout.fillWidth: true
+                        height: 1
+                        color: theme.dividerColor
+                    }
+
+                    RowLayout {
+                        Layout.topMargin: 10
+                        Layout.fillWidth: true
+                        Text {
+                            id: descriptionText
+                            text: description
+                            font.pixelSize: theme.fontSizeLarge
+                            Layout.fillWidth: true
+                            wrapMode: Text.WordWrap
+                            textFormat: Text.StyledText
+                            color: theme.textColor
+                            linkColor: theme.textColor
+                            Accessible.role: Accessible.Paragraph
+                            Accessible.name: qsTr("Description")
+                            Accessible.description: qsTr("File description")
+                            onLinkActivated: function(link) { Qt.openUrlExternally(link); }
+                            MouseArea {
+                                anchors.fill: parent
+                                acceptedButtons: Qt.NoButton // pass clicks to parent
+                                cursorShape: parent.hoveredLink ? Qt.PointingHandCursor : Qt.ArrowCursor
+                            }
+                        }
+
+                        // FIXME Need to overhaul design here which must take into account
+                        // features not present in current figma including:
+                        // * Ability to cancel a current download
+                        // * Ability to resume a download
+                        // * The presentation of an error if encountered
+                        // * Whether to show already installed models
+                        // * Install of remote models with API keys
+                        // * The presentation of the progress bar
+                        Rectangle {
+                            id: actionBox
+                            width: childrenRect.width + 20
+                            color: "transparent"
+                            border.width: 1
+                            border.color: theme.dividerColor
+                            radius: 10
+                            Layout.rightMargin: 20
+                            Layout.bottomMargin: 20
+                            Layout.minimumHeight: childrenRect.height + 20
+                            Layout.alignment: Qt.AlignRight | Qt.AlignTop
+
+                            ColumnLayout {
+                                spacing: 0
+                                MySettingsButton {
+                                    id: downloadButton
+                                    text: isDownloading ? qsTr("Cancel") : isIncomplete ? qsTr("Resume") : qsTr("Download")
+                                    font.pixelSize: theme.fontSizeLarge
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.fillWidth: true
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    visible: !isOnline && !installed && !calcHash && downloadError === ""
+                                    Accessible.description: qsTr("Stop/restart/start the download")
+                                    onClicked: {
+                                        if (!isDownloading) {
+                                            Download.downloadModel(filename);
+                                        } else {
+                                            Download.cancelDownload(filename);
+                                        }
+                                    }
+                                }
+
+                                MySettingsDestructiveButton {
+                                    id: removeButton
+                                    text: qsTr("Remove")
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.fillWidth: true
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    visible: !isDownloading && (installed || isIncomplete)
+                                    Accessible.description: qsTr("Remove model from filesystem")
+                                    onClicked: {
+                                        Download.removeModel(filename);
+                                    }
+                                }
+
+                                MySettingsButton {
+                                    id: installButton
+                                    visible: !installed && isOnline
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.fillWidth: true
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    text: qsTr("Install")
+                                    font.pixelSize: theme.fontSizeLarge
+                                    onClicked: {
+                                        var apiKeyText = apiKey.text.trim(),
+                                        baseUrlText = baseUrl.text.trim(),
+                                        modelNameText = modelName.text.trim();
+
+                                        var apiKeyOk = apiKeyText !== "",
+                                        baseUrlOk = !isCompatibleApi || baseUrlText !== "",
+                                        modelNameOk = !isCompatibleApi || modelNameText !== "";
+
+                                        if (!apiKeyOk)
+                                            apiKey.showError();
+                                        if (!baseUrlOk)
+                                            baseUrl.showError();
+                                        if (!modelNameOk)
+                                            modelName.showError();
+
+                                        if (!apiKeyOk || !baseUrlOk || !modelNameOk)
+                                            return;
+
+                                        if (!isCompatibleApi)
+                                            Download.installModel(
+                                                        filename,
+                                                        apiKeyText,
+                                                        );
+                                        else
+                                            Download.installCompatibleModel(
+                                                        modelNameText,
+                                                        apiKeyText,
+                                                        baseUrlText,
+                                                        );
+                                    }
+                                    Accessible.role: Accessible.Button
+                                    Accessible.name: qsTr("Install")
+                                    Accessible.description: qsTr("Install online model")
+                                }
+
+                                ColumnLayout {
+                                    spacing: 0
+                                    Label {
+                                        Layout.topMargin: 20
+                                        Layout.leftMargin: 20
+                                        visible: downloadError !== ""
+                                        textFormat: Text.StyledText
+                                        text: qsTr("<strong><font size=\"1\"><a href=\"#error\">Error</a></strong></font>")
+                                        color: theme.textColor
+                                        font.pixelSize: theme.fontSizeLarge
+                                        linkColor: theme.textErrorColor
+                                        Accessible.role: Accessible.Paragraph
+                                        Accessible.name: text
+                                        Accessible.description: qsTr("Describes an error that occurred when downloading")
+                                        onLinkActivated: {
+                                            downloadingErrorPopup.text = downloadError;
+                                            downloadingErrorPopup.open();
+                                        }
+                                    }
+
+                                    Label {
+                                        visible: LLM.systemTotalRAMInGB() < ramrequired
+                                        Layout.topMargin: 20
+                                        Layout.leftMargin: 20
+                                        Layout.maximumWidth: 300
+                                        textFormat: Text.StyledText
+                                        text: qsTr("<strong><font size=\"2\">WARNING: Not recommended for your hardware. Model requires more memory (%1 GB) than your system has available (%2).</strong></font>").arg(ramrequired).arg(LLM.systemTotalRAMInGBString())
+                                        color: theme.textErrorColor
+                                        font.pixelSize: theme.fontSizeLarge
+                                        wrapMode: Text.WordWrap
+                                        Accessible.role: Accessible.Paragraph
+                                        Accessible.name: text
+                                        Accessible.description: qsTr("Error for incompatible hardware")
+                                        onLinkActivated: {
+                                            downloadingErrorPopup.text = downloadError;
+                                            downloadingErrorPopup.open();
+                                        }
+                                    }
+                                }
+
+                                ColumnLayout {
+                                    visible: isDownloading && !calcHash
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.fillWidth: true
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    spacing: 20
+
+                                    ProgressBar {
+                                        id: itemProgressBar
+                                        Layout.fillWidth: true
+                                        width: 200
+                                        value: bytesReceived / bytesTotal
+                                        background: Rectangle {
+                                            implicitHeight: 45
+                                            color: theme.progressBackground
+                                            radius: 3
+                                        }
+                                        contentItem: Item {
+                                            implicitHeight: 40
+
+                                            Rectangle {
+                                                width: itemProgressBar.visualPosition * parent.width
+                                                height: parent.height
+                                                radius: 2
+                                                color: theme.progressForeground
+                                            }
+                                        }
+                                        Accessible.role: Accessible.ProgressBar
+                                        Accessible.name: qsTr("Download progressBar")
+                                        Accessible.description: qsTr("Shows the progress made in the download")
+                                    }
+
+                                    Label {
+                                        id: speedLabel
+                                        color: theme.textColor
+                                        Layout.alignment: Qt.AlignRight
+                                        text: speed
+                                        font.pixelSize: theme.fontSizeLarge
+                                        Accessible.role: Accessible.Paragraph
+                                        Accessible.name: qsTr("Download speed")
+                                        Accessible.description: qsTr("Download speed in bytes/kilobytes/megabytes per second")
+                                    }
+                                }
+
+                                RowLayout {
+                                    visible: calcHash
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.maximumWidth: 200
+                                    Layout.fillWidth: true
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    clip: true
+
+                                    Label {
+                                        id: calcHashLabel
+                                        color: theme.textColor
+                                        text: qsTr("Calculating...")
+                                        font.pixelSize: theme.fontSizeLarge
+                                        Accessible.role: Accessible.Paragraph
+                                        Accessible.name: text
+                                        Accessible.description: qsTr("Whether the file hash is being calculated")
+                                    }
+
+                                    MyBusyIndicator {
+                                        id: busyCalcHash
+                                        running: calcHash
+                                        Accessible.role: Accessible.Animation
+                                        Accessible.name: qsTr("Busy indicator")
+                                        Accessible.description: qsTr("Displayed when the file hash is being calculated")
+                                    }
+                                }
+
+                                MyTextField {
+                                    id: apiKey
+                                    visible: !installed && isOnline
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    wrapMode: Text.WrapAnywhere
+                                    function showError() {
+                                        messageToast.show(qsTr("ERROR: $API_KEY is empty."));
+                                        apiKey.placeholderTextColor = theme.textErrorColor;
+                                    }
+                                    onTextChanged: {
+                                        apiKey.placeholderTextColor = theme.mutedTextColor;
+                                    }
+                                    placeholderText: qsTr("enter $API_KEY")
+                                    Accessible.role: Accessible.EditableText
+                                    Accessible.name: placeholderText
+                                    Accessible.description: qsTr("Whether the file hash is being calculated")
+                                }
+
+                                MyTextField {
+                                    id: baseUrl
+                                    visible: !installed && isOnline && isCompatibleApi
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    wrapMode: Text.WrapAnywhere
+                                    function showError() {
+                                        messageToast.show(qsTr("ERROR: $BASE_URL is empty."));
+                                        baseUrl.placeholderTextColor = theme.textErrorColor;
+                                    }
+                                    onTextChanged: {
+                                        baseUrl.placeholderTextColor = theme.mutedTextColor;
+                                    }
+                                    placeholderText: qsTr("enter $BASE_URL")
+                                    Accessible.role: Accessible.EditableText
+                                    Accessible.name: placeholderText
+                                    Accessible.description: qsTr("Whether the file hash is being calculated")
+                                }
+
+                                MyTextField {
+                                    id: modelName
+                                    visible: !installed && isOnline && isCompatibleApi
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    wrapMode: Text.WrapAnywhere
+                                    function showError() {
+                                        messageToast.show(qsTr("ERROR: $MODEL_NAME is empty."))
+                                        modelName.placeholderTextColor = theme.textErrorColor;
+                                    }
+                                    onTextChanged: {
+                                        modelName.placeholderTextColor = theme.mutedTextColor;
+                                    }
+                                    placeholderText: qsTr("enter $MODEL_NAME")
+                                    Accessible.role: Accessible.EditableText
+                                    Accessible.name: placeholderText
+                                    Accessible.description: qsTr("Whether the file hash is being calculated")
+                                }
+                            }
+                        }
+                    }
+
+                    Item  {
+                        Layout.minimumWidth: childrenRect.width
+                        Layout.minimumHeight: childrenRect.height
+                        Layout.bottomMargin: 10
+                        RowLayout {
+                            id: paramRow
+                            anchors.centerIn: parent
+                            ColumnLayout {
+                                Layout.topMargin: 10
+                                Layout.bottomMargin: 10
+                                Layout.leftMargin: 20
+                                Layout.rightMargin: 20
+                                Text {
+                                    text: qsTr("File size")
+                                    font.pixelSize: theme.fontSizeSmall
+                                    color: theme.mutedDarkTextColor
+                                }
+                                Text {
+                                    text: filesize
+                                    color: theme.textColor
+                                    font.pixelSize: theme.fontSizeSmall
+                                    font.bold: true
+                                }
+                            }
+                            Rectangle {
+                                width: 1
+                                Layout.fillHeight: true
+                                color: theme.dividerColor
+                            }
+                            ColumnLayout {
+                                Layout.topMargin: 10
+                                Layout.bottomMargin: 10
+                                Layout.leftMargin: 20
+                                Layout.rightMargin: 20
+                                Text {
+                                    text: qsTr("RAM required")
+                                    font.pixelSize: theme.fontSizeSmall
+                                    color: theme.mutedDarkTextColor
+                                }
+                                Text {
+                                    text: ramrequired >= 0 ? qsTr("%1 GB").arg(ramrequired) : qsTr("?")
+                                    color: theme.textColor
+                                    font.pixelSize: theme.fontSizeSmall
+                                    font.bold: true
+                                }
+                            }
+                            Rectangle {
+                                width: 1
+                                Layout.fillHeight: true
+                                color: theme.dividerColor
+                            }
+                            ColumnLayout {
+                                Layout.topMargin: 10
+                                Layout.bottomMargin: 10
+                                Layout.leftMargin: 20
+                                Layout.rightMargin: 20
+                                Text {
+                                    text: qsTr("Parameters")
+                                    font.pixelSize: theme.fontSizeSmall
+                                    color: theme.mutedDarkTextColor
+                                }
+                                Text {
+                                    text: parameters !== "" ? parameters : qsTr("?")
+                                    color: theme.textColor
+                                    font.pixelSize: theme.fontSizeSmall
+                                    font.bold: true
+                                }
+                            }
+                            Rectangle {
+                                width: 1
+                                Layout.fillHeight: true
+                                color: theme.dividerColor
+                            }
+                            ColumnLayout {
+                                Layout.topMargin: 10
+                                Layout.bottomMargin: 10
+                                Layout.leftMargin: 20
+                                Layout.rightMargin: 20
+                                Text {
+                                    text: qsTr("Quant")
+                                    font.pixelSize: theme.fontSizeSmall
+                                    color: theme.mutedDarkTextColor
+                                }
+                                Text {
+                                    text: quant
+                                    color: theme.textColor
+                                    font.pixelSize: theme.fontSizeSmall
+                                    font.bold: true
+                                }
+                            }
+                            Rectangle {
+                                width: 1
+                                Layout.fillHeight: true
+                                color: theme.dividerColor
+                            }
+                            ColumnLayout {
+                                Layout.topMargin: 10
+                                Layout.bottomMargin: 10
+                                Layout.leftMargin: 20
+                                Layout.rightMargin: 20
+                                Text {
+                                    text: qsTr("Type")
+                                    font.pixelSize: theme.fontSizeSmall
+                                    color: theme.mutedDarkTextColor
+                                }
+                                Text {
+                                    text: type
+                                    color: theme.textColor
+                                    font.pixelSize: theme.fontSizeSmall
+                                    font.bold: true
+                                }
+                            }
+                        }
+
+                        Rectangle {
+                            color: "transparent"
+                            anchors.fill: paramRow
+                            border.color: theme.dividerColor
+                            border.width: 1
+                            radius: 10
+                        }
+                    }
+
+                    Rectangle {
+                        Layout.fillWidth: true
+                        height: 1
+                        color: theme.dividerColor
+                    }
+                }
+            }
+        }
+    }
+}

gpt4all-chat/qml/AddHFModelView.qml

@@ -0,0 +1,703 @@
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import QtQuick.Layouts
+import QtQuick.Dialogs
+import Qt.labs.folderlistmodel
+import Qt5Compat.GraphicalEffects
+
+import llm
+import chatlistmodel
+import download
+import modellist
+import network
+import gpt4all
+import mysettings
+import localdocs
+
+ColumnLayout {
+    Layout.fillWidth: true
+    Layout.fillHeight: true
+    Layout.alignment: Qt.AlignTop
+    spacing: 5
+
+    Label {
+        Layout.topMargin: 0
+        Layout.bottomMargin: 25
+        Layout.rightMargin: 150 * theme.fontScale
+        Layout.alignment: Qt.AlignTop
+        Layout.fillWidth: true
+        verticalAlignment: Text.AlignTop
+        text: qsTr("Use the search to find and download models from HuggingFace. There is NO GUARANTEE that these " +
+                   "will work. Many will require additional configuration before they can be used.")
+        font.pixelSize: theme.fontSizeLarger
+        color: theme.textColor
+        wrapMode: Text.WordWrap
+    }
+
+    RowLayout {
+        Layout.fillWidth: true
+        Layout.fillHeight: true
+        Layout.alignment: Qt.AlignCenter
+        Layout.margins: 0
+        spacing: 10
+        MyTextField {
+            id: discoverField
+            property string textBeingSearched: ""
+            readOnly: ModelList.discoverInProgress
+            Layout.alignment: Qt.AlignCenter
+            Layout.fillWidth: true
+            font.pixelSize: theme.fontSizeLarger
+            placeholderText: qsTr("Discover and download models by keyword search...")
+            Accessible.role: Accessible.EditableText
+            Accessible.name: placeholderText
+            Accessible.description: qsTr("Text field for discovering and filtering downloadable models")
+            Connections {
+                target: ModelList
+                function onDiscoverInProgressChanged() {
+                    if (ModelList.discoverInProgress) {
+                        discoverField.textBeingSearched = discoverField.text;
+                        discoverField.text = qsTr("Searching \u00B7 %1").arg(discoverField.textBeingSearched);
+                    } else {
+                        discoverField.text = discoverField.textBeingSearched;
+                        discoverField.textBeingSearched = "";
+                    }
+                }
+            }
+            background: ProgressBar {
+                id: discoverProgressBar
+                indeterminate: ModelList.discoverInProgress && ModelList.discoverProgress === 0.0
+                value: ModelList.discoverProgress
+                background: Rectangle {
+                    color: theme.controlBackground
+                    border.color: theme.controlBorder
+                    radius: 10
+                }
+                contentItem: Item {
+                    Rectangle {
+                        visible: ModelList.discoverInProgress
+                        anchors.bottom: parent.bottom
+                        width: discoverProgressBar.visualPosition * parent.width
+                        height: 10
+                        radius: 2
+                        color: theme.progressForeground
+                    }
+                }
+            }
+
+            Keys.onReturnPressed: (event)=> {
+                                      if (event.modifiers & Qt.ControlModifier || event.modifiers & Qt.ShiftModifier)
+                                      event.accepted = false;
+                                      else {
+                                          editingFinished();
+                                          sendDiscovery()
+                                      }
+                                  }
+            function sendDiscovery() {
+                ModelList.huggingFaceDownloadableModels.discoverAndFilter(discoverField.text);
+            }
+            RowLayout {
+                spacing: 0
+                anchors.right: discoverField.right
+                anchors.verticalCenter: discoverField.verticalCenter
+                anchors.rightMargin: 15
+                visible: !ModelList.discoverInProgress
+                MyMiniButton {
+                    id: clearDiscoverButton
+                    backgroundColor: theme.textColor
+                    backgroundColorHovered: theme.iconBackgroundDark
+                    visible: discoverField.text !== ""
+                    source: "qrc:/gpt4all/icons/close.svg"
+                    onClicked: {
+                        discoverField.text = ""
+                        discoverField.sendDiscovery() // should clear results
+                    }
+                }
+                MyMiniButton {
+                    backgroundColor: theme.textColor
+                    backgroundColorHovered: theme.iconBackgroundDark
+                    source: "qrc:/gpt4all/icons/settings.svg"
+                    onClicked: {
+                        discoveryTools.visible = !discoveryTools.visible
+                    }
+                }
+                MyMiniButton {
+                    id: sendButton
+                    enabled: !ModelList.discoverInProgress
+                    backgroundColor: theme.textColor
+                    backgroundColorHovered: theme.iconBackgroundDark
+                    source: "qrc:/gpt4all/icons/send_message.svg"
+                    Accessible.name: qsTr("Initiate model discovery and filtering")
+                    Accessible.description: qsTr("Triggers discovery and filtering of models")
+                    onClicked: {
+                        discoverField.sendDiscovery()
+                    }
+                }
+            }
+        }
+    }
+
+    RowLayout {
+        id: discoveryTools
+        Layout.fillWidth: true
+        Layout.alignment: Qt.AlignCenter
+        Layout.margins: 0
+        spacing: 20
+        visible: false
+        MyComboBox {
+            id: comboSort
+            model: ListModel {
+                ListElement { name: qsTr("Default") }
+                ListElement { name: qsTr("Likes") }
+                ListElement { name: qsTr("Downloads") }
+                ListElement { name: qsTr("Recent") }
+            }
+            currentIndex: ModelList.discoverSort
+            contentItem: Text {
+                anchors.horizontalCenter: parent.horizontalCenter
+                rightPadding: 30
+                color: theme.textColor
+                text: {
+                    return qsTr("Sort by: %1").arg(comboSort.displayText)
+                }
+                font.pixelSize: theme.fontSizeLarger
+                verticalAlignment: Text.AlignVCenter
+                horizontalAlignment: Text.AlignHCenter
+                elide: Text.ElideRight
+            }
+            onActivated: function (index) {
+                ModelList.discoverSort = index;
+            }
+        }
+        MyComboBox {
+            id: comboSortDirection
+            model: ListModel {
+                ListElement { name: qsTr("Asc") }
+                ListElement { name: qsTr("Desc") }
+            }
+            currentIndex: {
+                if (ModelList.discoverSortDirection === 1)
+                    return 0
+                else
+                    return 1;
+            }
+            contentItem: Text {
+                anchors.horizontalCenter: parent.horizontalCenter
+                rightPadding: 30
+                color: theme.textColor
+                text: {
+                    return qsTr("Sort dir: %1").arg(comboSortDirection.displayText)
+                }
+                font.pixelSize: theme.fontSizeLarger
+                verticalAlignment: Text.AlignVCenter
+                horizontalAlignment: Text.AlignHCenter
+                elide: Text.ElideRight
+            }
+            onActivated: function (index) {
+                if (index === 0)
+                    ModelList.discoverSortDirection = 1;
+                else
+                    ModelList.discoverSortDirection = -1;
+            }
+        }
+        MyComboBox {
+            id: comboLimit
+            model: ListModel {
+                ListElement { name: "5" }
+                ListElement { name: "10" }
+                ListElement { name: "20" }
+                ListElement { name: "50" }
+                ListElement { name: "100" }
+                ListElement { name: qsTr("None") }
+            }
+
+            currentIndex: {
+                if (ModelList.discoverLimit === 5)
+                    return 0;
+                else if (ModelList.discoverLimit === 10)
+                    return 1;
+                else if (ModelList.discoverLimit === 20)
+                    return 2;
+                else if (ModelList.discoverLimit === 50)
+                    return 3;
+                else if (ModelList.discoverLimit === 100)
+                    return 4;
+                else if (ModelList.discoverLimit === -1)
+                    return 5;
+            }
+            contentItem: Text {
+                anchors.horizontalCenter: parent.horizontalCenter
+                rightPadding: 30
+                color: theme.textColor
+                text: {
+                    return qsTr("Limit: %1").arg(comboLimit.displayText)
+                }
+                font.pixelSize: theme.fontSizeLarger
+                verticalAlignment: Text.AlignVCenter
+                horizontalAlignment: Text.AlignHCenter
+                elide: Text.ElideRight
+            }
+            onActivated: function (index) {
+                switch (index) {
+                case 0:
+                    ModelList.discoverLimit = 5; break;
+                case 1:
+                    ModelList.discoverLimit = 10; break;
+                case 2:
+                    ModelList.discoverLimit = 20; break;
+                case 3:
+                    ModelList.discoverLimit = 50; break;
+                case 4:
+                    ModelList.discoverLimit = 100; break;
+                case 5:
+                    ModelList.discoverLimit = -1; break;
+                }
+            }
+        }
+    }
+
+    ScrollView {
+        id: scrollView
+        ScrollBar.vertical.policy: ScrollBar.AsNeeded
+        Layout.fillWidth: true
+        Layout.fillHeight: true
+        clip: true
+
+        ListView {
+            id: modelListView
+            model: ModelList.huggingFaceDownloadableModels
+            boundsBehavior: Flickable.StopAtBounds
+            spacing: 30
+
+            delegate: Rectangle {
+                id: delegateItem
+                width: modelListView.width
+                height: childrenRect.height + 60
+                color: theme.conversationBackground
+                radius: 10
+                border.width: 1
+                border.color: theme.controlBorder
+
+                ColumnLayout {
+                    anchors.top: parent.top
+                    anchors.left: parent.left
+                    anchors.right: parent.right
+                    anchors.margins: 30
+
+                    Text {
+                        Layout.fillWidth: true
+                        Layout.alignment: Qt.AlignLeft
+                        text: name
+                        elide: Text.ElideRight
+                        color: theme.titleTextColor
+                        font.pixelSize: theme.fontSizeLargest
+                        font.bold: true
+                        Accessible.role: Accessible.Paragraph
+                        Accessible.name: qsTr("Model file")
+                        Accessible.description: qsTr("Model file to be downloaded")
+                    }
+
+
+                    Rectangle {
+                        Layout.fillWidth: true
+                        height: 1
+                        color: theme.dividerColor
+                    }
+
+                    RowLayout {
+                        Layout.topMargin: 10
+                        Layout.fillWidth: true
+                        Text {
+                            id: descriptionText
+                            text: description
+                            font.pixelSize: theme.fontSizeLarge
+                            Layout.fillWidth: true
+                            wrapMode: Text.WordWrap
+                            textFormat: Text.StyledText
+                            color: theme.textColor
+                            linkColor: theme.textColor
+                            Accessible.role: Accessible.Paragraph
+                            Accessible.name: qsTr("Description")
+                            Accessible.description: qsTr("File description")
+                            onLinkActivated: function(link) { Qt.openUrlExternally(link); }
+                            MouseArea {
+                                anchors.fill: parent
+                                acceptedButtons: Qt.NoButton // pass clicks to parent
+                                cursorShape: parent.hoveredLink ? Qt.PointingHandCursor : Qt.ArrowCursor
+                            }
+                        }
+
+                        // FIXME Need to overhaul design here which must take into account
+                        // features not present in current figma including:
+                        // * Ability to cancel a current download
+                        // * Ability to resume a download
+                        // * The presentation of an error if encountered
+                        // * Whether to show already installed models
+                        // * Install of remote models with API keys
+                        // * The presentation of the progress bar
+                        Rectangle {
+                            id: actionBox
+                            width: childrenRect.width + 20
+                            color: "transparent"
+                            border.width: 1
+                            border.color: theme.dividerColor
+                            radius: 10
+                            Layout.rightMargin: 20
+                            Layout.bottomMargin: 20
+                            Layout.minimumHeight: childrenRect.height + 20
+                            Layout.alignment: Qt.AlignRight | Qt.AlignTop
+
+                            ColumnLayout {
+                                spacing: 0
+                                MySettingsButton {
+                                    id: downloadButton
+                                    text: isDownloading ? qsTr("Cancel") : isIncomplete ? qsTr("Resume") : qsTr("Download")
+                                    font.pixelSize: theme.fontSizeLarge
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.fillWidth: true
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    visible: !isOnline && !installed && !calcHash && downloadError === ""
+                                    Accessible.description: qsTr("Stop/restart/start the download")
+                                    onClicked: {
+                                        if (!isDownloading) {
+                                            Download.downloadModel(filename);
+                                        } else {
+                                            Download.cancelDownload(filename);
+                                        }
+                                    }
+                                }
+
+                                MySettingsDestructiveButton {
+                                    id: removeButton
+                                    text: qsTr("Remove")
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.fillWidth: true
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    visible: !isDownloading && (installed || isIncomplete)
+                                    Accessible.description: qsTr("Remove model from filesystem")
+                                    onClicked: {
+                                        Download.removeModel(filename);
+                                    }
+                                }
+
+                                MySettingsButton {
+                                    id: installButton
+                                    visible: !installed && isOnline
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.fillWidth: true
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    text: qsTr("Install")
+                                    font.pixelSize: theme.fontSizeLarge
+                                    onClicked: {
+                                        var apiKeyText = apiKey.text.trim(),
+                                        baseUrlText = baseUrl.text.trim(),
+                                        modelNameText = modelName.text.trim();
+
+                                        var apiKeyOk = apiKeyText !== "",
+                                        baseUrlOk = !isCompatibleApi || baseUrlText !== "",
+                                        modelNameOk = !isCompatibleApi || modelNameText !== "";
+
+                                        if (!apiKeyOk)
+                                            apiKey.showError();
+                                        if (!baseUrlOk)
+                                            baseUrl.showError();
+                                        if (!modelNameOk)
+                                            modelName.showError();
+
+                                        if (!apiKeyOk || !baseUrlOk || !modelNameOk)
+                                            return;
+
+                                        if (!isCompatibleApi)
+                                            Download.installModel(
+                                                        filename,
+                                                        apiKeyText,
+                                                        );
+                                        else
+                                            Download.installCompatibleModel(
+                                                        modelNameText,
+                                                        apiKeyText,
+                                                        baseUrlText,
+                                                        );
+                                    }
+                                    Accessible.role: Accessible.Button
+                                    Accessible.name: qsTr("Install")
+                                    Accessible.description: qsTr("Install online model")
+                                }
+
+                                ColumnLayout {
+                                    spacing: 0
+                                    Label {
+                                        Layout.topMargin: 20
+                                        Layout.leftMargin: 20
+                                        visible: downloadError !== ""
+                                        textFormat: Text.StyledText
+                                        text: qsTr("<strong><font size=\"1\"><a href=\"#error\">Error</a></strong></font>")
+                                        color: theme.textColor
+                                        font.pixelSize: theme.fontSizeLarge
+                                        linkColor: theme.textErrorColor
+                                        Accessible.role: Accessible.Paragraph
+                                        Accessible.name: text
+                                        Accessible.description: qsTr("Describes an error that occurred when downloading")
+                                        onLinkActivated: {
+                                            downloadingErrorPopup.text = downloadError;
+                                            downloadingErrorPopup.open();
+                                        }
+                                    }
+
+                                    Label {
+                                        visible: LLM.systemTotalRAMInGB() < ramrequired
+                                        Layout.topMargin: 20
+                                        Layout.leftMargin: 20
+                                        Layout.maximumWidth: 300
+                                        textFormat: Text.StyledText
+                                        text: qsTr("<strong><font size=\"2\">WARNING: Not recommended for your hardware. Model requires more memory (%1 GB) than your system has available (%2).</strong></font>").arg(ramrequired).arg(LLM.systemTotalRAMInGBString())
+                                        color: theme.textErrorColor
+                                        font.pixelSize: theme.fontSizeLarge
+                                        wrapMode: Text.WordWrap
+                                        Accessible.role: Accessible.Paragraph
+                                        Accessible.name: text
+                                        Accessible.description: qsTr("Error for incompatible hardware")
+                                        onLinkActivated: {
+                                            downloadingErrorPopup.text = downloadError;
+                                            downloadingErrorPopup.open();
+                                        }
+                                    }
+                                }
+
+                                ColumnLayout {
+                                    visible: isDownloading && !calcHash
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.fillWidth: true
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    spacing: 20
+
+                                    ProgressBar {
+                                        id: itemProgressBar
+                                        Layout.fillWidth: true
+                                        width: 200
+                                        value: bytesReceived / bytesTotal
+                                        background: Rectangle {
+                                            implicitHeight: 45
+                                            color: theme.progressBackground
+                                            radius: 3
+                                        }
+                                        contentItem: Item {
+                                            implicitHeight: 40
+
+                                            Rectangle {
+                                                width: itemProgressBar.visualPosition * parent.width
+                                                height: parent.height
+                                                radius: 2
+                                                color: theme.progressForeground
+                                            }
+                                        }
+                                        Accessible.role: Accessible.ProgressBar
+                                        Accessible.name: qsTr("Download progressBar")
+                                        Accessible.description: qsTr("Shows the progress made in the download")
+                                    }
+
+                                    Label {
+                                        id: speedLabel
+                                        color: theme.textColor
+                                        Layout.alignment: Qt.AlignRight
+                                        text: speed
+                                        font.pixelSize: theme.fontSizeLarge
+                                        Accessible.role: Accessible.Paragraph
+                                        Accessible.name: qsTr("Download speed")
+                                        Accessible.description: qsTr("Download speed in bytes/kilobytes/megabytes per second")
+                                    }
+                                }
+
+                                RowLayout {
+                                    visible: calcHash
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.maximumWidth: 200
+                                    Layout.fillWidth: true
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    clip: true
+
+                                    Label {
+                                        id: calcHashLabel
+                                        color: theme.textColor
+                                        text: qsTr("Calculating...")
+                                        font.pixelSize: theme.fontSizeLarge
+                                        Accessible.role: Accessible.Paragraph
+                                        Accessible.name: text
+                                        Accessible.description: qsTr("Whether the file hash is being calculated")
+                                    }
+
+                                    MyBusyIndicator {
+                                        id: busyCalcHash
+                                        running: calcHash
+                                        Accessible.role: Accessible.Animation
+                                        Accessible.name: qsTr("Busy indicator")
+                                        Accessible.description: qsTr("Displayed when the file hash is being calculated")
+                                    }
+                                }
+
+                                MyTextField {
+                                    id: apiKey
+                                    visible: !installed && isOnline
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    wrapMode: Text.WrapAnywhere
+                                    function showError() {
+                                        messageToast.show(qsTr("ERROR: $API_KEY is empty."));
+                                        apiKey.placeholderTextColor = theme.textErrorColor;
+                                    }
+                                    onTextChanged: {
+                                        apiKey.placeholderTextColor = theme.mutedTextColor;
+                                    }
+                                    placeholderText: qsTr("enter $API_KEY")
+                                    Accessible.role: Accessible.EditableText
+                                    Accessible.name: placeholderText
+                                    Accessible.description: qsTr("Whether the file hash is being calculated")
+                                }
+
+                                MyTextField {
+                                    id: baseUrl
+                                    visible: !installed && isOnline && isCompatibleApi
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    wrapMode: Text.WrapAnywhere
+                                    function showError() {
+                                        messageToast.show(qsTr("ERROR: $BASE_URL is empty."));
+                                        baseUrl.placeholderTextColor = theme.textErrorColor;
+                                    }
+                                    onTextChanged: {
+                                        baseUrl.placeholderTextColor = theme.mutedTextColor;
+                                    }
+                                    placeholderText: qsTr("enter $BASE_URL")
+                                    Accessible.role: Accessible.EditableText
+                                    Accessible.name: placeholderText
+                                    Accessible.description: qsTr("Whether the file hash is being calculated")
+                                }
+
+                                MyTextField {
+                                    id: modelName
+                                    visible: !installed && isOnline && isCompatibleApi
+                                    Layout.topMargin: 20
+                                    Layout.leftMargin: 20
+                                    Layout.minimumWidth: 200
+                                    Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                    wrapMode: Text.WrapAnywhere
+                                    function showError() {
+                                        messageToast.show(qsTr("ERROR: $MODEL_NAME is empty."))
+                                        modelName.placeholderTextColor = theme.textErrorColor;
+                                    }
+                                    onTextChanged: {
+                                        modelName.placeholderTextColor = theme.mutedTextColor;
+                                    }
+                                    placeholderText: qsTr("enter $MODEL_NAME")
+                                    Accessible.role: Accessible.EditableText
+                                    Accessible.name: placeholderText
+                                    Accessible.description: qsTr("Whether the file hash is being calculated")
+                                }
+                            }
+                        }
+                    }
+
+                    Item  {
+                        Layout.minimumWidth: childrenRect.width
+                        Layout.minimumHeight: childrenRect.height
+                        Layout.bottomMargin: 10
+                        RowLayout {
+                            id: paramRow
+                            anchors.centerIn: parent
+                            ColumnLayout {
+                                Layout.topMargin: 10
+                                Layout.bottomMargin: 10
+                                Layout.leftMargin: 20
+                                Layout.rightMargin: 20
+                                Text {
+                                    text: qsTr("File size")
+                                    font.pixelSize: theme.fontSizeSmall
+                                    color: theme.mutedDarkTextColor
+                                }
+                                Text {
+                                    text: filesize
+                                    color: theme.textColor
+                                    font.pixelSize: theme.fontSizeSmall
+                                    font.bold: true
+                                }
+                            }
+                            Rectangle {
+                                width: 1
+                                Layout.fillHeight: true
+                                color: theme.dividerColor
+                            }
+                            ColumnLayout {
+                                Layout.topMargin: 10
+                                Layout.bottomMargin: 10
+                                Layout.leftMargin: 20
+                                Layout.rightMargin: 20
+                                Text {
+                                    text: qsTr("Quant")
+                                    font.pixelSize: theme.fontSizeSmall
+                                    color: theme.mutedDarkTextColor
+                                }
+                                Text {
+                                    text: quant
+                                    color: theme.textColor
+                                    font.pixelSize: theme.fontSizeSmall
+                                    font.bold: true
+                                }
+                            }
+                            Rectangle {
+                                width: 1
+                                Layout.fillHeight: true
+                                color: theme.dividerColor
+                            }
+                            ColumnLayout {
+                                Layout.topMargin: 10
+                                Layout.bottomMargin: 10
+                                Layout.leftMargin: 20
+                                Layout.rightMargin: 20
+                                Text {
+                                    text: qsTr("Type")
+                                    font.pixelSize: theme.fontSizeSmall
+                                    color: theme.mutedDarkTextColor
+                                }
+                                Text {
+                                    text: type
+                                    color: theme.textColor
+                                    font.pixelSize: theme.fontSizeSmall
+                                    font.bold: true
+                                }
+                            }
+                        }
+
+                        Rectangle {
+                            color: "transparent"
+                            anchors.fill: paramRow
+                            border.color: theme.dividerColor
+                            border.width: 1
+                            radius: 10
+                        }
+                    }
+
+                    Rectangle {
+                        Layout.fillWidth: true
+                        height: 1
+                        color: theme.dividerColor
+                    }
+                }
+            }
+        }
+    }
+}

gpt4all-chat/qml/AddModelView.qml

@@ -42,12 +42,12 @@ Rectangle {
         anchors.top: parent.top
         anchors.bottom: parent.bottom
         anchors.margins: 30
-        spacing: 30
+        spacing: 10
 
         ColumnLayout {
             Layout.fillWidth: true
             Layout.alignment: Qt.AlignTop
-            spacing: 30
+            spacing: 10
 
             MyButton {
                 id: backButton
@@ -76,732 +76,60 @@ Rectangle {
                 font.pixelSize: theme.fontSizeBanner
                 color: theme.titleTextColor
             }
+        }
 
-            RowLayout {
-                Layout.fillWidth: true
-                Layout.alignment: Qt.AlignCenter
-                Layout.margins: 0
-                spacing: 10
-                MyTextField {
-                    id: discoverField
-                    property string textBeingSearched: ""
-                    readOnly: ModelList.discoverInProgress
-                    Layout.alignment: Qt.AlignCenter
-                    Layout.fillWidth: true
-                    font.pixelSize: theme.fontSizeLarger
-                    placeholderText: qsTr("Discover and download models by keyword search...")
-                    Accessible.role: Accessible.EditableText
-                    Accessible.name: placeholderText
-                    Accessible.description: qsTr("Text field for discovering and filtering downloadable models")
-                    Connections {
-                        target: ModelList
-                        function onDiscoverInProgressChanged() {
-                            if (ModelList.discoverInProgress) {
-                                discoverField.textBeingSearched = discoverField.text;
-                                discoverField.text = qsTr("Searching \u00B7 %1").arg(discoverField.textBeingSearched);
-                            } else {
-                                discoverField.text = discoverField.textBeingSearched;
-                                discoverField.textBeingSearched = "";
-                            }
-                        }
-                    }
-                    background: ProgressBar {
-                        id: discoverProgressBar
-                        indeterminate: ModelList.discoverInProgress && ModelList.discoverProgress === 0.0
-                        value: ModelList.discoverProgress
-                        background: Rectangle {
-                            color: theme.controlBackground
-                            border.color: theme.controlBorder
-                            radius: 10
-                        }
-                        contentItem: Item {
-                            Rectangle {
-                                visible: ModelList.discoverInProgress
-                                anchors.bottom: parent.bottom
-                                width: discoverProgressBar.visualPosition * parent.width
-                                height: 10
-                                radius: 2
-                                color: theme.progressForeground
-                            }
-                        }
-                    }
-
-                    Keys.onReturnPressed: (event)=> {
-                                              if (event.modifiers & Qt.ControlModifier || event.modifiers & Qt.ShiftModifier)
-                                              event.accepted = false;
-                                              else {
-                                                  editingFinished();
-                                                  sendDiscovery()
-                                              }
-                                          }
-                    function sendDiscovery() {
-                        ModelList.downloadableModels.discoverAndFilter(discoverField.text);
-                    }
-                    RowLayout {
-                        spacing: 0
-                        anchors.right: discoverField.right
-                        anchors.verticalCenter: discoverField.verticalCenter
-                        anchors.rightMargin: 15
-                        visible: !ModelList.discoverInProgress
-                        MyMiniButton {
-                            id: clearDiscoverButton
-                            backgroundColor: theme.textColor
-                            backgroundColorHovered: theme.iconBackgroundDark
-                            visible: discoverField.text !== ""
-                            source: "qrc:/gpt4all/icons/close.svg"
-                            onClicked: {
-                                discoverField.text = ""
-                                discoverField.sendDiscovery() // should clear results
-                            }
-                        }
-                        MyMiniButton {
-                            backgroundColor: theme.textColor
-                            backgroundColorHovered: theme.iconBackgroundDark
-                            source: "qrc:/gpt4all/icons/settings.svg"
-                            onClicked: {
-                                discoveryTools.visible = !discoveryTools.visible
-                            }
-                        }
-                        MyMiniButton {
-                            id: sendButton
-                            enabled: !ModelList.discoverInProgress
-                            backgroundColor: theme.textColor
-                            backgroundColorHovered: theme.iconBackgroundDark
-                            source: "qrc:/gpt4all/icons/send_message.svg"
-                            Accessible.name: qsTr("Initiate model discovery and filtering")
-                            Accessible.description: qsTr("Triggers discovery and filtering of models")
-                            onClicked: {
-                                discoverField.sendDiscovery()
-                            }
-                        }
-                    }
+        RowLayout {
+            id: bar
+            implicitWidth: 600
+            spacing: 10
+            MyTabButton {
+                text: qsTr("GPT4All")
+                isSelected: gpt4AllModelView.isShown()
+                onPressed: {
+                    gpt4AllModelView.show();
                 }
             }
-
-            RowLayout {
-                id: discoveryTools
-                Layout.fillWidth: true
-                Layout.alignment: Qt.AlignCenter
-                Layout.margins: 0
-                spacing: 20
-                visible: false
-                MyComboBox {
-                    id: comboSort
-                    model: ListModel {
-                        ListElement { name: qsTr("Default") }
-                        ListElement { name: qsTr("Likes") }
-                        ListElement { name: qsTr("Downloads") }
-                        ListElement { name: qsTr("Recent") }
-                    }
-                    currentIndex: ModelList.discoverSort
-                    contentItem: Text {
-                        anchors.horizontalCenter: parent.horizontalCenter
-                        rightPadding: 30
-                        color: theme.textColor
-                        text: {
-                            return qsTr("Sort by: %1").arg(comboSort.displayText)
-                        }
-                        font.pixelSize: theme.fontSizeLarger
-                        verticalAlignment: Text.AlignVCenter
-                        horizontalAlignment: Text.AlignHCenter
-                        elide: Text.ElideRight
-                    }
-                    onActivated: function (index) {
-                        ModelList.discoverSort = index;
-                    }
-                }
-                MyComboBox {
-                    id: comboSortDirection
-                    model: ListModel {
-                        ListElement { name: qsTr("Asc") }
-                        ListElement { name: qsTr("Desc") }
-                    }
-                    currentIndex: {
-                        if (ModelList.discoverSortDirection === 1)
-                            return 0
-                        else
-                            return 1;
-                    }
-                    contentItem: Text {
-                        anchors.horizontalCenter: parent.horizontalCenter
-                        rightPadding: 30
-                        color: theme.textColor
-                        text: {
-                            return qsTr("Sort dir: %1").arg(comboSortDirection.displayText)
-                        }
-                        font.pixelSize: theme.fontSizeLarger
-                        verticalAlignment: Text.AlignVCenter
-                        horizontalAlignment: Text.AlignHCenter
-                        elide: Text.ElideRight
-                    }
-                    onActivated: function (index) {
-                        if (index === 0)
-                            ModelList.discoverSortDirection = 1;
-                        else
-                            ModelList.discoverSortDirection = -1;
-                    }
-                }
-                MyComboBox {
-                    id: comboLimit
-                    model: ListModel {
-                        ListElement { name: "5" }
-                        ListElement { name: "10" }
-                        ListElement { name: "20" }
-                        ListElement { name: "50" }
-                        ListElement { name: "100" }
-                        ListElement { name: qsTr("None") }
-                    }
-
-                    currentIndex: {
-                        if (ModelList.discoverLimit === 5)
-                            return 0;
-                        else if (ModelList.discoverLimit === 10)
-                            return 1;
-                        else if (ModelList.discoverLimit === 20)
-                            return 2;
-                        else if (ModelList.discoverLimit === 50)
-                            return 3;
-                        else if (ModelList.discoverLimit === 100)
-                            return 4;
-                        else if (ModelList.discoverLimit === -1)
-                            return 5;
-                    }
-                    contentItem: Text {
-                        anchors.horizontalCenter: parent.horizontalCenter
-                        rightPadding: 30
-                        color: theme.textColor
-                        text: {
-                            return qsTr("Limit: %1").arg(comboLimit.displayText)
-                        }
-                        font.pixelSize: theme.fontSizeLarger
-                        verticalAlignment: Text.AlignVCenter
-                        horizontalAlignment: Text.AlignHCenter
-                        elide: Text.ElideRight
-                    }
-                    onActivated: function (index) {
-                        switch (index) {
-                        case 0:
-                            ModelList.discoverLimit = 5; break;
-                        case 1:
-                            ModelList.discoverLimit = 10; break;
-                        case 2:
-                            ModelList.discoverLimit = 20; break;
-                        case 3:
-                            ModelList.discoverLimit = 50; break;
-                        case 4:
-                            ModelList.discoverLimit = 100; break;
-                        case 5:
-                            ModelList.discoverLimit = -1; break;
-                        }
-                    }
+            MyTabButton {
+                text: qsTr("HuggingFace")
+                isSelected: huggingfaceModelView.isShown()
+                onPressed: {
+                    huggingfaceModelView.show();
                 }
             }
         }
 
-        Label {
-            visible: !ModelList.downloadableModels.count && !ModelList.asyncModelRequestOngoing
+        StackLayout {
+            id: stackLayout
             Layout.fillWidth: true
             Layout.fillHeight: true
-            horizontalAlignment: Qt.AlignHCenter
-            verticalAlignment: Qt.AlignVCenter
-            text: qsTr("Network error: could not retrieve %1").arg("http://gpt4all.io/models/models3.json")
-            font.pixelSize: theme.fontSizeLarge
-            color: theme.mutedTextColor
-        }
 
-        MyBusyIndicator {
-            visible: !ModelList.downloadableModels.count && ModelList.asyncModelRequestOngoing
-            running: ModelList.asyncModelRequestOngoing
-            Accessible.role: Accessible.Animation
-            Layout.alignment: Qt.AlignCenter
-            Accessible.name: qsTr("Busy indicator")
-            Accessible.description: qsTr("Displayed when the models request is ongoing")
-        }
+            AddGPT4AllModelView {
+                id: gpt4AllModelView
+                Layout.fillWidth: true
+                Layout.fillHeight: true
 
-        ScrollView {
-            id: scrollView
-            ScrollBar.vertical.policy: ScrollBar.AsNeeded
-            Layout.fillWidth: true
-            Layout.fillHeight: true
-            clip: true
+                function show() {
+                    stackLayout.currentIndex = 0;
+                }
+                function isShown() {
+                    return stackLayout.currentIndex === 0
+                }
+            }
 
-            ListView {
-                id: modelListView
-                model: ModelList.downloadableModels
-                boundsBehavior: Flickable.StopAtBounds
-                spacing: 30
+            AddHFModelView {
+                id: huggingfaceModelView
+                Layout.fillWidth: true
+                Layout.fillHeight: true
+                // FIXME: This generates a warning and should not be used inside a layout, but without
+                // it the text field inside this qml does not display at full width so it looks like
+                // a bug in stacklayout
+                anchors.fill: parent
 
-                delegate: Rectangle {
-                    id: delegateItem
-                    width: modelListView.width
-                    height: childrenRect.height + 60
-                    color: theme.conversationBackground
-                    radius: 10
-                    border.width: 1
-                    border.color: theme.controlBorder
-
-                    ColumnLayout {
-                        anchors.top: parent.top
-                        anchors.left: parent.left
-                        anchors.right: parent.right
-                        anchors.margins: 30
-
-                        Text {
-                            Layout.fillWidth: true
-                            Layout.alignment: Qt.AlignLeft
-                            text: name
-                            elide: Text.ElideRight
-                            color: theme.titleTextColor
-                            font.pixelSize: theme.fontSizeLargest
-                            font.bold: true
-                            Accessible.role: Accessible.Paragraph
-                            Accessible.name: qsTr("Model file")
-                            Accessible.description: qsTr("Model file to be downloaded")
-                        }
-
-
-                        Rectangle {
-                            Layout.fillWidth: true
-                            height: 1
-                            color: theme.dividerColor
-                        }
-
-                        RowLayout {
-                            Layout.topMargin: 10
-                            Layout.fillWidth: true
-                            Text {
-                                id: descriptionText
-                                text: description
-                                font.pixelSize: theme.fontSizeLarge
-                                Layout.fillWidth: true
-                                wrapMode: Text.WordWrap
-                                textFormat: Text.StyledText
-                                color: theme.textColor
-                                linkColor: theme.textColor
-                                Accessible.role: Accessible.Paragraph
-                                Accessible.name: qsTr("Description")
-                                Accessible.description: qsTr("File description")
-                                onLinkActivated: function(link) { Qt.openUrlExternally(link); }
-                                MouseArea {
-                                    anchors.fill: parent
-                                    acceptedButtons: Qt.NoButton // pass clicks to parent
-                                    cursorShape: parent.hoveredLink ? Qt.PointingHandCursor : Qt.ArrowCursor
-                                }
-                            }
-
-                            // FIXME Need to overhaul design here which must take into account
-                            // features not present in current figma including:
-                            // * Ability to cancel a current download
-                            // * Ability to resume a download
-                            // * The presentation of an error if encountered
-                            // * Whether to show already installed models
-                            // * Install of remote models with API keys
-                            // * The presentation of the progress bar
-                            Rectangle {
-                                id: actionBox
-                                width: childrenRect.width + 20
-                                color: "transparent"
-                                border.width: 1
-                                border.color: theme.dividerColor
-                                radius: 10
-                                Layout.rightMargin: 20
-                                Layout.bottomMargin: 20
-                                Layout.minimumHeight: childrenRect.height + 20
-                                Layout.alignment: Qt.AlignRight | Qt.AlignTop
-
-                                ColumnLayout {
-                                    spacing: 0
-                                    MySettingsButton {
-                                        id: downloadButton
-                                        text: isDownloading ? qsTr("Cancel") : isIncomplete ? qsTr("Resume") : qsTr("Download")
-                                        font.pixelSize: theme.fontSizeLarge
-                                        Layout.topMargin: 20
-                                        Layout.leftMargin: 20
-                                        Layout.minimumWidth: 200
-                                        Layout.fillWidth: true
-                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
-                                        visible: !isOnline && !installed && !calcHash && downloadError === ""
-                                        Accessible.description: qsTr("Stop/restart/start the download")
-                                        onClicked: {
-                                            if (!isDownloading) {
-                                                Download.downloadModel(filename);
-                                            } else {
-                                                Download.cancelDownload(filename);
-                                            }
-                                        }
-                                    }
-
-                                    MySettingsDestructiveButton {
-                                        id: removeButton
-                                        text: qsTr("Remove")
-                                        Layout.topMargin: 20
-                                        Layout.leftMargin: 20
-                                        Layout.minimumWidth: 200
-                                        Layout.fillWidth: true
-                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
-                                        visible: !isDownloading && (installed || isIncomplete)
-                                        Accessible.description: qsTr("Remove model from filesystem")
-                                        onClicked: {
-                                            Download.removeModel(filename);
-                                        }
-                                    }
-
-                                    MySettingsButton {
-                                        id: installButton
-                                        visible: !installed && isOnline
-                                        Layout.topMargin: 20
-                                        Layout.leftMargin: 20
-                                        Layout.minimumWidth: 200
-                                        Layout.fillWidth: true
-                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
-                                        text: qsTr("Install")
-                                        font.pixelSize: theme.fontSizeLarge
-                                        onClicked: {
-                                            var apiKeyText = apiKey.text.trim(),
-                                                baseUrlText = baseUrl.text.trim(),
-                                                modelNameText = modelName.text.trim();
-
-                                            var apiKeyOk = apiKeyText !== "",
-                                                baseUrlOk = !isCompatibleApi || baseUrlText !== "",
-                                                modelNameOk = !isCompatibleApi || modelNameText !== "";
-
-                                            if (!apiKeyOk)
-                                                apiKey.showError();
-                                            if (!baseUrlOk)
-                                                baseUrl.showError();
-                                            if (!modelNameOk)
-                                                modelName.showError();
-
-                                            if (!apiKeyOk || !baseUrlOk || !modelNameOk)
-                                                return;
-
-                                            if (!isCompatibleApi)
-                                                Download.installModel(
-                                                    filename,
-                                                    apiKeyText,
-                                                );
-                                            else
-                                                Download.installCompatibleModel(
-                                                    modelNameText,
-                                                    apiKeyText,
-                                                    baseUrlText,
-                                                );
-                                        }
-                                        Accessible.role: Accessible.Button
-                                        Accessible.name: qsTr("Install")
-                                        Accessible.description: qsTr("Install online model")
-                                    }
-
-                                    ColumnLayout {
-                                        spacing: 0
-                                        Label {
-                                            Layout.topMargin: 20
-                                            Layout.leftMargin: 20
-                                            visible: downloadError !== ""
-                                            textFormat: Text.StyledText
-                                            text: qsTr("<strong><font size=\"1\"><a href=\"#error\">Error</a></strong></font>")
-                                            color: theme.textColor
-                                            font.pixelSize: theme.fontSizeLarge
-                                            linkColor: theme.textErrorColor
-                                            Accessible.role: Accessible.Paragraph
-                                            Accessible.name: text
-                                            Accessible.description: qsTr("Describes an error that occurred when downloading")
-                                            onLinkActivated: {
-                                                downloadingErrorPopup.text = downloadError;
-                                                downloadingErrorPopup.open();
-                                            }
-                                        }
-
-                                        Label {
-                                            visible: LLM.systemTotalRAMInGB() < ramrequired
-                                            Layout.topMargin: 20
-                                            Layout.leftMargin: 20
-                                            Layout.maximumWidth: 300
-                                            textFormat: Text.StyledText
-                                            text: qsTr("<strong><font size=\"2\">WARNING: Not recommended for your hardware. Model requires more memory (%1 GB) than your system has available (%2).</strong></font>").arg(ramrequired).arg(LLM.systemTotalRAMInGBString())
-                                            color: theme.textErrorColor
-                                            font.pixelSize: theme.fontSizeLarge
-                                            wrapMode: Text.WordWrap
-                                            Accessible.role: Accessible.Paragraph
-                                            Accessible.name: text
-                                            Accessible.description: qsTr("Error for incompatible hardware")
-                                            onLinkActivated: {
-                                                downloadingErrorPopup.text = downloadError;
-                                                downloadingErrorPopup.open();
-                                            }
-                                        }
-                                    }
-
-                                    ColumnLayout {
-                                        visible: isDownloading && !calcHash
-                                        Layout.topMargin: 20
-                                        Layout.leftMargin: 20
-                                        Layout.minimumWidth: 200
-                                        Layout.fillWidth: true
-                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
-                                        spacing: 20
-
-                                        ProgressBar {
-                                            id: itemProgressBar
-                                            Layout.fillWidth: true
-                                            width: 200
-                                            value: bytesReceived / bytesTotal
-                                            background: Rectangle {
-                                                implicitHeight: 45
-                                                color: theme.progressBackground
-                                                radius: 3
-                                            }
-                                            contentItem: Item {
-                                                implicitHeight: 40
-
-                                                Rectangle {
-                                                    width: itemProgressBar.visualPosition * parent.width
-                                                    height: parent.height
-                                                    radius: 2
-                                                    color: theme.progressForeground
-                                                }
-                                            }
-                                            Accessible.role: Accessible.ProgressBar
-                                            Accessible.name: qsTr("Download progressBar")
-                                            Accessible.description: qsTr("Shows the progress made in the download")
-                                        }
-
-                                        Label {
-                                            id: speedLabel
-                                            color: theme.textColor
-                                            Layout.alignment: Qt.AlignRight
-                                            text: speed
-                                            font.pixelSize: theme.fontSizeLarge
-                                            Accessible.role: Accessible.Paragraph
-                                            Accessible.name: qsTr("Download speed")
-                                            Accessible.description: qsTr("Download speed in bytes/kilobytes/megabytes per second")
-                                        }
-                                    }
-
-                                    RowLayout {
-                                        visible: calcHash
-                                        Layout.topMargin: 20
-                                        Layout.leftMargin: 20
-                                        Layout.minimumWidth: 200
-                                        Layout.maximumWidth: 200
-                                        Layout.fillWidth: true
-                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
-                                        clip: true
-
-                                        Label {
-                                            id: calcHashLabel
-                                            color: theme.textColor
-                                            text: qsTr("Calculating...")
-                                            font.pixelSize: theme.fontSizeLarge
-                                            Accessible.role: Accessible.Paragraph
-                                            Accessible.name: text
-                                            Accessible.description: qsTr("Whether the file hash is being calculated")
-                                        }
-
-                                        MyBusyIndicator {
-                                            id: busyCalcHash
-                                            running: calcHash
-                                            Accessible.role: Accessible.Animation
-                                            Accessible.name: qsTr("Busy indicator")
-                                            Accessible.description: qsTr("Displayed when the file hash is being calculated")
-                                        }
-                                    }
-
-                                    MyTextField {
-                                        id: apiKey
-                                        visible: !installed && isOnline
-                                        Layout.topMargin: 20
-                                        Layout.leftMargin: 20
-                                        Layout.minimumWidth: 200
-                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
-                                        wrapMode: Text.WrapAnywhere
-                                        function showError() {
-                                            messageToast.show(qsTr("ERROR: $API_KEY is empty."));
-                                            apiKey.placeholderTextColor = theme.textErrorColor;
-                                        }
-                                        onTextChanged: {
-                                            apiKey.placeholderTextColor = theme.mutedTextColor;
-                                        }
-                                        placeholderText: qsTr("enter $API_KEY")
-                                        Accessible.role: Accessible.EditableText
-                                        Accessible.name: placeholderText
-                                        Accessible.description: qsTr("Whether the file hash is being calculated")
-                                    }
-
-                                    MyTextField {
-                                        id: baseUrl
-                                        visible: !installed && isOnline && isCompatibleApi
-                                        Layout.topMargin: 20
-                                        Layout.leftMargin: 20
-                                        Layout.minimumWidth: 200
-                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
-                                        wrapMode: Text.WrapAnywhere
-                                        function showError() {
-                                            messageToast.show(qsTr("ERROR: $BASE_URL is empty."));
-                                            baseUrl.placeholderTextColor = theme.textErrorColor;
-                                        }
-                                        onTextChanged: {
-                                            baseUrl.placeholderTextColor = theme.mutedTextColor;
-                                        }
-                                        placeholderText: qsTr("enter $BASE_URL")
-                                        Accessible.role: Accessible.EditableText
-                                        Accessible.name: placeholderText
-                                        Accessible.description: qsTr("Whether the file hash is being calculated")
-                                    }
-
-                                    MyTextField {
-                                        id: modelName
-                                        visible: !installed && isOnline && isCompatibleApi
-                                        Layout.topMargin: 20
-                                        Layout.leftMargin: 20
-                                        Layout.minimumWidth: 200
-                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
-                                        wrapMode: Text.WrapAnywhere
-                                        function showError() {
-                                            messageToast.show(qsTr("ERROR: $MODEL_NAME is empty."))
-                                            modelName.placeholderTextColor = theme.textErrorColor;
-                                        }
-                                        onTextChanged: {
-                                            modelName.placeholderTextColor = theme.mutedTextColor;
-                                        }
-                                        placeholderText: qsTr("enter $MODEL_NAME")
-                                        Accessible.role: Accessible.EditableText
-                                        Accessible.name: placeholderText
-                                        Accessible.description: qsTr("Whether the file hash is being calculated")
-                                    }
-                                }
-                            }
-                        }
-
-                        Item  {
-                            Layout.minimumWidth: childrenRect.width
-                            Layout.minimumHeight: childrenRect.height
-                            Layout.bottomMargin: 10
-                            RowLayout {
-                                id: paramRow
-                                anchors.centerIn: parent
-                                ColumnLayout {
-                                    Layout.topMargin: 10
-                                    Layout.bottomMargin: 10
-                                    Layout.leftMargin: 20
-                                    Layout.rightMargin: 20
-                                    Text {
-                                        text: qsTr("File size")
-                                        font.pixelSize: theme.fontSizeSmall
-                                        color: theme.mutedDarkTextColor
-                                    }
-                                    Text {
-                                        text: filesize
-                                        color: theme.textColor
-                                        font.pixelSize: theme.fontSizeSmall
-                                        font.bold: true
-                                    }
-                                }
-                                Rectangle {
-                                    width: 1
-                                    Layout.fillHeight: true
-                                    color: theme.dividerColor
-                                }
-                                ColumnLayout {
-                                    Layout.topMargin: 10
-                                    Layout.bottomMargin: 10
-                                    Layout.leftMargin: 20
-                                    Layout.rightMargin: 20
-                                    Text {
-                                        text: qsTr("RAM required")
-                                        font.pixelSize: theme.fontSizeSmall
-                                        color: theme.mutedDarkTextColor
-                                    }
-                                    Text {
-                                        text: ramrequired >= 0 ? qsTr("%1 GB").arg(ramrequired) : qsTr("?")
-                                        color: theme.textColor
-                                        font.pixelSize: theme.fontSizeSmall
-                                        font.bold: true
-                                    }
-                                }
-                                Rectangle {
-                                    width: 1
-                                    Layout.fillHeight: true
-                                    color: theme.dividerColor
-                                }
-                                ColumnLayout {
-                                    Layout.topMargin: 10
-                                    Layout.bottomMargin: 10
-                                    Layout.leftMargin: 20
-                                    Layout.rightMargin: 20
-                                    Text {
-                                        text: qsTr("Parameters")
-                                        font.pixelSize: theme.fontSizeSmall
-                                        color: theme.mutedDarkTextColor
-                                    }
-                                    Text {
-                                        text: parameters !== "" ? parameters : qsTr("?")
-                                        color: theme.textColor
-                                        font.pixelSize: theme.fontSizeSmall
-                                        font.bold: true
-                                    }
-                                }
-                                Rectangle {
-                                    width: 1
-                                    Layout.fillHeight: true
-                                    color: theme.dividerColor
-                                }
-                                ColumnLayout {
-                                    Layout.topMargin: 10
-                                    Layout.bottomMargin: 10
-                                    Layout.leftMargin: 20
-                                    Layout.rightMargin: 20
-                                    Text {
-                                        text: qsTr("Quant")
-                                        font.pixelSize: theme.fontSizeSmall
-                                        color: theme.mutedDarkTextColor
-                                    }
-                                    Text {
-                                        text: quant
-                                        color: theme.textColor
-                                        font.pixelSize: theme.fontSizeSmall
-                                        font.bold: true
-                                    }
-                                }
-                                Rectangle {
-                                    width: 1
-                                    Layout.fillHeight: true
-                                    color: theme.dividerColor
-                                }
-                                ColumnLayout {
-                                    Layout.topMargin: 10
-                                    Layout.bottomMargin: 10
-                                    Layout.leftMargin: 20
-                                    Layout.rightMargin: 20
-                                    Text {
-                                        text: qsTr("Type")
-                                        font.pixelSize: theme.fontSizeSmall
-                                        color: theme.mutedDarkTextColor
-                                    }
-                                    Text {
-                                        text: type
-                                        color: theme.textColor
-                                        font.pixelSize: theme.fontSizeSmall
-                                        font.bold: true
-                                    }
-                                }
-                            }
-
-                            Rectangle {
-                                color: "transparent"
-                                anchors.fill: paramRow
-                                border.color: theme.dividerColor
-                                border.width: 1
-                                radius: 10
-                            }
-                        }
-
-                        Rectangle {
-                            Layout.fillWidth: true
-                            height: 1
-                            color: theme.dividerColor
-                        }
-                    }
+                function show() {
+                    stackLayout.currentIndex = 1;
+                }
+                function isShown() {
+                    return stackLayout.currentIndex === 1
                 }
             }
         }

gpt4all-chat/qml/ApplicationSettings.qml

@@ -10,7 +10,7 @@ import network
 import llm
 
 MySettingsTab {
-    onRestoreDefaultsClicked: {
+    onRestoreDefaults: {
         MySettings.restoreApplicationDefaults();
     }
     title: qsTr("Application")
@@ -487,32 +487,32 @@ MySettingsTab {
             Accessible.description: ToolTip.text
         }
         MySettingsLabel {
-            id: saveChatsContextLabel
-            text: qsTr("Save Chat Context")
-            helpText: qsTr("Save the chat model's state to disk for faster loading. WARNING: Uses ~2GB per chat.")
-            Layout.row: 12
+            id: trayLabel
+            text: qsTr("Enable System Tray")
+            helpText: qsTr("The application will minimize to the system tray when the window is closed.")
+            Layout.row: 13
             Layout.column: 0
         }
         MyCheckBox {
-            id: saveChatsContextBox
-            Layout.row: 12
+            id: trayBox
+            Layout.row: 13
             Layout.column: 2
             Layout.alignment: Qt.AlignRight
-            checked: MySettings.saveChatsContext
+            checked: MySettings.systemTray
             onClicked: {
-                MySettings.saveChatsContext = !MySettings.saveChatsContext
+                MySettings.systemTray = !MySettings.systemTray
             }
         }
         MySettingsLabel {
             id: serverChatLabel
             text: qsTr("Enable Local API Server")
             helpText: qsTr("Expose an OpenAI-Compatible server to localhost. WARNING: Results in increased resource usage.")
-            Layout.row: 13
+            Layout.row: 14
             Layout.column: 0
         }
         MyCheckBox {
             id: serverChatBox
-            Layout.row: 13
+            Layout.row: 14
             Layout.column: 2
             Layout.alignment: Qt.AlignRight
             checked: MySettings.serverChat
@@ -524,7 +524,7 @@ MySettingsTab {
             id: serverPortLabel
             text: qsTr("API Server Port")
             helpText: qsTr("The port to use for the local server. Requires restart.")
-            Layout.row: 14
+            Layout.row: 15
             Layout.column: 0
         }
         MyTextField {
@@ -532,7 +532,7 @@ MySettingsTab {
             text: MySettings.networkPort
             color: theme.textColor
             font.pixelSize: theme.fontSizeLarge
-            Layout.row: 14
+            Layout.row: 15
             Layout.column: 2
             Layout.minimumWidth: 200
             Layout.maximumWidth: 200
@@ -577,12 +577,12 @@ MySettingsTab {
             id: updatesLabel
             text: qsTr("Check For Updates")
             helpText: qsTr("Manually check for an update to GPT4All.");
-            Layout.row: 15
+            Layout.row: 16
             Layout.column: 0
         }
 
         MySettingsButton {
-            Layout.row: 15
+            Layout.row: 16
             Layout.column: 2
             Layout.alignment: Qt.AlignRight
             text: qsTr("Updates");
@@ -593,7 +593,7 @@ MySettingsTab {
         }
 
         Rectangle {
-            Layout.row: 16
+            Layout.row: 17
             Layout.column: 0
             Layout.columnSpan: 3
             Layout.fillWidth: true

gpt4all-chat/qml/ChatCollapsibleItem.qml

@@ -0,0 +1,160 @@
+import Qt5Compat.GraphicalEffects
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import QtQuick.Layouts
+
+import gpt4all
+import mysettings
+import toolenums
+
+ColumnLayout {
+    property alias textContent: innerTextItem.textContent
+    property bool isCurrent: false
+    property bool isError: false
+
+    Layout.topMargin: 10
+    Layout.bottomMargin: 10
+
+    Item {
+        Layout.preferredWidth: childrenRect.width
+        Layout.preferredHeight: 38
+        RowLayout {
+            anchors.left: parent.left
+            anchors.top: parent.top
+            anchors.bottom: parent.bottom
+
+            Item {
+                width: myTextArea.width
+                height: myTextArea.height
+                TextArea {
+                    id: myTextArea
+                    text: {
+                        if (isError)
+                            return qsTr("Analysis encountered error");
+                        if (isCurrent)
+                            return qsTr("Analyzing");
+                        return qsTr("Analyzed");
+                    }
+                    padding: 0
+                    font.pixelSize: theme.fontSizeLarger
+                    enabled: false
+                    focus: false
+                    readOnly: true
+                    color: headerMA.containsMouse ? theme.mutedDarkTextColorHovered : theme.mutedTextColor
+                    hoverEnabled: false
+                }
+
+                Item {
+                    id: textColorOverlay
+                    anchors.fill: parent
+                    clip: true
+                    visible: false
+                    Rectangle {
+                        id: animationRec
+                        width: myTextArea.width * 0.3
+                        anchors.top: parent.top
+                        anchors.bottom: parent.bottom
+                        color: theme.textColor
+
+                        SequentialAnimation {
+                            running: isCurrent
+                            loops: Animation.Infinite
+                            NumberAnimation {
+                                target: animationRec;
+                                property: "x";
+                                from: -animationRec.width;
+                                to: myTextArea.width * 3;
+                                duration: 2000
+                            }
+                        }
+                    }
+                }
+                OpacityMask {
+                    visible: isCurrent
+                    anchors.fill: parent
+                    maskSource: myTextArea
+                    source: textColorOverlay
+                }
+            }
+
+            Item {
+                id: caret
+                Layout.preferredWidth: contentCaret.width
+                Layout.preferredHeight: contentCaret.height
+                Image {
+                    id: contentCaret
+                    anchors.centerIn: parent
+                    visible: false
+                    sourceSize.width: theme.fontSizeLarge
+                    sourceSize.height: theme.fontSizeLarge
+                    mipmap: true
+                    source: {
+                        if (contentLayout.state === "collapsed")
+                            return "qrc:/gpt4all/icons/caret_right.svg";
+                        else
+                            return "qrc:/gpt4all/icons/caret_down.svg";
+                    }
+                }
+
+                ColorOverlay {
+                    anchors.fill: contentCaret
+                    source: contentCaret
+                    color: headerMA.containsMouse ? theme.mutedDarkTextColorHovered : theme.mutedTextColor
+                }
+            }
+        }
+
+        MouseArea {
+            id: headerMA
+            hoverEnabled: true
+            anchors.fill: parent
+            onClicked: {
+                if (contentLayout.state === "collapsed")
+                    contentLayout.state = "expanded";
+                else
+                    contentLayout.state = "collapsed";
+            }
+        }
+    }
+
+    ColumnLayout {
+        id: contentLayout
+        spacing: 0
+        state: "collapsed"
+        clip: true
+
+        states: [
+            State {
+                name: "expanded"
+                PropertyChanges { target: contentLayout; Layout.preferredHeight: innerContentLayout.height }
+            },
+            State {
+                name: "collapsed"
+                PropertyChanges { target: contentLayout; Layout.preferredHeight: 0 }
+            }
+        ]
+
+        transitions: [
+            Transition {
+                SequentialAnimation {
+                    PropertyAnimation {
+                        target: contentLayout
+                        property: "Layout.preferredHeight"
+                        duration: 300
+                        easing.type: Easing.InOutQuad
+                    }
+                }
+            }
+        ]
+
+        ColumnLayout {
+            id: innerContentLayout
+            Layout.leftMargin: 30
+            ChatTextItem {
+                id: innerTextItem
+            }
+        }
+    }
+}

gpt4all-chat/qml/ChatItemView.qml

@@ -0,0 +1,819 @@
+import Qt5Compat.GraphicalEffects
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import QtQuick.Layouts
+import Qt.labs.qmlmodels
+
+import gpt4all
+import mysettings
+import toolenums
+
+ColumnLayout {
+
+property var inputBoxText: null
+signal setInputBoxText(text: string)
+
+Item {
+
+Layout.fillWidth: true
+Layout.maximumWidth: parent.width
+Layout.preferredHeight: gridLayout.height
+
+HoverHandler { id: hoverArea }
+
+GridLayout {
+    id: gridLayout
+    anchors.left: parent.left
+    anchors.right: parent.right
+    columns: 2
+
+    Item {
+        Layout.row: 0
+        Layout.column: 0
+        Layout.alignment: Qt.AlignVCenter | Qt.AlignRight
+        Layout.preferredWidth: 32
+        Layout.preferredHeight: 32
+        Layout.topMargin: model.index > 0 ? 25 : 0
+
+        Image {
+            id: logo
+            sourceSize: Qt.size(32, 32)
+            fillMode: Image.PreserveAspectFit
+            mipmap: true
+            visible: false
+            source: name !== "Response: " ? "qrc:/gpt4all/icons/you.svg" : "qrc:/gpt4all/icons/gpt4all_transparent.svg"
+        }
+
+        ColorOverlay {
+            id: colorOver
+            anchors.fill: logo
+            source: logo
+            color: theme.conversationHeader
+            RotationAnimation {
+                id: rotationAnimation
+                target: colorOver
+                property: "rotation"
+                from: 0
+                to: 360
+                duration: 1000
+                loops: Animation.Infinite
+                running: isCurrentResponse && currentChat.responseInProgress
+            }
+        }
+    }
+
+    Item {
+        Layout.row: 0
+        Layout.column: 1
+        Layout.fillWidth: true
+        Layout.preferredHeight: 38
+        Layout.topMargin: model.index > 0 ? 25 : 0
+
+        RowLayout {
+            spacing: 5
+            anchors.left: parent.left
+            anchors.top: parent.top
+            anchors.bottom: parent.bottom
+
+            TextArea {
+                text: {
+                    if (name === "Response: ")
+                        return qsTr("GPT4All");
+                    return qsTr("You");
+                }
+                padding: 0
+                font.pixelSize: theme.fontSizeLarger
+                font.bold: true
+                color: theme.conversationHeader
+                enabled: false
+                focus: false
+                readOnly: true
+            }
+            Text {
+                visible: name === "Response: "
+                font.pixelSize: theme.fontSizeLarger
+                text: currentModelName()
+                color: theme.mutedTextColor
+            }
+            RowLayout {
+                visible: isCurrentResponse && (content === "" && currentChat.responseInProgress)
+                Text {
+                    color: theme.mutedTextColor
+                    font.pixelSize: theme.fontSizeLarger
+                    text: {
+                        switch (currentChat.responseState) {
+                        case Chat.ResponseStopped: return qsTr("response stopped ...");
+                        case Chat.LocalDocsRetrieval: return qsTr("retrieving localdocs: %1 ...").arg(currentChat.collectionList.join(", "));
+                        case Chat.LocalDocsProcessing: return qsTr("searching localdocs: %1 ...").arg(currentChat.collectionList.join(", "));
+                        case Chat.PromptProcessing: return qsTr("processing ...")
+                        case Chat.ResponseGeneration: return qsTr("generating response ...");
+                        case Chat.GeneratingQuestions: return qsTr("generating questions ...");
+                        default: return ""; // handle unexpected values
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    ColumnLayout {
+        Layout.row: 1
+        Layout.column: 1
+        Layout.fillWidth: true
+        spacing: 10
+        Flow {
+            id: attachedUrlsFlow
+            Layout.fillWidth: true
+            Layout.bottomMargin: 10
+            spacing: 10
+            visible: promptAttachments.length !== 0
+            Repeater {
+                model: promptAttachments
+
+                delegate: Rectangle {
+                    width: 350
+                    height: 50
+                    radius: 5
+                    color: theme.attachmentBackground
+                    border.color: theme.controlBorder
+
+                    Row {
+                        spacing: 5
+                        anchors.fill: parent
+                        anchors.margins: 5
+
+                        MyFileIcon {
+                            iconSize: 40
+                            fileName: modelData.file
+                        }
+
+                        Text {
+                            width: 295
+                            height: 40
+                            text: modelData.file
+                            color: theme.textColor
+                            horizontalAlignment: Text.AlignHLeft
+                            verticalAlignment: Text.AlignVCenter
+                            font.pixelSize: theme.fontSizeMedium
+                            font.bold: true
+                            wrapMode: Text.WrapAnywhere
+                            elide: Qt.ElideRight
+                        }
+                    }
+                }
+            }
+        }
+
+        Repeater {
+            model: childItems
+
+            DelegateChooser {
+                id: chooser
+                role: "name"
+                DelegateChoice {
+                    roleValue: "Text: ";
+                    ChatTextItem {
+                        Layout.fillWidth: true
+                        textContent: modelData.content
+                    }
+                }
+                DelegateChoice {
+                    roleValue: "ToolCall: ";
+                    ChatCollapsibleItem {
+                        Layout.fillWidth: true
+                        textContent: modelData.content
+                        isCurrent: modelData.isCurrentResponse
+                        isError: modelData.isToolCallError
+                    }
+                }
+            }
+
+            delegate: chooser
+        }
+
+        ChatTextItem {
+            Layout.fillWidth: true
+            textContent: content
+        }
+
+        ThumbsDownDialog {
+            id: thumbsDownDialog
+            x: Math.round((parent.width - width) / 2)
+            y: Math.round((parent.height - height) / 2)
+            width: 640
+            height: 300
+            property string text: content
+            response: newResponse === undefined || newResponse === "" ? text : newResponse
+            onAccepted: {
+                var responseHasChanged = response !== text && response !== newResponse
+                if (thumbsDownState && !thumbsUpState && !responseHasChanged)
+                    return
+
+                chatModel.updateNewResponse(model.index, response)
+                chatModel.updateThumbsUpState(model.index, false)
+                chatModel.updateThumbsDownState(model.index, true)
+                Network.sendConversation(currentChat.id, getConversationJson());
+            }
+        }
+    }
+
+    Item {
+        Layout.row: 2
+        Layout.column: 1
+        Layout.topMargin: 5
+        Layout.alignment: Qt.AlignVCenter
+        Layout.preferredWidth: childrenRect.width
+        Layout.preferredHeight: childrenRect.height
+        visible: {
+            if (name !== "Response: ")
+                return false
+            if (consolidatedSources.length === 0)
+                return false
+            if (!MySettings.localDocsShowReferences)
+                return false
+            if (isCurrentResponse && currentChat.responseInProgress
+                    && currentChat.responseState !== Chat.GeneratingQuestions )
+                return false
+            return true
+        }
+
+        MyButton {
+            backgroundColor: theme.sourcesBackground
+            backgroundColorHovered: theme.sourcesBackgroundHovered
+            contentItem: RowLayout {
+                anchors.centerIn: parent
+
+                Item {
+                    Layout.preferredWidth: 24
+                    Layout.preferredHeight: 24
+
+                    Image {
+                        id: sourcesIcon
+                        visible: false
+                        anchors.fill: parent
+                        sourceSize.width: 24
+                        sourceSize.height: 24
+                        mipmap: true
+                        source: "qrc:/gpt4all/icons/db.svg"
+                    }
+
+                    ColorOverlay {
+                        anchors.fill: sourcesIcon
+                        source: sourcesIcon
+                        color: theme.textColor
+                    }
+                }
+
+                Text {
+                    text: qsTr("%n Source(s)", "", consolidatedSources.length)
+                    padding: 0
+                    font.pixelSize: theme.fontSizeLarge
+                    font.bold: true
+                    color: theme.styledTextColor
+                }
+
+                Item {
+                    Layout.preferredWidth: caret.width
+                    Layout.preferredHeight: caret.height
+                    Image {
+                        id: caret
+                        anchors.centerIn: parent
+                        visible: false
+                        sourceSize.width: theme.fontSizeLarge
+                        sourceSize.height: theme.fontSizeLarge
+                        mipmap: true
+                        source: {
+                            if (sourcesLayout.state === "collapsed")
+                                return "qrc:/gpt4all/icons/caret_right.svg";
+                            else
+                                return "qrc:/gpt4all/icons/caret_down.svg";
+                        }
+                    }
+
+                    ColorOverlay {
+                        anchors.fill: caret
+                        source: caret
+                        color: theme.textColor
+                    }
+                }
+            }
+
+            onClicked: {
+                if (sourcesLayout.state === "collapsed")
+                    sourcesLayout.state = "expanded";
+                else
+                    sourcesLayout.state = "collapsed";
+            }
+        }
+    }
+
+    ColumnLayout {
+        id: sourcesLayout
+        Layout.row: 3
+        Layout.column: 1
+        Layout.topMargin: 5
+        visible: {
+            if (consolidatedSources.length === 0)
+                return false
+            if (!MySettings.localDocsShowReferences)
+                return false
+            if (isCurrentResponse && currentChat.responseInProgress
+                    && currentChat.responseState !== Chat.GeneratingQuestions )
+                return false
+            return true
+        }
+        clip: true
+        Layout.fillWidth: true
+        Layout.preferredHeight: 0
+        state: "collapsed"
+        states: [
+            State {
+                name: "expanded"
+                PropertyChanges { target: sourcesLayout; Layout.preferredHeight: sourcesFlow.height }
+            },
+            State {
+                name: "collapsed"
+                PropertyChanges { target: sourcesLayout; Layout.preferredHeight: 0 }
+            }
+        ]
+
+        transitions: [
+            Transition {
+                SequentialAnimation {
+                    PropertyAnimation {
+                        target: sourcesLayout
+                        property: "Layout.preferredHeight"
+                        duration: 300
+                        easing.type: Easing.InOutQuad
+                    }
+                }
+            }
+        ]
+
+        Flow {
+            id: sourcesFlow
+            Layout.fillWidth: true
+            spacing: 10
+            visible: consolidatedSources.length !== 0
+            Repeater {
+                model: consolidatedSources
+
+                delegate: Rectangle {
+                    radius: 10
+                    color: ma.containsMouse ? theme.sourcesBackgroundHovered : theme.sourcesBackground
+                    width: 200
+                    height: 75
+
+                    MouseArea {
+                        id: ma
+                        enabled: modelData.path !== ""
+                        anchors.fill: parent
+                        hoverEnabled: true
+                        onClicked: function() {
+                            Qt.openUrlExternally(modelData.fileUri)
+                        }
+                    }
+
+                    Rectangle {
+                        id: debugTooltip
+                        anchors.right: parent.right
+                        anchors.bottom: parent.bottom
+                        width: 24
+                        height: 24
+                        color: "transparent"
+                        ToolTip {
+                            parent: debugTooltip
+                            visible: debugMouseArea.containsMouse
+                            text: modelData.text
+                            contentWidth: 900
+                            delay: 500
+                        }
+                        MouseArea {
+                            id: debugMouseArea
+                            anchors.fill: parent
+                            hoverEnabled: true
+                        }
+                    }
+
+                    ColumnLayout {
+                        anchors.left: parent.left
+                        anchors.top: parent.top
+                        anchors.margins: 10
+                        spacing: 0
+                        RowLayout {
+                            id: title
+                            spacing: 5
+                            Layout.maximumWidth: 180
+                            MyFileIcon {
+                                iconSize: 24
+                                fileName: modelData.file
+                                Layout.preferredWidth: iconSize
+                                Layout.preferredHeight: iconSize
+                            }
+                            Text {
+                                Layout.maximumWidth: 156
+                                text: modelData.collection !== "" ? modelData.collection : qsTr("LocalDocs")
+                                font.pixelSize: theme.fontSizeLarge
+                                font.bold: true
+                                color: theme.styledTextColor
+                                elide: Qt.ElideRight
+                            }
+                            Rectangle {
+                                Layout.fillWidth: true
+                                color: "transparent"
+                                height: 1
+                            }
+                        }
+                        Text {
+                            Layout.fillHeight: true
+                            Layout.maximumWidth: 180
+                            Layout.maximumHeight: 55 - title.height
+                            text: modelData.file
+                            color: theme.textColor
+                            font.pixelSize: theme.fontSizeSmall
+                            elide: Qt.ElideRight
+                            wrapMode: Text.WrapAnywhere
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    ConfirmationDialog {
+        id: editPromptDialog
+        dialogTitle: qsTr("Edit this message?")
+        description: qsTr("All following messages will be permanently erased.")
+        onAccepted: {
+            const msg = currentChat.popPrompt(index);
+            if (msg !== null)
+                setInputBoxText(msg);
+        }
+    }
+
+    ConfirmationDialog {
+        id: redoResponseDialog
+        dialogTitle: qsTr("Redo this response?")
+        description: qsTr("All following messages will be permanently erased.")
+        onAccepted: currentChat.regenerateResponse(index)
+    }
+
+    RowLayout {
+        id: buttonRow
+        Layout.row: 4
+        Layout.column: 1
+        Layout.maximumWidth: parent.width
+        Layout.fillWidth: false
+        Layout.alignment: Qt.AlignLeft | Qt.AlignTop
+        spacing: 3
+        visible: !isCurrentResponse || !currentChat.responseInProgress
+        enabled: opacity > 0
+        opacity: hoverArea.hovered
+
+        Behavior on opacity {
+            OpacityAnimator { duration: 30 }
+        }
+
+        ChatMessageButton {
+            readonly property var editingDisabledReason: {
+                if (!currentChat.isModelLoaded)
+                    return qsTr("Cannot edit chat without a loaded model.");
+                if (currentChat.responseInProgress)
+                    return qsTr("Cannot edit chat while the model is generating.");
+                return null;
+            }
+            visible: !currentChat.isServer && model.name === "Prompt: "
+            enabled: editingDisabledReason === null
+            Layout.maximumWidth: 24
+            Layout.maximumHeight: 24
+            Layout.alignment: Qt.AlignVCenter
+            Layout.fillWidth: false
+            name: editingDisabledReason ?? qsTr("Edit")
+            source: "qrc:/gpt4all/icons/edit.svg"
+            onClicked: {
+                if (inputBoxText === "")
+                    editPromptDialog.open();
+            }
+        }
+
+        ChatMessageButton {
+            readonly property var editingDisabledReason: {
+                if (!currentChat.isModelLoaded)
+                    return qsTr("Cannot redo response without a loaded model.");
+                if (currentChat.responseInProgress)
+                    return qsTr("Cannot redo response while the model is generating.");
+                return null;
+            }
+            visible: !currentChat.isServer && model.name === "Response: "
+            enabled: editingDisabledReason === null
+            Layout.maximumWidth: 24
+            Layout.maximumHeight: 24
+            Layout.alignment: Qt.AlignVCenter
+            Layout.fillWidth: false
+            name: editingDisabledReason ?? qsTr("Redo")
+            source: "qrc:/gpt4all/icons/regenerate.svg"
+            onClicked: {
+                if (index == chatModel.count - 1) {
+                    // regenerate last message without confirmation
+                    currentChat.regenerateResponse(index);
+                    return;
+                }
+                redoResponseDialog.open();
+            }
+        }
+
+        ChatMessageButton {
+            Layout.maximumWidth: 24
+            Layout.maximumHeight: 24
+            Layout.alignment: Qt.AlignVCenter
+            Layout.fillWidth: false
+            name: qsTr("Copy")
+            source: "qrc:/gpt4all/icons/copy.svg"
+            onClicked: {
+                chatModel.copyToClipboard(index);
+            }
+        }
+
+        Item {
+            visible: name === "Response: " && MySettings.networkIsActive
+            Layout.alignment: Qt.AlignVCenter
+            Layout.preferredWidth: childrenRect.width
+            Layout.preferredHeight: childrenRect.height
+            Layout.fillWidth: false
+
+            ChatMessageButton {
+                id: thumbsUp
+                anchors.left: parent.left
+                anchors.verticalCenter: parent.verticalCenter
+                opacity: thumbsUpState || thumbsUpState == thumbsDownState ? 1.0 : 0.2
+                source: "qrc:/gpt4all/icons/thumbs_up.svg"
+                name: qsTr("Like response")
+                onClicked: {
+                    if (thumbsUpState && !thumbsDownState)
+                        return
+
+                    chatModel.updateNewResponse(index, "")
+                    chatModel.updateThumbsUpState(index, true)
+                    chatModel.updateThumbsDownState(index, false)
+                    Network.sendConversation(currentChat.id, getConversationJson());
+                }
+            }
+
+            ChatMessageButton {
+                id: thumbsDown
+                anchors.top: thumbsUp.top
+                anchors.topMargin: buttonRow.spacing
+                anchors.left: thumbsUp.right
+                anchors.leftMargin: buttonRow.spacing
+                checked: thumbsDownState
+                opacity: thumbsDownState || thumbsUpState == thumbsDownState ? 1.0 : 0.2
+                bgTransform: [
+                    Matrix4x4 {
+                        matrix: Qt.matrix4x4(-1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1)
+                    },
+                    Translate {
+                        x: thumbsDown.width
+                    }
+                ]
+                source: "qrc:/gpt4all/icons/thumbs_down.svg"
+                name: qsTr("Dislike response")
+                onClicked: {
+                    thumbsDownDialog.open()
+                }
+            }
+        }
+    }
+} // GridLayout
+
+} // Item
+
+GridLayout {
+    Layout.fillWidth: true
+    Layout.maximumWidth: parent.width
+
+    function shouldShowSuggestions() {
+        if (!isCurrentResponse)
+            return false;
+        if (MySettings.suggestionMode === 2) // Off
+            return false;
+        if (MySettings.suggestionMode === 0 && consolidatedSources.length === 0) // LocalDocs only
+            return false;
+        return currentChat.responseState === Chat.GeneratingQuestions || currentChat.generatedQuestions.length !== 0;
+    }
+
+    Item {
+        visible: parent.shouldShowSuggestions()
+        Layout.row: 5
+        Layout.column: 0
+        Layout.topMargin: 20
+        Layout.alignment: Qt.AlignVCenter | Qt.AlignRight
+        Layout.preferredWidth: 28
+        Layout.preferredHeight: 28
+        Image {
+            id: stack
+            sourceSize: Qt.size(28, 28)
+            fillMode: Image.PreserveAspectFit
+            mipmap: true
+            visible: false
+            source: "qrc:/gpt4all/icons/stack.svg"
+        }
+
+        ColorOverlay {
+            anchors.fill: stack
+            source: stack
+            color: theme.conversationHeader
+        }
+    }
+
+    Item {
+        visible: parent.shouldShowSuggestions()
+        Layout.row: 5
+        Layout.column: 1
+        Layout.topMargin: 20
+        Layout.fillWidth: true
+        Layout.preferredHeight: 38
+        RowLayout {
+            spacing: 5
+            anchors.left: parent.left
+            anchors.top: parent.top
+            anchors.bottom: parent.bottom
+
+            TextArea {
+                text: qsTr("Suggested follow-ups")
+                padding: 0
+                font.pixelSize: theme.fontSizeLarger
+                font.bold: true
+                color: theme.conversationHeader
+                enabled: false
+                focus: false
+                readOnly: true
+            }
+        }
+    }
+
+    ColumnLayout {
+        visible: parent.shouldShowSuggestions()
+        Layout.row: 6
+        Layout.column: 1
+        Layout.fillWidth: true
+        Layout.minimumHeight: 1
+        spacing: 10
+        Repeater {
+            model: currentChat.generatedQuestions
+            TextArea {
+                id: followUpText
+                Layout.fillWidth: true
+                Layout.alignment: Qt.AlignLeft
+                rightPadding: 40
+                topPadding: 10
+                leftPadding: 20
+                bottomPadding: 10
+                text: modelData
+                focus: false
+                readOnly: true
+                wrapMode: Text.WordWrap
+                hoverEnabled: !currentChat.responseInProgress
+                color: theme.textColor
+                font.pixelSize: theme.fontSizeLarge
+                background: Rectangle {
+                    color: hovered ? theme.sourcesBackgroundHovered : theme.sourcesBackground
+                    radius: 10
+                }
+                MouseArea {
+                    id: maFollowUp
+                    anchors.fill: parent
+                    enabled: !currentChat.responseInProgress
+                    onClicked: function() {
+                        var chat = window.currentChat
+                        var followup = modelData
+                        chat.stopGenerating()
+                        chat.newPromptResponsePair(followup)
+                    }
+                }
+                Item {
+                    anchors.right: parent.right
+                    anchors.verticalCenter: parent.verticalCenter
+                    width: 40
+                    height: 40
+                    visible: !currentChat.responseInProgress
+                    Image {
+                        id: plusImage
+                        anchors.verticalCenter: parent.verticalCenter
+                        sourceSize.width: 20
+                        sourceSize.height: 20
+                        mipmap: true
+                        visible: false
+                        source: "qrc:/gpt4all/icons/plus.svg"
+                    }
+
+                    ColorOverlay {
+                        anchors.fill: plusImage
+                        source: plusImage
+                        color: theme.styledTextColor
+                    }
+                }
+            }
+        }
+
+        Rectangle {
+            Layout.fillWidth: true
+            color: "transparent"
+            radius: 10
+            Layout.preferredHeight: currentChat.responseInProgress ? 40 : 0
+            clip: true
+            ColumnLayout {
+                id: followUpLayout
+                anchors.fill: parent
+                Rectangle {
+                    id: myRect1
+                    Layout.preferredWidth: 0
+                    Layout.minimumWidth: 0
+                    Layout.maximumWidth: parent.width
+                    height: 12
+                    color: theme.sourcesBackgroundHovered
+                }
+
+                Rectangle {
+                    id: myRect2
+                    Layout.preferredWidth: 0
+                    Layout.minimumWidth: 0
+                    Layout.maximumWidth: parent.width
+                    height: 12
+                    color: theme.sourcesBackgroundHovered
+                }
+
+                SequentialAnimation {
+                    id: followUpProgressAnimation
+                    ParallelAnimation {
+                        PropertyAnimation {
+                            target: myRect1
+                            property: "Layout.preferredWidth"
+                            from: 0
+                            to: followUpLayout.width
+                            duration: 1000
+                        }
+                        PropertyAnimation {
+                            target: myRect2
+                            property: "Layout.preferredWidth"
+                            from: 0
+                            to: followUpLayout.width / 2
+                            duration: 1000
+                        }
+                    }
+                    SequentialAnimation {
+                        loops: Animation.Infinite
+                        ParallelAnimation {
+                            PropertyAnimation {
+                                target: myRect1
+                                property: "opacity"
+                                from: 1
+                                to: 0.2
+                                duration: 1500
+                            }
+                            PropertyAnimation {
+                                target: myRect2
+                                property: "opacity"
+                                from: 1
+                                to: 0.2
+                                duration: 1500
+                            }
+                        }
+                        ParallelAnimation {
+                            PropertyAnimation {
+                                target: myRect1
+                                property: "opacity"
+                                from: 0.2
+                                to: 1
+                                duration: 1500
+                            }
+                            PropertyAnimation {
+                                target: myRect2
+                                property: "opacity"
+                                from: 0.2
+                                to: 1
+                                duration: 1500
+                            }
+                        }
+                    }
+                }
+
+                onVisibleChanged: {
+                    if (visible)
+                        followUpProgressAnimation.start();
+                }
+            }
+
+            Behavior on Layout.preferredHeight {
+                NumberAnimation {
+                    duration: 300
+                    easing.type: Easing.InOutQuad
+                }
+            }
+        }
+    }
+
+} // GridLayout
+
+} // ColumnLayout

gpt4all-chat/qml/ChatMessageButton.qml

@@ -0,0 +1,20 @@
+import QtQuick
+import QtQuick.Controls
+
+import gpt4all
+
+MyToolButton {
+    property string name
+
+    width: 24
+    height: 24
+    imageWidth: width
+    imageHeight: height
+    ToolTip {
+        visible: parent.hovered
+        y: parent.height * 1.5
+        text: name
+        delay: Qt.styleHints.mousePressAndHoldInterval
+    }
+    Accessible.name: name
+}

gpt4all-chat/qml/ChatTextItem.qml

@@ -0,0 +1,139 @@
+import Qt5Compat.GraphicalEffects
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import QtQuick.Layouts
+
+import gpt4all
+import mysettings
+import toolenums
+
+TextArea {
+    id: myTextArea
+    property string textContent: ""
+    visible: textContent != ""
+    Layout.fillWidth: true
+    padding: 0
+    color: {
+        if (!currentChat.isServer)
+            return theme.textColor
+        return theme.white
+    }
+    wrapMode: Text.WordWrap
+    textFormat: TextEdit.PlainText
+    focus: false
+    readOnly: true
+    font.pixelSize: theme.fontSizeLarge
+    cursorVisible: isCurrentResponse ? currentChat.responseInProgress : false
+    cursorPosition: text.length
+    TapHandler {
+        id: tapHandler
+        onTapped: function(eventPoint, button) {
+            var clickedPos = myTextArea.positionAt(eventPoint.position.x, eventPoint.position.y);
+            var success = textProcessor.tryCopyAtPosition(clickedPos);
+            if (success)
+                copyCodeMessage.open();
+        }
+    }
+
+    MouseArea {
+        id: conversationMouseArea
+        anchors.fill: parent
+        acceptedButtons: Qt.RightButton
+
+        onClicked: (mouse) => {
+                       if (mouse.button === Qt.RightButton) {
+                           conversationContextMenu.x = conversationMouseArea.mouseX
+                           conversationContextMenu.y = conversationMouseArea.mouseY
+                           conversationContextMenu.open()
+                       }
+                   }
+    }
+
+    onLinkActivated: function(link) {
+        if (!isCurrentResponse || !currentChat.responseInProgress)
+            Qt.openUrlExternally(link)
+    }
+
+    onLinkHovered: function (link) {
+        if (!isCurrentResponse || !currentChat.responseInProgress)
+            statusBar.externalHoveredLink = link
+    }
+
+    MyMenu {
+        id: conversationContextMenu
+        MyMenuItem {
+            text: qsTr("Copy")
+            enabled: myTextArea.selectedText !== ""
+            height: enabled ? implicitHeight : 0
+            onTriggered: myTextArea.copy()
+        }
+        MyMenuItem {
+            text: qsTr("Copy Message")
+            enabled: myTextArea.selectedText === ""
+            height: enabled ? implicitHeight : 0
+            onTriggered: {
+                myTextArea.selectAll()
+                myTextArea.copy()
+                myTextArea.deselect()
+            }
+        }
+        MyMenuItem {
+            text: textProcessor.shouldProcessText ? qsTr("Disable markdown") : qsTr("Enable markdown")
+            height: enabled ? implicitHeight : 0
+            onTriggered: {
+                textProcessor.shouldProcessText = !textProcessor.shouldProcessText;
+                textProcessor.setValue(textContent);
+            }
+        }
+    }
+
+    ChatViewTextProcessor {
+        id: textProcessor
+    }
+
+    function resetChatViewTextProcessor() {
+        textProcessor.fontPixelSize                = myTextArea.font.pixelSize
+        textProcessor.codeColors.defaultColor      = theme.codeDefaultColor
+        textProcessor.codeColors.keywordColor      = theme.codeKeywordColor
+        textProcessor.codeColors.functionColor     = theme.codeFunctionColor
+        textProcessor.codeColors.functionCallColor = theme.codeFunctionCallColor
+        textProcessor.codeColors.commentColor      = theme.codeCommentColor
+        textProcessor.codeColors.stringColor       = theme.codeStringColor
+        textProcessor.codeColors.numberColor       = theme.codeNumberColor
+        textProcessor.codeColors.headerColor       = theme.codeHeaderColor
+        textProcessor.codeColors.backgroundColor   = theme.codeBackgroundColor
+        textProcessor.textDocument                 = textDocument
+        textProcessor.setValue(textContent);
+    }
+
+    property bool textProcessorReady: false
+
+    Component.onCompleted: {
+        resetChatViewTextProcessor();
+        textProcessorReady = true;
+    }
+
+    Connections {
+        target: myTextArea
+        function onTextContentChanged() {
+            if (myTextArea.textProcessorReady)
+                textProcessor.setValue(textContent);
+        }
+    }
+
+    Connections {
+        target: MySettings
+        function onFontSizeChanged() {
+            myTextArea.resetChatViewTextProcessor();
+        }
+        function onChatThemeChanged() {
+            myTextArea.resetChatViewTextProcessor();
+        }
+    }
+
+    Accessible.role: Accessible.Paragraph
+    Accessible.name: text
+    Accessible.description: name === "Response: " ? "The response by the model" : "The prompt by the user"
+}

gpt4all-chat/qml/ConfirmationDialog.qml

@@ -0,0 +1,59 @@
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import QtQuick.Layouts
+
+MyDialog {
+    id: confirmationDialog
+    anchors.centerIn: parent
+    modal: true
+    padding: 20
+    property alias dialogTitle: titleText.text
+    property alias description: descriptionText.text
+
+    Theme { id: theme }
+
+    contentItem: ColumnLayout {
+        Text {
+            id: titleText
+            Layout.alignment: Qt.AlignHCenter
+            textFormat: Text.StyledText
+            color: theme.textColor
+            font.pixelSize: theme.fontSizeLarger
+            font.bold: true
+        }
+
+        Text {
+            id: descriptionText
+            Layout.alignment: Qt.AlignHCenter
+            textFormat: Text.StyledText
+            color: theme.textColor
+            font.pixelSize: theme.fontSizeMedium
+        }
+    }
+
+    footer: DialogButtonBox {
+        id: dialogBox
+        padding: 20
+        alignment: Qt.AlignRight
+        spacing: 10
+        MySettingsButton {
+            text: qsTr("OK")
+            textColor: theme.mediumButtonText
+            backgroundColor: theme.mediumButtonBackground
+            backgroundColorHovered: theme.mediumButtonBackgroundHovered
+            DialogButtonBox.buttonRole: DialogButtonBox.AcceptRole
+        }
+        MySettingsButton {
+            text: qsTr("Cancel")
+            DialogButtonBox.buttonRole: DialogButtonBox.RejectRole
+        }
+        background: Rectangle {
+            color: "transparent"
+        }
+        Keys.onEnterPressed: confirmationDialog.accept()
+        Keys.onReturnPressed: confirmationDialog.accept()
+    }
+    Component.onCompleted: dialogBox.forceActiveFocus()
+}

gpt4all-chat/qml/HomeView.qml

@@ -47,7 +47,7 @@ Rectangle {
                     id: welcome
                     Layout.alignment: Qt.AlignHCenter
                     text: qsTr("Welcome to GPT4All")
-                    font.pixelSize: theme.fontSizeBanner
+                    font.pixelSize: theme.fontSizeBannerLarge
                     color: theme.titleTextColor
                 }
 

gpt4all-chat/qml/LocalDocsSettings.qml

@@ -10,7 +10,7 @@ import mysettings
 import network
 
 MySettingsTab {
-    onRestoreDefaultsClicked: {
+    onRestoreDefaults: {
         MySettings.restoreLocalDocsDefaults();
     }
 
@@ -29,37 +29,6 @@ MySettingsTab {
             text: qsTr("LocalDocs Settings")
         }
 
-        ColumnLayout {
-            spacing: 10
-            Label {
-                color: theme.styledTextColor
-                font.pixelSize: theme.fontSizeLarge
-                font.bold: true
-                text: qsTr("Behavior")
-            }
-
-            Rectangle {
-                Layout.fillWidth: true
-                height: 1
-                color: theme.settingsDivider
-            }
-        }
-
-        RowLayout {
-            MySettingsLabel {
-                id: automaticUpdateLabel
-                text: qsTr("Automatic Update")
-                helpText: qsTr("Whenever a file or folder changes it should automatically be re-indexed/embedded.")
-            }
-            MyCheckBox {
-                id: automaticUpdateBox
-                checked: MySettings.localDocsAutomaticUpdate
-                onClicked: {
-                    MySettings.localDocsAutomaticUpdate = !MySettings.localDocsAutomaticUpdate
-                }
-            }
-        }
-
         ColumnLayout {
             spacing: 10
             Label {
@@ -207,6 +176,7 @@ MySettingsTab {
                     ListElement { text: qsTr("Application default") }
                     Component.onCompleted: {
                         MySettings.embeddingsDeviceList.forEach(d => append({"text": d}));
+                        deviceBox.updateModel();
                     }
                 }
                 Accessible.name: deviceLabel.text

gpt4all-chat/qml/LocalDocsView.qml

@@ -210,9 +210,6 @@ Rectangle {
                                             if (model.currentEmbeddingsToIndex !== 0)
                                                  return theme.altProgressBackground
 
-                                            if (model.outOfDate)
-                                                 return theme.altProgressBackground
-
                                             if (model.forceIndexing)
                                                 return theme.red200
 
@@ -250,9 +247,6 @@ Rectangle {
                                         if (model.currentEmbeddingsToIndex !== 0)
                                             return theme.altProgressText
 
-                                        if (model.outOfDate)
-                                            return theme.altProgressText
-
                                         if (model.forceIndexing)
                                             return theme.textErrorColor
 
@@ -273,9 +267,6 @@ Rectangle {
                                         if (model.forceIndexing)
                                             return qsTr("REQUIRES UPDATE")
 
-                                        if (model.outOfDate)
-                                            return qsTr("OUT OF DATE")
-
                                         if (model.installed)
                                             return qsTr("READY")
 
@@ -314,11 +305,8 @@ Rectangle {
                                     if (model.forceIndexing)
                                         return qsTr("This collection requires an update after version change")
 
-                                    if (model.outOfDate)
-                                        return qsTr("This collection is out of date and needs to be rebuilt")
-
                                     if (model.installed)
-                                        return qsTr("This collection is up to date")
+                                        return qsTr("Automatically reindexes upon changes to the folder")
 
                                     return qsTr("Installation in progress")
                                 }

gpt4all-chat/qml/ModelSettings.qml

@@ -8,10 +8,34 @@ import mysettings
 import chatlistmodel
 
 MySettingsTab {
-    onRestoreDefaultsClicked: {
+    onRestoreDefaults: {
         MySettings.restoreModelDefaults(root.currentModelInfo);
     }
     title: qsTr("Model")
+
+    ConfirmationDialog {
+        id: resetSystemMessageDialog
+        property var index: null
+        property bool resetClears: false
+        dialogTitle: qsTr("%1 system message?").arg(resetClears ? qsTr("Clear") : qsTr("Reset"))
+        description: qsTr("The system message will be %1.").arg(resetClears ? qsTr("removed") : qsTr("reset to the default"))
+        onAccepted: MySettings.resetModelSystemMessage(ModelList.modelInfo(index))
+        function show(index_, resetClears_) { index = index_; resetClears = resetClears_; open(); }
+    }
+
+    ConfirmationDialog {
+        id: resetChatTemplateDialog
+        property bool resetClears: false
+        property var index: null
+        dialogTitle: qsTr("%1 chat template?").arg(resetClears ? qsTr("Clear") : qsTr("Reset"))
+        description: qsTr("The chat template will be %1.").arg(resetClears ? qsTr("erased") : qsTr("reset to the default"))
+        onAccepted: {
+            MySettings.resetModelChatTemplate(ModelList.modelInfo(index));
+            templateTextArea.resetText();
+        }
+        function show(index_, resetClears_) { index = index_; resetClears = resetClears_; open(); }
+    }
+
     contentItem: GridLayout {
         id: root
         columns: 3
@@ -35,6 +59,7 @@ MySettingsTab {
 
         RowLayout {
             Layout.fillWidth: true
+            Layout.maximumWidth: parent.width
             Layout.row: 2
             Layout.column: 0
             Layout.columnSpan: 2
@@ -153,69 +178,154 @@ MySettingsTab {
             Layout.fillWidth: true
         }
 
-        MySettingsLabel {
-            visible: !root.currentModelInfo.isOnline
-            text: qsTr("System Prompt")
-            helpText: qsTr("Prefixed at the beginning of every conversation. Must contain the appropriate framing tokens.")
+        RowLayout {
             Layout.row: 7
-            Layout.column: 0
+            Layout.columnSpan: 2
             Layout.topMargin: 15
+            Layout.fillWidth: true
+            Layout.maximumWidth: parent.width
+            spacing: 10
+            MySettingsLabel {
+                id: systemMessageLabel
+                text: qsTr("System Message")
+                helpText: qsTr("A message to set the context or guide the behavior of the model. Leave blank for " +
+                               "none. NOTE: Since GPT4All 3.5, this should not contain control tokens.")
+                onReset: () => resetSystemMessageDialog.show(root.currentModelId, resetClears)
+                function updateResetButton() {
+                    const info = root.currentModelInfo;
+                    // NOTE: checks if the *override* is set, regardless of whether there is a default
+                    canReset = !!info.id && MySettings.isModelSystemMessageSet(info);
+                    resetClears = !info.defaultSystemMessage;
+                }
+                Component.onCompleted: updateResetButton()
+                Connections {
+                    target: root
+                    function onCurrentModelIdChanged() { systemMessageLabel.updateResetButton(); }
+                }
+                Connections {
+                    target: MySettings
+                    function onSystemMessageChanged(info)
+                    { if (info.id === root.currentModelId) systemMessageLabel.updateResetButton(); }
+                }
+            }
+            Label {
+                id: systemMessageLabelHelp
+                visible: systemMessageArea.errState !== "ok"
+                Layout.alignment: Qt.AlignBottom
+                Layout.fillWidth: true
+                Layout.rightMargin: 5
+                Layout.maximumHeight: systemMessageLabel.height
+                text: qsTr("System message is not " +
+                           "<a href=\"https://docs.gpt4all.io/gpt4all_desktop/chat_templates.html\">plain text</a>.")
+                color: systemMessageArea.errState === "error" ? theme.textErrorColor : theme.textWarningColor
+                font.pixelSize: theme.fontSizeLarger
+                font.bold: true
+                wrapMode: Text.Wrap
+                elide: Text.ElideRight
+                onLinkActivated: function(link) { Qt.openUrlExternally(link) }
+            }
         }
 
         Rectangle {
-            id: systemPrompt
-            visible: !root.currentModelInfo.isOnline
+            id: systemMessage
             Layout.row: 8
             Layout.column: 0
             Layout.columnSpan: 2
             Layout.fillWidth: true
             color: "transparent"
-            Layout.minimumHeight: Math.max(100, systemPromptArea.contentHeight + 20)
+            Layout.minimumHeight: Math.max(100, systemMessageArea.contentHeight + 20)
             MyTextArea {
-                id: systemPromptArea
+                id: systemMessageArea
                 anchors.fill: parent
-                text: root.currentModelInfo.systemPrompt
+                property bool isBeingReset: false
+                function resetText() {
+                    const info = root.currentModelInfo;
+                    isBeingReset = true;
+                    text = (info.id ? info.systemMessage.value : null) ?? "";
+                    isBeingReset = false;
+                }
+                Component.onCompleted: resetText()
                 Connections {
                     target: MySettings
-                    function onSystemPromptChanged() {
-                        systemPromptArea.text = root.currentModelInfo.systemPrompt;
-                    }
+                    function onSystemMessageChanged(info)
+                    { if (info.id === root.currentModelId) systemMessageArea.resetText(); }
                 }
                 Connections {
                     target: root
-                    function onCurrentModelInfoChanged() {
-                        systemPromptArea.text = root.currentModelInfo.systemPrompt;
-                    }
+                    function onCurrentModelIdChanged() { systemMessageArea.resetText(); }
                 }
+                // strict validation, because setModelSystemMessage clears isLegacy
+                readonly property var reLegacyCheck: (
+                    /(?:^|\s)(?:### *System\b|S(?:ystem|YSTEM):)|<\|(?:im_(?:start|end)|(?:start|end)_header_id|eot_id|SYSTEM_TOKEN)\|>|<<SYS>>/m
+                )
                 onTextChanged: {
-                    MySettings.setModelSystemPrompt(root.currentModelInfo, text)
+                    const info = root.currentModelInfo;
+                    if (!info.id) {
+                        errState = "ok";
+                    } else if (info.systemMessage.isLegacy && (isBeingReset || reLegacyCheck.test(text))) {
+                        errState = "error";
+                    } else
+                        errState = reLegacyCheck.test(text) ? "warning" : "ok";
+                    if (info.id && errState !== "error" && !isBeingReset)
+                        MySettings.setModelSystemMessage(info, text);
+                    systemMessageLabel.updateResetButton();
                 }
                 Accessible.role: Accessible.EditableText
+                Accessible.name: systemMessageLabel.text
+                Accessible.description: systemMessageLabelHelp.text
             }
         }
 
         RowLayout {
             Layout.row: 9
-            Layout.column: 0
             Layout.columnSpan: 2
             Layout.topMargin: 15
+            Layout.fillWidth: true
+            Layout.maximumWidth: parent.width
             spacing: 10
             MySettingsLabel {
-                id: promptTemplateLabel
-                text: qsTr("Prompt Template")
-                helpText: qsTr("The template that wraps every prompt.")
+                id: chatTemplateLabel
+                text: qsTr("Chat Template")
+                helpText: qsTr("This Jinja template turns the chat into input for the model.")
+                onReset: () => resetChatTemplateDialog.show(root.currentModelId, resetClears)
+                function updateResetButton() {
+                    const info = root.currentModelInfo;
+                    canReset = !!info.id && (
+                        MySettings.isModelChatTemplateSet(info)
+                        || templateTextArea.text !== (info.chatTemplate.value ?? "")
+                    );
+                    resetClears = !info.defaultChatTemplate;
+                }
+                Component.onCompleted: updateResetButton()
+                Connections {
+                    target: root
+                    function onCurrentModelIdChanged() { chatTemplateLabel.updateResetButton(); }
+                }
+                Connections {
+                    target: MySettings
+                    function onChatTemplateChanged(info)
+                    { if (info.id === root.currentModelId) chatTemplateLabel.updateResetButton(); }
+                }
             }
-            MySettingsLabel {
-                id: promptTemplateLabelHelp
-                text: qsTr("Must contain the string \"%1\" to be replaced with the user's input.")
-                color: theme.textErrorColor
-                visible: templateTextArea.text.indexOf("%1") === -1
-                wrapMode: TextArea.Wrap
+            Label {
+                id: chatTemplateLabelHelp
+                visible: templateTextArea.errState !== "ok"
+                Layout.alignment: Qt.AlignBottom
+                Layout.fillWidth: true
+                Layout.rightMargin: 5
+                Layout.maximumHeight: chatTemplateLabel.height
+                text: templateTextArea.errMsg
+                color: templateTextArea.errState === "error" ? theme.textErrorColor : theme.textWarningColor
+                font.pixelSize: theme.fontSizeLarger
+                font.bold: true
+                wrapMode: Text.Wrap
+                elide: Text.ElideRight
+                onLinkActivated: function(link) { Qt.openUrlExternally(link) }
             }
         }
 
         Rectangle {
-            id: promptTemplate
+            id: chatTemplate
             Layout.row: 10
             Layout.column: 0
             Layout.columnSpan: 2
@@ -226,27 +336,71 @@ MySettingsTab {
             MyTextArea {
                 id: templateTextArea
                 anchors.fill: parent
-                text: root.currentModelInfo.promptTemplate
+                font: fixedFont
+                property bool isBeingReset: false
+                property var errMsg: null
+                function resetText() {
+                    const info = root.currentModelInfo;
+                    isBeingReset = true;
+                    text = (info.id ? info.chatTemplate.value : null) ?? "";
+                    isBeingReset = false;
+                }
+                Component.onCompleted: resetText()
                 Connections {
                     target: MySettings
-                    function onPromptTemplateChanged() {
-                        templateTextArea.text = root.currentModelInfo.promptTemplate;
-                    }
+                    function onChatTemplateChanged() { templateTextArea.resetText(); }
                 }
                 Connections {
                     target: root
-                    function onCurrentModelInfoChanged() {
-                        templateTextArea.text = root.currentModelInfo.promptTemplate;
-                    }
+                    function onCurrentModelIdChanged() { templateTextArea.resetText(); }
+                }
+                function legacyCheck() {
+                    return /%[12]\b/.test(text) || !/\{%.*%\}.*\{\{.*\}\}.*\{%.*%\}/.test(text.replace(/\n/g, ''))
+                        || !/\bcontent\b/.test(text);
                 }
                 onTextChanged: {
-                    if (templateTextArea.text.indexOf("%1") !== -1) {
-                        MySettings.setModelPromptTemplate(root.currentModelInfo, text)
+                    const info = root.currentModelInfo;
+                    let jinjaError;
+                    if (!info.id) {
+                        errMsg = null;
+                        errState = "ok";
+                    } else if (info.chatTemplate.isLegacy && (isBeingReset || legacyCheck())) {
+                        errMsg = null;
+                        errState = "error";
+                    } else if (text === "" && !info.chatTemplate.isSet) {
+                        errMsg = qsTr("No <a href=\"https://docs.gpt4all.io/gpt4all_desktop/chat_templates.html\">" +
+                                      "chat template</a> configured.");
+                        errState = "error";
+                    } else if (/^\s*$/.test(text)) {
+                        errMsg = qsTr("The <a href=\"https://docs.gpt4all.io/gpt4all_desktop/chat_templates.html\">" +
+                                      "chat template</a> cannot be blank.");
+                        errState = "error";
+                    } else if ((jinjaError = MySettings.checkJinjaTemplateError(text)) !== null) {
+                        errMsg = qsTr("<a href=\"https://docs.gpt4all.io/gpt4all_desktop/chat_templates.html\">Syntax" +
+                                      " error</a>: %1").arg(jinjaError);
+                        errState = "error";
+                    } else if (legacyCheck()) {
+                        errMsg = qsTr("Chat template is not in " +
+                                      "<a href=\"https://docs.gpt4all.io/gpt4all_desktop/chat_templates.html\">" +
+                                      "Jinja format</a>.")
+                        errState = "warning";
+                    } else {
+                        errState = "ok";
+                    }
+                    if (info.id && errState !== "error" && !isBeingReset)
+                        MySettings.setModelChatTemplate(info, text);
+                    chatTemplateLabel.updateResetButton();
+                }
+                Keys.onPressed: event => {
+                    if (event.key === Qt.Key_Tab) {
+                        const a = templateTextArea;
+                        event.accepted = true;              // suppress tab
+                        a.insert(a.cursorPosition, '    '); // four spaces
                     }
                 }
                 Accessible.role: Accessible.EditableText
-                Accessible.name: promptTemplateLabel.text
-                Accessible.description: promptTemplateLabelHelp.text
+                Accessible.name: chatTemplateLabel.text
+                Accessible.description: chatTemplateLabelHelp.text
             }
         }
 

gpt4all-chat/qml/MyFileIcon.qml

@@ -0,0 +1,41 @@
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import Qt5Compat.GraphicalEffects
+
+Item {
+    id: fileIcon
+    property real iconSize: 24
+    property string fileName: ""
+    implicitWidth: iconSize
+    implicitHeight: iconSize
+
+    Image {
+        id: fileImage
+        anchors.fill: parent
+        visible: false
+        sourceSize.width: iconSize
+        sourceSize.height: iconSize
+        mipmap: true
+        source: {
+            if (fileIcon.fileName.toLowerCase().endsWith(".txt"))
+                return "qrc:/gpt4all/icons/file-txt.svg"
+            else if (fileIcon.fileName.toLowerCase().endsWith(".pdf"))
+                return "qrc:/gpt4all/icons/file-pdf.svg"
+            else if (fileIcon.fileName.toLowerCase().endsWith(".md"))
+                return "qrc:/gpt4all/icons/file-md.svg"
+            else if (fileIcon.fileName.toLowerCase().endsWith(".xlsx"))
+                return "qrc:/gpt4all/icons/file-xls.svg"
+            else if (fileIcon.fileName.toLowerCase().endsWith(".docx"))
+                return "qrc:/gpt4all/icons/file-docx.svg"
+            else
+                return "qrc:/gpt4all/icons/file.svg"
+        }
+    }
+    ColorOverlay {
+        anchors.fill: fileImage
+        source: fileImage
+        color: theme.textColor
+    }
+}

gpt4all-chat/qml/MySettingsButton.qml

@@ -17,6 +17,7 @@ Button {
     property color  borderColor: "transparent"
     property real   fontPixelSize: theme.fontSizeLarge
     property string toolTip
+    property alias backgroundRadius: background.radius
 
     contentItem: Text {
         text: myButton.text
@@ -28,6 +29,7 @@ Button {
         Accessible.name: text
     }
     background: Rectangle {
+        id: background
         radius: 10
         border.width: borderWidth
         border.color: borderColor

gpt4all-chat/qml/MySettingsLabel.qml

@@ -17,13 +17,42 @@ ColumnLayout {
     property alias color: mainTextLabel.color
     property alias linkColor: mainTextLabel.linkColor
 
-    Label {
-        id: mainTextLabel
-        color: theme.settingsTitleTextColor
-        font.pixelSize: theme.fontSizeLarger
-        font.bold: true
-        onLinkActivated: function(link) {
-            root.linkActivated(link);
+    property var onReset: null
+    property alias canReset: resetButton.enabled
+    property bool resetClears: false
+
+    Item {
+        anchors.margins: 5
+        width: childrenRect.width
+        height: mainTextLabel.contentHeight
+
+        Label {
+            id: mainTextLabel
+            anchors.left: parent.left
+            anchors.top: parent.top
+            anchors.bottom: parent.bottom
+            color: theme.settingsTitleTextColor
+            font.pixelSize: theme.fontSizeLarger
+            font.bold: true
+            verticalAlignment: Text.AlignVCenter
+            onLinkActivated: function(link) {
+                root.linkActivated(link);
+            }
+        }
+
+        MySettingsButton {
+            id: resetButton
+            anchors.baseline: mainTextLabel.baseline
+            anchors.left: mainTextLabel.right
+            height: mainTextLabel.contentHeight
+            anchors.leftMargin: 10
+            padding: 2
+            leftPadding: 10
+            rightPadding: 10
+            backgroundRadius: 5
+            text: resetClears ? qsTr("Clear") : qsTr("Reset")
+            visible: root.onReset !== null
+            onClicked: root.onReset()
         }
     }
     Label {

gpt4all-chat/qml/MySettingsStack.qml

@@ -2,6 +2,7 @@ import QtCore
 import QtQuick
 import QtQuick.Controls
 import QtQuick.Controls.Basic
+import QtQuick.Controls.impl
 import QtQuick.Layouts
 import QtQuick.Dialogs
 import Qt.labs.folderlistmodel

gpt4all-chat/qml/MySettingsTab.qml

@@ -9,7 +9,7 @@ Item {
     property string title: ""
     property Item contentItem: null
     property bool showRestoreDefaultsButton: true
-    signal restoreDefaultsClicked
+    signal restoreDefaults
 
     onContentItemChanged: function() {
         if (contentItem) {
@@ -19,6 +19,13 @@ Item {
         }
     }
 
+    ConfirmationDialog {
+        id: restoreDefaultsDialog
+        dialogTitle: qsTr("Restore defaults?")
+        description: qsTr("This page of settings will be reset to the defaults.")
+        onAccepted: root.restoreDefaults()
+    }
+
     ScrollView {
         id: scrollView
         width: parent.width
@@ -47,6 +54,7 @@ Item {
             Column {
                 id: contentInner
                 Layout.fillWidth: true
+                Layout.maximumWidth: parent.width
             }
 
             Item {
@@ -63,9 +71,7 @@ Item {
                     Accessible.role: Accessible.Button
                     Accessible.name: text
                     Accessible.description: qsTr("Restores settings dialog to a default state")
-                    onClicked: {
-                        root.restoreDefaultsClicked();
-                    }
+                    onClicked: restoreDefaultsDialog.open()
                 }
             }
         }

gpt4all-chat/qml/MyTabButton.qml

@@ -0,0 +1,26 @@
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import mysettings
+import mysettingsenums
+
+MySettingsButton {
+    property bool isSelected: false
+    contentItem: Text {
+        text: parent.text
+        horizontalAlignment: Qt.AlignCenter
+        color: isSelected ? theme.titleTextColor : theme.styledTextColor
+        font.pixelSize: theme.fontSizeLarger
+    }
+    background: Item {
+        visible: isSelected || hovered
+        Rectangle {
+            anchors.bottom: parent.bottom
+            anchors.left: parent.left
+            anchors.right: parent.right
+            height: 3
+            color: isSelected ? theme.titleTextColor : theme.styledTextColorLighter
+        }
+    }
+}

gpt4all-chat/qml/MyTextArea.qml

@@ -5,18 +5,27 @@ import QtQuick.Controls.Basic
 
 TextArea {
     id: myTextArea
+
+    property string errState: "ok"  // one of "ok", "error", "warning"
+
     color: enabled ? theme.textColor : theme.mutedTextColor
     placeholderTextColor: theme.mutedTextColor
     font.pixelSize: theme.fontSizeLarge
     background: Rectangle {
         implicitWidth: 150
         color: theme.controlBackground
-        border.width: 1
-        border.color: theme.controlBorder
+        border.width: errState === "ok" ? 1 : 2
+        border.color: {
+            switch (errState) {
+                case "ok":      return theme.controlBorder;
+                case "warning": return theme.textWarningColor;
+                case "error":   return theme.textErrorColor;
+            }
+        }
         radius: 10
     }
     padding: 10
     wrapMode: TextArea.Wrap
 
     ToolTip.delay: Qt.styleHints.mousePressAndHoldInterval
-}
+}

gpt4all-chat/qml/MyToolButton.qml

@@ -16,6 +16,7 @@ Button {
     property alias fillMode: image.fillMode
     property alias imageWidth: image.sourceSize.width
     property alias imageHeight: image.sourceSize.height
+    property alias bgTransform: background.transform
     contentItem: Text {
         text: myButton.text
         horizontalAlignment: Text.AlignHCenter
@@ -26,6 +27,7 @@ Button {
     }
 
     background: Item {
+        id: background
         anchors.fill: parent
         Rectangle {
             anchors.fill: parent
@@ -47,7 +49,7 @@ Button {
         ColorOverlay {
             anchors.fill: image
             source: image
-            color: myButton.hovered ? backgroundColorHovered : backgroundColor
+            color: !myButton.enabled ? theme.mutedTextColor : myButton.hovered ? backgroundColorHovered : backgroundColor
         }
     }
     Accessible.role: Accessible.Button

gpt4all-chat/qml/StartupDialog.qml

@@ -115,7 +115,7 @@ model release that uses your data!")
             anchors.right: parent.right
             Label {
                 id: optInStatistics
-                text: "Opt-in to anonymous usage analytics used to improve GPT4All"
+                text: qsTr("Opt-in to anonymous usage analytics used to improve GPT4All")
                 Layout.row: 0
                 Layout.column: 0
                 color: theme.textColor
@@ -229,7 +229,7 @@ model release that uses your data!")
 
             Label {
                 id: optInNetwork
-                text: "Opt-in to anonymous sharing of chats to the GPT4All Datalake"
+                text: qsTr("Opt-in to anonymous sharing of chats to the GPT4All Datalake")
                 Layout.row: 1
                 Layout.column: 0
                 color: theme.textColor

gpt4all-chat/qml/SwitchModelDialog.qml

@@ -1,46 +0,0 @@
-import QtCore
-import QtQuick
-import QtQuick.Controls
-import QtQuick.Controls.Basic
-import QtQuick.Layouts
-import llm
-import mysettings
-
-MyDialog {
-    id: switchModelDialog
-    anchors.centerIn: parent
-    modal: true
-    padding: 20
-    property int index: -1
-
-    Theme {
-        id: theme
-    }
-
-    contentItem: Text {
-        textFormat: Text.StyledText
-        text: qsTr("<b>Warning:</b> changing the model will erase the current conversation. Do you wish to continue?")
-        color: theme.textColor
-        font.pixelSize: theme.fontSizeLarge
-    }
-
-    footer: DialogButtonBox {
-        id: dialogBox
-        padding: 20
-        alignment: Qt.AlignRight
-        spacing: 10
-        MySettingsButton {
-            text: qsTr("Continue")
-            Accessible.description: qsTr("Continue with model loading")
-            DialogButtonBox.buttonRole: DialogButtonBox.AcceptRole
-        }
-        MySettingsButton {
-            text: qsTr("Cancel")
-            Accessible.description: qsTr("Cancel")
-            DialogButtonBox.buttonRole: DialogButtonBox.RejectRole
-        }
-        background: Rectangle {
-            color: "transparent"
-        }
-    }
-}

gpt4all-chat/qml/Theme.qml

@@ -64,6 +64,9 @@ QtObject {
     property color green800: Qt.hsla(123/360, 0.17, 0.24)
     property color green900: Qt.hsla(124/360, 0.17, 0.20)
     property color green950: Qt.hsla(125/360, 0.22, 0.10)
+    property color green300_sat: Qt.hsla(122/360, 0.24, 0.73)
+    property color green400_sat: Qt.hsla(122/360, 0.23, 0.58)
+    property color green450_sat: Qt.hsla(122/360, 0.23, 0.52)
 
     // yellow
     property color yellow0: Qt.hsla(47/360, 0.90, 0.99)
@@ -99,6 +102,7 @@ QtObject {
     property color purple200: Qt.hsla(279/360, 1.0, 0.91)
     property color purple300: Qt.hsla(279/360, 1.0, 0.84)
     property color purple400: Qt.hsla(279/360, 1.0, 0.73)
+    property color purple450: Qt.hsla(279/360, 1.0, 0.68)
     property color purple500: Qt.hsla(279/360, 1.0, 0.63)
     property color purple600: Qt.hsla(279/360, 1.0, 0.53)
     property color purple700: Qt.hsla(279/360, 1.0, 0.47)
@@ -408,6 +412,39 @@ QtObject {
         }
     }
 
+    property color mediumButtonBackground: {
+        switch (MySettings.chatTheme) {
+            case MySettingsEnums.ChatTheme.LegacyDark:
+                return purple400
+            case MySettingsEnums.ChatTheme.Dark:
+                return green400_sat
+            default:
+                return green400_sat
+        }
+    }
+
+    property color mediumButtonBackgroundHovered: {
+        switch (MySettings.chatTheme) {
+            case MySettingsEnums.ChatTheme.LegacyDark:
+                return purple450
+            case MySettingsEnums.ChatTheme.Dark:
+                return green450_sat
+            default:
+                return green300_sat
+        }
+    }
+
+    property color mediumButtonText: {
+        switch (MySettings.chatTheme) {
+            case MySettingsEnums.ChatTheme.LegacyDark:
+                return textColor
+            case MySettingsEnums.ChatTheme.Dark:
+                return textColor
+            default:
+                return white
+        }
+    }
+
     property color darkButtonText: {
         switch (MySettings.chatTheme) {
             case MySettingsEnums.ChatTheme.LegacyDark:
@@ -922,16 +959,8 @@ QtObject {
         }
     }
 
-    property color textErrorColor: {
-        switch (MySettings.chatTheme) {
-            case MySettingsEnums.ChatTheme.LegacyDark:
-                return red400
-            case MySettingsEnums.ChatTheme.Dark:
-                return red400
-            default:
-                return red400
-        }
-    }
+    readonly property color textErrorColor:   red400
+    readonly property color textWarningColor: yellow400
 
     property color settingsTitleTextColor: {
         switch (MySettings.chatTheme) {
@@ -988,6 +1017,17 @@ QtObject {
         }
     }
 
+    property color styledTextColorLighter: {
+        switch (MySettings.chatTheme) {
+            case MySettingsEnums.ChatTheme.LegacyDark:
+                return purple50
+            case MySettingsEnums.ChatTheme.Dark:
+                return yellow0
+            default:
+                return grayRed400
+        }
+    }
+
     property color styledTextColor2: {
         switch (MySettings.chatTheme) {
             case MySettingsEnums.ChatTheme.LegacyDark:
@@ -1227,5 +1267,6 @@ QtObject {
     property real fontSizeLarger:      14 * fontScale
     property real fontSizeLargest:     18 * fontScale
     property real fontSizeBannerSmall: 24 * fontScale**.8
-    property real fontSizeBanner:      48 * fontScale**.8
+    property real fontSizeBanner:      32 * fontScale**.8
+    property real fontSizeBannerLarge: 48 * fontScale**.8
 }

gpt4all-chat/src/chat.cpp

@@ -1,23 +1,30 @@
 #include "chat.h"
 
 #include "chatlistmodel.h"
-#include "mysettings.h"
 #include "network.h"
 #include "server.h"
+#include "tool.h"
+#include "toolcallparser.h"
+#include "toolmodel.h"
 
 #include <QBuffer>
 #include <QDataStream>
 #include <QDebug>
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QJsonValue>
 #include <QLatin1String>
 #include <QMap>
+#include <QRegularExpression>
 #include <QString>
-#include <QStringList>
 #include <QVariant>
 #include <Qt>
 #include <QtLogging>
 
 #include <utility>
 
+using namespace ToolEnums;
+
 Chat::Chat(QObject *parent)
     : QObject(parent)
     , m_id(Network::globalInstance()->generateUniqueId())
@@ -31,7 +38,7 @@ Chat::Chat(QObject *parent)
     connectLLM();
 }
 
-Chat::Chat(bool isServer, QObject *parent)
+Chat::Chat(server_tag_t, QObject *parent)
     : QObject(parent)
     , m_id(Network::globalInstance()->generateUniqueId())
     , m_name(tr("Server Chat"))
@@ -61,13 +68,12 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::responseStopped, this, &Chat::responseStopped, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelLoadingError, this, &Chat::handleModelLoadingError, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelLoadingWarning, this, &Chat::modelLoadingWarning, Qt::QueuedConnection);
-    connect(m_llmodel, &ChatLLM::restoringFromTextChanged, this, &Chat::handleRestoringFromText, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::generatedNameChanged, this, &Chat::generatedNameChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::generatedQuestionFinished, this, &Chat::generatedQuestionFinished, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportSpeed, this, &Chat::handleTokenSpeedChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::loadedModelInfoChanged, this, &Chat::loadedModelInfoChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::databaseResultsChanged, this, &Chat::handleDatabaseResultsChanged, Qt::QueuedConnection);
-    connect(m_llmodel, &ChatLLM::modelInfoChanged, this, &Chat::handleModelInfoChanged, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::modelInfoChanged, this, &Chat::handleModelChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::trySwitchContextOfLoadedModelCompleted, this, &Chat::handleTrySwitchContextOfLoadedModelCompleted, Qt::QueuedConnection);
 
     connect(this, &Chat::promptRequested, m_llmodel, &ChatLLM::prompt, Qt::QueuedConnection);
@@ -75,11 +81,10 @@ void Chat::connectLLM()
     connect(this, &Chat::loadDefaultModelRequested, m_llmodel, &ChatLLM::loadDefaultModel, Qt::QueuedConnection);
     connect(this, &Chat::generateNameRequested, m_llmodel, &ChatLLM::generateName, Qt::QueuedConnection);
     connect(this, &Chat::regenerateResponseRequested, m_llmodel, &ChatLLM::regenerateResponse, Qt::QueuedConnection);
-    connect(this, &Chat::resetResponseRequested, m_llmodel, &ChatLLM::resetResponse, Qt::QueuedConnection);
-    connect(this, &Chat::resetContextRequested, m_llmodel, &ChatLLM::resetContext, Qt::QueuedConnection);
-    connect(this, &Chat::processSystemPromptRequested, m_llmodel, &ChatLLM::processSystemPrompt, Qt::QueuedConnection);
 
     connect(this, &Chat::collectionListChanged, m_collectionModel, &LocalDocsCollectionsModel::setCollections);
+
+    connect(ModelList::globalInstance(), &ModelList::modelInfoChanged, this, &Chat::handleModelInfoChanged);
 }
 
 void Chat::reset()
@@ -87,28 +92,17 @@ void Chat::reset()
     stopGenerating();
     // Erase our current on disk representation as we're completely resetting the chat along with id
     ChatListModel::globalInstance()->removeChatFile(this);
-    emit resetContextRequested();
     m_id = Network::globalInstance()->generateUniqueId();
     emit idChanged(m_id);
     // NOTE: We deliberately do no reset the name or creation date to indicate that this was originally
     // an older chat that was reset for another purpose. Resetting this data will lead to the chat
     // name label changing back to 'New Chat' and showing up in the chat model list as a 'New Chat'
     // further down in the list. This might surprise the user. In the future, we might get rid of
-    // the "reset context" button in the UI. Right now, by changing the model in the combobox dropdown
-    // we effectively do a reset context. We *have* to do this right now when switching between different
-    // types of models. The only way to get rid of that would be a very long recalculate where we rebuild
-    // the context if we switch between different types of models. Probably the right way to fix this
-    // is to allow switching models but throwing up a dialog warning users if we switch between types
-    // of models that a long recalculation will ensue.
+    // the "reset context" button in the UI.
     m_chatModel->clear();
     m_needsSave = true;
 }
 
-void Chat::processSystemPrompt()
-{
-    emit processSystemPromptRequested();
-}
-
 void Chat::resetResponseState()
 {
     if (m_responseInProgress && m_responseState == Chat::LocalDocsRetrieval)
@@ -132,7 +126,13 @@ void Chat::newPromptResponsePair(const QString &prompt, const QList<QUrl> &attac
         Q_ASSERT(url.isLocalFile());
         const QString localFilePath = url.toLocalFile();
         const QFileInfo info(localFilePath);
-        Q_ASSERT(info.suffix() == "xlsx"); // We only support excel right now
+
+        Q_ASSERT(
+            info.suffix().toLower() == "xlsx" ||
+            info.suffix().toLower() == "txt" ||
+            info.suffix().toLower() == "md" ||
+            info.suffix().toLower() == "rst"
+        );
 
         PromptAttachment attached;
         attached.url = url;
@@ -154,25 +154,29 @@ void Chat::newPromptResponsePair(const QString &prompt, const QList<QUrl> &attac
     if (!attachedContexts.isEmpty())
         promptPlusAttached = attachedContexts.join("\n\n") + "\n\n" + prompt;
 
-    newPromptResponsePairInternal(prompt, attachments);
-    emit resetResponseRequested();
+    resetResponseState();
+    if (int count = m_chatModel->count())
+        m_chatModel->updateCurrentResponse(count - 1, false);
+    m_chatModel->appendPrompt(prompt, attachments);
+    m_chatModel->appendResponse();
 
-    this->prompt(promptPlusAttached);
+    emit promptRequested(m_collections);
+    m_needsSave = true;
 }
 
-void Chat::prompt(const QString &prompt)
+void Chat::regenerateResponse(int index)
 {
     resetResponseState();
-    emit promptRequested(m_collections, prompt);
+    emit regenerateResponseRequested(index);
     m_needsSave = true;
 }
 
-void Chat::regenerateResponse()
+QVariant Chat::popPrompt(int index)
 {
-    const int index = m_chatModel->count() - 1;
-    m_chatModel->updateSources(index, QList<ResultInfo>());
-    emit regenerateResponseRequested();
+    auto content = m_llmodel->popPrompt(index);
     m_needsSave = true;
+    if (content) return *content;
+    return QVariant::fromValue(nullptr);
 }
 
 void Chat::stopGenerating()
@@ -180,27 +184,17 @@ void Chat::stopGenerating()
     m_llmodel->stopGenerating();
 }
 
-QString Chat::response() const
-{
-    return m_response;
-}
-
 Chat::ResponseState Chat::responseState() const
 {
     return m_responseState;
 }
 
-void Chat::handleResponseChanged(const QString &response)
+void Chat::handleResponseChanged()
 {
     if (m_responseState != Chat::ResponseGeneration) {
         m_responseState = Chat::ResponseGeneration;
         emit responseStateChanged();
     }
-
-    m_response = response;
-    const int index = m_chatModel->count() - 1;
-    m_chatModel->updateValue(index, this->response());
-    emit responseChanged();
 }
 
 void Chat::handleModelLoadingPercentageChanged(float loadingPercentage)
@@ -240,15 +234,59 @@ void Chat::responseStopped(qint64 promptResponseMs)
 {
     m_tokenSpeed = QString();
     emit tokenSpeedChanged();
-    emit responseChanged();
 
     m_responseInProgress = false;
     m_responseState = Chat::ResponseStopped;
     emit responseInProgressChanged();
     emit responseStateChanged();
+
+    const QString possibleToolcall = m_chatModel->possibleToolcall();
+
+    ToolCallParser parser;
+    parser.update(possibleToolcall);
+
+    if (parser.state() == ToolEnums::ParseState::Complete) {
+        const QString toolCall = parser.toolCall();
+
+        // Regex to remove the formatting around the code
+        static const QRegularExpression regex("^\\s*```javascript\\s*|\\s*```\\s*$");
+        QString code = toolCall;
+        code.remove(regex);
+        code = code.trimmed();
+
+        // Right now the code interpreter is the only available tool
+        Tool *toolInstance = ToolModel::globalInstance()->get(ToolCallConstants::CodeInterpreterFunction);
+        Q_ASSERT(toolInstance);
+
+        // The param is the code
+        const ToolParam param = { "code", ToolEnums::ParamType::String, code };
+        const QString result = toolInstance->run({param}, 10000 /*msecs to timeout*/);
+        const ToolEnums::Error error = toolInstance->error();
+        const QString errorString = toolInstance->errorString();
+
+        // Update the current response with meta information about toolcall and re-parent
+        m_chatModel->updateToolCall({
+            ToolCallConstants::CodeInterpreterFunction,
+            { param },
+            result,
+            error,
+            errorString
+        });
+
+        ++m_consecutiveToolCalls;
+
+        // We limit the number of consecutive toolcalls otherwise we get into a potentially endless loop
+        if (m_consecutiveToolCalls < 3 || error == ToolEnums::Error::NoError) {
+            resetResponseState();
+            emit promptRequested(m_collections); // triggers a new response
+            return;
+        }
+    }
+
     if (m_generatedName.isEmpty())
         emit generateNameRequested();
 
+    m_consecutiveToolCalls = 0;
     Network::globalInstance()->trackChatEvent("response_complete", {
         {"first", m_firstResponse},
         {"message_count", chatModel()->count()},
@@ -274,25 +312,6 @@ void Chat::setModelInfo(const ModelInfo &modelInfo)
     emit modelChangeRequested(modelInfo);
 }
 
-// the server needs to block until response is reset, so it calls resetResponse on its own m_llmThread
-void Chat::serverNewPromptResponsePair(const QString &prompt, const QList<PromptAttachment> &attachments)
-{
-    newPromptResponsePairInternal(prompt, attachments);
-}
-
-void Chat::newPromptResponsePairInternal(const QString &prompt, const QList<PromptAttachment> &attachments)
-{
-    resetResponseState();
-    m_chatModel->updateCurrentResponse(m_chatModel->count() - 1, false);
-    m_chatModel->appendPrompt("Prompt: ", prompt, attachments);
-    m_chatModel->appendResponse("Response: ");
-}
-
-bool Chat::restoringFromText() const
-{
-    return m_llmodel->restoringFromText();
-}
-
 void Chat::unloadAndDeleteLater()
 {
     if (!isModelLoaded()) {
@@ -358,12 +377,6 @@ void Chat::generatedQuestionFinished(const QString &question)
     m_needsSave = true;
 }
 
-void Chat::handleRestoringFromText()
-{
-    Network::globalInstance()->trackChatEvent("recalc_context", { {"length", m_chatModel->count()} });
-    emit restoringFromTextChanged();
-}
-
 void Chat::handleModelLoadingError(const QString &error)
 {
     if (!error.isEmpty()) {
@@ -398,12 +411,19 @@ QString Chat::fallbackReason() const
 void Chat::handleDatabaseResultsChanged(const QList<ResultInfo> &results)
 {
     m_databaseResults = results;
-    const int index = m_chatModel->count() - 1;
-    m_chatModel->updateSources(index, m_databaseResults);
     m_needsSave = true;
 }
 
+// we need to notify listeners of the modelInfo property when its properties are updated,
+// since it's a gadget and can't do that on its own
 void Chat::handleModelInfoChanged(const ModelInfo &modelInfo)
+{
+    if (!m_modelInfo.id().isNull() && modelInfo.id() == m_modelInfo.id())
+        emit modelInfoChanged();
+}
+
+// react if a new model is loaded
+void Chat::handleModelChanged(const ModelInfo &modelInfo)
 {
     if (m_modelInfo == modelInfo)
         return;
@@ -432,10 +452,7 @@ bool Chat::serialize(QDataStream &stream, int version) const
     if (version >= 3)
         stream << m_collections;
 
-    const bool serializeKV = MySettings::globalInstance()->saveChatsContext();
-    if (version >= 6)
-        stream << serializeKV;
-    if (!m_llmodel->serialize(stream, version, serializeKV))
+    if (!m_llmodel->serialize(stream, version))
         return false;
     if (!m_chatModel->serialize(stream, version))
         return false;
@@ -464,19 +481,13 @@ bool Chat::deserialize(QDataStream &stream, int version)
     if (!m_modelInfo.id().isEmpty())
         emit modelInfoChanged();
 
-    bool discardKV = m_modelInfo.id().isEmpty();
-
     if (version >= 3) {
         stream >> m_collections;
         emit collectionListChanged(m_collections);
     }
 
-    bool deserializeKV = true;
-    if (version >= 6)
-        stream >> deserializeKV;
-
     m_llmodel->setModelInfo(m_modelInfo);
-    if (!m_llmodel->deserialize(stream, version, deserializeKV, discardKV))
+    if (!m_llmodel->deserialize(stream, version))
         return false;
     if (!m_chatModel->deserialize(stream, version))
         return false;

gpt4all-chat/src/chat.h

@@ -12,6 +12,8 @@
 #include <QObject>
 #include <QQmlEngine>
 #include <QString>
+#include <QStringList> // IWYU pragma: keep
+#include <QStringView>
 #include <QtGlobal>
 
 class QDataStream;
@@ -25,10 +27,8 @@ class Chat : public QObject
     Q_PROPERTY(bool isModelLoaded READ isModelLoaded NOTIFY isModelLoadedChanged)
     Q_PROPERTY(bool isCurrentlyLoading READ isCurrentlyLoading NOTIFY isCurrentlyLoadingChanged)
     Q_PROPERTY(float modelLoadingPercentage READ modelLoadingPercentage NOTIFY modelLoadingPercentageChanged)
-    Q_PROPERTY(QString response READ response NOTIFY responseChanged)
     Q_PROPERTY(ModelInfo modelInfo READ modelInfo WRITE setModelInfo NOTIFY modelInfoChanged)
     Q_PROPERTY(bool responseInProgress READ responseInProgress NOTIFY responseInProgressChanged)
-    Q_PROPERTY(bool restoringFromText READ restoringFromText NOTIFY restoringFromTextChanged)
     Q_PROPERTY(bool isServer READ isServer NOTIFY isServerChanged)
     Q_PROPERTY(ResponseState responseState READ responseState NOTIFY responseStateChanged)
     Q_PROPERTY(QList<QString> collectionList READ collectionList NOTIFY collectionListChanged)
@@ -45,6 +45,10 @@ class Chat : public QObject
     QML_UNCREATABLE("Only creatable from c++!")
 
 public:
+    // tag for constructing a server chat
+    struct server_tag_t { explicit server_tag_t() = default; };
+    static inline constexpr server_tag_t server_tag = server_tag_t();
+
     enum ResponseState {
         ResponseStopped,
         LocalDocsRetrieval,
@@ -56,7 +60,7 @@ public:
     Q_ENUM(ResponseState)
 
     explicit Chat(QObject *parent = nullptr);
-    explicit Chat(bool isServer, QObject *parent = nullptr);
+    explicit Chat(server_tag_t, QObject *parent = nullptr);
     virtual ~Chat();
     void destroy() { m_llmodel->destroy(); }
     void connectLLM();
@@ -74,23 +78,20 @@ public:
     bool isNewChat() const { return m_name == tr("New Chat") && !m_chatModel->count(); }
 
     Q_INVOKABLE void reset();
-    Q_INVOKABLE void processSystemPrompt();
     bool  isModelLoaded()          const { return m_modelLoadingPercentage == 1.0f; }
     bool  isCurrentlyLoading()     const { return m_modelLoadingPercentage > 0.0f && m_modelLoadingPercentage < 1.0f; }
     float modelLoadingPercentage() const { return m_modelLoadingPercentage; }
     Q_INVOKABLE void newPromptResponsePair(const QString &prompt, const QList<QUrl> &attachedUrls = {});
-    Q_INVOKABLE void prompt(const QString &prompt);
-    Q_INVOKABLE void regenerateResponse();
+    Q_INVOKABLE void regenerateResponse(int index);
+    Q_INVOKABLE QVariant popPrompt(int index);
     Q_INVOKABLE void stopGenerating();
 
     QList<ResultInfo> databaseResults() const { return m_databaseResults; }
 
-    QString response() const;
     bool responseInProgress() const { return m_responseInProgress; }
     ResponseState responseState() const;
     ModelInfo modelInfo() const;
     void setModelInfo(const ModelInfo &modelInfo);
-    bool restoringFromText() const;
 
     Q_INVOKABLE void unloadModel();
     Q_INVOKABLE void reloadModel();
@@ -111,7 +112,6 @@ public:
     Q_INVOKABLE bool hasCollection(const QString &collection) const;
     Q_INVOKABLE void addCollection(const QString &collection);
     Q_INVOKABLE void removeCollection(const QString &collection);
-    void resetResponseState();
 
     QString modelLoadingError() const { return m_modelLoadingError; }
 
@@ -126,9 +126,10 @@ public:
     QList<QString> generatedQuestions() const { return m_generatedQuestions; }
 
     bool needsSave() const { return m_needsSave; }
+    void setNeedsSave(bool n) { m_needsSave = n; }
 
 public Q_SLOTS:
-    void serverNewPromptResponsePair(const QString &prompt, const QList<PromptAttachment> &attachments = {});
+    void resetResponseState();
 
 Q_SIGNALS:
     void idChanged(const QString &id);
@@ -138,17 +139,14 @@ Q_SIGNALS:
     void isCurrentlyLoadingChanged();
     void modelLoadingPercentageChanged();
     void modelLoadingWarning(const QString &warning);
-    void responseChanged();
     void responseInProgressChanged();
     void responseStateChanged();
-    void promptRequested(const QList<QString> &collectionList, const QString &prompt);
-    void regenerateResponseRequested();
+    void promptRequested(const QStringList &enabledCollections);
+    void regenerateResponseRequested(int index);
     void resetResponseRequested();
     void resetContextRequested();
-    void processSystemPromptRequested();
     void modelChangeRequested(const ModelInfo &modelInfo);
     void modelInfoChanged();
-    void restoringFromTextChanged();
     void loadDefaultModelRequested();
     void generateNameRequested();
     void modelLoadingErrorChanged();
@@ -163,23 +161,20 @@ Q_SIGNALS:
     void generatedQuestionsChanged();
 
 private Q_SLOTS:
-    void handleResponseChanged(const QString &response);
+    void handleResponseChanged();
     void handleModelLoadingPercentageChanged(float);
     void promptProcessing();
     void generatingQuestions();
     void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
     void generatedQuestionFinished(const QString &question);
-    void handleRestoringFromText();
     void handleModelLoadingError(const QString &error);
     void handleTokenSpeedChanged(const QString &tokenSpeed);
     void handleDatabaseResultsChanged(const QList<ResultInfo> &results);
     void handleModelInfoChanged(const ModelInfo &modelInfo);
+    void handleModelChanged(const ModelInfo &modelInfo);
     void handleTrySwitchContextOfLoadedModelCompleted(int value);
 
-private:
-    void newPromptResponsePairInternal(const QString &prompt, const QList<PromptAttachment> &attachments);
-
 private:
     QString m_id;
     QString m_name;
@@ -190,7 +185,6 @@ private:
     QString m_tokenSpeed;
     QString m_device;
     QString m_fallbackReason;
-    QString m_response;
     QList<QString> m_collections;
     QList<QString> m_generatedQuestions;
     ChatModel *m_chatModel;
@@ -210,6 +204,7 @@ private:
     // - The chat was freshly created during this launch.
     // - The chat was changed after loading it from disk.
     bool m_needsSave = true;
+    int m_consecutiveToolCalls = 0;
 };
 
 #endif // CHAT_H

gpt4all-chat/src/chatapi.cpp

@@ -1,10 +1,10 @@
 #include "chatapi.h"
 
-#include <gpt4all-backend/llmodel.h>
+#include "utils.h"
 
 #include <QCoreApplication>
-#include <QGuiApplication>
 #include <QDebug>
+#include <QGuiApplication>
 #include <QJsonArray>
 #include <QJsonDocument>
 #include <QJsonObject>
@@ -13,12 +13,17 @@
 #include <QNetworkRequest>
 #include <QThread>
 #include <QUrl>
+#include <QUtf8StringView>
 #include <QVariant>
+#include <QXmlStreamReader>
 #include <Qt>
 #include <QtGlobal>
 #include <QtLogging>
 
+#include <expected>
+#include <functional>
 #include <iostream>
+#include <utility>
 
 using namespace Qt::Literals::StringLiterals;
 
@@ -51,7 +56,6 @@ bool ChatAPI::loadModel(const std::string &modelPath, int n_ctx, int ngl)
 void ChatAPI::setThreadCount(int32_t n_threads)
 {
     Q_UNUSED(n_threads);
-    qt_noop();
 }
 
 int32_t ChatAPI::threadCount() const
@@ -68,89 +72,119 @@ bool ChatAPI::isModelLoaded() const
     return true;
 }
 
-// All three of the state virtual functions are handled custom inside of chatllm save/restore
-size_t ChatAPI::stateSize() const
+static auto parsePrompt(QXmlStreamReader &xml) -> std::expected<QJsonArray, QString>
 {
-    throw std::logic_error("not implemented");
+    QJsonArray messages;
+
+    auto xmlError = [&xml] {
+        return std::unexpected(u"%1:%2: %3"_s.arg(xml.lineNumber()).arg(xml.columnNumber()).arg(xml.errorString()));
+    };
+
+    if (xml.hasError())
+        return xmlError();
+    if (xml.atEnd())
+        return messages;
+
+    // skip header
+    bool foundElement = false;
+    do {
+        switch (xml.readNext()) {
+            using enum QXmlStreamReader::TokenType;
+        case Invalid:
+            return xmlError();
+        case EndDocument:
+            return messages;
+        default:
+            foundElement = true;
+        case StartDocument:
+        case Comment:
+        case DTD:
+        case ProcessingInstruction:
+            ;
+        }
+    } while (!foundElement);
+
+    // document body loop
+    bool foundRoot = false;
+    for (;;) {
+        switch (xml.tokenType()) {
+            using enum QXmlStreamReader::TokenType;
+        case StartElement:
+            {
+                auto name = xml.name();
+                if (!foundRoot) {
+                    if (name != "chat"_L1)
+                        return std::unexpected(u"unexpected tag: %1"_s.arg(name));
+                    foundRoot = true;
+                } else {
+                    if (name != "user"_L1 && name != "assistant"_L1 && name != "system"_L1)
+                        return std::unexpected(u"unknown role: %1"_s.arg(name));
+                    auto content = xml.readElementText();
+                    if (xml.tokenType() != EndElement)
+                        return xmlError();
+                    messages << makeJsonObject({
+                        { "role"_L1,    name.toString().trimmed() },
+                        { "content"_L1, content                   },
+                    });
+                }
+                break;
+            }
+        case Characters:
+            if (!xml.isWhitespace())
+                return std::unexpected(u"unexpected text: %1"_s.arg(xml.text()));
+        case Comment:
+        case ProcessingInstruction:
+        case EndElement:
+            break;
+        case EndDocument:
+            return messages;
+        case Invalid:
+            return xmlError();
+        default:
+            return std::unexpected(u"unexpected token: %1"_s.arg(xml.tokenString()));
+        }
+        xml.readNext();
+    }
 }
 
-size_t ChatAPI::saveState(std::span<uint8_t> dest) const
-{
-    Q_UNUSED(dest);
-    throw std::logic_error("not implemented");
-}
+void ChatAPI::prompt(
+    std::string_view        prompt,
+    const PromptCallback   &promptCallback,
+    const ResponseCallback &responseCallback,
+    const PromptContext    &promptCtx
+) {
+    Q_UNUSED(promptCallback)
 
-size_t ChatAPI::restoreState(std::span<const uint8_t> src)
-{
-    Q_UNUSED(src);
-    throw std::logic_error("not implemented");
-}
-
-void ChatAPI::prompt(const std::string &prompt,
-                     const std::string &promptTemplate,
-                     std::function<bool(int32_t)> promptCallback,
-                     std::function<bool(int32_t, const std::string&)> responseCallback,
-                     bool allowContextShift,
-                     PromptContext &promptCtx,
-                     bool special,
-                     std::optional<std::string_view> fakeReply) {
-
-    Q_UNUSED(promptCallback);
-    Q_UNUSED(allowContextShift);
-    Q_UNUSED(special);
-
-    if (!isModelLoaded()) {
-        std::cerr << "ChatAPI ERROR: prompt won't work with an unloaded model!\n";
-        return;
-    }
-
-    if (!promptCtx.n_past) { m_queuedPrompts.clear(); }
-    Q_ASSERT(promptCtx.n_past <= m_context.size());
-    m_context.resize(promptCtx.n_past);
-
-    // FIXME(cebtenzzre): We're assuming people don't try to use %2 with ChatGPT. What would that even mean?
-    m_queuedPrompts << QString::fromStdString(promptTemplate).arg(QString::fromStdString(prompt));
-
-    if (!promptCtx.n_predict && !fakeReply) {
-        return; // response explicitly suppressed, queue prompt for later
-    }
-
-    QString formattedPrompt = m_queuedPrompts.join("");
-    m_queuedPrompts.clear();
-
-    if (fakeReply) {
-        promptCtx.n_past += 1;
-        m_context.append(formattedPrompt);
-        m_context.append(QString::fromUtf8(fakeReply->data(), fakeReply->size()));
-        return;
-    }
+    if (!isModelLoaded())
+        throw std::invalid_argument("Attempted to prompt an unloaded model.");
+    if (!promptCtx.n_predict)
+        return; // nothing requested
 
     // FIXME: We don't set the max_tokens on purpose because in order to do so safely without encountering
     // an error we need to be able to count the tokens in our prompt. The only way to do this is to use
-    // the OpenAI tiktokken library or to implement our own tokenization function that matches precisely
+    // the OpenAI tiktoken library or to implement our own tokenization function that matches precisely
     // the tokenization used by the OpenAI model we're calling. OpenAI has not introduced any means of
     // using the REST API to count tokens in a prompt.
-    QJsonObject root;
-    root.insert("model", m_modelName);
-    root.insert("stream", true);
-    root.insert("temperature", promptCtx.temp);
-    root.insert("top_p", promptCtx.top_p);
+    auto root = makeJsonObject({
+        { "model"_L1,       m_modelName     },
+        { "stream"_L1,      true            },
+        { "temperature"_L1, promptCtx.temp  },
+        { "top_p"_L1,       promptCtx.top_p },
+    });
 
     // conversation history
-    QJsonArray messages;
-    for (int i = 0; i < m_context.count(); ++i) {
-        QJsonObject message;
-        message.insert("role", i % 2 == 0 ? "user" : "assistant");
-        message.insert("content", m_context.at(i));
-        messages.append(message);
+    {
+        QUtf8StringView promptUtf8(prompt);
+        QXmlStreamReader xml(promptUtf8);
+        auto messages = parsePrompt(xml);
+        if (!messages) {
+            auto error = fmt::format("Failed to parse API model prompt: {}", messages.error());
+            qDebug().noquote() << "ChatAPI ERROR:" << error << "Prompt:\n\n" << promptUtf8 << '\n';
+            throw std::invalid_argument(error);
+        }
+        root.insert("messages"_L1, *messages);
     }
 
-    QJsonObject promptObject;
-    promptObject.insert("role", "user");
-    promptObject.insert("content", formattedPrompt);
-    messages.append(promptObject);
-    root.insert("messages", messages);
-
     QJsonDocument doc(root);
 
 #if defined(DEBUG)
@@ -167,12 +201,9 @@ void ChatAPI::prompt(const std::string &prompt,
     connect(&worker, &ChatAPIWorker::finished, &workerThread, &QThread::quit, Qt::DirectConnection);
     connect(this, &ChatAPI::request, &worker, &ChatAPIWorker::request, Qt::QueuedConnection);
     workerThread.start();
-    emit request(m_apiKey, &promptCtx, doc.toJson(QJsonDocument::Compact));
+    emit request(m_apiKey, doc.toJson(QJsonDocument::Compact));
     workerThread.wait();
 
-    promptCtx.n_past += 1;
-    m_context.append(formattedPrompt);
-    m_context.append(worker.currentResponse());
     m_responseCallback = nullptr;
 
 #if defined(DEBUG)
@@ -190,12 +221,8 @@ bool ChatAPI::callResponse(int32_t token, const std::string& string)
     return m_responseCallback(token, string);
 }
 
-void ChatAPIWorker::request(const QString &apiKey,
-                            LLModel::PromptContext *promptCtx,
-                            const QByteArray &array)
+void ChatAPIWorker::request(const QString &apiKey, const QByteArray &array)
 {
-    m_ctx = promptCtx;
-
     QUrl apiUrl(m_chat->url());
     const QString authorization = u"Bearer %1"_s.arg(apiKey).trimmed();
     QNetworkRequest request(apiUrl);
@@ -302,7 +329,6 @@ void ChatAPIWorker::handleReadyRead()
         const QJsonObject choice = choices.first().toObject();
         const QJsonObject delta = choice.value("delta").toObject();
         const QString content = delta.value("content").toString();
-        Q_ASSERT(m_ctx);
         m_currentResponse += content;
         if (!m_chat->callResponse(0, content.toStdString())) {
             reply->abort();

gpt4all-chat/src/chatapi.h

@@ -3,19 +3,18 @@
 
 #include <gpt4all-backend/llmodel.h>
 
-#include <QByteArray>
+#include <QByteArray> // IWYU pragma: keep
 #include <QNetworkReply>
 #include <QObject>
 #include <QString>
-#include <QStringList>
-#include <QList>
 
 #include <cstddef>
 #include <cstdint>
-#include <functional>
+#include <span>
 #include <stdexcept>
 #include <string>
 #include <string_view>
+#include <unordered_map>
 #include <vector>
 
 class QNetworkAccessManager;
@@ -26,16 +25,13 @@ class ChatAPIWorker : public QObject {
 public:
     ChatAPIWorker(ChatAPI *chatAPI)
         : QObject(nullptr)
-        , m_ctx(nullptr)
         , m_networkManager(nullptr)
         , m_chat(chatAPI) {}
     virtual ~ChatAPIWorker() {}
 
     QString currentResponse() const { return m_currentResponse; }
 
-    void request(const QString &apiKey,
-                 LLModel::PromptContext *promptCtx,
-                 const QByteArray &array);
+    void request(const QString &apiKey, const QByteArray &array);
 
 Q_SIGNALS:
     void finished();
@@ -47,7 +43,6 @@ private Q_SLOTS:
 
 private:
     ChatAPI *m_chat;
-    LLModel::PromptContext *m_ctx;
     QNetworkAccessManager *m_networkManager;
     QString m_currentResponse;
 };
@@ -63,17 +58,23 @@ public:
     bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
-    size_t stateSize() const override;
-    size_t saveState(std::span<uint8_t> dest) const override;
-    size_t restoreState(std::span<const uint8_t> src) override;
-    void prompt(const std::string &prompt,
-                const std::string &promptTemplate,
-                std::function<bool(int32_t)> promptCallback,
-                std::function<bool(int32_t, const std::string&)> responseCallback,
-                bool allowContextShift,
-                PromptContext &ctx,
-                bool special,
-                std::optional<std::string_view> fakeReply) override;
+
+    // All three of the state virtual functions are handled custom inside of chatllm save/restore
+    size_t stateSize() const override
+    { throwNotImplemented(); }
+    size_t saveState(std::span<uint8_t> stateOut, std::vector<Token> &inputTokensOut) const override
+    { Q_UNUSED(stateOut); Q_UNUSED(inputTokensOut); throwNotImplemented(); }
+    size_t restoreState(std::span<const uint8_t> state, std::span<const Token> inputTokens) override
+    { Q_UNUSED(state); Q_UNUSED(inputTokens); throwNotImplemented(); }
+
+    void prompt(std::string_view        prompt,
+                const PromptCallback   &promptCallback,
+                const ResponseCallback &responseCallback,
+                const PromptContext    &ctx) override;
+
+    [[noreturn]]
+    int32_t countPromptTokens(std::string_view prompt) const override
+    { Q_UNUSED(prompt); throwNotImplemented(); }
 
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
@@ -83,84 +84,87 @@ public:
     void setRequestURL(const QString &requestURL) { m_requestURL = requestURL; }
     QString url() const { return m_requestURL; }
 
-    QList<QString> context() const { return m_context; }
-    void setContext(const QList<QString> &context) { m_context = context; }
-
     bool callResponse(int32_t token, const std::string &string);
 
+    [[noreturn]]
+    int32_t contextLength() const override
+    { throwNotImplemented(); }
+
+    auto specialTokens() -> std::unordered_map<std::string, std::string> const override
+    { return {}; }
+
 Q_SIGNALS:
-    void request(const QString &apiKey,
-                 LLModel::PromptContext *ctx,
-                 const QByteArray &array);
+    void request(const QString &apiKey, const QByteArray &array);
 
 protected:
     // We have to implement these as they are pure virtual in base class, but we don't actually use
     // them as they are only called from the default implementation of 'prompt' which we override and
     // completely replace
 
-    std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special) override
-    {
-        (void)ctx;
-        (void)str;
-        (void)special;
-        throw std::logic_error("not implemented");
-    }
+    [[noreturn]]
+    static void throwNotImplemented() { throw std::logic_error("not implemented"); }
 
+    [[noreturn]]
+    std::vector<Token> tokenize(std::string_view str) const override
+    { Q_UNUSED(str); throwNotImplemented(); }
+
+    [[noreturn]]
     bool isSpecialToken(Token id) const override
-    {
-        (void)id;
-        throw std::logic_error("not implemented");
-    }
+    { Q_UNUSED(id); throwNotImplemented(); }
 
+    [[noreturn]]
     std::string tokenToString(Token id) const override
-    {
-        (void)id;
-        throw std::logic_error("not implemented");
-    }
+    { Q_UNUSED(id); throwNotImplemented(); }
 
-    void initSampler(PromptContext &ctx) override
-    {
-        (void)ctx;
-        throw std::logic_error("not implemented");
-    }
+    [[noreturn]]
+    void initSampler(const PromptContext &ctx) override
+    { Q_UNUSED(ctx); throwNotImplemented(); }
 
-    Token sampleToken() const override { throw std::logic_error("not implemented"); }
+    [[noreturn]]
+    Token sampleToken() const override
+    { throwNotImplemented(); }
 
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override
-    {
-        (void)ctx;
-        (void)tokens;
-        throw std::logic_error("not implemented");
-    }
+    [[noreturn]]
+    bool evalTokens(int32_t nPast, std::span<const Token> tokens) const override
+    { Q_UNUSED(nPast); Q_UNUSED(tokens); throwNotImplemented(); }
 
-    void shiftContext(PromptContext &promptCtx) override
-    {
-        (void)promptCtx;
-        throw std::logic_error("not implemented");
-    }
+    [[noreturn]]
+    void shiftContext(const PromptContext &promptCtx, int32_t *nPast) override
+    { Q_UNUSED(promptCtx); Q_UNUSED(nPast); throwNotImplemented(); }
 
-    int32_t contextLength() const override
-    {
-        throw std::logic_error("not implemented");
-    }
+    [[noreturn]]
+    int32_t inputLength() const override
+    { throwNotImplemented(); }
 
+    [[noreturn]]
+    int32_t computeModelInputPosition(std::span<const Token> input) const override
+    { Q_UNUSED(input); throwNotImplemented(); }
+
+    [[noreturn]]
+    void setModelInputPosition(int32_t pos) override
+    { Q_UNUSED(pos); throwNotImplemented(); }
+
+    [[noreturn]]
+    void appendInputToken(Token tok) override
+    { Q_UNUSED(tok); throwNotImplemented(); }
+
+    [[noreturn]]
     const std::vector<Token> &endTokens() const override
-    {
-        throw std::logic_error("not implemented");
-    }
+    { throwNotImplemented(); }
 
+    [[noreturn]]
     bool shouldAddBOS() const override
-    {
-        throw std::logic_error("not implemented");
-    }
+    { throwNotImplemented(); }
+
+    [[noreturn]]
+    std::span<const Token> inputTokens() const override
+    { throwNotImplemented(); }
 
 private:
-    std::function<bool(int32_t, const std::string&)> m_responseCallback;
-    QString m_modelName;
-    QString m_apiKey;
-    QString m_requestURL;
-    QList<QString> m_context;
-    QStringList m_queuedPrompts;
+    ResponseCallback m_responseCallback;
+    QString          m_modelName;
+    QString          m_apiKey;
+    QString          m_requestURL;
 };
 
 #endif // CHATAPI_H

gpt4all-chat/src/chatlistmodel.cpp

@@ -17,9 +17,10 @@
 #include <Qt>
 
 #include <algorithm>
+#include <memory>
 
-#define CHAT_FORMAT_MAGIC 0xF5D553CC
-#define CHAT_FORMAT_VERSION 10
+static constexpr quint32 CHAT_FORMAT_MAGIC   = 0xF5D553CC;
+static constexpr qint32  CHAT_FORMAT_VERSION = 12;
 
 class MyChatListModel: public ChatListModel { };
 Q_GLOBAL_STATIC(MyChatListModel, chatListModelInstance)
@@ -51,6 +52,10 @@ void ChatListModel::loadChats()
     connect(thread, &ChatsRestoreThread::finished, thread, &QObject::deleteLater);
     thread->start();
 
+    ChatSaver *saver = new ChatSaver;
+    connect(this, &ChatListModel::requestSaveChats, saver, &ChatSaver::saveChats, Qt::QueuedConnection);
+    connect(saver, &ChatSaver::saveChatsFinished, this, &ChatListModel::saveChatsFinished, Qt::QueuedConnection);
+
     connect(MySettings::globalInstance(), &MySettings::serverChatChanged, this, &ChatListModel::handleServerEnabledChanged);
 }
 
@@ -88,9 +93,6 @@ void ChatListModel::saveChats()
         return;
     }
 
-    ChatSaver *saver = new ChatSaver;
-    connect(this, &ChatListModel::requestSaveChats, saver, &ChatSaver::saveChats, Qt::QueuedConnection);
-    connect(saver, &ChatSaver::saveChatsFinished, this, &ChatListModel::saveChatsFinished, Qt::QueuedConnection);
     emit requestSaveChats(toSave);
 }
 
@@ -117,8 +119,8 @@ void ChatSaver::saveChats(const QVector<Chat *> &chats)
         }
         QDataStream out(&tempFile);
 
-        out << (quint32)CHAT_FORMAT_MAGIC;
-        out << (qint32)CHAT_FORMAT_VERSION;
+        out << CHAT_FORMAT_MAGIC;
+        out << CHAT_FORMAT_VERSION;
         out.setVersion(QDataStream::Qt_6_2);
 
         qDebug() << "serializing chat" << fileName;
@@ -128,6 +130,7 @@ void ChatSaver::saveChats(const QVector<Chat *> &chats)
             continue;
         }
 
+        chat->setNeedsSave(false);
         if (originalFile.exists())
             originalFile.remove();
         tempFile.rename(filePath);
@@ -255,12 +258,15 @@ void ChatsRestoreThread::run()
 
         qDebug() << "deserializing chat" << f.file;
 
-        Chat *chat = new Chat;
+        auto chat = std::make_unique<Chat>();
         chat->moveToThread(qGuiApp->thread());
-        if (!chat->deserialize(in, version)) {
+        bool ok = chat->deserialize(in, version);
+        if (!ok) {
             qWarning() << "ERROR: Couldn't deserialize chat from file:" << file.fileName();
+        } else if (!in.atEnd()) {
+            qWarning().nospace() << "error loading chat from " << file.fileName() << ": extra data at end of file";
         } else {
-            emit chatRestored(chat);
+            emit chatRestored(chat.release());
         }
         if (f.oldFile)
            file.remove(); // No longer storing in this directory

gpt4all-chat/src/chatlistmodel.h

@@ -33,7 +33,6 @@ class ChatSaver : public QObject
     Q_OBJECT
 public:
     explicit ChatSaver();
-    void stop();
 
 Q_SIGNALS:
     void saveChatsFinished();
@@ -147,7 +146,7 @@ public:
         if (m_serverChat)
             return;
 
-        m_serverChat = new Chat(true /*isServer*/, this);
+        m_serverChat = new Chat(Chat::server_tag, this);
         beginInsertRows(QModelIndex(), m_chats.size(), m_chats.size());
         m_chats.append(m_serverChat);
         endInsertRows();
@@ -238,7 +237,6 @@ public Q_SLOTS:
 Q_SIGNALS:
     void countChanged();
     void currentChatChanged();
-    void chatsSavedFinished();
     void requestSaveChats(const QVector<Chat*> &);
     void saveChatsFinished();
 

gpt4all-chat/src/chatllm.h

@@ -1,6 +1,7 @@
 #ifndef CHATLLM_H
 #define CHATLLM_H
 
+#include "chatmodel.h"
 #include "database.h" // IWYU pragma: keep
 #include "modellist.h"
 
@@ -9,19 +10,23 @@
 #include <QByteArray>
 #include <QElapsedTimer>
 #include <QFileInfo>
-#include <QList>
+#include <QList>      // IWYU pragma: keep
 #include <QObject>
 #include <QPointer>
 #include <QString>
+#include <QStringList> // IWYU pragma: keep
+#include <QStringView>
 #include <QThread>
-#include <QVariantMap>
+#include <QVariantMap> // IWYU pragma: keep
 #include <QtGlobal>
 
 #include <atomic>
 #include <cstdint>
 #include <memory>
 #include <optional>
+#include <span>
 #include <string>
+#include <variant>
 
 using namespace Qt::Literals::StringLiterals;
 
@@ -51,7 +56,7 @@ enum class LLModelTypeV1 { // since chat version 6 (v2.5.0)
     NONE      = -1, // no state
 };
 
-static LLModelTypeV1 parseLLModelTypeV1(int type)
+inline LLModelTypeV1 parseLLModelTypeV1(int type)
 {
     switch (LLModelTypeV1(type)) {
     case LLModelTypeV1::GPTJ:
@@ -68,7 +73,7 @@ static LLModelTypeV1 parseLLModelTypeV1(int type)
     }
 }
 
-static LLModelTypeV1 parseLLModelTypeV0(int v0)
+inline LLModelTypeV1 parseLLModelTypeV0(int v0)
 {
     switch (LLModelTypeV0(v0)) {
     case LLModelTypeV0::MPT:       return LLModelTypeV1::MPT;
@@ -141,7 +146,6 @@ class Chat;
 class ChatLLM : public QObject
 {
     Q_OBJECT
-    Q_PROPERTY(bool restoringFromText READ restoringFromText NOTIFY restoringFromTextChanged)
     Q_PROPERTY(QString deviceBackend READ deviceBackend NOTIFY loadedModelInfoChanged)
     Q_PROPERTY(QString device READ device NOTIFY loadedModelInfoChanged)
     Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY loadedModelInfoChanged)
@@ -149,12 +153,14 @@ public:
     ChatLLM(Chat *parent, bool isServer = false);
     virtual ~ChatLLM();
 
-    void destroy();
     static void destroyStore();
+    static std::optional<std::string> checkJinjaTemplateError(const std::string &source);
+
+    void destroy();
     bool isModelLoaded() const;
-    void regenerateResponse();
-    void resetResponse();
-    void resetContext();
+    void regenerateResponse(int index);
+    // used to implement edit functionality
+    std::optional<QString> popPrompt(int index);
 
     void stopGenerating() { m_stopGenerating = true; }
 
@@ -164,13 +170,9 @@ public:
     void setForceUnloadModel(bool b) { m_forceUnloadModel = b; }
     void setMarkedForDeletion(bool b) { m_markedForDeletion = b; }
 
-    QString response(bool trim = true) const;
-
     ModelInfo modelInfo() const;
     void setModelInfo(const ModelInfo &info);
 
-    bool restoringFromText() const { return m_restoringFromText; }
-
     void acquireModel();
     void resetModel();
 
@@ -195,13 +197,11 @@ public:
         return m_llModelInfo.fallbackReason.value_or(u""_s);
     }
 
-    QString generatedName() const { return QString::fromStdString(m_nameResponse); }
-
-    bool serialize(QDataStream &stream, int version, bool serializeKV);
-    bool deserialize(QDataStream &stream, int version, bool deserializeKV, bool discardKV);
+    bool serialize(QDataStream &stream, int version);
+    bool deserialize(QDataStream &stream, int version);
 
 public Q_SLOTS:
-    bool prompt(const QList<QString> &collectionList, const QString &prompt);
+    void prompt(const QStringList &enabledCollections);
     bool loadDefaultModel();
     void trySwitchContextOfLoadedModel(const ModelInfo &modelInfo);
     bool loadModel(const ModelInfo &modelInfo);
@@ -209,22 +209,19 @@ public Q_SLOTS:
     void unloadModel();
     void reloadModel();
     void generateName();
-    void generateQuestions(qint64 elapsed);
     void handleChatIdChanged(const QString &id);
     void handleShouldBeLoadedChanged();
     void handleThreadStarted();
     void handleForceMetalChanged(bool forceMetal);
     void handleDeviceChanged();
-    void processSystemPrompt();
-    void processRestoreStateFromText();
 
 Q_SIGNALS:
-    void restoringFromTextChanged();
     void loadedModelInfoChanged();
     void modelLoadingPercentageChanged(float);
     void modelLoadingError(const QString &error);
     void modelLoadingWarning(const QString &warning);
-    void responseChanged(const QString &response);
+    void responseChanged();
+    void responseFailed();
     void promptProcessing();
     void generatingQuestions();
     void responseStopped(qint64 promptResponseMs);
@@ -243,56 +240,51 @@ Q_SIGNALS:
     void modelInfoChanged(const ModelInfo &modelInfo);
 
 protected:
-    bool promptInternal(const QList<QString> &collectionList, const QString &prompt, const QString &promptTemplate,
-        int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
-        int32_t repeat_penalty_tokens, std::optional<QString> fakeReply = {});
-    bool handlePrompt(int32_t token);
-    bool handleResponse(int32_t token, const std::string &response);
-    bool handleNamePrompt(int32_t token);
-    bool handleNameResponse(int32_t token, const std::string &response);
-    bool handleSystemPrompt(int32_t token);
-    bool handleSystemResponse(int32_t token, const std::string &response);
-    bool handleRestoreStateFromTextPrompt(int32_t token);
-    bool handleRestoreStateFromTextResponse(int32_t token, const std::string &response);
-    bool handleQuestionPrompt(int32_t token);
-    bool handleQuestionResponse(int32_t token, const std::string &response);
-    void saveState();
-    void restoreState();
+    struct PromptResult {
+        QByteArray response;       // raw UTF-8
+        int        promptTokens;   // note: counts *entire* history, even if cached
+        int        responseTokens;
+    };
 
-protected:
-    LLModel::PromptContext m_ctx;
-    quint32 m_promptTokens;
-    quint32 m_promptResponseTokens;
+    struct ChatPromptResult : PromptResult {
+        QList<ResultInfo> databaseResults;
+    };
+
+    ChatPromptResult promptInternalChat(const QStringList &enabledCollections, const LLModel::PromptContext &ctx,
+                                        qsizetype startOffset = 0);
+    // passing a string_view directly skips templating and uses the raw string
+    PromptResult promptInternal(const std::variant<std::span<const MessageItem>, std::string_view> &prompt,
+                                const LLModel::PromptContext &ctx,
+                                bool usedLocalDocs);
 
 private:
     bool loadNewModel(const ModelInfo &modelInfo, QVariantMap &modelLoadProps);
 
+    std::vector<MessageItem> forkConversation(const QString &prompt) const;
+
+    // Applies the Jinja template. Query mode returns only the last message without special tokens.
+    // Returns a (# of messages, rendered prompt) pair.
+    std::string applyJinjaTemplate(std::span<const MessageItem> items) const;
+
+    void generateQuestions(qint64 elapsed);
+
+protected:
+    QPointer<ChatModel> m_chatModel;
+
+private:
     const Chat *m_chat;
-    std::string m_response;
-    std::string m_trimmedResponse;
-    std::string m_nameResponse;
-    QString m_questionResponse;
     LLModelInfo m_llModelInfo;
     LLModelTypeV1 m_llModelType = LLModelTypeV1::NONE;
     ModelInfo m_modelInfo;
     TokenTimer *m_timer;
-    QByteArray m_state;
     QThread m_llmThread;
     std::atomic<bool> m_stopGenerating;
     std::atomic<bool> m_shouldBeLoaded;
-    std::atomic<bool> m_restoringFromText; // status indication
     std::atomic<bool> m_forceUnloadModel;
     std::atomic<bool> m_markedForDeletion;
     bool m_isServer;
     bool m_forceMetal;
     bool m_reloadingToChangeVariant;
-    bool m_processedSystemPrompt;
-    bool m_restoreStateFromText;
-    // m_pristineLoadedState is set if saveSate is unnecessary, either because:
-    // - an unload was queued during LLModel::restoreState()
-    // - the chat will be restored from text and hasn't been interacted with yet
-    bool m_pristineLoadedState = false;
-    QPointer<ChatModel> m_chatModel;
 };
 
 #endif // CHATLLM_H

gpt4all-chat/src/chatmodel.cpp

@@ -0,0 +1,351 @@
+#include "chatmodel.h"
+
+#include <QDebug>
+#include <QMap>
+#include <QtGlobal>
+#include <QtLogging>
+
+
+QList<ResultInfo> ChatItem::consolidateSources(const QList<ResultInfo> &sources)
+{
+    QMap<QString, ResultInfo> groupedData;
+    for (const ResultInfo &info : sources) {
+        if (groupedData.contains(info.file)) {
+            groupedData[info.file].text += "\n---\n" + info.text;
+        } else {
+            groupedData[info.file] = info;
+        }
+    }
+    QList<ResultInfo> consolidatedSources = groupedData.values();
+    return consolidatedSources;
+}
+
+void ChatItem::serializeResponse(QDataStream &stream, int version)
+{
+    stream << value;
+}
+
+void ChatItem::serializeToolCall(QDataStream &stream, int version)
+{
+    stream << value;
+    toolCallInfo.serialize(stream, version);
+}
+
+void ChatItem::serializeToolResponse(QDataStream &stream, int version)
+{
+    stream << value;
+}
+
+void ChatItem::serializeText(QDataStream &stream, int version)
+{
+    stream << value;
+}
+
+void ChatItem::serializeSubItems(QDataStream &stream, int version)
+{
+    stream << name;
+    switch (auto typ = type()) {
+        using enum ChatItem::Type;
+        case Response:      { serializeResponse(stream, version);       break; }
+        case ToolCall:      { serializeToolCall(stream, version);       break; }
+        case ToolResponse:  { serializeToolResponse(stream, version);   break; }
+        case Text:          { serializeText(stream, version);           break; }
+        case System:
+        case Prompt:
+            throw std::invalid_argument(fmt::format("cannot serialize subitem type {}", int(typ)));
+    }
+
+    stream << qsizetype(subItems.size());
+    for (ChatItem *item :subItems)
+        item->serializeSubItems(stream, version);
+}
+
+void ChatItem::serialize(QDataStream &stream, int version)
+{
+    stream << name;
+    stream << value;
+    stream << newResponse;
+    stream << isCurrentResponse;
+    stream << stopped;
+    stream << thumbsUpState;
+    stream << thumbsDownState;
+    if (version >= 11 && type() == ChatItem::Type::Response)
+        stream << isError;
+    if (version >= 8) {
+        stream << sources.size();
+        for (const ResultInfo &info : sources) {
+            Q_ASSERT(!info.file.isEmpty());
+            stream << info.collection;
+            stream << info.path;
+            stream << info.file;
+            stream << info.title;
+            stream << info.author;
+            stream << info.date;
+            stream << info.text;
+            stream << info.page;
+            stream << info.from;
+            stream << info.to;
+        }
+    } else if (version >= 3) {
+        QList<QString> references;
+        QList<QString> referencesContext;
+        int validReferenceNumber = 1;
+        for (const ResultInfo &info : sources) {
+            if (info.file.isEmpty())
+                continue;
+
+            QString reference;
+            {
+                QTextStream stream(&reference);
+                stream << (validReferenceNumber++) << ". ";
+                if (!info.title.isEmpty())
+                    stream << "\"" << info.title << "\". ";
+                if (!info.author.isEmpty())
+                    stream << "By " << info.author << ". ";
+                if (!info.date.isEmpty())
+                    stream << "Date: " << info.date << ". ";
+                stream << "In " << info.file << ". ";
+                if (info.page != -1)
+                    stream << "Page " << info.page << ". ";
+                if (info.from != -1) {
+                    stream << "Lines " << info.from;
+                    if (info.to != -1)
+                        stream << "-" << info.to;
+                    stream << ". ";
+                }
+                stream << "[Context](context://" << validReferenceNumber - 1 << ")";
+            }
+            references.append(reference);
+            referencesContext.append(info.text);
+        }
+
+        stream << references.join("\n");
+        stream << referencesContext;
+    }
+    if (version >= 10) {
+        stream << promptAttachments.size();
+        for (const PromptAttachment &a : promptAttachments) {
+            Q_ASSERT(!a.url.isEmpty());
+            stream << a.url;
+            stream << a.content;
+        }
+    }
+
+    if (version >= 12) {
+        stream << qsizetype(subItems.size());
+        for (ChatItem *item :subItems)
+            item->serializeSubItems(stream, version);
+    }
+}
+
+bool ChatItem::deserializeToolCall(QDataStream &stream, int version)
+{
+    stream >> value;
+    return toolCallInfo.deserialize(stream, version);;
+}
+
+bool ChatItem::deserializeToolResponse(QDataStream &stream, int version)
+{
+    stream >> value;
+    return true;
+}
+
+bool ChatItem::deserializeText(QDataStream &stream, int version)
+{
+    stream >> value;
+    return true;
+}
+
+bool ChatItem::deserializeResponse(QDataStream &stream, int version)
+{
+    stream >> value;
+    return true;
+}
+
+bool ChatItem::deserializeSubItems(QDataStream &stream, int version)
+{
+    stream >> name;
+    try {
+        type(); // check name
+    } catch (const std::exception &e) {
+        qWarning() << "ChatModel ERROR:" << e.what();
+        return false;
+    }
+    switch (auto typ = type()) {
+        using enum ChatItem::Type;
+        case Response:      { deserializeResponse(stream, version); break; }
+        case ToolCall:      { deserializeToolCall(stream, version); break; }
+        case ToolResponse:  { deserializeToolResponse(stream, version); break; }
+        case Text:          { deserializeText(stream, version); break; }
+        case System:
+        case Prompt:
+            throw std::invalid_argument(fmt::format("cannot serialize subitem type {}", int(typ)));
+    }
+
+    qsizetype count;
+    stream >> count;
+    for (int i = 0; i < count; ++i) {
+        ChatItem *c = new ChatItem(this);
+        if (!c->deserializeSubItems(stream, version)) {
+            delete c;
+            return false;
+        }
+        subItems.push_back(c);
+    }
+
+    return true;
+}
+
+bool ChatItem::deserialize(QDataStream &stream, int version)
+{
+    if (version < 12) {
+        int id;
+        stream >> id;
+    }
+    stream >> name;
+    try {
+        type(); // check name
+    } catch (const std::exception &e) {
+        qWarning() << "ChatModel ERROR:" << e.what();
+        return false;
+    }
+    stream >> value;
+    if (version < 10) {
+        // This is deprecated and no longer used
+        QString prompt;
+        stream >> prompt;
+    }
+    stream >> newResponse;
+    stream >> isCurrentResponse;
+    stream >> stopped;
+    stream >> thumbsUpState;
+    stream >> thumbsDownState;
+    if (version >= 11 && type() == ChatItem::Type::Response)
+        stream >> isError;
+    if (version >= 8) {
+        qsizetype count;
+        stream >> count;
+        for (int i = 0; i < count; ++i) {
+            ResultInfo info;
+            stream >> info.collection;
+            stream >> info.path;
+            stream >> info.file;
+            stream >> info.title;
+            stream >> info.author;
+            stream >> info.date;
+            stream >> info.text;
+            stream >> info.page;
+            stream >> info.from;
+            stream >> info.to;
+            sources.append(info);
+        }
+        consolidatedSources = ChatItem::consolidateSources(sources);
+    } else if (version >= 3) {
+        QString references;
+        QList<QString> referencesContext;
+        stream >> references;
+        stream >> referencesContext;
+
+        if (!references.isEmpty()) {
+            QList<QString> referenceList = references.split("\n");
+
+            // Ignore empty lines and those that begin with "---" which is no longer used
+            for (auto it = referenceList.begin(); it != referenceList.end();) {
+                if (it->trimmed().isEmpty() || it->trimmed().startsWith("---"))
+                    it = referenceList.erase(it);
+                else
+                    ++it;
+            }
+
+            Q_ASSERT(referenceList.size() == referencesContext.size());
+            for (int j = 0; j < referenceList.size(); ++j) {
+                QString reference = referenceList[j];
+                QString context = referencesContext[j];
+                ResultInfo info;
+                QTextStream refStream(&reference);
+                QString dummy;
+                int validReferenceNumber;
+                refStream >> validReferenceNumber >> dummy;
+                // Extract title (between quotes)
+                if (reference.contains("\"")) {
+                    int startIndex = reference.indexOf('"') + 1;
+                    int endIndex = reference.indexOf('"', startIndex);
+                    info.title = reference.mid(startIndex, endIndex - startIndex);
+                }
+
+                // Extract author (after "By " and before the next period)
+                if (reference.contains("By ")) {
+                    int startIndex = reference.indexOf("By ") + 3;
+                    int endIndex = reference.indexOf('.', startIndex);
+                    info.author = reference.mid(startIndex, endIndex - startIndex).trimmed();
+                }
+
+                // Extract date (after "Date: " and before the next period)
+                if (reference.contains("Date: ")) {
+                    int startIndex = reference.indexOf("Date: ") + 6;
+                    int endIndex = reference.indexOf('.', startIndex);
+                    info.date = reference.mid(startIndex, endIndex - startIndex).trimmed();
+                }
+
+                // Extract file name (after "In " and before the "[Context]")
+                if (reference.contains("In ") && reference.contains(". [Context]")) {
+                    int startIndex = reference.indexOf("In ") + 3;
+                    int endIndex = reference.indexOf(". [Context]", startIndex);
+                    info.file = reference.mid(startIndex, endIndex - startIndex).trimmed();
+                }
+
+                // Extract page number (after "Page " and before the next space)
+                if (reference.contains("Page ")) {
+                    int startIndex = reference.indexOf("Page ") + 5;
+                    int endIndex = reference.indexOf(' ', startIndex);
+                    if (endIndex == -1) endIndex = reference.length();
+                    info.page = reference.mid(startIndex, endIndex - startIndex).toInt();
+                }
+
+                // Extract lines (after "Lines " and before the next space or hyphen)
+                if (reference.contains("Lines ")) {
+                    int startIndex = reference.indexOf("Lines ") + 6;
+                    int endIndex = reference.indexOf(' ', startIndex);
+                    if (endIndex == -1) endIndex = reference.length();
+                    int hyphenIndex = reference.indexOf('-', startIndex);
+                    if (hyphenIndex != -1 && hyphenIndex < endIndex) {
+                        info.from = reference.mid(startIndex, hyphenIndex - startIndex).toInt();
+                        info.to = reference.mid(hyphenIndex + 1, endIndex - hyphenIndex - 1).toInt();
+                    } else {
+                        info.from = reference.mid(startIndex, endIndex - startIndex).toInt();
+                    }
+                }
+                info.text = context;
+                sources.append(info);
+            }
+
+            consolidatedSources = ChatItem::consolidateSources(sources);
+        }
+    }
+    if (version >= 10) {
+        qsizetype count;
+        stream >> count;
+        QList<PromptAttachment> attachments;
+        for (int i = 0; i < count; ++i) {
+            PromptAttachment a;
+            stream >> a.url;
+            stream >> a.content;
+            attachments.append(a);
+        }
+        promptAttachments = attachments;
+    }
+
+    if (version >= 12) {
+        qsizetype count;
+        stream >> count;
+        for (int i = 0; i < count; ++i) {
+            ChatItem *c = new ChatItem(this);
+            if (!c->deserializeSubItems(stream, version)) {
+                delete c;
+                return false;
+            }
+            subItems.push_back(c);
+        }
+    }
+    return true;
+}

gpt4all-chat/src/chatviewtextprocessor.cpp

@@ -967,8 +967,6 @@ void ChatViewTextProcessor::handleCodeBlocks()
         cursor.setPosition(matchesCode[index].capturedEnd(), QTextCursor::KeepAnchor);
         cursor.removeSelectedText();
 
-        int startPos = cursor.position();
-
         QTextFrameFormat frameFormat = frameFormatBase;
         QString capturedText = matchesCode[index].captured(1);
         QString codeLanguage;
@@ -1004,7 +1002,7 @@ void ChatViewTextProcessor::handleCodeBlocks()
         QTextFrame *mainFrame = cursor.currentFrame();
         cursor.setCharFormat(textFormat);
 
-        QTextFrame *frame = cursor.insertFrame(frameFormat);
+        cursor.insertFrame(frameFormat);
         QTextTable *table = cursor.insertTable(codeLanguage.isEmpty() ? 1 : 2, 1, tableFormat);
 
         if (!codeLanguage.isEmpty()) {
@@ -1016,7 +1014,6 @@ void ChatViewTextProcessor::handleCodeBlocks()
             headerCursor.insertText(codeLanguage);
             QTextTableCell copy = headerTable->cellAt(0, 1);
             QTextCursor copyCursor = copy.firstCursorPosition();
-            int startPos = copyCursor.position();
             CodeCopy newCopy;
             newCopy.text = lines.join("\n");
             newCopy.startPos = copyCursor.position();

gpt4all-chat/src/codeinterpreter.cpp

@@ -0,0 +1,125 @@
+#include "codeinterpreter.h"
+
+#include <QJSValue>
+#include <QStringList>
+#include <QThread>
+#include <QVariant>
+
+QString CodeInterpreter::run(const QList<ToolParam> &params, qint64 timeout)
+{
+    m_error = ToolEnums::Error::NoError;
+    m_errorString = QString();
+
+    Q_ASSERT(params.count() == 1
+          && params.first().name == "code"
+          && params.first().type == ToolEnums::ParamType::String);
+
+    const QString code = params.first().value.toString();
+
+    QThread workerThread;
+    CodeInterpreterWorker worker;
+    worker.moveToThread(&workerThread);
+    connect(&worker, &CodeInterpreterWorker::finished, &workerThread, &QThread::quit, Qt::DirectConnection);
+    connect(&workerThread, &QThread::started, [&worker, code]() {
+        worker.request(code);
+    });
+    workerThread.start();
+    bool timedOut = !workerThread.wait(timeout);
+    if (timedOut) {
+        worker.interrupt(); // thread safe
+        m_error = ToolEnums::Error::TimeoutError;
+    }
+    workerThread.quit();
+    workerThread.wait();
+    if (!timedOut) {
+        m_error = worker.error();
+        m_errorString = worker.errorString();
+    }
+    return worker.response();
+}
+
+QList<ToolParamInfo> CodeInterpreter::parameters() const
+{
+    return {{
+        "code",
+        ToolEnums::ParamType::String,
+        "javascript code to compute",
+        true
+    }};
+}
+
+QString CodeInterpreter::symbolicFormat() const
+{
+    return "{human readable plan to complete the task}\n" + ToolCallConstants::CodeInterpreterPrefix + "{code}\n" + ToolCallConstants::CodeInterpreterSuffix;
+}
+
+QString CodeInterpreter::examplePrompt() const
+{
+    return R"(Write code to check if a number is prime, use that to see if the number 7 is prime)";
+}
+
+QString CodeInterpreter::exampleCall() const
+{
+    static const QString example = R"(function isPrime(n) {
+    if (n <= 1) {
+        return false;
+    }
+    for (let i = 2; i <= Math.sqrt(n); i++) {
+        if (n % i === 0) {
+            return false;
+        }
+    }
+    return true;
+}
+
+const number = 7;
+console.log(`The number ${number} is prime: ${isPrime(number)}`);
+)";
+
+    return "Certainly! Let's compute the answer to whether the number 7 is prime.\n" + ToolCallConstants::CodeInterpreterPrefix + example + ToolCallConstants::CodeInterpreterSuffix;
+}
+
+QString CodeInterpreter::exampleReply() const
+{
+    return R"("The computed result shows that 7 is a prime number.)";
+}
+
+CodeInterpreterWorker::CodeInterpreterWorker()
+    : QObject(nullptr)
+{
+}
+
+void CodeInterpreterWorker::request(const QString &code)
+{
+    JavaScriptConsoleCapture consoleCapture;
+    QJSValue consoleObject = m_engine.newQObject(&consoleCapture);
+    m_engine.globalObject().setProperty("console", consoleObject);
+
+    const QJSValue result = m_engine.evaluate(code);
+    QString resultString = result.isUndefined() ? QString() : result.toString();
+
+    // NOTE: We purposely do not set the m_error or m_errorString for the code interpreter since
+    // we *want* the model to see the response has an error so it can hopefully correct itself. The
+    // error member variables are intended for tools that have error conditions that cannot be corrected.
+    // For instance, a tool depending upon the network might set these error variables if the network
+    // is not available.
+    if (result.isError()) {
+        const QStringList lines = code.split('\n');
+        const int line = result.property("lineNumber").toInt();
+        const int index = line - 1;
+        const QString lineContent = (index >= 0 && index < lines.size()) ? lines.at(index) : "Line not found in code.";
+            resultString = QString("Uncaught exception at line %1: %2\n\t%3")
+                .arg(line)
+                .arg(result.toString())
+                .arg(lineContent);
+        m_error = ToolEnums::Error::UnknownError;
+        m_errorString = resultString;
+    }
+
+    if (resultString.isEmpty())
+        resultString = consoleCapture.output;
+    else if (!consoleCapture.output.isEmpty())
+        resultString += "\n" + consoleCapture.output;
+    m_response = resultString;
+    emit finished();
+}

gpt4all-chat/src/codeinterpreter.h

@@ -0,0 +1,84 @@
+#ifndef CODEINTERPRETER_H
+#define CODEINTERPRETER_H
+
+#include "tool.h"
+#include "toolcallparser.h"
+
+#include <QJSEngine>
+#include <QObject>
+#include <QString>
+#include <QtGlobal>
+
+class JavaScriptConsoleCapture : public QObject
+{
+    Q_OBJECT
+public:
+    QString output;
+    Q_INVOKABLE void log(const QString &message)
+    {
+        const int maxLength = 1024;
+        if (output.length() >= maxLength)
+            return;
+
+        if (output.length() + message.length() + 1 > maxLength) {
+            static const QString trunc = "\noutput truncated at " + QString::number(maxLength) + " characters...";
+            int remainingLength = maxLength - output.length();
+            if (remainingLength > 0)
+                output.append(message.left(remainingLength));
+            output.append(trunc);
+            Q_ASSERT(output.length() > maxLength);
+        } else {
+            output.append(message + "\n");
+        }
+    }
+};
+
+class CodeInterpreterWorker : public QObject {
+    Q_OBJECT
+public:
+    CodeInterpreterWorker();
+    virtual ~CodeInterpreterWorker() {}
+
+    QString response() const { return m_response; }
+
+    void request(const QString &code);
+    void interrupt() { m_engine.setInterrupted(true); }
+    ToolEnums::Error error() const { return m_error; }
+    QString errorString() const { return m_errorString; }
+
+Q_SIGNALS:
+    void finished();
+
+private:
+    QJSEngine m_engine;
+    QString m_response;
+    ToolEnums::Error m_error = ToolEnums::Error::NoError;
+    QString m_errorString;
+};
+
+class CodeInterpreter : public Tool
+{
+    Q_OBJECT
+public:
+    explicit CodeInterpreter() : Tool(), m_error(ToolEnums::Error::NoError) {}
+    virtual ~CodeInterpreter() {}
+
+    QString run(const QList<ToolParam> &params, qint64 timeout = 2000) override;
+    ToolEnums::Error error() const override { return m_error; }
+    QString errorString() const override { return m_errorString; }
+
+    QString name() const override { return tr("Code Interpreter"); }
+    QString description() const override { return tr("compute javascript code using console.log as output"); }
+    QString function() const override { return ToolCallConstants::CodeInterpreterFunction; }
+    QList<ToolParamInfo> parameters() const override;
+    virtual QString symbolicFormat() const override;
+    QString examplePrompt() const override;
+    QString exampleCall() const override;
+    QString exampleReply() const override;
+
+private:
+    ToolEnums::Error m_error = ToolEnums::Error::NoError;
+    QString m_errorString;
+};
+
+#endif // CODEINTERPRETER_H

gpt4all-chat/src/database.cpp

@@ -233,12 +233,17 @@ static const QString SELECT_COUNT_CHUNKS_SQL = uR"(
 )"_s;
 
 static const QString SELECT_CHUNKS_FTS_SQL = uR"(
-    select id, bm25(chunks_fts) as score
-    from chunks_fts
+    select fts.id, bm25(chunks_fts) as score
+    from chunks_fts fts
+    join documents d on fts.document_id = d.id
+    join collection_items ci on d.folder_id = ci.folder_id
+    join collections co on ci.collection_id = co.id
     where chunks_fts match ?
-    order by score limit %1;
+    and co.name in ('%1')
+    order by score limit %2;
 )"_s;
 
+
 #define NAMED_PAIR(name, typea, a, typeb, b) \
     struct name { typea a; typeb b; }; \
     static bool operator==(const name &x, const name &y) { return x.a == y.a && x.b == y.b; } \
@@ -285,7 +290,7 @@ static bool selectCountChunks(QSqlQuery &q, int folder_id, int &count)
     return true;
 }
 
-static bool selectChunk(QSqlQuery &q, const QList<int> &chunk_ids, int retrievalSize)
+static bool selectChunk(QSqlQuery &q, const QList<int> &chunk_ids)
 {
     QString chunk_ids_str = QString::number(chunk_ids[0]);
     for (size_t i = 1; i < chunk_ids.size(); ++i)
@@ -302,10 +307,6 @@ static const QString INSERT_COLLECTION_SQL = uR"(
         returning id;
     )"_s;
 
-static const QString DELETE_COLLECTION_SQL = uR"(
-    delete from collections where name = ? and folder_id = ?;
-    )"_s;
-
 static const QString SELECT_FOLDERS_FROM_COLLECTIONS_SQL = uR"(
     select f.id, f.path
     from collections c
@@ -349,6 +350,14 @@ static const QString UPDATE_LAST_UPDATE_TIME_SQL = uR"(
     update collections set last_update_time = ? where id = ?;
 )"_s;
 
+static const QString FTS_INTEGRITY_SQL = uR"(
+    insert into chunks_fts(chunks_fts, rank) values('integrity-check', 1);
+)"_s;
+
+static const QString FTS_REBUILD_SQL = uR"(
+    insert into chunks_fts(chunks_fts) values('rebuild');
+)"_s;
+
 static bool addCollection(QSqlQuery &q, const QString &collection_name, const QDateTime &start_update,
                           const QDateTime &last_update, const QString &embedding_model, CollectionItem &item)
 {
@@ -366,15 +375,6 @@ static bool addCollection(QSqlQuery &q, const QString &collection_name, const QD
     return true;
 }
 
-static bool removeCollection(QSqlQuery &q, const QString &collection_name, int folder_id)
-{
-    if (!q.prepare(DELETE_COLLECTION_SQL))
-        return false;
-    q.addBindValue(collection_name);
-    q.addBindValue(folder_id);
-    return q.exec();
-}
-
 static bool selectFoldersFromCollection(QSqlQuery &q, const QString &collection_name, QList<QPair<int, QString>> *folders)
 {
     if (!q.prepare(SELECT_FOLDERS_FROM_COLLECTIONS_SQL))
@@ -507,10 +507,6 @@ static const QString GET_FOLDER_EMBEDDING_MODEL_SQL = uR"(
     where ci.folder_id = ?;
     )"_s;
 
-static const QString SELECT_ALL_FOLDERPATHS_SQL = uR"(
-    select path from folders;
-    )"_s;
-
 static const QString FOLDER_REMOVE_ALL_DOCS_SQL[] = {
     uR"(
         delete from embeddings
@@ -585,17 +581,6 @@ static bool sqlGetFolderEmbeddingModel(QSqlQuery &q, int id, QString &embedding_
     return true;
 }
 
-static bool selectAllFolderPaths(QSqlQuery &q, QList<QString> *folder_paths)
-{
-    if (!q.prepare(SELECT_ALL_FOLDERPATHS_SQL))
-        return false;
-    if (!q.exec())
-        return false;
-    while (q.next())
-        folder_paths->append(q.value(0).toString());
-    return true;
-}
-
 static const QString INSERT_COLLECTION_ITEM_SQL = uR"(
     insert into collection_items(collection_id, folder_id)
     values(?, ?)
@@ -1014,9 +999,8 @@ bool Database::initDb(const QString &modelPath, const QList<CollectionItem> &old
     return true;
 }
 
-Database::Database(bool automaticUpdate, int chunkSize, QStringList extensions)
+Database::Database(int chunkSize, QStringList extensions)
     : QObject(nullptr)
-    , m_automaticUpdate(automaticUpdate)
     , m_chunkSize(chunkSize)
     , m_scannedFileExtensions(std::move(extensions))
     , m_scanIntervalTimer(new QTimer(this))
@@ -1101,7 +1085,6 @@ void Database::updateFolderToIndex(int folder_id, size_t countForFolder, bool se
             sendChunkList(); // send any remaining embedding chunks to llm
         item.indexing = false;
         item.installed = true;
-        item.outOfDate = false;
 
         // Set the last update if we are done
         if (item.startUpdate > item.lastUpdate && item.currentEmbeddingsToIndex == 0)
@@ -1118,9 +1101,12 @@ static void handleDocumentError(const QString &errorMessage, int document_id, co
 
 class DocumentReader {
 public:
+    struct Metadata { QString title, author, subject, keywords; };
+
     static std::unique_ptr<DocumentReader> fromDocument(const DocumentInfo &info);
 
     const DocumentInfo           &doc     () const { return *m_info; }
+    const Metadata               &metadata() const { return m_metadata; }
     const std::optional<QString> &word    () const { return m_word; }
     const std::optional<QString> &nextWord()       { m_word = advance(); return m_word; }
     virtual std::optional<ChunkStreamer::Status> getError() const { return std::nullopt; }
@@ -1132,11 +1118,16 @@ protected:
     explicit DocumentReader(const DocumentInfo &info)
         : m_info(&info) {}
 
-    void postInit() { m_word = advance(); }
+    void postInit(Metadata &&metadata = {})
+    {
+        m_metadata = std::move(metadata);
+        m_word = advance();
+    }
 
     virtual std::optional<QString> advance() = 0;
 
     const DocumentInfo     *m_info;
+    Metadata                m_metadata;
     std::optional<QString>  m_word;
 };
 
@@ -1150,7 +1141,13 @@ public:
         QString path = info.file.canonicalFilePath();
         if (m_doc.load(path) != QPdfDocument::Error::None)
             throw std::runtime_error(fmt::format("Failed to load PDF: {}", path));
-        postInit();
+        Metadata metadata {
+            .title    = m_doc.metaData(QPdfDocument::MetaDataField::Title   ).toString(),
+            .author   = m_doc.metaData(QPdfDocument::MetaDataField::Author  ).toString(),
+            .subject  = m_doc.metaData(QPdfDocument::MetaDataField::Subject ).toString(),
+            .keywords = m_doc.metaData(QPdfDocument::MetaDataField::Keywords).toString(),
+        };
+        postInit(std::move(metadata));
     }
 
     int page() const override { return m_currentPage; }
@@ -1189,6 +1186,7 @@ public:
 
         m_paragraph = &m_doc.paragraphs();
         m_run       = &m_paragraph->runs();
+        // TODO(jared): metadata for Word documents?
         postInit();
     }
 
@@ -1210,11 +1208,14 @@ protected:
         qsizetype wordEnd = wordStart + 1;
         while (wordEnd >= m_buffer.size() || !m_buffer[wordEnd].isSpace()) {
             if (wordEnd >= m_buffer.size() && !fillBuffer())
-                return std::nullopt;
+                break;
             if (!m_buffer[wordEnd].isSpace())
                 ++wordEnd;
         }
 
+        if (wordStart == wordEnd)
+            return std::nullopt;
+
         auto size = wordEnd - wordStart;
         QString word = std::move(m_buffer);
         m_buffer = word.sliced(wordStart + size);
@@ -1222,7 +1223,6 @@ protected:
             word.resize(size);
         else
             word = word.sliced(wordStart, size);
-
         return word;
     }
 
@@ -1234,18 +1234,30 @@ protected:
                 // try next paragraph
                 if (!m_paragraph->has_next())
                     return false;
+
                 m_paragraph->next();
                 m_buffer += u'\n';
             }
+
+            bool foundText = false;
             auto &run = m_run->get_node();
-            const char *text = run.child("w:t").text().get();
-            if (!*text && run.child("w:tab"))
-                text = "\t";
-            m_run->next();
-            if (*text) {
-                m_buffer += QUtf8StringView(text);
-                return true;
+            for (auto node = run.first_child(); node; node = node.next_sibling()) {
+                std::string node_name = node.name();
+                if (node_name == "w:t") {
+                    const char *text = node.text().get();
+                    if (*text) {
+                        foundText = true;
+                        m_buffer += QUtf8StringView(text);
+                    }
+                } else if (node_name == "w:br") {
+                    m_buffer += u'\n';
+                } else if (node_name == "w:tab") {
+                    m_buffer += u'\t';
+                }
             }
+
+            m_run->next();
+            if (foundText) return true;
         }
     }
 
@@ -1313,9 +1325,7 @@ ChunkStreamer::ChunkStreamer(Database *database)
 
 ChunkStreamer::~ChunkStreamer() = default;
 
-void ChunkStreamer::setDocument(const DocumentInfo &doc, int documentId, const QString &embeddingModel,
-                                const QString &title, const QString &author, const QString &subject,
-                                const QString &keywords)
+void ChunkStreamer::setDocument(const DocumentInfo &doc, int documentId, const QString &embeddingModel)
 {
     auto docKey = doc.key();
     if (!m_docKey || *m_docKey != docKey) {
@@ -1323,10 +1333,6 @@ void ChunkStreamer::setDocument(const DocumentInfo &doc, int documentId, const Q
         m_reader         = DocumentReader::fromDocument(doc);
         m_documentId     = documentId;
         m_embeddingModel = embeddingModel;
-        m_title          = title;
-        m_author         = author;
-        m_subject        = subject;
-        m_keywords       = keywords;
         m_chunk.clear();
         m_page = 0;
 
@@ -1363,10 +1369,7 @@ ChunkStreamer::Status ChunkStreamer::step()
     for (;;) {
         if (auto error = m_reader->getError()) {
             m_docKey.reset(); // done processing
-            return *error;
-        }
-        if (m_database->scanQueueInterrupted()) {
-            retval = Status::INTERRUPTED;
+            retval = *error;
             break;
         }
 
@@ -1427,14 +1430,15 @@ ChunkStreamer::Status ChunkStreamer::step()
 
                 QSqlQuery q(m_database->m_db);
                 int chunkId = 0;
+                auto &metadata = m_reader->metadata();
                 if (!m_database->addChunk(q,
                     m_documentId,
                     chunk,
                     m_reader->doc().file.fileName(), // basename
-                    m_title,
-                    m_author,
-                    m_subject,
-                    m_keywords,
+                    metadata.title,
+                    metadata.author,
+                    metadata.subject,
+                    metadata.keywords,
                     m_page,
                     line_from,
                     line_to,
@@ -1461,6 +1465,11 @@ ChunkStreamer::Status ChunkStreamer::step()
                 break;
             }
         }
+
+        if (m_database->scanQueueInterrupted()) {
+            retval = Status::INTERRUPTED;
+            break;
+        }
     }
 
     if (nChunks) {
@@ -1521,8 +1530,22 @@ void Database::handleEmbeddingsGenerated(const QVector<EmbeddingResult> &embeddi
     for (const auto &[key, stat]: std::as_const(stats).asKeyValueRange()) {
         if (!m_collectionMap.contains(key.folder_id)) continue;
         CollectionItem item = guiCollectionItem(key.folder_id);
-        item.currentEmbeddingsToIndex -= stat.nAdded + stat.nSkipped;
-        item.totalEmbeddingsToIndex -= stat.nSkipped;
+        Q_ASSERT(item.currentEmbeddingsToIndex >= stat.nAdded + stat.nSkipped);
+        if (item.currentEmbeddingsToIndex < stat.nAdded + stat.nSkipped) {
+            qWarning() << "Database ERROR: underflow in current embeddings to index statistics";
+            item.currentEmbeddingsToIndex = 0;
+        } else {
+            item.currentEmbeddingsToIndex -= stat.nAdded + stat.nSkipped;
+        }
+
+        Q_ASSERT(item.totalEmbeddingsToIndex >= stat.nSkipped);
+        if (item.totalEmbeddingsToIndex < stat.nSkipped) {
+            qWarning() << "Database ERROR: underflow in total embeddings to index statistics";
+            item.totalEmbeddingsToIndex = 0;
+        } else {
+            item.totalEmbeddingsToIndex -= stat.nSkipped;
+        }
+
         if (!stat.lastFile.isNull())
             item.fileCurrentlyProcessing = stat.lastFile;
 
@@ -1601,41 +1624,6 @@ void Database::enqueueDocumentInternal(DocumentInfo &&info, bool prepend)
     queue.insert(prepend ? queue.begin() : queue.end(), std::move(info));
 }
 
-bool Database::isOutOfDate(int folder_id, std::list<DocumentInfo> &&infos) const
-{
-    for (auto &info : infos) {
-        // Update info
-        info.file.stat();
-
-        // If the doc has since been deleted or no longer readable, then we schedule more work and return
-        // leaving the cleanup for the cleanup handler
-        if (!info.file.exists() || !info.file.isReadable())
-            return true;
-
-        const qint64 document_time = info.file.fileTime(QFile::FileModificationTime).toMSecsSinceEpoch();
-        const QString document_path = info.file.canonicalFilePath();
-        const bool currentlyProcessing = info.currentlyProcessing;
-
-        // Check and see if we already have this document
-        QSqlQuery q(m_db);
-        int existing_id = -1;
-        qint64 existing_time = -1;
-        if (!selectDocument(q, document_path, &existing_id, &existing_time)) {
-            handleDocumentError("ERROR: Cannot select document", existing_id, document_path, q.lastError());
-            continue;
-        }
-
-        // If not, then we are out of date
-        if (existing_id == -1)
-            return true;
-
-        // If we have the document and the time does not match, then we are out of date
-        if (document_time != existing_time)
-            return true;
-    }
-    return false;
-}
-
 void Database::enqueueDocuments(int folder_id, std::list<DocumentInfo> &&infos)
 {
     // enqueue all documents
@@ -1659,13 +1647,16 @@ bool Database::scanQueueInterrupted() const
 
 void Database::scanQueueBatch()
 {
-    m_scanDurationTimer.start();
-
     transaction();
 
-    // scan for up to 100ms or until we run out of documents
-    while (!m_docsToScan.empty() && !scanQueueInterrupted())
+    m_scanDurationTimer.start();
+
+    // scan for up to the maximum scan duration or until we run out of documents
+    while (!m_docsToScan.empty()) {
         scanQueue();
+        if (scanQueueInterrupted())
+            break;
+    }
 
     commit();
 
@@ -1751,22 +1742,8 @@ void Database::scanQueue()
     Q_ASSERT(document_id != -1);
 
     {
-        QString title, author, subject, keywords;
-        if (info.isPdf()) {
-            QPdfDocument doc;
-            if (doc.load(document_path) != QPdfDocument::Error::None) {
-                qWarning() << "ERROR: Could not load pdf" << document_id << document_path;
-                return updateFolderToIndex(folder_id, countForFolder);
-            }
-            title    = doc.metaData(QPdfDocument::MetaDataField::Title).toString();
-            author   = doc.metaData(QPdfDocument::MetaDataField::Author).toString();
-            subject  = doc.metaData(QPdfDocument::MetaDataField::Subject).toString();
-            keywords = doc.metaData(QPdfDocument::MetaDataField::Keywords).toString();
-            // TODO(jared): metadata for Word documents?
-        }
-
         try {
-            m_chunkStreamer.setDocument(info, document_id, embedding_model, title, author, subject, keywords);
+            m_chunkStreamer.setDocument(info, document_id, embedding_model);
         } catch (const std::runtime_error &e) {
             qWarning() << "LocalDocs ERROR:" << e.what();
             goto dequeue;
@@ -1798,12 +1775,18 @@ void Database::scanQueue()
 
 dequeue:
     auto item = guiCollectionItem(folder_id);
-    item.currentBytesToIndex -= info.file.size();
+    Q_ASSERT(item.currentBytesToIndex >= info.file.size());
+    if (item.currentBytesToIndex < info.file.size()) {
+        qWarning() << "Database ERROR: underflow in current bytes to index statistics";
+        item.currentBytesToIndex = 0;
+    } else {
+        item.currentBytesToIndex -= info.file.size();
+    }
     updateGuiForCollectionItem(item);
     return updateFolderToIndex(folder_id, countForFolder);
 }
 
-void Database::scanDocuments(int folder_id, const QString &folder_path, bool forceIndexing)
+void Database::scanDocuments(int folder_id, const QString &folder_path)
 {
 #if defined(DEBUG)
     qDebug() << "scanning folder for documents" << folder_path;
@@ -1828,14 +1811,9 @@ void Database::scanDocuments(int folder_id, const QString &folder_path, bool for
 
     if (!infos.empty()) {
         CollectionItem item = guiCollectionItem(folder_id);
-        const bool shouldIndex = m_automaticUpdate || forceIndexing;
-        if (shouldIndex)
-            item.indexing = true;
-        else
-            item.outOfDate = isOutOfDate(folder_id, std::move(infos));
+        item.indexing = true;
         updateGuiForCollectionItem(item);
-        if (shouldIndex)
-            enqueueDocuments(folder_id, std::move(infos));
+        enqueueDocuments(folder_id, std::move(infos));
     } else {
         updateFolderToIndex(folder_id, 0, false);
     }
@@ -1857,6 +1835,7 @@ void Database::start()
         m_databaseValid = false;
     } else {
         cleanDB();
+        ftsIntegrityCheck();
         QSqlQuery q(m_db);
         if (!refreshDocumentIdCache(q)) {
             m_databaseValid = false;
@@ -1889,7 +1868,7 @@ void Database::addCurrentFolders()
     for (const auto &i : collections) {
         if (!i.forceIndexing) {
             addFolderToWatch(i.folder_path);
-            scanDocuments(i.folder_id, i.folder_path, false /*forceIndexing*/);
+            scanDocuments(i.folder_id, i.folder_path);
         }
     }
 
@@ -2024,7 +2003,7 @@ void Database::forceIndexing(const QString &collection, const QString &embedding
         item.forceIndexing = false;
         updateGuiForCollectionItem(item);
         addFolderToWatch(folder.second);
-        scanDocuments(folder.first, folder.second, true /*forceIndexing*/);
+        scanDocuments(folder.first, folder.second);
     }
 }
 
@@ -2056,7 +2035,7 @@ void Database::forceRebuildFolder(const QString &path)
     item.currentEmbeddingsToIndex = item.totalEmbeddingsToIndex = 0;
     updateGuiForCollectionItem(item);
 
-    scanDocuments(folder_id, path, true /*forceIndexing*/);
+    scanDocuments(folder_id, path);
 }
 
 bool Database::addFolder(const QString &collection, const QString &path, const QString &embedding_model)
@@ -2100,7 +2079,7 @@ bool Database::addFolder(const QString &collection, const QString &path, const Q
         // note: this is the existing embedding model if the collection was found
         if (!item->embeddingModel.isNull()) {
             addFolderToWatch(path);
-            scanDocuments(folder_id, path, true /*forceIndexing*/);
+            scanDocuments(folder_id, path);
         }
     }
     return true;
@@ -2370,7 +2349,7 @@ QList<int> Database::searchBM25(const QString &query, const QList<QString> &coll
     QList<BM25Query> bm25Queries = queriesForFTS5(query);
 
     QSqlQuery sqlQuery(m_db);
-    sqlQuery.prepare(SELECT_CHUNKS_FTS_SQL.arg(k));
+    sqlQuery.prepare(SELECT_CHUNKS_FTS_SQL.arg(collections.join("', '"), QString::number(k)));
 
     QList<SearchResult> results;
     for (auto &bm25Query : std::as_const(bm25Queries)) {
@@ -2388,11 +2367,13 @@ QList<int> Database::searchBM25(const QString &query, const QList<QString> &coll
         }
     }
 
-    do {
-        const int chunkId = sqlQuery.value(0).toInt();
-        const float score = sqlQuery.value(1).toFloat();
-        results.append({chunkId, score});
-    } while (sqlQuery.next());
+    if (sqlQuery.at() != QSql::AfterLastRow) {
+        do {
+            const int chunkId = sqlQuery.value(0).toInt();
+            const float score = sqlQuery.value(1).toFloat();
+            results.append({chunkId, score});
+        } while (sqlQuery.next());
+    }
 
     k = qMin(k, results.size());
     std::partial_sort(
@@ -2525,7 +2506,7 @@ void Database::retrieveFromDB(const QList<QString> &collections, const QString &
         return;
 
     QSqlQuery q(m_db);
-    if (!selectChunk(q, searchResults, retrievalSize)) {
+    if (!selectChunk(q, searchResults)) {
         qDebug() << "ERROR: selecting chunks:" << q.lastError();
         return;
     }
@@ -2566,6 +2547,26 @@ void Database::retrieveFromDB(const QList<QString> &collections, const QString &
             results->append(tempResults.value(id));
 }
 
+bool Database::ftsIntegrityCheck()
+{
+    QSqlQuery q(m_db);
+
+    // Returns an error executing sql if it the integrity check fails
+    // See: https://www.sqlite.org/fts5.html#the_integrity_check_command
+    const bool success = q.exec(FTS_INTEGRITY_SQL);
+    if (!success && q.lastError().nativeErrorCode() != "267" /*SQLITE_CORRUPT_VTAB from sqlite header*/) {
+        qWarning() << "ERROR: Cannot prepare sql for fts integrity check" << q.lastError();
+        return false;
+    }
+
+    if (!success && !q.exec(FTS_REBUILD_SQL)) {
+        qWarning() << "ERROR: Cannot exec sql for fts rebuild" << q.lastError();
+        return false;
+    }
+
+    return true;
+}
+
 // FIXME This is very slow and non-interruptible and when we close the application and we're
 // cleaning a large table this can cause the app to take forever to shut down. This would ideally be
 // interruptible and we'd continue 'cleaning' when we restart
@@ -2616,7 +2617,7 @@ bool Database::cleanDB()
         int document_id = q.value(0).toInt();
         QString document_path = q.value(1).toString();
         QFileInfo info(document_path);
-        if (info.exists() && info.isReadable() && m_scannedFileExtensions.contains(info.suffix()))
+        if (info.exists() && info.isReadable() && m_scannedFileExtensions.contains(info.suffix(), Qt::CaseInsensitive))
             continue;
 
 #if defined(DEBUG)
@@ -2707,7 +2708,7 @@ void Database::changeFileExtensions(const QStringList &extensions)
 
     for (const auto &i: std::as_const(collections)) {
         if (!i.forceIndexing)
-            scanDocuments(i.folder_id, i.folder_path, false /*forceIndexing*/);
+            scanDocuments(i.folder_id, i.folder_path);
     }
 }
 
@@ -2744,13 +2745,6 @@ void Database::directoryChanged(const QString &path)
         updateCollectionStatistics();
 
     // Rescan the documents associated with the folder
-    if (folder_id != -1) {
-        if (m_automaticUpdate) {
-            scanDocuments(folder_id, path, false /*forceIndexing*/);
-        } else {
-            CollectionItem item = guiCollectionItem(folder_id);
-            item.outOfDate = true;
-            updateGuiForCollectionItem(item);
-        }
-    }
+    if (folder_id != -1)
+        scanDocuments(folder_id, path);
 }

gpt4all-chat/src/database.h

@@ -41,10 +41,20 @@ class QTimer;
 
 /* Version 0: GPT4All v2.4.3, full-text search
  * Version 1: GPT4All v2.5.3, embeddings in hsnwlib
- * Version 2: GPT4All v3.0.0, embeddings in sqlite */
+ * Version 2: GPT4All v3.0.0, embeddings in sqlite
+ * Version 3: GPT4All v3.4.0, hybrid search
+ */
 
 // minimum supported version
 static const int LOCALDOCS_MIN_VER = 1;
+
+// FIXME: (Adam) The next time we bump the version we should add triggers to manage the fts external
+// content table as recommended in the official documentation to keep the fts index in sync
+// See: https://www.sqlite.org/fts5.html#external_content_tables
+
+// FIXME: (Adam) The fts virtual table should include the chunk_id explicitly instead of relying upon
+// the id of the two tables to be in sync
+
 // current version
 static const int LOCALDOCS_VERSION = 3;
 
@@ -134,7 +144,6 @@ struct CollectionItem {
     bool installed = false;
     bool indexing = false;
     bool forceIndexing = false;
-    bool outOfDate = false;
     QString error;
 
     // progress
@@ -162,8 +171,7 @@ public:
     explicit ChunkStreamer(Database *database);
     ~ChunkStreamer();
 
-    void setDocument(const DocumentInfo &doc, int documentId, const QString &embeddingModel, const QString &title,
-                     const QString &author, const QString &subject, const QString &keywords);
+    void setDocument(const DocumentInfo &doc, int documentId, const QString &embeddingModel);
     std::optional<DocumentInfo::key_type> currentDocKey() const;
     void reset();
 
@@ -190,7 +198,7 @@ class Database : public QObject
 {
     Q_OBJECT
 public:
-    Database(bool automaticUpdate, int chunkSize, QStringList extensions);
+    Database(int chunkSize, QStringList extensions);
     ~Database() override;
 
     bool isValid() const { return m_databaseValid; }
@@ -199,7 +207,7 @@ public Q_SLOTS:
     void start();
     bool scanQueueInterrupted() const;
     void scanQueueBatch();
-    void scanDocuments(int folder_id, const QString &folder_path, bool forceIndexing);
+    void scanDocuments(int folder_id, const QString &folder_path);
     void forceIndexing(const QString &collection, const QString &embedding_model);
     void forceRebuildFolder(const QString &path);
     bool addFolder(const QString &collection, const QString &path, const QString &embedding_model);
@@ -251,9 +259,9 @@ private:
     DocumentInfo dequeueDocument();
     void removeFolderFromDocumentQueue(int folder_id);
     void enqueueDocumentInternal(DocumentInfo &&info, bool prepend = false);
-    bool isOutOfDate(int folder_id, std::list<DocumentInfo> &&infos) const;
     void enqueueDocuments(int folder_id, std::list<DocumentInfo> &&infos);
     void scanQueue();
+    bool ftsIntegrityCheck();
     bool cleanDB();
     void addFolderToWatch(const QString &path);
     void removeFolderFromWatch(const QString &path);
@@ -289,7 +297,6 @@ private:
 
 private:
     QSqlDatabase m_db;
-    bool m_automaticUpdate;
     int m_chunkSize;
     QStringList m_scannedFileExtensions;
     QTimer *m_scanIntervalTimer;

gpt4all-chat/src/download.cpp

@@ -58,11 +58,6 @@ Download::Download()
     m_startTime = QDateTime::currentDateTime();
 }
 
-static bool operator==(const ReleaseInfo& lhs, const ReleaseInfo& rhs)
-{
-    return lhs.version == rhs.version;
-}
-
 std::strong_ordering Download::compareAppVersions(const QString &a, const QString &b)
 {
     static QRegularExpression versionRegex(R"(^(\d+(?:\.\d+){0,2})(-.+)?$)");

gpt4all-chat/src/jinja_helpers.cpp

@@ -0,0 +1,117 @@
+#include "jinja_helpers.h"
+
+#include "utils.h"
+
+#include <fmt/format.h>
+
+#include <QString>
+#include <QUrl>
+
+#include <memory>
+#include <vector>
+
+using namespace std::literals::string_view_literals;
+
+
+JinjaResultInfo::~JinjaResultInfo() = default;
+
+const JinjaFieldMap<ResultInfo> JinjaResultInfo::s_fields = {
+    { "collection", [](auto &s) { return s.collection.toStdString(); } },
+    { "path",       [](auto &s) { return s.path      .toStdString(); } },
+    { "file",       [](auto &s) { return s.file      .toStdString(); } },
+    { "title",      [](auto &s) { return s.title     .toStdString(); } },
+    { "author",     [](auto &s) { return s.author    .toStdString(); } },
+    { "date",       [](auto &s) { return s.date      .toStdString(); } },
+    { "text",       [](auto &s) { return s.text      .toStdString(); } },
+    { "page",       [](auto &s) { return s.page;                     } },
+    { "file_uri",   [](auto &s) { return s.fileUri() .toStdString(); } },
+};
+
+JinjaPromptAttachment::~JinjaPromptAttachment() = default;
+
+const JinjaFieldMap<PromptAttachment> JinjaPromptAttachment::s_fields = {
+    { "url",               [](auto &s) { return s.url.toString()    .toStdString(); } },
+    { "file",              [](auto &s) { return s.file()            .toStdString(); } },
+    { "processed_content", [](auto &s) { return s.processedContent().toStdString(); } },
+};
+
+std::vector<std::string> JinjaMessage::GetKeys() const
+{
+    std::vector<std::string> result;
+    auto &keys = this->keys();
+    result.reserve(keys.size());
+    result.assign(keys.begin(), keys.end());
+    return result;
+}
+
+auto JinjaMessage::keys() const -> const std::unordered_set<std::string_view> &
+{
+    static const std::unordered_set<std::string_view> baseKeys
+        { "role", "content" };
+    static const std::unordered_set<std::string_view> userKeys
+        { "role", "content", "sources", "prompt_attachments" };
+    switch (m_item->type()) {
+        using enum MessageItem::Type;
+    case System:
+    case Response:
+    case ToolResponse:
+        return baseKeys;
+    case Prompt:
+        return userKeys;
+        break;
+    }
+    Q_UNREACHABLE();
+}
+
+bool operator==(const JinjaMessage &a, const JinjaMessage &b)
+{
+    if (a.m_item == b.m_item)
+        return true;
+    const auto &[ia, ib] = std::tie(*a.m_item, *b.m_item);
+    auto type = ia.type();
+    if (type != ib.type() || ia.content() != ib.content())
+        return false;
+
+    switch (type) {
+        using enum MessageItem::Type;
+    case System:
+    case Response:
+    case ToolResponse:
+        return true;
+    case Prompt:
+        return ia.sources() == ib.sources() && ia.promptAttachments() == ib.promptAttachments();
+        break;
+    }
+    Q_UNREACHABLE();
+}
+
+const JinjaFieldMap<JinjaMessage> JinjaMessage::s_fields = {
+    { "role", [](auto &m) {
+        switch (m.item().type()) {
+            using enum MessageItem::Type;
+            case System:   return "system"sv;
+            case Prompt:   return "user"sv;
+            case Response: return "assistant"sv;
+            case ToolResponse: return "tool"sv;
+                break;
+        }
+        Q_UNREACHABLE();
+    } },
+    { "content", [](auto &m) {
+        if (m.version() == 0 && m.item().type() == MessageItem::Type::Prompt)
+            return m.item().bakedPrompt().toStdString();
+        return m.item().content().toStdString();
+    } },
+    { "sources", [](auto &m) {
+        auto sources = m.item().sources() | views::transform([](auto &r) {
+            return jinja2::GenericMap([map = std::make_shared<JinjaResultInfo>(r)] { return map.get(); });
+        });
+        return jinja2::ValuesList(sources.begin(), sources.end());
+    } },
+    { "prompt_attachments", [](auto &m) {
+        auto attachments = m.item().promptAttachments() | views::transform([](auto &pa) {
+            return jinja2::GenericMap([map = std::make_shared<JinjaPromptAttachment>(pa)] { return map.get(); });
+        });
+        return jinja2::ValuesList(attachments.begin(), attachments.end());
+    } },
+};

gpt4all-chat/src/jinja_helpers.h

@@ -0,0 +1,116 @@
+#pragma once
+
+#include "chatmodel.h"
+#include "database.h"
+
+#include <jinja2cpp/value.h>
+
+#include <functional>
+#include <ranges>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <unordered_set>
+
+#include <QtGlobal>
+
+namespace views = std::views;
+
+
+template <typename T>
+using JinjaFieldMap = std::unordered_map<std::string_view, std::function<jinja2::Value (const T &)>>;
+
+template <typename Derived>
+class JinjaComparable : public jinja2::IMapItemAccessor {
+public:
+    JinjaComparable() = default;
+
+    bool IsEqual(const jinja2::IComparable &other) const override;
+
+private:
+    Q_DISABLE_COPY_MOVE(JinjaComparable)
+};
+
+template <typename Derived>
+class JinjaHelper : public JinjaComparable<Derived> {
+public:
+    size_t GetSize() const override
+    { return Derived::s_fields.size(); }
+
+    bool HasValue(const std::string &name) const override
+    { return Derived::s_fields.contains(name); }
+
+    jinja2::Value GetValueByName(const std::string &name) const override;
+
+    std::vector<std::string> GetKeys() const override
+    { auto keys = views::elements<0>(Derived::s_fields); return { keys.begin(), keys.end() }; }
+};
+
+class JinjaResultInfo : public JinjaHelper<JinjaResultInfo> {
+public:
+    explicit JinjaResultInfo(const ResultInfo &source) noexcept
+        : m_source(&source) {}
+
+    ~JinjaResultInfo() override;
+
+    const ResultInfo &value() const { return *m_source; }
+
+    friend bool operator==(const JinjaResultInfo &a, const JinjaResultInfo &b)
+    { return a.m_source == b.m_source || *a.m_source == *b.m_source; }
+
+private:
+    static const JinjaFieldMap<ResultInfo> s_fields;
+    const ResultInfo *m_source;
+
+    friend class JinjaHelper<JinjaResultInfo>;
+};
+
+class JinjaPromptAttachment : public JinjaHelper<JinjaPromptAttachment> {
+public:
+    explicit JinjaPromptAttachment(const PromptAttachment &attachment) noexcept
+        : m_attachment(&attachment) {}
+
+    ~JinjaPromptAttachment() override;
+
+    const PromptAttachment &value() const { return *m_attachment; }
+
+    friend bool operator==(const JinjaPromptAttachment &a, const JinjaPromptAttachment &b)
+    { return a.m_attachment == b.m_attachment || *a.m_attachment == *b.m_attachment; }
+
+private:
+    static const JinjaFieldMap<PromptAttachment> s_fields;
+    const PromptAttachment *m_attachment;
+
+    friend class JinjaHelper<JinjaPromptAttachment>;
+};
+
+class JinjaMessage : public JinjaHelper<JinjaMessage> {
+public:
+    explicit JinjaMessage(uint version, const MessageItem &item) noexcept
+        : m_version(version), m_item(&item) {}
+
+    const JinjaMessage &value  () const { return *this;     }
+    uint                version() const { return m_version; }
+    const MessageItem      &item  () const { return *m_item;   }
+
+    size_t GetSize() const override { return keys().size(); }
+    bool HasValue(const std::string &name) const override { return keys().contains(name); }
+
+    jinja2::Value GetValueByName(const std::string &name) const override
+    { return HasValue(name) ? JinjaHelper::GetValueByName(name) : jinja2::EmptyValue(); }
+
+    std::vector<std::string> GetKeys() const override;
+
+private:
+    auto keys() const -> const std::unordered_set<std::string_view> &;
+
+private:
+    static const JinjaFieldMap<JinjaMessage> s_fields;
+    uint            m_version;
+    const MessageItem *m_item;
+
+    friend class JinjaHelper<JinjaMessage>;
+    friend bool operator==(const JinjaMessage &a, const JinjaMessage &b);
+};
+
+#include "jinja_helpers.inl"

gpt4all-chat/src/jinja_helpers.inl

@@ -0,0 +1,17 @@
+template <typename D>
+bool JinjaComparable<D>::IsEqual(const jinja2::IComparable &other) const
+{
+    if (auto *omsg = dynamic_cast<const D *>(&other))
+        return *static_cast<const D *>(this) == *omsg;
+    return false;
+}
+
+template <typename D>
+jinja2::Value JinjaHelper<D>::GetValueByName(const std::string &name) const
+{
+    if (auto it = D::s_fields.find(name); it != D::s_fields.end()) {
+        auto [_, func] = *it;
+        return func(static_cast<const D *>(this)->value());
+    }
+    return jinja2::EmptyValue();
+}

gpt4all-chat/src/jinja_replacements.cpp

@@ -0,0 +1,480 @@
+// The map in this file is automatically generated by Jared. Do not hand-edit it.
+
+#include "jinja_replacements.h"
+
+// This is a list of prompt templates known to GPT4All and their associated replacements which are automatically used
+// instead when loading the chat template from GGUF. These exist for two primary reasons:
+// - HuggingFace model authors make ugly chat templates because they do not expect the end user to see them;
+// - and our Jinja2Cpp-based template parsing is not fully compatible with HuggingFace transformers and jinja2.
+
+// Below is a list of known incompatibilities with the official HF jinja2 implementation. These are not all necessarily
+// reflected in the below substitution list, and this cannot be an exhaustive list because there are a plethora of edge
+// cases in template parsing in which jinja2 and Jinja2Cpp differ. These are differences that could be reasonably
+// expected to affect chat templates that could be seen in the wild, or that cause a crash:
+// - Jinja2Cpp crashes (in debug builds) if given the template `a[""(`
+// - Jinja2Cpp does not support these jinja2 constructs:
+//   - `is not none`
+//   - list slicing, e.g. `messages[1:]`
+//   - the jinja2.ext.loopcontrols extension, which HF enables by default
+//   - a missing space after a quote in substitution (e.g. `{{ 'foo'}}`), which *has* been seen in the wild
+// - GPT4All does not currently support these HuggingFace template features:
+//   - customized "tojson" filter (we provide the built-in Jinja2Cpp one)
+//   - the AssistantTracker extension
+
+
+// The substitution list.
+// For templates that apply to models listed in models3.json, these should be copied there as well for best
+// compatibility with older versions of GPT4All.
+
+const std::unordered_map<std::string_view, std::string_view> CHAT_TEMPLATE_SUBSTITUTIONS {
+    // calme-2.1-phi3.5-4b.Q6_K.gguf (reported by ThilotE on Discord)
+    {
+        R"TEMPLATE({% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>
+' + message['content'] + '<|end|>
+'}}{% elif message['role'] == 'user' %}{{'<|user|>
+' + message['content'] + '<|end|>
+'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>
+' + message['content'] + '<|end|>
+'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>
+' }}{% else %}{{ eos_token }}{% endif %})TEMPLATE",
+        R"TEMPLATE({%- for message in messages %}
+    {%- if message['role'] == 'system' and message['content'] %}
+        {{- '<|system|>\n' + message['content'] + '<|end|>\n' }}
+    {%- elif message['role'] == 'user' %}
+        {{- '<|user|>\n' + message['content'] + '<|end|>\n' }}
+    {%- elif message['role'] == 'assistant' %}
+        {{- '<|assistant|>\n' + message['content'] + '<|end|>\n' }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|assistant|>\n' }}
+{%- else %}
+    {{- eos_token }}
+{%- endif %})TEMPLATE",
+    },
+    // gemma-2-9b-it-Q4_0.gguf (nomic-ai/gpt4all#3282)
+    {
+        R"TEMPLATE({{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %})TEMPLATE",
+        R"TEMPLATE({{- bos_token }}
+{%- if messages[0]['role'] == 'system' %}
+    {{- raise_exception('System role not supported') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
+        {{- raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
+    {%- endif %}
+    {%- if message['role'] == 'assistant' %}
+        {%- set role = 'model' %}
+    {%- else %}
+        {%- set role = message['role'] %}
+    {%- endif %}
+    {{- '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<start_of_turn>model\n' }}
+{%- endif %})TEMPLATE",
+    },
+    // ghost-7b-v0.9.1-Q4_0.gguf
+    {
+        R"TEMPLATE({% for message in messages %}
+{% if message['role'] == 'user' %}
+{{ '<|user|>
+' + message['content'] + eos_token }}
+{% elif message['role'] == 'system' %}
+{{ '<|system|>
+' + message['content'] + eos_token }}
+{% elif message['role'] == 'assistant' %}
+{{ '<|assistant|>
+'  + message['content'] + eos_token }}
+{% endif %}
+{% if loop.last and add_generation_prompt %}
+{{ '<|assistant|>' }}
+{% endif %}
+{% endfor %})TEMPLATE",
+        R"TEMPLATE({%- for message in messages %}
+    {%- if message['role'] == 'user' %}
+        {{- '<|user|>\n' + message['content'] + eos_token }}
+    {%- elif message['role'] == 'system' %}
+        {{- '<|system|>\n' + message['content'] + eos_token }}
+    {%- elif message['role'] == 'assistant' %}
+        {{- '<|assistant|>\n'  + message['content'] + eos_token }}
+    {%- endif %}
+    {%- if loop.last and add_generation_prompt %}
+        {{- '<|assistant|>' }}
+    {%- endif %}
+{%- endfor %})TEMPLATE",
+    },
+    // Llama-3.2-1B-Instruct-Q4_0.gguf, Llama-3.2-3B-Instruct-Q4_0.gguf, SummLlama3.2-3B-Q4_0.gguf (nomic-ai/gpt4all#3309)
+    {
+        R"TEMPLATE({{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %})TEMPLATE",
+        R"TEMPLATE({{- bos_token }}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now('%d %b %Y') %}
+    {%- else %}
+        {%- set date_string = '26 Jul 2024' %}
+    {%- endif %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content'] | trim %}
+    {%- set loop_start = 1 %}
+{%- else %}
+    {%- set system_message = '' %}
+    {%- set loop_start = 0 %}
+{%- endif %}
+
+{#- System message #}
+{{- '<|start_header_id|>system<|end_header_id|>\n\n' }}
+{{- 'Cutting Knowledge Date: December 2023\n' }}
+{{- 'Today Date: ' + date_string + '\n\n' }}
+{{- system_message }}
+{{- '<|eot_id|>' }}
+
+{%- for message in messages %}
+    {%- if loop.index0 >= loop_start %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %})TEMPLATE",
+    },
+    // Llama-3.3-70B-Instruct-Q4_0.gguf (nomic-ai/gpt4all#3305)
+    {
+        R"TEMPLATE({{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %})TEMPLATE",
+        R"TEMPLATE({{- bos_token }}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content'] | trim %}
+    {%- set loop_start = 1 %}
+{%- else %}
+    {%- set system_message = "" %}
+    {%- set loop_start = 0 %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{%- for message in messages %}
+    {%- if loop.index0 >= loop_start %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %})TEMPLATE",
+    },
+    // Meta-Llama-3.1-8B-Instruct-128k-Q4_0.gguf
+    {
+        R"TEMPLATE({% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }})TEMPLATE",
+        R"TEMPLATE({%- set loop_messages = messages %}
+{%- for message in loop_messages %}
+    {%- set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}
+    {%- if loop.index0 == 0 %}
+        {%- set content = bos_token + content %}
+    {%- endif %}
+    {{- content }}
+{%- endfor %}
+{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }})TEMPLATE",
+    },
+    // Meta-Llama-3-8B-Instruct.Q4_0.gguf
+    {
+        R"TEMPLATE({% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %})TEMPLATE",
+        R"TEMPLATE({%- set loop_messages = messages %}
+{%- for message in loop_messages %}
+    {%- set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}
+    {{- content }}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %})TEMPLATE",
+    },
+    // mistral-7b-openorca.gguf2.Q4_0.gguf, Hermes-3-Llama-3.2-3B.Q4_0.gguf
+    {
+        R"TEMPLATE({% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %})TEMPLATE",
+        R"TEMPLATE({%- if not add_generation_prompt is defined %}
+    {%- set add_generation_prompt = false %}
+{%- endif %}
+{%- for message in messages %}
+    {{- '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n' }}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %})TEMPLATE",
+    },
+    // Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf
+    {
+        R"TEMPLATE({% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %})TEMPLATE",
+        R"TEMPLATE({%- for message in messages %}
+    {{- '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n' }}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %})TEMPLATE",
+    },
+    // Phi-3-mini-4k-instruct.Q4_0.gguf
+    {
+        R"TEMPLATE({{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '
+' + message['content'] + '<|end|>
+' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>
+' }}{% else %}{{ eos_token }}{% endif %})TEMPLATE",
+        R"TEMPLATE({{- bos_token }}
+{%- for message in messages %}
+    {{- '<|' + message['role'] + '|>\n' + message['content'] + '<|end|>\n' }}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|assistant|>\n' }}
+{%- else %}
+    {{- eos_token }}
+{%- endif %})TEMPLATE",
+    },
+    // qwen2-1_5b-instruct-q4_0.gguf, Qwen2-1.5B-Instruct.Q6_K.gguf (nomic-ai/gpt4all#3263), Qwen2-72B-Instruct.Q4_K_M.gguf
+    {
+        R"TEMPLATE({% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful assistant.<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %})TEMPLATE",
+        R"TEMPLATE({%- for message in messages %}
+    {%- if loop.first and messages[0]['role'] != 'system' %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+    {{- '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>\n' }}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %})TEMPLATE",
+    },
+};

gpt4all-chat/src/jinja_replacements.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#include <string_view>
+#include <unordered_map>
+
+extern const std::unordered_map<std::string_view, std::string_view> CHAT_TEMPLATE_SUBSTITUTIONS;

gpt4all-chat/src/llm.cpp

@@ -19,6 +19,10 @@
 #   include "network.h"
 #endif
 
+#ifdef Q_OS_MAC
+#include "macosdock.h"
+#endif
+
 using namespace Qt::Literals::StringLiterals;
 
 class MyLLM: public LLM { };
@@ -105,3 +109,21 @@ bool LLM::isNetworkOnline() const
     auto * netinfo = QNetworkInformation::instance();
     return !netinfo || netinfo->reachability() == QNetworkInformation::Reachability::Online;
 }
+
+void LLM::showDockIcon() const
+{
+#ifdef Q_OS_MAC
+    MacOSDock::showIcon();
+#else
+    qt_noop();
+#endif
+}
+
+void LLM::hideDockIcon() const
+{
+#ifdef Q_OS_MAC
+    MacOSDock::hideIcon();
+#else
+    qt_noop();
+#endif
+}

gpt4all-chat/src/llm.h

@@ -23,6 +23,9 @@ public:
     Q_INVOKABLE QString systemTotalRAMInGBString() const;
     Q_INVOKABLE bool isNetworkOnline() const;
 
+    Q_INVOKABLE void showDockIcon() const;
+    Q_INVOKABLE void hideDockIcon() const;
+
 Q_SIGNALS:
     void isNetworkOnlineChanged();
 

gpt4all-chat/src/localdocs.cpp

@@ -26,8 +26,7 @@ LocalDocs::LocalDocs()
     connect(MySettings::globalInstance(), &MySettings::localDocsFileExtensionsChanged, this, &LocalDocs::handleFileExtensionsChanged);
 
     // Create the DB with the chunk size from settings
-    m_database = new Database(MySettings::globalInstance()->localDocsAutomaticUpdate(),
-                              MySettings::globalInstance()->localDocsChunkSize(),
+    m_database = new Database(MySettings::globalInstance()->localDocsChunkSize(),
                               MySettings::globalInstance()->localDocsFileExtensions());
 
     connect(this, &LocalDocs::requestStart, m_database,

gpt4all-chat/src/localdocsmodel.cpp

@@ -121,8 +121,6 @@ QVariant LocalDocsModel::data(const QModelIndex &index, int role) const
             return item.embeddingModel;
         case UpdatingRole:
             return item.indexing || item.currentEmbeddingsToIndex != 0;
-        case OutOfDateRole:
-            return item.outOfDate;
     }
 
     return QVariant();
@@ -151,7 +149,6 @@ QHash<int, QByteArray> LocalDocsModel::roleNames() const
     roles[FileCurrentlyProcessingRole] = "fileCurrentlyProcessing";
     roles[EmbeddingModelRole] = "embeddingModel";
     roles[UpdatingRole] = "updating";
-    roles[OutOfDateRole] = "outOfDate";
     return roles;
 }
 
@@ -203,8 +200,6 @@ void LocalDocsModel::updateCollectionItem(const CollectionItem &item)
             changed.append(FileCurrentlyProcessingRole);
         if (stored.embeddingModel != item.embeddingModel)
             changed.append(EmbeddingModelRole);
-        if (stored.outOfDate != item.outOfDate)
-            changed.append(OutOfDateRole);
 
         // preserve collection name as we ignore it for matching
         QString collection = stored.collection;

gpt4all-chat/src/localdocsmodel.h

@@ -70,8 +70,7 @@ public:
         LastUpdateRole,
         FileCurrentlyProcessingRole,
         EmbeddingModelRole,
-        UpdatingRole,
-        OutOfDateRole
+        UpdatingRole
     };
 
     explicit LocalDocsModel(QObject *parent = nullptr);

gpt4all-chat/src/macosdock.h

@@ -0,0 +1,9 @@
+#ifndef MACOSDOCK_H
+#define MACOSDOCK_H
+
+struct MacOSDock {
+static void showIcon();
+static void hideIcon();
+};
+
+#endif // MACOSDOCK_H

gpt4all-chat/src/macosdock.mm

@@ -0,0 +1,13 @@
+#include "macosdock.h"
+
+#include <Cocoa/Cocoa.h>
+
+void MacOSDock::showIcon()
+{
+    [[NSApplication sharedApplication] setActivationPolicy:NSApplicationActivationPolicyRegular];
+}
+
+void MacOSDock::hideIcon()
+{
+    [[NSApplication sharedApplication] setActivationPolicy:NSApplicationActivationPolicyProhibited];
+}

gpt4all-chat/src/main.cpp

@@ -7,17 +7,22 @@
 #include "modellist.h"
 #include "mysettings.h"
 #include "network.h"
+#include "toolmodel.h"
 
 #include <gpt4all-backend/llmodel.h>
 #include <singleapplication.h>
 
 #include <QCoreApplication>
+#include <QFont>
+#include <QFontDatabase>
 #include <QObject>
 #include <QQmlApplicationEngine>
+#include <QQmlContext>
 #include <QQuickWindow>
 #include <QSettings>
 #include <QString>
 #include <QUrl>
+#include <QVariant>
 #include <Qt>
 
 #ifdef Q_OS_LINUX
@@ -26,6 +31,8 @@
 
 #ifdef Q_OS_WINDOWS
 #   include <windows.h>
+#else
+#   include <signal.h>
 #endif
 
 using namespace Qt::Literals::StringLiterals;
@@ -42,6 +49,7 @@ static void raiseWindow(QWindow *window)
 
     SetForegroundWindow(hwnd);
 #else
+    LLM::globalInstance()->showDockIcon();
     window->show();
     window->raise();
     window->requestActivate();
@@ -88,25 +96,36 @@ int main(int argc, char *argv[])
 
     // Set the local and language translation before the qml engine has even been started. This will
     // use the default system locale unless the user has explicitly set it to use a different one.
-    MySettings::globalInstance()->setLanguageAndLocale();
+    auto *mySettings = MySettings::globalInstance();
+    mySettings->setLanguageAndLocale();
 
     QQmlApplicationEngine engine;
 
     // Add a connection here from MySettings::languageAndLocaleChanged signal to a lambda slot where I can call
     // engine.uiLanguage property
-    QObject::connect(MySettings::globalInstance(), &MySettings::languageAndLocaleChanged, [&engine]() {
+    QObject::connect(mySettings, &MySettings::languageAndLocaleChanged, [&engine]() {
         engine.setUiLanguage(MySettings::globalInstance()->languageAndLocale());
     });
 
-    qmlRegisterSingletonInstance("mysettings", 1, 0, "MySettings", MySettings::globalInstance());
-    qmlRegisterSingletonInstance("modellist", 1, 0, "ModelList", ModelList::globalInstance());
+    auto *modelList = ModelList::globalInstance();
+    QObject::connect(modelList, &ModelList::dataChanged, mySettings, &MySettings::onModelInfoChanged);
+
+    qmlRegisterSingletonInstance("mysettings", 1, 0, "MySettings", mySettings);
+    qmlRegisterSingletonInstance("modellist", 1, 0, "ModelList", modelList);
     qmlRegisterSingletonInstance("chatlistmodel", 1, 0, "ChatListModel", ChatListModel::globalInstance());
     qmlRegisterSingletonInstance("llm", 1, 0, "LLM", LLM::globalInstance());
     qmlRegisterSingletonInstance("download", 1, 0, "Download", Download::globalInstance());
     qmlRegisterSingletonInstance("network", 1, 0, "Network", Network::globalInstance());
     qmlRegisterSingletonInstance("localdocs", 1, 0, "LocalDocs", LocalDocs::globalInstance());
+    qmlRegisterSingletonInstance("toollist", 1, 0, "ToolList", ToolModel::globalInstance());
+    qmlRegisterUncreatableMetaObject(ToolEnums::staticMetaObject, "toolenums", 1, 0, "ToolEnums", "Error: only enums");
     qmlRegisterUncreatableMetaObject(MySettingsEnums::staticMetaObject, "mysettingsenums", 1, 0, "MySettingsEnums", "Error: only enums");
 
+    {
+        auto fixedFont = QFontDatabase::systemFont(QFontDatabase::FixedFont);
+        engine.rootContext()->setContextProperty("fixedFont", fixedFont);
+    }
+
     const QUrl url(u"qrc:/gpt4all/main.qml"_s);
 
     QObject::connect(&engine, &QQmlApplicationEngine::objectCreated,
@@ -130,6 +149,17 @@ int main(int argc, char *argv[])
     }
 #endif
 
+#ifndef Q_OS_WINDOWS
+    // handle signals gracefully
+    struct sigaction sa;
+    sa.sa_handler = [](int s) { QCoreApplication::exit(s == SIGINT ? 0 : 1); };
+    sa.sa_flags   = SA_RESETHAND;
+    sigemptyset(&sa.sa_mask);
+    sigaction(SIGINT,  &sa, nullptr);
+    sigaction(SIGTERM, &sa, nullptr);
+    sigaction(SIGHUP,  &sa, nullptr);
+#endif
+
     int res = app.exec();
 
     // Make sure ChatLLM threads are joined before global destructors run.

gpt4all-chat/src/modellist.cpp

@@ -1,6 +1,7 @@
 #include "modellist.h"
 
 #include "download.h"
+#include "jinja_replacements.h"
 #include "mysettings.h"
 #include "network.h"
 
@@ -47,8 +48,35 @@ using namespace Qt::Literals::StringLiterals;
 
 #define MODELS_JSON_VERSION "3"
 
+
 static const QStringList FILENAME_BLACKLIST { u"gpt4all-nomic-embed-text-v1.rmodel"_s };
 
+static const QString RMODEL_CHAT_TEMPLATE = uR"(<chat>
+{%- set loop_messages = messages %}
+{%- for message in loop_messages %}
+    {%- if not message['role'] in ['user', 'assistant', 'system'] %}
+        {{- raise_exception('Unknown role: ' + message['role']) }}
+    {%- endif %}
+    {{- '<' + message['role'] + '>' }}
+    {%- if message['role'] == 'user' %}
+        {%- for source in message.sources %}
+            {%- if loop.first %}
+                {{- '### Context:\n' }}
+            {%- endif %}
+            {{- ('Collection: ' + source.collection + '\n'    +
+                 'Path: '       + source.path       + '\n'    +
+                 'Excerpt: '    + source.text       + '\n\n') | escape }}
+        {%- endfor %}
+    {%- endif %}
+    {%- for attachment in message.prompt_attachments %}
+        {{- (attachment.processed_content + '\n\n') | escape }}
+    {%- endfor %}
+    {{- message.content | escape }}
+    {{- '</' + message['role'] + '>' }}
+{%- endfor %}
+</chat>)"_s;
+
+
 QString ModelInfo::id() const
 {
     return m_id;
@@ -316,26 +344,55 @@ void ModelInfo::setRepeatPenaltyTokens(int t)
     m_repeatPenaltyTokens = t;
 }
 
-QString ModelInfo::promptTemplate() const
+QVariant ModelInfo::defaultChatTemplate() const
 {
-    return MySettings::globalInstance()->modelPromptTemplate(*this);
+    auto res = m_chatTemplate.or_else([this]() -> std::optional<QString> {
+        if (!installed || isOnline)
+            return std::nullopt;
+        if (!m_modelChatTemplate) {
+            auto path = (dirpath + filename()).toUtf8();
+            auto res = LLModel::Implementation::chatTemplate(path.constData());
+            if (res) {
+                std::string ggufTmpl(std::move(*res));
+                if (ggufTmpl.size() >= 2 && ggufTmpl.end()[-2] != '\n' && ggufTmpl.end()[-1] == '\n')
+                    ggufTmpl.erase(ggufTmpl.end() - 1); // strip trailing newline for e.g. Llama-3.2-3B-Instruct
+                if (
+                    auto replacement = CHAT_TEMPLATE_SUBSTITUTIONS.find(ggufTmpl);
+                    replacement != CHAT_TEMPLATE_SUBSTITUTIONS.end()
+                ) {
+                    qWarning() << "automatically substituting chat template for" << filename();
+                    auto &[badTemplate, goodTemplate] = *replacement;
+                    ggufTmpl = goodTemplate;
+                }
+                m_modelChatTemplate = QString::fromStdString(ggufTmpl);
+            } else {
+                qWarning().nospace() << "failed to get chat template for " << filename() << ": " << res.error().c_str();
+                m_modelChatTemplate = QString(); // do not retry
+            }
+        }
+        if (m_modelChatTemplate->isNull())
+            return std::nullopt;
+        return m_modelChatTemplate;
+    });
+
+    if (res)
+        return std::move(*res);
+    return QVariant::fromValue(nullptr);
 }
 
-void ModelInfo::setPromptTemplate(const QString &t)
+auto ModelInfo::chatTemplate() const -> UpgradeableSetting
 {
-    if (shouldSaveMetadata()) MySettings::globalInstance()->setModelPromptTemplate(*this, t, true /*force*/);
-    m_promptTemplate = t;
+    return MySettings::globalInstance()->modelChatTemplate(*this);
 }
 
-QString ModelInfo::systemPrompt() const
+QString ModelInfo::defaultSystemMessage() const
 {
-    return MySettings::globalInstance()->modelSystemPrompt(*this);
+    return m_systemMessage;
 }
 
-void ModelInfo::setSystemPrompt(const QString &p)
+auto ModelInfo::systemMessage() const -> UpgradeableSetting
 {
-    if (shouldSaveMetadata()) MySettings::globalInstance()->setModelSystemPrompt(*this, p, true /*force*/);
-    m_systemPrompt = p;
+    return MySettings::globalInstance()->modelSystemMessage(*this);
 }
 
 QString ModelInfo::chatNamePrompt() const
@@ -360,39 +417,41 @@ void ModelInfo::setSuggestedFollowUpPrompt(const QString &p)
     m_suggestedFollowUpPrompt = p;
 }
 
+// FIXME(jared): this should not be used for model settings that have meaningful defaults, such as temperature
 bool ModelInfo::shouldSaveMetadata() const
 {
     return installed && (isClone() || isDiscovered() || description() == "" /*indicates sideloaded*/);
 }
 
-QVariantMap ModelInfo::getFields() const
+QVariant ModelInfo::getField(QLatin1StringView name) const
 {
-    return {
-        { "filename",            m_filename },
-        { "description",         m_description },
-        { "url",                 m_url },
-        { "quant",               m_quant },
-        { "type",                m_type },
-        { "isClone",             m_isClone },
-        { "isDiscovered",        m_isDiscovered },
-        { "likes",               m_likes },
-        { "downloads",           m_downloads },
-        { "recency",             m_recency },
-        { "temperature",         m_temperature },
-        { "topP",                m_topP },
-        { "minP",                m_minP },
-        { "topK",                m_topK },
-        { "maxLength",           m_maxLength },
-        { "promptBatchSize",     m_promptBatchSize },
-        { "contextLength",       m_contextLength },
-        { "gpuLayers",           m_gpuLayers },
-        { "repeatPenalty",       m_repeatPenalty },
-        { "repeatPenaltyTokens", m_repeatPenaltyTokens },
-        { "promptTemplate",      m_promptTemplate },
-        { "systemPrompt",        m_systemPrompt },
-        { "chatNamePrompt",      m_chatNamePrompt },
-        { "suggestedFollowUpPrompt", m_suggestedFollowUpPrompt },
+    static const std::unordered_map<QLatin1StringView, QVariant(*)(const ModelInfo &)> s_fields = {
+        { "filename"_L1,                [](auto &i) -> QVariant { return i.m_filename;                } },
+        { "description"_L1,             [](auto &i) -> QVariant { return i.m_description;             } },
+        { "url"_L1,                     [](auto &i) -> QVariant { return i.m_url;                     } },
+        { "quant"_L1,                   [](auto &i) -> QVariant { return i.m_quant;                   } },
+        { "type"_L1,                    [](auto &i) -> QVariant { return i.m_type;                    } },
+        { "isClone"_L1,                 [](auto &i) -> QVariant { return i.m_isClone;                 } },
+        { "isDiscovered"_L1,            [](auto &i) -> QVariant { return i.m_isDiscovered;            } },
+        { "likes"_L1,                   [](auto &i) -> QVariant { return i.m_likes;                   } },
+        { "downloads"_L1,               [](auto &i) -> QVariant { return i.m_downloads;               } },
+        { "recency"_L1,                 [](auto &i) -> QVariant { return i.m_recency;                 } },
+        { "temperature"_L1,             [](auto &i) -> QVariant { return i.m_temperature;             } },
+        { "topP"_L1,                    [](auto &i) -> QVariant { return i.m_topP;                    } },
+        { "minP"_L1,                    [](auto &i) -> QVariant { return i.m_minP;                    } },
+        { "topK"_L1,                    [](auto &i) -> QVariant { return i.m_topK;                    } },
+        { "maxLength"_L1,               [](auto &i) -> QVariant { return i.m_maxLength;               } },
+        { "promptBatchSize"_L1,         [](auto &i) -> QVariant { return i.m_promptBatchSize;         } },
+        { "contextLength"_L1,           [](auto &i) -> QVariant { return i.m_contextLength;           } },
+        { "gpuLayers"_L1,               [](auto &i) -> QVariant { return i.m_gpuLayers;               } },
+        { "repeatPenalty"_L1,           [](auto &i) -> QVariant { return i.m_repeatPenalty;           } },
+        { "repeatPenaltyTokens"_L1,     [](auto &i) -> QVariant { return i.m_repeatPenaltyTokens;     } },
+        { "chatTemplate"_L1,            [](auto &i) -> QVariant { return i.defaultChatTemplate();     } },
+        { "systemMessage"_L1,           [](auto &i) -> QVariant { return i.m_systemMessage;           } },
+        { "chatNamePrompt"_L1,          [](auto &i) -> QVariant { return i.m_chatNamePrompt;          } },
+        { "suggestedFollowUpPrompt"_L1, [](auto &i) -> QVariant { return i.m_suggestedFollowUpPrompt; } },
     };
+    return s_fields.at(name)(*this);
 }
 
 InstalledModels::InstalledModels(QObject *parent, bool selectable)
@@ -418,47 +477,64 @@ bool InstalledModels::filterAcceptsRow(int sourceRow,
     return (isInstalled || (!m_selectable && isDownloading)) && !isEmbeddingModel;
 }
 
-DownloadableModels::DownloadableModels(QObject *parent)
+GPT4AllDownloadableModels::GPT4AllDownloadableModels(QObject *parent)
     : QSortFilterProxyModel(parent)
-    , m_expanded(false)
-    , m_limit(5)
 {
-    connect(this, &DownloadableModels::rowsInserted, this, &DownloadableModels::countChanged);
-    connect(this, &DownloadableModels::rowsRemoved, this, &DownloadableModels::countChanged);
-    connect(this, &DownloadableModels::modelReset, this, &DownloadableModels::countChanged);
+    connect(this, &GPT4AllDownloadableModels::rowsInserted, this, &GPT4AllDownloadableModels::countChanged);
+    connect(this, &GPT4AllDownloadableModels::rowsRemoved, this, &GPT4AllDownloadableModels::countChanged);
+    connect(this, &GPT4AllDownloadableModels::modelReset, this, &GPT4AllDownloadableModels::countChanged);
 }
 
-bool DownloadableModels::filterAcceptsRow(int sourceRow,
+void GPT4AllDownloadableModels::filter(const QVector<QString> &keywords)
+{
+    m_keywords = keywords;
+    invalidateFilter();
+}
+
+bool GPT4AllDownloadableModels::filterAcceptsRow(int sourceRow,
                                           const QModelIndex &sourceParent) const
 {
-    // FIXME We can eliminate the 'expanded' code as the UI no longer uses this
-    bool withinLimit = sourceRow < (m_expanded ? sourceModel()->rowCount() : m_limit);
     QModelIndex index = sourceModel()->index(sourceRow, 0, sourceParent);
-    bool hasDescription = !sourceModel()->data(index, ModelList::DescriptionRole).toString().isEmpty();
+    const QString description = sourceModel()->data(index, ModelList::DescriptionRole).toString();
+    bool hasDescription = !description.isEmpty();
     bool isClone = sourceModel()->data(index, ModelList::IsCloneRole).toBool();
-    return withinLimit && hasDescription && !isClone;
+    bool isDiscovered = sourceModel()->data(index, ModelList::IsDiscoveredRole).toBool();
+    bool satisfiesKeyword = m_keywords.isEmpty();
+    for (const QString &k : m_keywords)
+        satisfiesKeyword = description.contains(k) ? true : satisfiesKeyword;
+    return !isDiscovered && hasDescription && !isClone && satisfiesKeyword;
 }
 
-int DownloadableModels::count() const
+int GPT4AllDownloadableModels::count() const
 {
     return rowCount();
 }
 
-bool DownloadableModels::isExpanded() const
+HuggingFaceDownloadableModels::HuggingFaceDownloadableModels(QObject *parent)
+    : QSortFilterProxyModel(parent)
+    , m_limit(5)
 {
-    return m_expanded;
+    connect(this, &HuggingFaceDownloadableModels::rowsInserted, this, &HuggingFaceDownloadableModels::countChanged);
+    connect(this, &HuggingFaceDownloadableModels::rowsRemoved, this, &HuggingFaceDownloadableModels::countChanged);
+    connect(this, &HuggingFaceDownloadableModels::modelReset, this, &HuggingFaceDownloadableModels::countChanged);
 }
 
-void DownloadableModels::setExpanded(bool expanded)
+bool HuggingFaceDownloadableModels::filterAcceptsRow(int sourceRow,
+                                          const QModelIndex &sourceParent) const
 {
-    if (m_expanded != expanded) {
-        m_expanded = expanded;
-        invalidateFilter();
-        emit expandedChanged(m_expanded);
-    }
+    QModelIndex index = sourceModel()->index(sourceRow, 0, sourceParent);
+    bool hasDescription = !sourceModel()->data(index, ModelList::DescriptionRole).toString().isEmpty();
+    bool isClone = sourceModel()->data(index, ModelList::IsCloneRole).toBool();
+    bool isDiscovered = sourceModel()->data(index, ModelList::IsDiscoveredRole).toBool();
+    return isDiscovered && hasDescription && !isClone;
 }
 
-void DownloadableModels::discoverAndFilter(const QString &discover)
+int HuggingFaceDownloadableModels::count() const
+{
+    return rowCount();
+}
+
+void HuggingFaceDownloadableModels::discoverAndFilter(const QString &discover)
 {
     m_discoverFilter = discover;
     ModelList *ml = qobject_cast<ModelList*>(parent());
@@ -476,7 +552,8 @@ ModelList::ModelList()
     : QAbstractListModel(nullptr)
     , m_installedModels(new InstalledModels(this))
     , m_selectableModels(new InstalledModels(this, /*selectable*/ true))
-    , m_downloadableModels(new DownloadableModels(this))
+    , m_gpt4AllDownloadableModels(new GPT4AllDownloadableModels(this))
+    , m_huggingFaceDownloadableModels(new HuggingFaceDownloadableModels(this))
     , m_asyncModelRequestOngoing(false)
     , m_discoverLimit(20)
     , m_discoverSortDirection(-1)
@@ -489,33 +566,51 @@ ModelList::ModelList()
 
     m_installedModels->setSourceModel(this);
     m_selectableModels->setSourceModel(this);
-    m_downloadableModels->setSourceModel(this);
+    m_gpt4AllDownloadableModels->setSourceModel(this);
+    m_huggingFaceDownloadableModels->setSourceModel(this);
+
+    auto *mySettings = MySettings::globalInstance();
+    connect(mySettings, &MySettings::nameChanged,                this, &ModelList::updateDataForSettings     );
+    connect(mySettings, &MySettings::temperatureChanged,         this, &ModelList::updateDataForSettings     );
+    connect(mySettings, &MySettings::topPChanged,                this, &ModelList::updateDataForSettings     );
+    connect(mySettings, &MySettings::minPChanged,                this, &ModelList::updateDataForSettings     );
+    connect(mySettings, &MySettings::topKChanged,                this, &ModelList::updateDataForSettings     );
+    connect(mySettings, &MySettings::maxLengthChanged,           this, &ModelList::updateDataForSettings     );
+    connect(mySettings, &MySettings::promptBatchSizeChanged,     this, &ModelList::updateDataForSettings     );
+    connect(mySettings, &MySettings::contextLengthChanged,       this, &ModelList::updateDataForSettings     );
+    connect(mySettings, &MySettings::gpuLayersChanged,           this, &ModelList::updateDataForSettings     );
+    connect(mySettings, &MySettings::repeatPenaltyChanged,       this, &ModelList::updateDataForSettings     );
+    connect(mySettings, &MySettings::repeatPenaltyTokensChanged, this, &ModelList::updateDataForSettings     );
+    connect(mySettings, &MySettings::chatTemplateChanged,        this, &ModelList::maybeUpdateDataForSettings);
+    connect(mySettings, &MySettings::systemMessageChanged,       this, &ModelList::maybeUpdateDataForSettings);
+
+    connect(this, &ModelList::dataChanged, this, &ModelList::onDataChanged);
 
-    connect(MySettings::globalInstance(), &MySettings::modelPathChanged, this, &ModelList::updateModelsFromDirectory);
-    connect(MySettings::globalInstance(), &MySettings::modelPathChanged, this, &ModelList::updateModelsFromJson);
-    connect(MySettings::globalInstance(), &MySettings::modelPathChanged, this, &ModelList::updateModelsFromSettings);
-    connect(MySettings::globalInstance(), &MySettings::nameChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::temperatureChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::topPChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::minPChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::topKChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::maxLengthChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::promptBatchSizeChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::contextLengthChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::gpuLayersChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::repeatPenaltyChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::repeatPenaltyTokensChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::promptTemplateChanged, this, &ModelList::updateDataForSettings);
-    connect(MySettings::globalInstance(), &MySettings::systemPromptChanged, this, &ModelList::updateDataForSettings);
     connect(&m_networkManager, &QNetworkAccessManager::sslErrors, this, &ModelList::handleSslErrors);
 
     updateModelsFromJson();
     updateModelsFromSettings();
     updateModelsFromDirectory();
 
+    connect(mySettings, &MySettings::modelPathChanged, this, &ModelList::updateModelsFromDirectory);
+    connect(mySettings, &MySettings::modelPathChanged, this, &ModelList::updateModelsFromJson     );
+    connect(mySettings, &MySettings::modelPathChanged, this, &ModelList::updateModelsFromSettings );
+
     QCoreApplication::instance()->installEventFilter(this);
 }
 
+// an easier way to listen for model info and setting changes
+void ModelList::onDataChanged(const QModelIndex &topLeft, const QModelIndex &bottomRight, const QList<int> &roles)
+{
+    Q_UNUSED(roles)
+    for (int row = topLeft.row(); row <= bottomRight.row(); row++) {
+        auto index = topLeft.siblingAtRow(row);
+        auto id = index.data(ModelList::IdRole).toString();
+        if (auto info = modelInfo(id); !info.id().isNull())
+            emit modelInfoChanged(info);
+    }
+}
+
 QString ModelList::compatibleModelNameHash(QUrl baseUrl, QString modelName) {
     QCryptographicHash sha256(QCryptographicHash::Sha256);
     sha256.addData((baseUrl.toString() + "_" + modelName).toUtf8());
@@ -776,10 +871,10 @@ QVariant ModelList::dataInternal(const ModelInfo *info, int role) const
             return info->repeatPenalty();
         case RepeatPenaltyTokensRole:
             return info->repeatPenaltyTokens();
-        case PromptTemplateRole:
-            return info->promptTemplate();
-        case SystemPromptRole:
-            return info->systemPrompt();
+        case ChatTemplateRole:
+            return QVariant::fromValue(info->chatTemplate());
+        case SystemMessageRole:
+            return QVariant::fromValue(info->systemMessage());
         case ChatNamePromptRole:
             return info->chatNamePrompt();
         case SuggestedFollowUpPromptRole:
@@ -952,10 +1047,10 @@ void ModelList::updateData(const QString &id, const QVector<QPair<int, QVariant>
                 info->setRepeatPenalty(value.toDouble()); break;
             case RepeatPenaltyTokensRole:
                 info->setRepeatPenaltyTokens(value.toInt()); break;
-            case PromptTemplateRole:
-                info->setPromptTemplate(value.toString()); break;
-            case SystemPromptRole:
-                info->setSystemPrompt(value.toString()); break;
+            case ChatTemplateRole:
+                info->m_chatTemplate = value.toString(); break;
+            case SystemMessageRole:
+                info->m_systemMessage = value.toString(); break;
             case ChatNamePromptRole:
                 info->setChatNamePrompt(value.toString()); break;
             case SuggestedFollowUpPromptRole:
@@ -1056,11 +1151,11 @@ ModelInfo ModelList::modelInfo(const QString &id) const
     return *m_modelMap.value(id);
 }
 
-ModelInfo ModelList::modelInfoByFilename(const QString &filename) const
+ModelInfo ModelList::modelInfoByFilename(const QString &filename, bool allowClone) const
 {
     QMutexLocker locker(&m_mutex);
     for (ModelInfo *info : m_models)
-        if (info->filename() == filename)
+        if (info->filename() == filename && (allowClone || !info->isClone()))
             return *info;
     return ModelInfo();
 }
@@ -1077,9 +1172,25 @@ bool ModelList::isUniqueName(const QString &name) const
 
 QString ModelList::clone(const ModelInfo &model)
 {
+    auto *mySettings = MySettings::globalInstance();
+
     const QString id = Network::globalInstance()->generateUniqueId();
     addModel(id);
 
+    QString tmplSetting, sysmsgSetting;
+    if (auto tmpl = model.chatTemplate().asModern()) {
+        tmplSetting = *tmpl;
+    } else {
+        qWarning("ModelList Warning: attempted to clone model with legacy chat template");
+        return {};
+    }
+    if (auto msg = model.systemMessage().asModern()) {
+        sysmsgSetting = *msg;
+    } else {
+        qWarning("ModelList Warning: attempted to clone model with legacy system message");
+        return {};
+    }
+
     QVector<QPair<int, QVariant>> data {
         { ModelList::InstalledRole, model.installed },
         { ModelList::IsCloneRole, true },
@@ -1099,12 +1210,22 @@ QString ModelList::clone(const ModelInfo &model)
         { ModelList::GpuLayersRole, model.gpuLayers() },
         { ModelList::RepeatPenaltyRole, model.repeatPenalty() },
         { ModelList::RepeatPenaltyTokensRole, model.repeatPenaltyTokens() },
-        { ModelList::PromptTemplateRole, model.promptTemplate() },
-        { ModelList::SystemPromptRole, model.systemPrompt() },
+        { ModelList::SystemMessageRole, model.m_systemMessage },
         { ModelList::ChatNamePromptRole, model.chatNamePrompt() },
         { ModelList::SuggestedFollowUpPromptRole, model.suggestedFollowUpPrompt() },
     };
+    if (auto tmpl = model.m_chatTemplate)
+        data.emplace_back(ModelList::ChatTemplateRole, *tmpl); // copy default chat template, if known
     updateData(id, data);
+
+    // Ensure setting overrides are copied in case the base model overrides change.
+    // This is necessary because setting these roles on ModelInfo above does not write to settings.
+    auto cloneInfo = modelInfo(id);
+    if (mySettings->isModelChatTemplateSet (model))
+        mySettings->setModelChatTemplate (cloneInfo, tmplSetting  );
+    if (mySettings->isModelSystemMessageSet(model))
+        mySettings->setModelSystemMessage(cloneInfo, sysmsgSetting);
+
     return id;
 }
 
@@ -1125,21 +1246,23 @@ void ModelList::removeInstalled(const ModelInfo &model)
     removeInternal(model);
 }
 
+int ModelList::indexByModelId(const QString &id) const
+{
+    QMutexLocker locker(&m_mutex);
+    if (auto it = m_modelMap.find(id); it != m_modelMap.cend())
+        return m_models.indexOf(*it);
+    return -1;
+}
+
 void ModelList::removeInternal(const ModelInfo &model)
 {
-    const bool hasModel = contains(model.id());
-    Q_ASSERT(hasModel);
-    if (!hasModel) {
+    int indexOfModel = indexByModelId(model.id());
+    Q_ASSERT(indexOfModel != -1);
+    if (indexOfModel == -1) {
         qWarning() << "ERROR: model list does not contain" << model.id();
         return;
     }
 
-    int indexOfModel = 0;
-    {
-        QMutexLocker locker(&m_mutex);
-        ModelInfo *info = m_modelMap.value(model.id());
-        indexOfModel = m_models.indexOf(info);
-    }
     beginRemoveRows(QModelIndex(), indexOfModel, indexOfModel);
     {
         QMutexLocker locker(&m_mutex);
@@ -1314,8 +1437,7 @@ void ModelList::processModelDirectory(const QString &path)
                 // The description is hard-coded into "GPT4All.ini" due to performance issue.
                 // If the description goes to be dynamic from its .rmodel file, it will get high I/O usage while using the ModelList.
                 data.append({ DescriptionRole, description });
-                // Prompt template should be clear while using ChatML format which is using in most of OpenAI-Compatible API server.
-                data.append({ PromptTemplateRole, "%1" });
+                data.append({ ChatTemplateRole, RMODEL_CHAT_TEMPLATE });
             }
             updateData(id, data);
         }
@@ -1451,9 +1573,20 @@ void ModelList::handleSslErrors(QNetworkReply *reply, const QList<QSslError> &er
         qWarning() << "ERROR: Received ssl error:" << e.errorString() << "for" << url;
 }
 
+void ModelList::maybeUpdateDataForSettings(const ModelInfo &info, bool fromInfo)
+{
+    // ignore updates that were *because* of a dataChanged - would cause a circular dependency
+    int idx;
+    if (!fromInfo && (idx = indexByModelId(info.id())) != -1) {
+        emit dataChanged(index(idx, 0), index(idx, 0));
+        emit selectableModelListChanged();
+    }
+}
+
 void ModelList::updateDataForSettings()
 {
     emit dataChanged(index(0, 0), index(m_models.size() - 1, 0));
+    emit selectableModelListChanged();
 }
 
 void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
@@ -1560,10 +1693,10 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             data.append({ ModelList::RepeatPenaltyRole, obj["repeatPenalty"].toDouble() });
         if (obj.contains("repeatPenaltyTokens"))
             data.append({ ModelList::RepeatPenaltyTokensRole, obj["repeatPenaltyTokens"].toInt() });
-        if (obj.contains("promptTemplate"))
-            data.append({ ModelList::PromptTemplateRole, obj["promptTemplate"].toString() });
-        if (obj.contains("systemPrompt"))
-            data.append({ ModelList::SystemPromptRole, obj["systemPrompt"].toString() });
+        if (auto it = obj.find("chatTemplate"_L1); it != obj.end())
+            data.append({ ModelList::ChatTemplateRole, it->toString() });
+        if (auto it = obj.find("systemMessage"_L1); it != obj.end())
+            data.append({ ModelList::SystemMessageRole, it->toString() });
         updateData(id, data);
     }
 
@@ -1593,7 +1726,8 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             { ModelList::ParametersRole, "?" },
             { ModelList::QuantRole, "NA" },
             { ModelList::TypeRole, "GPT" },
-            { ModelList::UrlRole, "https://api.openai.com/v1/chat/completions"},
+            { ModelList::UrlRole, "https://api.openai.com/v1/chat/completions" },
+            { ModelList::ChatTemplateRole, RMODEL_CHAT_TEMPLATE },
         };
         updateData(id, data);
     }
@@ -1621,7 +1755,8 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             { ModelList::ParametersRole, "?" },
             { ModelList::QuantRole, "NA" },
             { ModelList::TypeRole, "GPT" },
-            { ModelList::UrlRole, "https://api.openai.com/v1/chat/completions"},
+            { ModelList::UrlRole, "https://api.openai.com/v1/chat/completions" },
+            { ModelList::ChatTemplateRole, RMODEL_CHAT_TEMPLATE },
         };
         updateData(id, data);
     }
@@ -1652,7 +1787,8 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             { ModelList::ParametersRole, "?" },
             { ModelList::QuantRole, "NA" },
             { ModelList::TypeRole, "Mistral" },
-            { ModelList::UrlRole, "https://api.mistral.ai/v1/chat/completions"},
+            { ModelList::UrlRole, "https://api.mistral.ai/v1/chat/completions" },
+            { ModelList::ChatTemplateRole, RMODEL_CHAT_TEMPLATE },
         };
         updateData(id, data);
     }
@@ -1677,7 +1813,8 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             { ModelList::ParametersRole, "?" },
             { ModelList::QuantRole, "NA" },
             { ModelList::TypeRole, "Mistral" },
-            { ModelList::UrlRole, "https://api.mistral.ai/v1/chat/completions"},
+            { ModelList::UrlRole, "https://api.mistral.ai/v1/chat/completions" },
+            { ModelList::ChatTemplateRole, RMODEL_CHAT_TEMPLATE },
         };
         updateData(id, data);
     }
@@ -1703,7 +1840,8 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             { ModelList::ParametersRole, "?" },
             { ModelList::QuantRole, "NA" },
             { ModelList::TypeRole, "Mistral" },
-            { ModelList::UrlRole, "https://api.mistral.ai/v1/chat/completions"},
+            { ModelList::UrlRole, "https://api.mistral.ai/v1/chat/completions" },
+            { ModelList::ChatTemplateRole, RMODEL_CHAT_TEMPLATE },
         };
         updateData(id, data);
     }
@@ -1732,6 +1870,7 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             { ModelList::ParametersRole, "?" },
             { ModelList::QuantRole, "NA" },
             { ModelList::TypeRole, "NA" },
+            { ModelList::ChatTemplateRole, RMODEL_CHAT_TEMPLATE },
         };
         updateData(id, data);
     }
@@ -1755,6 +1894,9 @@ void ModelList::updateDiscoveredInstalled(const ModelInfo &info)
     updateData(info.id(), data);
 }
 
+// FIXME(jared): This should only contain fields without reasonable defaults such as name, description, and URL.
+//               For other settings, there is no authoritative value and we should load the setting lazily like we do
+//               for any other override.
 void ModelList::updateModelsFromSettings()
 {
     QSettings settings;
@@ -1769,12 +1911,27 @@ void ModelList::updateModelsFromSettings()
 
         // If we can't find the corresponding file, then ignore it as this reflects a stale model.
         // The file could have been deleted manually by the user for instance or temporarily renamed.
-        if (!settings.contains(g + "/filename") || !modelExists(settings.value(g + "/filename").toString()))
-            continue;
+        QString filename;
+        {
+            auto value = settings.value(u"%1/filename"_s.arg(g));
+            if (!value.isValid() || !modelExists(filename = value.toString()))
+                continue;
+        }
+
+        QVector<QPair<int, QVariant>> data;
+
+        // load data from base model
+        // FIXME(jared): how does "Restore Defaults" work for other settings of clones which we don't do this for?
+        if (auto base = modelInfoByFilename(filename, /*allowClone*/ false); !base.id().isNull()) {
+            if (auto tmpl = base.m_chatTemplate)
+                data.append({ ModelList::ChatTemplateRole,  *tmpl });
+            if (auto msg  = base.m_systemMessage; !msg.isNull())
+                data.append({ ModelList::SystemMessageRole,  msg  });
+        }
 
         addModel(id);
 
-        QVector<QPair<int, QVariant>> data;
+        // load data from settings
         if (settings.contains(g + "/name")) {
             const QString name = settings.value(g + "/name").toString();
             data.append({ ModelList::NameRole, name });
@@ -1859,14 +2016,6 @@ void ModelList::updateModelsFromSettings()
             const int repeatPenaltyTokens = settings.value(g + "/repeatPenaltyTokens").toInt();
             data.append({ ModelList::RepeatPenaltyTokensRole, repeatPenaltyTokens });
         }
-        if (settings.contains(g + "/promptTemplate")) {
-            const QString promptTemplate = settings.value(g + "/promptTemplate").toString();
-            data.append({ ModelList::PromptTemplateRole, promptTemplate });
-        }
-        if (settings.contains(g + "/systemPrompt")) {
-            const QString systemPrompt = settings.value(g + "/systemPrompt").toString();
-            data.append({ ModelList::SystemPromptRole, systemPrompt });
-        }
         if (settings.contains(g + "/chatNamePrompt")) {
             const QString chatNamePrompt = settings.value(g + "/chatNamePrompt").toString();
             data.append({ ModelList::ChatNamePromptRole, chatNamePrompt });

gpt4all-chat/src/modellist.h

@@ -5,12 +5,14 @@
 #include <QByteArray>
 #include <QDateTime>
 #include <QHash>
+#include <QLatin1StringView>
 #include <QList>
 #include <QMutex>
 #include <QNetworkAccessManager>
 #include <QNetworkReply>
 #include <QObject>
 #include <QPair>
+#include <QQmlEngine>
 #include <QSortFilterProxyModel>
 #include <QSslError>
 #include <QString>
@@ -19,11 +21,53 @@
 #include <Qt>
 #include <QtGlobal>
 
+#include <optional>
 #include <utility>
 
 using namespace Qt::Literals::StringLiterals;
 
 
+class UpgradeableSetting {
+    Q_GADGET
+    QML_ANONYMOUS
+
+    // NOTE: Unset implies there is neither a value nor a default
+    enum class State { Unset, Legacy, Modern };
+
+    Q_PROPERTY(bool     isSet     READ isSet   )
+    Q_PROPERTY(bool     isLegacy  READ isLegacy)
+    Q_PROPERTY(bool     isModern  READ isModern)
+    Q_PROPERTY(QVariant value READ value) // string or null
+
+public:
+    struct legacy_tag_t { explicit legacy_tag_t() = default; };
+    static inline constexpr legacy_tag_t legacy_tag = legacy_tag_t();
+
+    UpgradeableSetting()                           : m_state(State::Unset ) {}
+    UpgradeableSetting(legacy_tag_t, QString value): m_state(State::Legacy), m_value(std::move(value)) {}
+    UpgradeableSetting(              QString value): m_state(State::Modern), m_value(std::move(value)) {}
+
+    bool     isSet   () const { return m_state != State::Unset;  }
+    bool     isLegacy() const { return m_state == State::Legacy; }
+    bool     isModern() const { return m_state == State::Modern; }
+    QVariant value   () const { return m_state == State::Unset ? QVariant::fromValue(nullptr) : m_value; }
+
+    friend bool operator==(const UpgradeableSetting &a, const UpgradeableSetting &b)
+    { return a.m_state == b.m_state && (a.m_state == State::Unset || a.m_value == b.m_value); }
+
+    // returns std::nullopt if there is a legacy template or it is not set
+    std::optional<QString> asModern() const
+    {
+        if (m_state == State::Modern)
+            return m_value;
+        return std::nullopt;
+    }
+
+private:
+    State   m_state;
+    QString m_value;
+};
+
 struct ModelInfo {
     Q_GADGET
     Q_PROPERTY(QString id READ id WRITE setId)
@@ -69,8 +113,11 @@ struct ModelInfo {
     Q_PROPERTY(int maxGpuLayers READ maxGpuLayers)
     Q_PROPERTY(double repeatPenalty READ repeatPenalty WRITE setRepeatPenalty)
     Q_PROPERTY(int repeatPenaltyTokens READ repeatPenaltyTokens WRITE setRepeatPenaltyTokens)
-    Q_PROPERTY(QString promptTemplate READ promptTemplate WRITE setPromptTemplate)
-    Q_PROPERTY(QString systemPrompt READ systemPrompt WRITE setSystemPrompt)
+    // user-defined chat template and system message must be written through settings because of their legacy compat
+    Q_PROPERTY(QVariant           defaultChatTemplate  READ defaultChatTemplate )
+    Q_PROPERTY(UpgradeableSetting chatTemplate         READ chatTemplate        )
+    Q_PROPERTY(QString            defaultSystemMessage READ defaultSystemMessage)
+    Q_PROPERTY(UpgradeableSetting systemMessage        READ systemMessage       )
     Q_PROPERTY(QString chatNamePrompt READ chatNamePrompt WRITE setChatNamePrompt)
     Q_PROPERTY(QString suggestedFollowUpPrompt READ suggestedFollowUpPrompt WRITE setSuggestedFollowUpPrompt)
     Q_PROPERTY(int likes READ likes WRITE setLikes)
@@ -178,19 +225,22 @@ public:
     void setRepeatPenalty(double p);
     int repeatPenaltyTokens() const;
     void setRepeatPenaltyTokens(int t);
-    QString promptTemplate() const;
-    void setPromptTemplate(const QString &t);
-    QString systemPrompt() const;
-    void setSystemPrompt(const QString &p);
+    QVariant defaultChatTemplate() const;
+    UpgradeableSetting chatTemplate() const;
+    QString defaultSystemMessage() const;
+    UpgradeableSetting systemMessage() const;
     QString chatNamePrompt() const;
     void setChatNamePrompt(const QString &p);
     QString suggestedFollowUpPrompt() const;
     void setSuggestedFollowUpPrompt(const QString &p);
 
+    // Some metadata must be saved to settings because it does not have a meaningful default from some other source.
+    // This is useful for fields such as name, description, and URL.
+    // It is true for any models that have not been installed from models.json.
     bool shouldSaveMetadata() const;
 
 private:
-    QVariantMap getFields() const;
+    QVariant getField(QLatin1StringView name) const;
 
     QString m_id;
     QString m_name;
@@ -216,11 +266,13 @@ private:
     mutable int m_maxGpuLayers        = -1;
     double  m_repeatPenalty           = 1.18;
     int     m_repeatPenaltyTokens     = 64;
-    QString m_promptTemplate          = "### Human:\n%1\n\n### Assistant:\n";
-    QString m_systemPrompt            = "### System:\nYou are an AI assistant who gives a quality response to whatever humans ask of you.\n\n";
+            std::optional<QString> m_chatTemplate;
+    mutable std::optional<QString> m_modelChatTemplate;
+    QString m_systemMessage;
     QString m_chatNamePrompt          = "Describe the above conversation in seven words or less.";
     QString m_suggestedFollowUpPrompt = "Suggest three very short factual follow-up questions that have not been answered yet or cannot be found inspired by the previous conversation and excerpts.";
     friend class MySettings;
+    friend class ModelList;
 };
 Q_DECLARE_METATYPE(ModelInfo)
 
@@ -242,17 +294,33 @@ private:
     bool m_selectable;
 };
 
-class DownloadableModels : public QSortFilterProxyModel
+class GPT4AllDownloadableModels : public QSortFilterProxyModel
 {
     Q_OBJECT
     Q_PROPERTY(int count READ count NOTIFY countChanged)
-    Q_PROPERTY(bool expanded READ isExpanded WRITE setExpanded NOTIFY expandedChanged)
 public:
-    explicit DownloadableModels(QObject *parent);
+    explicit GPT4AllDownloadableModels(QObject *parent);
     int count() const;
 
-    bool isExpanded() const;
-    void setExpanded(bool expanded);
+    Q_INVOKABLE void filter(const QVector<QString> &keywords);
+
+Q_SIGNALS:
+    void countChanged();
+
+protected:
+    bool filterAcceptsRow(int sourceRow, const QModelIndex &sourceParent) const override;
+
+private:
+    QVector<QString> m_keywords;
+};
+
+class HuggingFaceDownloadableModels : public QSortFilterProxyModel
+{
+    Q_OBJECT
+    Q_PROPERTY(int count READ count NOTIFY countChanged)
+public:
+    explicit HuggingFaceDownloadableModels(QObject *parent);
+    int count() const;
 
     Q_INVOKABLE void discoverAndFilter(const QString &discover);
 
@@ -262,11 +330,7 @@ Q_SIGNALS:
 protected:
     bool filterAcceptsRow(int sourceRow, const QModelIndex &sourceParent) const override;
 
-Q_SIGNALS:
-    void expandedChanged(bool expanded);
-
 private:
-    bool m_expanded;
     int m_limit;
     QString m_discoverFilter;
 };
@@ -277,7 +341,8 @@ class ModelList : public QAbstractListModel
     Q_PROPERTY(int count READ count NOTIFY countChanged)
     Q_PROPERTY(InstalledModels* installedModels READ installedModels NOTIFY installedModelsChanged)
     Q_PROPERTY(InstalledModels* selectableModels READ selectableModels NOTIFY selectableModelsChanged)
-    Q_PROPERTY(DownloadableModels* downloadableModels READ downloadableModels NOTIFY downloadableModelsChanged)
+    Q_PROPERTY(GPT4AllDownloadableModels* gpt4AllDownloadableModels READ gpt4AllDownloadableModels CONSTANT)
+    Q_PROPERTY(HuggingFaceDownloadableModels* huggingFaceDownloadableModels READ huggingFaceDownloadableModels CONSTANT)
     Q_PROPERTY(QList<ModelInfo> selectableModelList READ selectableModelList NOTIFY  selectableModelListChanged)
     Q_PROPERTY(bool asyncModelRequestOngoing READ asyncModelRequestOngoing NOTIFY asyncModelRequestOngoingChanged)
     Q_PROPERTY(int discoverLimit READ discoverLimit WRITE setDiscoverLimit NOTIFY discoverLimitChanged)
@@ -340,8 +405,8 @@ public:
         GpuLayersRole,
         RepeatPenaltyRole,
         RepeatPenaltyTokensRole,
-        PromptTemplateRole,
-        SystemPromptRole,
+        ChatTemplateRole,
+        SystemMessageRole,
         ChatNamePromptRole,
         SuggestedFollowUpPromptRole,
         MinPRole,
@@ -394,8 +459,8 @@ public:
         roles[GpuLayersRole] = "gpuLayers";
         roles[RepeatPenaltyRole] = "repeatPenalty";
         roles[RepeatPenaltyTokensRole] = "repeatPenaltyTokens";
-        roles[PromptTemplateRole] = "promptTemplate";
-        roles[SystemPromptRole] = "systemPrompt";
+        roles[ChatTemplateRole] = "chatTemplate";
+        roles[SystemMessageRole] = "systemMessage";
         roles[ChatNamePromptRole] = "chatNamePrompt";
         roles[SuggestedFollowUpPromptRole] = "suggestedFollowUpPrompt";
         roles[LikesRole] = "likes";
@@ -416,7 +481,7 @@ public:
     bool contains(const QString &id) const;
     bool containsByFilename(const QString &filename) const;
     Q_INVOKABLE ModelInfo modelInfo(const QString &id) const;
-    Q_INVOKABLE ModelInfo modelInfoByFilename(const QString &filename) const;
+    Q_INVOKABLE ModelInfo modelInfoByFilename(const QString &filename, bool allowClone = true) const;
     Q_INVOKABLE bool isUniqueName(const QString &name) const;
     Q_INVOKABLE QString clone(const ModelInfo &model);
     Q_INVOKABLE void removeClone(const ModelInfo &model);
@@ -430,7 +495,8 @@ public:
 
     InstalledModels *installedModels() const { return m_installedModels; }
     InstalledModels *selectableModels() const { return m_selectableModels; }
-    DownloadableModels *downloadableModels() const { return m_downloadableModels; }
+    GPT4AllDownloadableModels *gpt4AllDownloadableModels() const { return m_gpt4AllDownloadableModels; }
+    HuggingFaceDownloadableModels *huggingFaceDownloadableModels() const { return m_huggingFaceDownloadableModels; }
 
     static inline QString toFileSize(quint64 sz) {
         if (sz < 1024) {
@@ -468,7 +534,6 @@ Q_SIGNALS:
     void countChanged();
     void installedModelsChanged();
     void selectableModelsChanged();
-    void downloadableModelsChanged();
     void selectableModelListChanged();
     void asyncModelRequestOngoingChanged();
     void discoverLimitChanged();
@@ -476,15 +541,18 @@ Q_SIGNALS:
     void discoverSortChanged();
     void discoverProgressChanged();
     void discoverInProgressChanged();
+    void modelInfoChanged(const ModelInfo &info);
 
 protected:
     bool eventFilter(QObject *obj, QEvent *ev) override;
 
 private Q_SLOTS:
+    void onDataChanged(const QModelIndex &topLeft, const QModelIndex &bottomRight, const QList<int> &roles);
     void resortModel();
     void updateModelsFromJson();
     void updateModelsFromJsonAsync();
     void updateModelsFromSettings();
+    void maybeUpdateDataForSettings(const ModelInfo &info, bool fromInfo);
     void updateDataForSettings();
     void handleModelsJsonDownloadFinished();
     void handleModelsJsonDownloadErrorOccurred(QNetworkReply::NetworkError code);
@@ -495,6 +563,9 @@ private Q_SLOTS:
     void handleSslErrors(QNetworkReply *reply, const QList<QSslError> &errors);
 
 private:
+    // Return the index of the model with the given id, or -1 if not found.
+    int indexByModelId(const QString &id) const;
+
     void removeInternal(const ModelInfo &model);
     void clearDiscoveredModels();
     bool modelExists(const QString &fileName) const;
@@ -512,7 +583,8 @@ private:
     QNetworkAccessManager m_networkManager;
     InstalledModels *m_installedModels;
     InstalledModels *m_selectableModels;
-    DownloadableModels *m_downloadableModels;
+    GPT4AllDownloadableModels *m_gpt4AllDownloadableModels;
+    HuggingFaceDownloadableModels *m_huggingFaceDownloadableModels;
     QList<ModelInfo*> m_models;
     QHash<QString, ModelInfo*> m_modelMap;
     bool m_asyncModelRequestOngoing;

gpt4all-chat/src/mysettings.cpp

@@ -1,5 +1,8 @@
 #include "mysettings.h"
 
+#include "chatllm.h"
+#include "modellist.h"
+
 #include <gpt4all-backend/llmodel.h>
 
 #include <QDebug>
@@ -29,8 +32,13 @@ static const QStringList suggestionModeNames { "LocalDocsOnly", "On", "Off" };
 static const QStringList chatThemeNames      { "Light", "Dark", "LegacyDark" };
 static const QStringList fontSizeNames       { "Small", "Medium", "Large" };
 
-// FIXME: All of these default strings that are shown in the UI for settings need to be marked as
-// translatable
+// psuedo-enum
+namespace ModelSettingsKey { namespace {
+    auto ChatTemplate   = "chatTemplate"_L1;
+    auto PromptTemplate = "promptTemplate"_L1; // legacy
+    auto SystemMessage  = "systemMessage"_L1;
+    auto SystemPrompt   = "systemPrompt"_L1;   // legacy
+} } // namespace ModelSettingsKey::(anonymous)
 
 namespace defaults {
 
@@ -48,7 +56,7 @@ static const QVariantMap basicDefaults {
     { "fontSize",                 QVariant::fromValue(FontSize::Small) },
     { "lastVersionStarted",       "" },
     { "networkPort",              4891, },
-    { "saveChatsContext",         false },
+    { "systemTray",               false },
     { "serverChat",               false },
     { "userDefaultModel",         "Application default" },
     { "suggestionMode",           QVariant::fromValue(SuggestionMode::LocalDocsOnly) },
@@ -59,7 +67,6 @@ static const QVariantMap basicDefaults {
     { "localdocs/useRemoteEmbed", false },
     { "localdocs/nomicAPIKey",    "" },
     { "localdocs/embedDevice",    "Auto" },
-    { "localdocs/automaticUpdate",false },
     { "network/attribution",      "" },
 };
 
@@ -147,6 +154,11 @@ static QStringList getUiLanguages(const QString &modelPath)
     return languageList;
 }
 
+static QString modelSettingName(const ModelInfo &info, auto &&name)
+{
+    return u"model-%1/%2"_s.arg(info.id(), name);
+}
+
 class MyPrivateSettings: public MySettings { };
 Q_GLOBAL_STATIC(MyPrivateSettings, settingsInstance)
 MySettings *MySettings::globalInstance()
@@ -162,6 +174,34 @@ MySettings::MySettings()
 {
 }
 
+QVariant MySettings::checkJinjaTemplateError(const QString &tmpl)
+{
+    if (auto err = ChatLLM::checkJinjaTemplateError(tmpl.toStdString()))
+        return QString::fromStdString(*err);
+    return QVariant::fromValue(nullptr);
+}
+
+// Unset settings come from ModelInfo. Listen for changes so we can emit our own setting-specific signals.
+void MySettings::onModelInfoChanged(const QModelIndex &topLeft, const QModelIndex &bottomRight, const QList<int> &roles)
+{
+    auto settingChanged = [&](const auto &info, auto role, const auto &name) {
+        return (roles.isEmpty() || roles.contains(role)) && !m_settings.contains(modelSettingName(info, name));
+    };
+
+    auto &modelList = dynamic_cast<const ModelList &>(*QObject::sender());
+    for (int row = topLeft.row(); row <= bottomRight.row(); row++) {
+        using enum ModelList::Roles;
+        using namespace ModelSettingsKey;
+        auto index = topLeft.siblingAtRow(row);
+        if (auto info = modelList.modelInfo(index.data(IdRole).toString()); !info.id().isNull()) {
+            if (settingChanged(info, ChatTemplateRole, ChatTemplate))
+                emit chatTemplateChanged(info, /*fromInfo*/ true);
+            if (settingChanged(info, SystemMessageRole, SystemMessage))
+                emit systemMessageChanged(info, /*fromInfo*/ true);
+        }
+    }
+}
+
 QVariant MySettings::getBasicSetting(const QString &name) const
 {
     return m_settings.value(name, basicDefaults.value(name));
@@ -194,8 +234,8 @@ void MySettings::restoreModelDefaults(const ModelInfo &info)
     setModelGpuLayers(info, info.m_gpuLayers);
     setModelRepeatPenalty(info, info.m_repeatPenalty);
     setModelRepeatPenaltyTokens(info, info.m_repeatPenaltyTokens);
-    setModelPromptTemplate(info, info.m_promptTemplate);
-    setModelSystemPrompt(info, info.m_systemPrompt);
+    resetModelChatTemplate (info);
+    resetModelSystemMessage(info);
     setModelChatNamePrompt(info, info.m_chatNamePrompt);
     setModelSuggestedFollowUpPrompt(info, info.m_suggestedFollowUpPrompt);
 }
@@ -206,7 +246,7 @@ void MySettings::restoreApplicationDefaults()
     setFontSize(basicDefaults.value("fontSize").value<FontSize>());
     setDevice(defaults::device);
     setThreadCount(defaults::threadCount);
-    setSaveChatsContext(basicDefaults.value("saveChatsContext").toBool());
+    setSystemTray(basicDefaults.value("systemTray").toBool());
     setServerChat(basicDefaults.value("serverChat").toBool());
     setNetworkPort(basicDefaults.value("networkPort").toInt());
     setModelPath(defaultLocalModelsPath());
@@ -225,7 +265,6 @@ void MySettings::restoreLocalDocsDefaults()
     setLocalDocsUseRemoteEmbed(basicDefaults.value("localdocs/useRemoteEmbed").toBool());
     setLocalDocsNomicAPIKey(basicDefaults.value("localdocs/nomicAPIKey").toString());
     setLocalDocsEmbedDevice(basicDefaults.value("localdocs/embedDevice").toString());
-    setLocalDocsAutomaticUpdate(basicDefaults.value("localdocs/automaticUpdate").toBool());
 }
 
 void MySettings::eraseModel(const ModelInfo &info)
@@ -252,29 +291,37 @@ void MySettings::setModelName(const ModelInfo &info, const QString &value, bool
         emit nameChanged(info);
 }
 
-static QString modelSettingName(const ModelInfo &info, const QString &name)
+QVariant MySettings::getModelSetting(QLatin1StringView name, const ModelInfo &info) const
 {
-    return u"model-%1/%2"_s.arg(info.id(), name);
+    QLatin1StringView nameL1(name);
+    return m_settings.value(modelSettingName(info, nameL1), info.getField(nameL1));
 }
 
-QVariant MySettings::getModelSetting(const QString &name, const ModelInfo &info) const
+QVariant MySettings::getModelSetting(const char *name, const ModelInfo &info) const
 {
-    return m_settings.value(modelSettingName(info, name), info.getFields().value(name));
+    return getModelSetting(QLatin1StringView(name), info);
 }
 
-void MySettings::setModelSetting(const QString &name, const ModelInfo &info, const QVariant &value, bool force,
+void MySettings::setModelSetting(QLatin1StringView name, const ModelInfo &info, const QVariant &value, bool force,
                                  bool signal)
 {
     if (!force && (info.id().isEmpty() || getModelSetting(name, info) == value))
         return;
 
-    QString settingName = modelSettingName(info, name);
-    if (info.getFields().value(name) == value && !info.shouldSaveMetadata())
+    QLatin1StringView nameL1(name);
+    QString settingName = modelSettingName(info, nameL1);
+    if (info.getField(nameL1) == value && !info.shouldSaveMetadata())
         m_settings.remove(settingName);
     else
         m_settings.setValue(settingName, value);
     if (signal && !force)
-        QMetaObject::invokeMethod(this, u"%1Changed"_s.arg(name).toLatin1().constData(), Q_ARG(ModelInfo, info));
+        QMetaObject::invokeMethod(this, u"%1Changed"_s.arg(nameL1).toLatin1().constData(), Q_ARG(ModelInfo, info));
+}
+
+void MySettings::setModelSetting(const char *name, const ModelInfo &info, const QVariant &value, bool force,
+                                 bool signal)
+{
+    setModelSetting(QLatin1StringView(name), info, value, force, signal);
 }
 
 QString   MySettings::modelFilename               (const ModelInfo &info) const { return getModelSetting("filename",                info).toString(); }
@@ -297,11 +344,68 @@ int       MySettings::modelContextLength          (const ModelInfo &info) const
 int       MySettings::modelGpuLayers              (const ModelInfo &info) const { return getModelSetting("gpuLayers",               info).toInt(); }
 double    MySettings::modelRepeatPenalty          (const ModelInfo &info) const { return getModelSetting("repeatPenalty",           info).toDouble(); }
 int       MySettings::modelRepeatPenaltyTokens    (const ModelInfo &info) const { return getModelSetting("repeatPenaltyTokens",     info).toInt(); }
-QString   MySettings::modelPromptTemplate         (const ModelInfo &info) const { return getModelSetting("promptTemplate",          info).toString(); }
-QString   MySettings::modelSystemPrompt           (const ModelInfo &info) const { return getModelSetting("systemPrompt",            info).toString(); }
 QString   MySettings::modelChatNamePrompt         (const ModelInfo &info) const { return getModelSetting("chatNamePrompt",          info).toString(); }
 QString   MySettings::modelSuggestedFollowUpPrompt(const ModelInfo &info) const { return getModelSetting("suggestedFollowUpPrompt", info).toString(); }
 
+auto MySettings::getUpgradeableModelSetting(
+    const ModelInfo &info, QLatin1StringView legacyKey, QLatin1StringView newKey
+) const -> UpgradeableSetting
+{
+    if (info.id().isEmpty()) {
+        qWarning("%s: got null model", Q_FUNC_INFO);
+        return {};
+    }
+
+    auto value = m_settings.value(modelSettingName(info, legacyKey));
+    if (value.isValid())
+        return { UpgradeableSetting::legacy_tag, value.toString() };
+
+    value = getModelSetting(newKey, info);
+    if (!value.isNull())
+        return value.toString();
+    return {}; // neither a default nor an override
+}
+
+bool MySettings::isUpgradeableModelSettingSet(
+    const ModelInfo &info, QLatin1StringView legacyKey, QLatin1StringView newKey
+) const
+{
+    if (info.id().isEmpty()) {
+        qWarning("%s: got null model", Q_FUNC_INFO);
+        return false;
+    }
+
+    if (m_settings.contains(modelSettingName(info, legacyKey)))
+        return true;
+
+    // NOTE: unlike getUpgradeableSetting(), this ignores the default
+    return m_settings.contains(modelSettingName(info, newKey));
+}
+
+auto MySettings::modelChatTemplate(const ModelInfo &info) const -> UpgradeableSetting
+{
+    using namespace ModelSettingsKey;
+    return getUpgradeableModelSetting(info, PromptTemplate, ChatTemplate);
+}
+
+bool MySettings::isModelChatTemplateSet(const ModelInfo &info) const
+{
+    using namespace ModelSettingsKey;
+    return isUpgradeableModelSettingSet(info, PromptTemplate, ChatTemplate);
+}
+
+auto MySettings::modelSystemMessage(const ModelInfo &info) const -> UpgradeableSetting
+{
+    using namespace ModelSettingsKey;
+    return getUpgradeableModelSetting(info, SystemPrompt, SystemMessage);
+}
+
+bool MySettings::isModelSystemMessageSet(const ModelInfo &info) const
+{
+    using namespace ModelSettingsKey;
+    return isUpgradeableModelSettingSet(info, SystemPrompt, SystemMessage);
+}
+
 void MySettings::setModelFilename(const ModelInfo &info, const QString &value, bool force)
 {
     setModelSetting("filename", info, value, force, true);
@@ -402,14 +506,77 @@ void MySettings::setModelRepeatPenaltyTokens(const ModelInfo &info, int value, b
     setModelSetting("repeatPenaltyTokens", info, value, force, true);
 }
 
-void MySettings::setModelPromptTemplate(const ModelInfo &info, const QString &value, bool force)
-{
-    setModelSetting("promptTemplate", info, value, force, true);
+bool MySettings::setUpgradeableModelSetting(
+    const ModelInfo &info, const QString &value, QLatin1StringView legacyKey, QLatin1StringView newKey
+) {
+    if (info.id().isEmpty()) {
+        qWarning("%s: got null model", Q_FUNC_INFO);
+        return false;
+    }
+
+    auto legacyModelKey = modelSettingName(info, legacyKey);
+    auto newModelKey    = modelSettingName(info, newKey   );
+    bool changed = false;
+    if (m_settings.contains(legacyModelKey)) {
+        m_settings.remove(legacyModelKey);
+        changed = true;
+    }
+    auto oldValue = m_settings.value(newModelKey);
+    if (!oldValue.isValid() || oldValue.toString() != value) {
+        m_settings.setValue(newModelKey, value);
+        changed = true;
+    }
+    return changed;
 }
 
-void MySettings::setModelSystemPrompt(const ModelInfo &info, const QString &value, bool force)
+bool MySettings::resetUpgradeableModelSetting(
+    const ModelInfo &info, QLatin1StringView legacyKey, QLatin1StringView newKey
+) {
+    if (info.id().isEmpty()) {
+        qWarning("%s: got null model", Q_FUNC_INFO);
+        return false;
+    }
+
+    auto legacyModelKey = modelSettingName(info, legacyKey);
+    auto newModelKey    = modelSettingName(info, newKey   );
+    bool changed        = false;
+    if (m_settings.contains(legacyModelKey)) {
+        m_settings.remove(legacyModelKey);
+        changed = true;
+    }
+    if (m_settings.contains(newModelKey)) {
+        m_settings.remove(newModelKey);
+        changed = true;
+    }
+    return changed;
+}
+
+void MySettings::setModelChatTemplate(const ModelInfo &info, const QString &value)
 {
-    setModelSetting("systemPrompt", info, value, force, true);
+    using namespace ModelSettingsKey;
+    if (setUpgradeableModelSetting(info, value, PromptTemplate, ChatTemplate))
+        emit chatTemplateChanged(info);
+}
+
+void MySettings::resetModelChatTemplate(const ModelInfo &info)
+{
+    using namespace ModelSettingsKey;
+    if (resetUpgradeableModelSetting(info, PromptTemplate, ChatTemplate))
+        emit chatTemplateChanged(info);
+}
+
+void MySettings::setModelSystemMessage(const ModelInfo &info, const QString &value)
+{
+    using namespace ModelSettingsKey;
+    if (setUpgradeableModelSetting(info, value, SystemPrompt, SystemMessage))
+        emit systemMessageChanged(info);
+}
+
+void MySettings::resetModelSystemMessage(const ModelInfo &info)
+{
+    using namespace ModelSettingsKey;
+    if (resetUpgradeableModelSetting(info, SystemPrompt, SystemMessage))
+        emit systemMessageChanged(info);
 }
 
 void MySettings::setModelChatNamePrompt(const ModelInfo &info, const QString &value, bool force)
@@ -445,7 +612,7 @@ void MySettings::setThreadCount(int value)
     emit threadCountChanged();
 }
 
-bool        MySettings::saveChatsContext() const        { return getBasicSetting("saveChatsContext"        ).toBool(); }
+bool        MySettings::systemTray() const              { return getBasicSetting("systemTray"              ).toBool(); }
 bool        MySettings::serverChat() const              { return getBasicSetting("serverChat"              ).toBool(); }
 int         MySettings::networkPort() const             { return getBasicSetting("networkPort"             ).toInt(); }
 QString     MySettings::userDefaultModel() const        { return getBasicSetting("userDefaultModel"        ).toString(); }
@@ -457,14 +624,13 @@ QStringList MySettings::localDocsFileExtensions() const { return getBasicSetting
 bool        MySettings::localDocsUseRemoteEmbed() const { return getBasicSetting("localdocs/useRemoteEmbed").toBool(); }
 QString     MySettings::localDocsNomicAPIKey() const    { return getBasicSetting("localdocs/nomicAPIKey"   ).toString(); }
 QString     MySettings::localDocsEmbedDevice() const    { return getBasicSetting("localdocs/embedDevice"   ).toString(); }
-bool        MySettings::localDocsAutomaticUpdate() const{ return getBasicSetting("localdocs/automaticUpdate").toBool(); }
 QString     MySettings::networkAttribution() const      { return getBasicSetting("network/attribution"     ).toString(); }
 
 ChatTheme      MySettings::chatTheme() const      { return ChatTheme     (getEnumSetting("chatTheme", chatThemeNames)); }
 FontSize       MySettings::fontSize() const       { return FontSize      (getEnumSetting("fontSize",  fontSizeNames)); }
 SuggestionMode MySettings::suggestionMode() const { return SuggestionMode(getEnumSetting("suggestionMode", suggestionModeNames)); }
 
-void MySettings::setSaveChatsContext(bool value)                      { setBasicSetting("saveChatsContext",         value); }
+void MySettings::setSystemTray(bool value)                            { setBasicSetting("systemTray",               value); }
 void MySettings::setServerChat(bool value)                            { setBasicSetting("serverChat",               value); }
 void MySettings::setNetworkPort(int value)                            { setBasicSetting("networkPort",              value); }
 void MySettings::setUserDefaultModel(const QString &value)            { setBasicSetting("userDefaultModel",         value); }
@@ -476,7 +642,6 @@ void MySettings::setLocalDocsFileExtensions(const QStringList &value) { setBasic
 void MySettings::setLocalDocsUseRemoteEmbed(bool value)               { setBasicSetting("localdocs/useRemoteEmbed", value, "localDocsUseRemoteEmbed"); }
 void MySettings::setLocalDocsNomicAPIKey(const QString &value)        { setBasicSetting("localdocs/nomicAPIKey",    value, "localDocsNomicAPIKey"); }
 void MySettings::setLocalDocsEmbedDevice(const QString &value)        { setBasicSetting("localdocs/embedDevice",    value, "localDocsEmbedDevice"); }
-void MySettings::setLocalDocsAutomaticUpdate(bool value)              { setBasicSetting("localdocs/automaticUpdate",value, "localDocsAutomaticUpdate"); }
 void MySettings::setNetworkAttribution(const QString &value)          { setBasicSetting("network/attribution",      value, "networkAttribution"); }
 
 void MySettings::setChatTheme(ChatTheme value)           { setBasicSetting("chatTheme",      chatThemeNames     .value(int(value))); }

gpt4all-chat/src/mysettings.h

@@ -4,6 +4,9 @@
 #include "modellist.h" // IWYU pragma: keep
 
 #include <QDateTime>
+#include <QLatin1StringView>
+#include <QList>
+#include <QModelIndex>
 #include <QObject>
 #include <QSettings>
 #include <QString>
@@ -48,7 +51,7 @@ class MySettings : public QObject
 {
     Q_OBJECT
     Q_PROPERTY(int threadCount READ threadCount WRITE setThreadCount NOTIFY threadCountChanged)
-    Q_PROPERTY(bool saveChatsContext READ saveChatsContext WRITE setSaveChatsContext NOTIFY saveChatsContextChanged)
+    Q_PROPERTY(bool systemTray READ systemTray WRITE setSystemTray NOTIFY systemTrayChanged)
     Q_PROPERTY(bool serverChat READ serverChat WRITE setServerChat NOTIFY serverChatChanged)
     Q_PROPERTY(QString modelPath READ modelPath WRITE setModelPath NOTIFY modelPathChanged)
     Q_PROPERTY(QString userDefaultModel READ userDefaultModel WRITE setUserDefaultModel NOTIFY userDefaultModelChanged)
@@ -64,7 +67,6 @@ class MySettings : public QObject
     Q_PROPERTY(bool localDocsUseRemoteEmbed READ localDocsUseRemoteEmbed WRITE setLocalDocsUseRemoteEmbed NOTIFY localDocsUseRemoteEmbedChanged)
     Q_PROPERTY(QString localDocsNomicAPIKey READ localDocsNomicAPIKey WRITE setLocalDocsNomicAPIKey NOTIFY localDocsNomicAPIKeyChanged)
     Q_PROPERTY(QString localDocsEmbedDevice READ localDocsEmbedDevice WRITE setLocalDocsEmbedDevice NOTIFY localDocsEmbedDeviceChanged)
-    Q_PROPERTY(bool localDocsAutomaticUpdate READ localDocsAutomaticUpdate WRITE setLocalDocsAutomaticUpdate NOTIFY localDocsAutomaticUpdateChanged)
     Q_PROPERTY(QString networkAttribution READ networkAttribution WRITE setNetworkAttribution NOTIFY networkAttributionChanged)
     Q_PROPERTY(bool networkIsActive READ networkIsActive WRITE setNetworkIsActive NOTIFY networkIsActiveChanged)
     Q_PROPERTY(bool networkUsageStatsActive READ networkUsageStatsActive WRITE setNetworkUsageStatsActive NOTIFY networkUsageStatsActiveChanged)
@@ -75,9 +77,18 @@ class MySettings : public QObject
     Q_PROPERTY(SuggestionMode suggestionMode READ suggestionMode WRITE setSuggestionMode NOTIFY suggestionModeChanged)
     Q_PROPERTY(QStringList uiLanguages MEMBER m_uiLanguages CONSTANT)
 
+private:
+    explicit MySettings();
+    ~MySettings() override = default;
+
+public Q_SLOTS:
+    void onModelInfoChanged(const QModelIndex &topLeft, const QModelIndex &bottomRight, const QList<int> &roles = {});
+
 public:
     static MySettings *globalInstance();
 
+    Q_INVOKABLE static QVariant checkJinjaTemplateError(const QString &tmpl);
+
     // Restore methods
     Q_INVOKABLE void restoreModelDefaults(const ModelInfo &info);
     Q_INVOKABLE void restoreApplicationDefaults();
@@ -125,10 +136,14 @@ public:
     Q_INVOKABLE void setModelRepeatPenalty(const ModelInfo &info, double value, bool force = false);
     int modelRepeatPenaltyTokens(const ModelInfo &info) const;
     Q_INVOKABLE void setModelRepeatPenaltyTokens(const ModelInfo &info, int value, bool force = false);
-    QString modelPromptTemplate(const ModelInfo &info) const;
-    Q_INVOKABLE void setModelPromptTemplate(const ModelInfo &info, const QString &value, bool force = false);
-    QString modelSystemPrompt(const ModelInfo &info) const;
-    Q_INVOKABLE void setModelSystemPrompt(const ModelInfo &info, const QString &value, bool force = false);
+    auto modelChatTemplate(const ModelInfo &info) const -> UpgradeableSetting;
+    Q_INVOKABLE bool isModelChatTemplateSet(const ModelInfo &info) const;
+    Q_INVOKABLE void setModelChatTemplate(const ModelInfo &info, const QString &value);
+    Q_INVOKABLE void resetModelChatTemplate(const ModelInfo &info);
+    auto modelSystemMessage(const ModelInfo &info) const -> UpgradeableSetting;
+    Q_INVOKABLE bool isModelSystemMessageSet(const ModelInfo &info) const;
+    Q_INVOKABLE void setModelSystemMessage(const ModelInfo &info, const QString &value);
+    Q_INVOKABLE void resetModelSystemMessage(const ModelInfo &info);
     int modelContextLength(const ModelInfo &info) const;
     Q_INVOKABLE void setModelContextLength(const ModelInfo &info, int value, bool force = false);
     int modelGpuLayers(const ModelInfo &info) const;
@@ -141,8 +156,8 @@ public:
     // Application settings
     int threadCount() const;
     void setThreadCount(int value);
-    bool saveChatsContext() const;
-    void setSaveChatsContext(bool value);
+    bool systemTray() const;
+    void setSystemTray(bool value);
     bool serverChat() const;
     void setServerChat(bool value);
     QString modelPath();
@@ -186,8 +201,6 @@ public:
     void setLocalDocsNomicAPIKey(const QString &value);
     QString localDocsEmbedDevice() const;
     void setLocalDocsEmbedDevice(const QString &value);
-    bool localDocsAutomaticUpdate() const;
-    void setLocalDocsAutomaticUpdate(bool value);
 
     // Network settings
     QString networkAttribution() const;
@@ -215,12 +228,12 @@ Q_SIGNALS:
     void gpuLayersChanged(const ModelInfo &info);
     void repeatPenaltyChanged(const ModelInfo &info);
     void repeatPenaltyTokensChanged(const ModelInfo &info);
-    void promptTemplateChanged(const ModelInfo &info);
-    void systemPromptChanged(const ModelInfo &info);
+    void chatTemplateChanged(const ModelInfo &info, bool fromInfo = false);
+    void systemMessageChanged(const ModelInfo &info, bool fromInfo = false);
     void chatNamePromptChanged(const ModelInfo &info);
     void suggestedFollowUpPromptChanged(const ModelInfo &info);
     void threadCountChanged();
-    void saveChatsContextChanged();
+    void systemTrayChanged();
     void serverChatChanged();
     void modelPathChanged();
     void userDefaultModelChanged();
@@ -235,7 +248,6 @@ Q_SIGNALS:
     void localDocsUseRemoteEmbedChanged();
     void localDocsNomicAPIKeyChanged();
     void localDocsEmbedDeviceChanged();
-    void localDocsAutomaticUpdateChanged();
     void networkAttributionChanged();
     void networkIsActiveChanged();
     void networkPortChanged();
@@ -245,6 +257,30 @@ Q_SIGNALS:
     void suggestionModeChanged();
     void languageAndLocaleChanged();
 
+private:
+    QVariant getBasicSetting(const QString &name) const;
+    void setBasicSetting(const QString &name, const QVariant &value, std::optional<QString> signal = std::nullopt);
+    int getEnumSetting(const QString &setting, const QStringList &valueNames) const;
+    QVariant getModelSetting(QLatin1StringView name, const ModelInfo &info) const;
+    QVariant getModelSetting(const char *name, const ModelInfo &info) const;
+    void setModelSetting(QLatin1StringView name, const ModelInfo &info, const QVariant &value, bool force,
+                         bool signal = false);
+    void setModelSetting(const char *name, const ModelInfo &info, const QVariant &value, bool force,
+                         bool signal = false);
+    auto getUpgradeableModelSetting(
+        const ModelInfo &info, QLatin1StringView legacyKey, QLatin1StringView newKey
+    ) const -> UpgradeableSetting;
+    bool isUpgradeableModelSettingSet(
+        const ModelInfo &info, QLatin1StringView legacyKey, QLatin1StringView newKey
+    ) const;
+    bool setUpgradeableModelSetting(
+        const ModelInfo &info, const QString &value, QLatin1StringView legacyKey, QLatin1StringView newKey
+    );
+    bool resetUpgradeableModelSetting(
+        const ModelInfo &info, QLatin1StringView legacyKey, QLatin1StringView newKey
+    );
+    QString filePathForLocale(const QLocale &locale);
+
 private:
     QSettings m_settings;
     bool m_forceMetal;
@@ -253,18 +289,7 @@ private:
     const QStringList m_uiLanguages;
     std::unique_ptr<QTranslator> m_translator;
 
-private:
-    explicit MySettings();
-    ~MySettings() {}
     friend class MyPrivateSettings;
-
-    QVariant getBasicSetting(const QString &name) const;
-    void setBasicSetting(const QString &name, const QVariant &value, std::optional<QString> signal = std::nullopt);
-    int getEnumSetting(const QString &setting, const QStringList &valueNames) const;
-    QVariant getModelSetting(const QString &name, const ModelInfo &info) const;
-    void setModelSetting(const QString &name, const ModelInfo &info, const QVariant &value, bool force,
-                         bool signal = false);
-    QString filePathForLocale(const QLocale &locale);
 };
 
 #endif // MYSETTINGS_H

gpt4all-chat/src/network.cpp

@@ -8,6 +8,7 @@
 #include "localdocsmodel.h"
 #include "modellist.h"
 #include "mysettings.h"
+#include "utils.h"
 
 #include <gpt4all-backend/llmodel.h>
 
@@ -192,11 +193,14 @@ bool Network::packageAndSendJson(const QString &ingestId, const QString &json)
         return false;
     }
 
+    auto *currentChat = ChatListModel::globalInstance()->currentChat();
+    Q_ASSERT(currentChat);
+    auto modelInfo = currentChat->modelInfo();
+
     Q_ASSERT(doc.isObject());
-    Q_ASSERT(ChatListModel::globalInstance()->currentChat());
     QJsonObject object = doc.object();
     object.insert("source", "gpt4all-chat");
-    object.insert("agent_id", ChatListModel::globalInstance()->currentChat()->modelInfo().filename());
+    object.insert("agent_id", modelInfo.filename());
     object.insert("submitter_id", m_uniqueId);
     object.insert("ingest_id", ingestId);
 
@@ -204,8 +208,9 @@ bool Network::packageAndSendJson(const QString &ingestId, const QString &json)
     if (!attribution.isEmpty())
         object.insert("network/attribution", attribution);
 
-    QString promptTemplate = ChatListModel::globalInstance()->currentChat()->modelInfo().promptTemplate();
-    object.insert("prompt_template", promptTemplate);
+    if (!modelInfo.id().isNull())
+        if (auto tmpl = modelInfo.chatTemplate().asModern())
+            object.insert("chat_template"_L1, *tmpl);
 
     QJsonDocument newDoc;
     newDoc.setObject(object);
@@ -358,7 +363,8 @@ void Network::sendStartup()
 
 void Network::trackChatEvent(const QString &ev, QVariantMap props)
 {
-    const auto &curChat = ChatListModel::globalInstance()->currentChat();
+    auto *curChat = ChatListModel::globalInstance()->currentChat();
+    Q_ASSERT(curChat);
     if (!props.contains("model"))
         props.insert("model", curChat->modelInfo().filename());
     props.insert("device_backend", curChat->deviceBackend());
@@ -366,7 +372,7 @@ void Network::trackChatEvent(const QString &ev, QVariantMap props)
     props.insert("doc_collections_enabled", curChat->collectionList().count());
     props.insert("doc_collections_total", LocalDocs::globalInstance()->localDocsModel()->rowCount());
     props.insert("datalake_active", MySettings::globalInstance()->networkIsActive());
-    props.insert("using_server", ChatListModel::globalInstance()->currentChat()->isServer());
+    props.insert("using_server", curChat->isServer());
     trackEvent(ev, props);
 }
 

gpt4all-chat/src/server.cpp

@@ -1,6 +1,7 @@
 #include "server.h"
 
 #include "chat.h"
+#include "chatmodel.h"
 #include "modellist.h"
 #include "mysettings.h"
 #include "utils.h"
@@ -37,6 +38,10 @@
 #include <unordered_map>
 #include <utility>
 
+#if QT_VERSION >= QT_VERSION_CHECK(6, 8, 0)
+#   include <QTcpServer>
+#endif
+
 using namespace std::string_literals;
 using namespace Qt::Literals::StringLiterals;
 
@@ -309,11 +314,8 @@ const std::unordered_map<BaseCompletionRequest::Type, const char *> BaseCompleti
 class ChatRequest : public BaseCompletionRequest {
 public:
     struct Message {
-        enum class Role : uint8_t {
-            User,
-            Assistant,
-        };
-        Role role;
+        enum class Role { System, User, Assistant };
+        Role    role;
         QString content;
     };
 
@@ -345,7 +347,6 @@ protected:
         this->messages.clear();
         {
             QCborArray arr = value.toArray();
-            Message::Role nextRole = Message::Role::User;
             for (qsizetype i = 0; i < arr.size(); i++) {
                 const auto &elem = arr[i];
                 if (!elem.isMap())
@@ -356,9 +357,9 @@ protected:
                 QCborMap msg = elem.toMap();
                 Message res;
                 QString role = takeValue(msg, "role", String, /*required*/ true).toString();
-                if (role == u"system"_s)
-                    continue; // FIXME(jared): don't ignore these
-                if (role == u"user"_s) {
+                if (role == u"system"_s) {
+                    res.role = Message::Role::System;
+                } else if (role == u"user"_s) {
                     res.role = Message::Role::User;
                 } else if (role == u"assistant"_s) {
                     res.role = Message::Role::Assistant;
@@ -370,13 +371,7 @@ protected:
                     ));
                 }
                 res.content = takeValue(msg, "content", String, /*required*/ true).toString();
-                if (res.role != nextRole)
-                    throw InvalidRequestError(fmt::format(
-                        "Invalid 'messages[{}].role': did not expect '{}' here", i, role
-                    ));
                 this->messages.append(res);
-                nextRole = res.role == Message::Role::User ? Message::Role::Assistant
-                                                           : Message::Role::User;
 
                 if (!msg.isEmpty())
                     throw InvalidRequestError(fmt::format(
@@ -435,7 +430,6 @@ T &parseRequest(T &request, QJsonObject &&obj)
 Server::Server(Chat *chat)
     : ChatLLM(chat, true /*isServer*/)
     , m_chat(chat)
-    , m_server(nullptr)
 {
     connect(this, &Server::threadStarted, this, &Server::start);
     connect(this, &Server::databaseResultsChanged, this, &Server::handleDatabaseResultsChanged);
@@ -457,10 +451,23 @@ static QJsonObject requestFromJson(const QByteArray &request)
 void Server::start()
 {
     m_server = std::make_unique<QHttpServer>(this);
-    if (!m_server->listen(QHostAddress::LocalHost, MySettings::globalInstance()->networkPort())) {
-        qWarning() << "ERROR: Unable to start the server";
+#if QT_VERSION >= QT_VERSION_CHECK(6, 8, 0)
+    auto *tcpServer = new QTcpServer(m_server.get());
+#else
+    auto *tcpServer = m_server.get();
+#endif
+
+    auto port = MySettings::globalInstance()->networkPort();
+    if (!tcpServer->listen(QHostAddress::LocalHost, port)) {
+        qWarning() << "Server ERROR: Failed to listen on port" << port;
         return;
     }
+#if QT_VERSION >= QT_VERSION_CHECK(6, 8, 0)
+    if (!m_server->bind(tcpServer)) {
+        qWarning() << "Server ERROR: Failed to HTTP server to socket" << port;
+        return;
+    }
+#endif
 
     m_server->route("/v1/models", QHttpServerRequest::Method::Get,
         [](const QHttpServerRequest &) {
@@ -600,13 +607,21 @@ void Server::start()
         }
     );
 
-    m_server->afterRequest([] (QHttpServerResponse &&resp) {
+#if QT_VERSION >= QT_VERSION_CHECK(6, 8, 0)
+    m_server->addAfterRequestHandler(this, [](const QHttpServerRequest &req, QHttpServerResponse &resp) {
+        Q_UNUSED(req);
+        auto headers = resp.headers();
+        headers.append("Access-Control-Allow-Origin"_L1, "*"_L1);
+        resp.setHeaders(std::move(headers));
+    });
+#else
+    m_server->afterRequest([](QHttpServerResponse &&resp) {
         resp.addHeader("Access-Control-Allow-Origin", "*");
         return std::move(resp);
     });
+#endif
 
-    connect(this, &Server::requestServerNewPromptResponsePair, m_chat,
-        &Chat::serverNewPromptResponsePair, Qt::BlockingQueuedConnection);
+    connect(this, &Server::requestResetResponseState, m_chat, &Chat::resetResponseState, Qt::BlockingQueuedConnection);
 }
 
 static auto makeError(auto &&...args) -> std::pair<QHttpServerResponse, std::optional<QJsonObject>>
@@ -617,6 +632,10 @@ static auto makeError(auto &&...args) -> std::pair<QHttpServerResponse, std::opt
 auto Server::handleCompletionRequest(const CompletionRequest &request)
     -> std::pair<QHttpServerResponse, std::optional<QJsonObject>>
 {
+    Q_ASSERT(m_chatModel);
+
+    auto *mySettings = MySettings::globalInstance();
+
     ModelInfo modelInfo = ModelList::globalInstance()->defaultModelInfo();
     const QList<ModelInfo> modelList = ModelList::globalInstance()->selectableModelList();
     for (const ModelInfo &info : modelList) {
@@ -629,10 +648,6 @@ auto Server::handleCompletionRequest(const CompletionRequest &request)
         }
     }
 
-    // adds prompt/response items to GUI
-    emit requestServerNewPromptResponsePair(request.prompt); // blocks
-    resetResponse();
-
     // load the new model if necessary
     setShouldBeLoaded(true);
 
@@ -641,47 +656,56 @@ auto Server::handleCompletionRequest(const CompletionRequest &request)
         return makeError(QHttpServerResponder::StatusCode::InternalServerError);
     }
 
+    emit requestResetResponseState(); // blocks
+    qsizetype prevMsgIndex = m_chatModel->count() - 1;
+    if (prevMsgIndex >= 0)
+        m_chatModel->updateCurrentResponse(prevMsgIndex, false);
+
     // NB: this resets the context, regardless of whether this model is already loaded
     if (!loadModel(modelInfo)) {
         std::cerr << "ERROR: couldn't load model " << modelInfo.name().toStdString() << std::endl;
         return makeError(QHttpServerResponder::StatusCode::InternalServerError);
     }
 
-    // FIXME(jared): taking parameters from the UI inhibits reproducibility of results
-    const int  top_k          = modelInfo.topK();
-    const int  n_batch        = modelInfo.promptBatchSize();
-    const auto repeat_penalty = float(modelInfo.repeatPenalty());
-    const int  repeat_last_n  = modelInfo.repeatPenaltyTokens();
+    // add prompt/response items to GUI
+    m_chatModel->appendPrompt(request.prompt);
+    m_chatModel->appendResponse();
 
+    // FIXME(jared): taking parameters from the UI inhibits reproducibility of results
+    LLModel::PromptContext promptCtx {
+        .n_predict      = request.max_tokens,
+        .top_k          = mySettings->modelTopK(modelInfo),
+        .top_p          = request.top_p,
+        .min_p          = request.min_p,
+        .temp           = request.temperature,
+        .n_batch        = mySettings->modelPromptBatchSize(modelInfo),
+        .repeat_penalty = float(mySettings->modelRepeatPenalty(modelInfo)),
+        .repeat_last_n  = mySettings->modelRepeatPenaltyTokens(modelInfo),
+    };
+
+    auto promptUtf8 = request.prompt.toUtf8();
     int promptTokens = 0;
     int responseTokens = 0;
-    QList<QPair<QString, QList<ResultInfo>>> responses;
+    QStringList responses;
     for (int i = 0; i < request.n; ++i) {
-        if (!promptInternal(
-            m_collections,
-            request.prompt,
-            /*promptTemplate*/ u"%1"_s,
-            request.max_tokens,
-            top_k,
-            request.top_p,
-            request.min_p,
-            request.temperature,
-            n_batch,
-            repeat_penalty,
-            repeat_last_n)) {
-
-            std::cerr << "ERROR: couldn't prompt model " << modelInfo.name().toStdString() << std::endl;
+        PromptResult result;
+        try {
+            result = promptInternal(std::string_view(promptUtf8.cbegin(), promptUtf8.cend()),
+                                    promptCtx,
+                                    /*usedLocalDocs*/ false);
+        } catch (const std::exception &e) {
+            m_chatModel->setResponseValue(e.what());
+            m_chatModel->setError();
+            emit responseStopped(0);
             return makeError(QHttpServerResponder::StatusCode::InternalServerError);
         }
-        QString resp = response(/*trim*/ false);
+        QString resp = QString::fromUtf8(result.response);
         if (request.echo)
             resp = request.prompt + resp;
-        responses.append({resp, m_databaseResults});
-        if (!promptTokens)
-            promptTokens = m_promptTokens;
-        responseTokens += m_promptResponseTokens - m_promptTokens;
-        if (i < request.n - 1)
-            resetResponse();
+        responses << resp;
+        if (i == 0)
+            promptTokens = result.promptTokens;
+        responseTokens += result.responseTokens;
     }
 
     QJsonObject responseObject {
@@ -692,25 +716,13 @@ auto Server::handleCompletionRequest(const CompletionRequest &request)
     };
 
     QJsonArray choices;
-    {
-        int index = 0;
-        for (const auto &r : responses) {
-            QString result = r.first;
-            QList<ResultInfo> infos = r.second;
-            QJsonObject choice {
-                { "text",          result                                                   },
-                { "index",         index++                                                  },
-                { "logprobs",      QJsonValue::Null                                         },
-                { "finish_reason", responseTokens == request.max_tokens ? "length" : "stop" },
-            };
-            if (MySettings::globalInstance()->localDocsShowReferences()) {
-                QJsonArray references;
-                for (const auto &ref : infos)
-                    references.append(resultToJson(ref));
-                choice.insert("references", references.isEmpty() ? QJsonValue::Null : QJsonValue(references));
-            }
-            choices.append(choice);
-        }
+    for (qsizetype i = 0; auto &resp : std::as_const(responses)) {
+        choices << QJsonObject {
+            { "text",          resp                                                     },
+            { "index",         i++                                                      },
+            { "logprobs",      QJsonValue::Null                                         },
+            { "finish_reason", responseTokens == request.max_tokens ? "length" : "stop" },
+        };
     }
 
     responseObject.insert("choices", choices);
@@ -726,6 +738,8 @@ auto Server::handleCompletionRequest(const CompletionRequest &request)
 auto Server::handleChatRequest(const ChatRequest &request)
     -> std::pair<QHttpServerResponse, std::optional<QJsonObject>>
 {
+    auto *mySettings = MySettings::globalInstance();
+
     ModelInfo modelInfo = ModelList::globalInstance()->defaultModelInfo();
     const QList<ModelInfo> modelList = ModelList::globalInstance()->selectableModelList();
     for (const ModelInfo &info : modelList) {
@@ -746,83 +760,59 @@ auto Server::handleChatRequest(const ChatRequest &request)
         return makeError(QHttpServerResponder::StatusCode::InternalServerError);
     }
 
+    emit requestResetResponseState(); // blocks
+
     // NB: this resets the context, regardless of whether this model is already loaded
     if (!loadModel(modelInfo)) {
         std::cerr << "ERROR: couldn't load model " << modelInfo.name().toStdString() << std::endl;
         return makeError(QHttpServerResponder::StatusCode::InternalServerError);
     }
 
-    const QString promptTemplate = modelInfo.promptTemplate();
-    const int     top_k          = modelInfo.topK();
-    const int     n_batch        = modelInfo.promptBatchSize();
-    const auto    repeat_penalty = float(modelInfo.repeatPenalty());
-    const int     repeat_last_n  = modelInfo.repeatPenaltyTokens();
+    m_chatModel->updateCurrentResponse(m_chatModel->count() - 1, false);
 
-    int promptTokens = 0;
+    Q_ASSERT(!request.messages.isEmpty());
+
+    // adds prompt/response items to GUI
+    std::vector<MessageInput> messages;
+    for (auto &message : request.messages) {
+        using enum ChatRequest::Message::Role;
+        switch (message.role) {
+            case System:    messages.push_back({ MessageInput::Type::System,   message.content }); break;
+            case User:      messages.push_back({ MessageInput::Type::Prompt,   message.content }); break;
+            case Assistant: messages.push_back({ MessageInput::Type::Response, message.content }); break;
+        }
+    }
+    auto startOffset = m_chatModel->appendResponseWithHistory(messages);
+
+    // FIXME(jared): taking parameters from the UI inhibits reproducibility of results
+    LLModel::PromptContext promptCtx {
+        .n_predict      = request.max_tokens,
+        .top_k          = mySettings->modelTopK(modelInfo),
+        .top_p          = request.top_p,
+        .min_p          = request.min_p,
+        .temp           = request.temperature,
+        .n_batch        = mySettings->modelPromptBatchSize(modelInfo),
+        .repeat_penalty = float(mySettings->modelRepeatPenalty(modelInfo)),
+        .repeat_last_n  = mySettings->modelRepeatPenaltyTokens(modelInfo),
+    };
+
+    int promptTokens   = 0;
     int responseTokens = 0;
     QList<QPair<QString, QList<ResultInfo>>> responses;
-    Q_ASSERT(!request.messages.isEmpty());
-    Q_ASSERT(request.messages.size() % 2 == 1);
-    for (int i = 0; i < request.messages.size() - 2; i += 2) {
-        using enum ChatRequest::Message::Role;
-        auto &user      = request.messages[i];
-        auto &assistant = request.messages[i + 1];
-        Q_ASSERT(user.role      == User);
-        Q_ASSERT(assistant.role == Assistant);
-
-        // adds prompt/response items to GUI
-        emit requestServerNewPromptResponsePair(user.content); // blocks
-        resetResponse();
-
-        if (!promptInternal(
-            {},
-            user.content,
-            promptTemplate,
-            request.max_tokens,
-            top_k,
-            request.top_p,
-            request.min_p,
-            request.temperature,
-            n_batch,
-            repeat_penalty,
-            repeat_last_n,
-            assistant.content)
-        ) {
-            std::cerr << "ERROR: couldn't prompt model " << modelInfo.name().toStdString() << std::endl;
-            return makeError(QHttpServerResponder::StatusCode::InternalServerError);
-        }
-        promptTokens += m_promptResponseTokens; // previous responses are part of current prompt
-    }
-
-    QString lastMessage = request.messages.last().content;
-    // adds prompt/response items to GUI
-    emit requestServerNewPromptResponsePair(lastMessage); // blocks
-    resetResponse();
-
     for (int i = 0; i < request.n; ++i) {
-        if (!promptInternal(
-            m_collections,
-            lastMessage,
-            promptTemplate,
-            request.max_tokens,
-            top_k,
-            request.top_p,
-            request.min_p,
-            request.temperature,
-            n_batch,
-            repeat_penalty,
-            repeat_last_n)
-        ) {
-            std::cerr << "ERROR: couldn't prompt model " << modelInfo.name().toStdString() << std::endl;
+        ChatPromptResult result;
+        try {
+            result = promptInternalChat(m_collections, promptCtx, startOffset);
+        } catch (const std::exception &e) {
+            m_chatModel->setResponseValue(e.what());
+            m_chatModel->setError();
+            emit responseStopped(0);
             return makeError(QHttpServerResponder::StatusCode::InternalServerError);
         }
-        responses.append({response(), m_databaseResults});
-        // FIXME(jared): these are UI counts and do not include framing tokens, which they should
+        responses.emplace_back(result.response, result.databaseResults);
         if (i == 0)
-            promptTokens += m_promptTokens;
-        responseTokens += m_promptResponseTokens - m_promptTokens;
-        if (i != request.n - 1)
-            resetResponse();
+            promptTokens = result.promptTokens;
+        responseTokens += result.responseTokens;
     }
 
     QJsonObject responseObject {

gpt4all-chat/src/server.h

@@ -2,7 +2,6 @@
 #define SERVER_H
 
 #include "chatllm.h"
-#include "chatmodel.h"
 #include "database.h"
 
 #include <QHttpServer>
@@ -33,7 +32,7 @@ public Q_SLOTS:
     void start();
 
 Q_SIGNALS:
-    void requestServerNewPromptResponsePair(const QString &prompt, const QList<PromptAttachment> &attachments = {});
+    void requestResetResponseState();
 
 private:
     auto handleCompletionRequest(const CompletionRequest &request) -> std::pair<QHttpServerResponse, std::optional<QJsonObject>>;

gpt4all-chat/src/tool.cpp

@@ -0,0 +1,74 @@
+#include "tool.h"
+
+#include <jinja2cpp/value.h>
+
+#include <string>
+
+jinja2::Value Tool::jinjaValue() const
+{
+    jinja2::ValuesList paramList;
+    const QList<ToolParamInfo> p = parameters();
+    for (auto &info : p) {
+        std::string typeStr;
+        switch (info.type) {
+        using enum ToolEnums::ParamType;
+        case String:   typeStr = "string"; break;
+        case Number:   typeStr = "number"; break;
+        case Integer:  typeStr = "integer"; break;
+        case Object:   typeStr = "object"; break;
+        case Array:    typeStr = "array"; break;
+        case Boolean:  typeStr = "boolean"; break;
+        case Null:     typeStr = "null"; break;
+        }
+        jinja2::ValuesMap infoMap {
+            { "name", info.name.toStdString() },
+            { "type", typeStr},
+            { "description", info.description.toStdString() },
+            { "required", info.required }
+        };
+        paramList.push_back(infoMap);
+    }
+
+    jinja2::ValuesMap params {
+        { "name", name().toStdString() },
+        { "description", description().toStdString() },
+        { "function", function().toStdString() },
+        { "parameters", paramList },
+        { "symbolicFormat", symbolicFormat().toStdString() },
+        { "examplePrompt", examplePrompt().toStdString() },
+        { "exampleCall", exampleCall().toStdString() },
+        { "exampleReply", exampleReply().toStdString() }
+    };
+    return params;
+}
+
+void ToolCallInfo::serialize(QDataStream &stream, int version)
+{
+    stream << name;
+    stream << params.size();
+    for (auto param : params) {
+        stream << param.name;
+        stream << param.type;
+        stream << param.value;
+    }
+    stream << result;
+    stream << error;
+    stream << errorString;
+}
+
+bool ToolCallInfo::deserialize(QDataStream &stream, int version)
+{
+    stream >> name;
+    qsizetype count;
+    stream >> count;
+    for (int i = 0; i < count; ++i) {
+        ToolParam p;
+        stream >> p.name;
+        stream >> p.type;
+        stream >> p.value;
+    }
+    stream >> result;
+    stream >> error;
+    stream >> errorString;
+    return true;
+}