python: fix CUDA dependency version (#2858)

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/continue_config.yml

@@ -29,7 +29,7 @@ jobs:
       - run: echo "CircleCI pipeline triggered"
   build-offline-chat-installer-macos:
     macos:
-      xcode: 14.0.0
+      xcode: 15.4.0
     steps:
       - checkout
       - run:
@@ -56,29 +56,242 @@ jobs:
           key: macos-qt-cache-v3
           paths:
             - ~/Qt
+      - run:
+          name: Setup Keychain
+          command: |
+            echo $MAC_SIGNING_CERT | base64 --decode > cert.p12
+            security create-keychain -p "$MAC_KEYCHAIN_KEY" sign.keychain
+            security default-keychain -s sign.keychain
+            security unlock-keychain -p "$MAC_KEYCHAIN_KEY" sign.keychain
+            security import cert.p12 -k sign.keychain -P "$MAC_SIGNING_CERT_PWD" -T /usr/bin/codesign
+            security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$MAC_KEYCHAIN_KEY" sign.keychain
       - run:
           name: Build
+          no_output_timeout: 30m
           command: |
             mkdir build
             cd build
             export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
-              -DCMAKE_GENERATOR:STRING=Ninja \
-              -DBUILD_UNIVERSAL=ON \
-              -DMACDEPLOYQT=~/Qt/6.5.1/macos/bin/macdeployqt \
-              -DGPT4ALL_OFFLINE_INSTALLER=ON \
+              -S ../gpt4all-chat -B . -G Ninja \
               -DCMAKE_BUILD_TYPE=Release \
               -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
               -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-              -S ../gpt4all-chat \
-              -B .
+              -DBUILD_UNIVERSAL=ON \
+              -DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
+              -DGGML_METAL_MACOSX_VERSION_MIN=12.6 \
+              -DMACDEPLOYQT=~/Qt/6.5.1/macos/bin/macdeployqt \
+              -DGPT4ALL_OFFLINE_INSTALLER=ON \
+              -DGPT4ALL_SIGN_INSTALL=ON
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package
             mkdir upload
             cp gpt4all-installer-* upload
+      # persist the unsigned installer
       - store_artifacts:
           path: build/upload
+      # add workspace so signing jobs can connect & obtain dmg
+      - persist_to_workspace:
+          root: build
+          # specify path to only include components we want to persist
+          # accross builds
+          paths:
+            - upload
+
+  sign-offline-chat-installer-macos:
+    macos:
+      xcode: 15.4.0
+    steps:
+      - checkout
+      # attach to a workspace containing unsigned dmg
+      - attach_workspace:
+          at: build
+      - run:
+          name: "Setup Keychain"
+          command: |
+            echo $MAC_SIGNING_CERT | base64 --decode > cert.p12
+            security create-keychain -p "$MAC_KEYCHAIN_KEY" sign.keychain
+            security default-keychain -s sign.keychain
+            security unlock-keychain -p "$MAC_KEYCHAIN_KEY" sign.keychain
+            security import cert.p12 -k sign.keychain -P "$MAC_SIGNING_CERT_PWD" -T /usr/bin/codesign
+            security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$MAC_KEYCHAIN_KEY" sign.keychain
+            rm cert.p12
+      - run:
+          name: "Sign App Bundle"
+          command: |
+            python3 -m pip install click
+            python3 gpt4all-chat/cmake/sign_dmg.py --input-dmg build/upload/gpt4all-installer-darwin.dmg --output-dmg build/upload/gpt4all-installer-darwin-signed.dmg --signing-identity "$MAC_SIGNING_CERT_NAME"
+      - run:
+          name: "Sign DMG"
+          command: |
+            codesign --options runtime --timestamp -s "$MAC_SIGNING_CERT_NAME" build/upload/gpt4all-installer-darwin-signed.dmg
+      # add workspace so signing jobs can connect & obtain dmg
+      - persist_to_workspace:
+          root: build
+          # specify path to only include components we want to persist
+          # accross builds
+          paths:
+            - upload
+
+  notarize-offline-chat-installer-macos:
+    macos:
+      xcode: 15.4.0
+    steps:
+      - checkout
+      - attach_workspace:
+          at: build
+      - run:
+          name: "Notarize"
+          command: |
+            xcrun notarytool submit build/upload/gpt4all-installer-darwin-signed.dmg --apple-id "$MAC_NOTARIZATION_ID" --team-id "$MAC_NOTARIZATION_TID" --password "$MAC_NOTARIZATION_KEY" --wait  | tee notarize_log.txt
+      - run:
+          name: "Report Notarization Failure"
+          command: |
+            NID=`python3 .circleci/grab_notary_id.py notarize_log.txt` && export NID
+            xcrun notarytool log $NID --keychain-profile "notary-profile"
+            exit 1
+          when: on_fail
+      - run:
+          name: "Staple"
+          command: |
+            xcrun stapler staple build/upload/gpt4all-installer-darwin-signed.dmg
+      - store_artifacts:
+          path: build/upload
+
+  build-online-chat-installer-macos:
+    macos:
+      xcode: 15.4.0
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - macos-qt-cache-v3
+      - run:
+          name: Install Rosetta
+          command: softwareupdate --install-rosetta --agree-to-license  # needed for QtIFW
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
+              hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: macos-qt-cache-v3
+          paths:
+            - ~/Qt
+      - run:
+          name: Setup Keychain
+          command: |
+            echo $MAC_SIGNING_CERT | base64 --decode > cert.p12
+            security create-keychain -p "$MAC_KEYCHAIN_KEY" sign.keychain
+            security default-keychain -s sign.keychain
+            security unlock-keychain -p "$MAC_KEYCHAIN_KEY" sign.keychain
+            security import cert.p12 -k sign.keychain -P "$MAC_SIGNING_CERT_PWD" -T /usr/bin/codesign
+            security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$MAC_KEYCHAIN_KEY" sign.keychain
+      - run:
+          name: Build
+          no_output_timeout: 30m
+          command: |
+            mkdir build
+            cd build
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
+              -S ../gpt4all-chat -B . -G Ninja \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
+              -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
+              -DBUILD_UNIVERSAL=ON \
+              -DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
+              -DGGML_METAL_MACOSX_VERSION_MIN=12.6 \
+              -DMACDEPLOYQT=~/Qt/6.5.1/macos/bin/macdeployqt \
+              -DGPT4ALL_OFFLINE_INSTALLER=OFF \
+              -DGPT4ALL_SIGN_INSTALL=ON
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package
+            mkdir upload
+            cp gpt4all-installer-* upload
+            tar -cvzf upload/repository.tar.gz -C _CPack_Packages/Darwin/IFW/gpt4all-installer-darwin repository
+      # persist the unsigned installer
+      - store_artifacts:
+          path: build/upload
+      # add workspace so signing jobs can connect & obtain dmg
+      - persist_to_workspace:
+          root: build
+          # specify path to only include components we want to persist
+          # accross builds
+          paths:
+            - upload
+
+  sign-online-chat-installer-macos:
+    macos:
+      xcode: 15.4.0
+    steps:
+      - checkout
+      # attach to a workspace containing unsigned dmg
+      - attach_workspace:
+          at: build
+      - run:
+          name: "Setup Keychain"
+          command: |
+            echo $MAC_SIGNING_CERT | base64 --decode > cert.p12
+            security create-keychain -p "$MAC_KEYCHAIN_KEY" sign.keychain
+            security default-keychain -s sign.keychain
+            security unlock-keychain -p "$MAC_KEYCHAIN_KEY" sign.keychain
+            security import cert.p12 -k sign.keychain -P "$MAC_SIGNING_CERT_PWD" -T /usr/bin/codesign
+            security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$MAC_KEYCHAIN_KEY" sign.keychain
+            rm cert.p12
+      - run:
+          name: "Sign App Bundle"
+          command: |
+            python3 -m pip install click
+            python3 gpt4all-chat/cmake/sign_dmg.py --input-dmg build/upload/gpt4all-installer-darwin.dmg --output-dmg build/upload/gpt4all-installer-darwin-signed.dmg --signing-identity "$MAC_SIGNING_CERT_NAME"
+      - run:
+          name: "Sign DMG"
+          command: |
+            codesign --options runtime --timestamp -s "$MAC_SIGNING_CERT_NAME" build/upload/gpt4all-installer-darwin-signed.dmg
+      # add workspace so signing jobs can connect & obtain dmg
+      - persist_to_workspace:
+          root: build
+          # specify path to only include components we want to persist
+          # accross builds
+          paths:
+            - upload
+
+  notarize-online-chat-installer-macos:
+    macos:
+      xcode: 15.4.0
+    steps:
+      - checkout
+      - attach_workspace:
+          at: build
+      - run:
+          name: "Notarize"
+          command: |
+            xcrun notarytool submit build/upload/gpt4all-installer-darwin-signed.dmg --apple-id "$MAC_NOTARIZATION_ID" --team-id "$MAC_NOTARIZATION_TID" --password "$MAC_NOTARIZATION_KEY" --wait  | tee notarize_log.txt
+      - run:
+          name: "Report Notarization Failure"
+          command: |
+            NID=`python3 .circleci/grab_notary_id.py notarize_log.txt` && export NID
+            xcrun notarytool log $NID --keychain-profile "notary-profile"
+            exit 1
+          when: on_fail
+      - run:
+          name: "Staple"
+          command: |
+            xcrun stapler staple build/upload/gpt4all-installer-darwin-signed.dmg
+      - store_artifacts:
+          path: build/upload
+
   build-offline-chat-installer-linux:
     machine:
       image: ubuntu-2204:2023.04.2
@@ -99,7 +312,7 @@ jobs:
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
             wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
             sudo dpkg -i cuda-keyring_1.1-1_all.deb
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Installing Qt
           command: |
@@ -119,6 +332,7 @@ jobs:
             cd linuxdeployqt && qmake && sudo make install
       - run:
           name: Build
+          no_output_timeout: 30m
           command: |
             set -eo pipefail
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
@@ -127,13 +341,78 @@ jobs:
             mkdir build
             cd build
             mkdir upload
-            ~/Qt/Tools/CMake/bin/cmake -DGPT4ALL_OFFLINE_INSTALLER=ON -DCMAKE_BUILD_TYPE=Release -S ../gpt4all-chat -B .
-            ~/Qt/Tools/CMake/bin/cmake --build . --target all
-            ~/Qt/Tools/CMake/bin/cmake --build . --target install
-            ~/Qt/Tools/CMake/bin/cmake --build . --target package
+            ~/Qt/Tools/CMake/bin/cmake \
+              -S ../gpt4all-chat -B . \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DGPT4ALL_OFFLINE_INSTALLER=ON
+            ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target all
+            ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target install
+            ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target package
             cp gpt4all-installer-* upload
       - store_artifacts:
           path: build/upload
+
+  build-online-chat-installer-linux:
+    machine:
+      image: ubuntu-2204:2023.04.2
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - linux-qt-cache-v2
+      - run:
+          name: Setup Linux and Dependencies
+          command: |
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
+              chmod +x qt-unified-linux-x64-4.6.0-online.run
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: linux-qt-cache-v2
+          paths:
+            - ~/Qt
+      - run:
+          name: Build linuxdeployqt
+          command: |
+            git clone https://github.com/nomic-ai/linuxdeployqt
+            cd linuxdeployqt && qmake && sudo make install
+      - run:
+          name: Build
+          no_output_timeout: 30m
+          command: |
+            set -eo pipefail
+            export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
+            export PATH=$PATH:/usr/local/cuda/bin
+            mkdir build
+            cd build
+            mkdir upload
+            ~/Qt/Tools/CMake/bin/cmake \
+              -S ../gpt4all-chat -B . \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DGPT4ALL_OFFLINE_INSTALLER=OFF
+            ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target all
+            ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target install
+            ~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target package
+            cp gpt4all-installer-* upload
+            tar -cvzf upload/repository.tar.gz -C _CPack_Packages/Linux/IFW/gpt4all-installer-linux repository
+      - store_artifacts:
+          path: build/upload
+
   build-offline-chat-installer-windows:
     machine:
       image: 'windows-server-2019-vs2019:2022.08.1'
@@ -168,10 +447,23 @@ jobs:
       - run:
           name: Install CUDA Toolkit
           command: |
-            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
-            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe -OutFile cuda_11.8.0_windows_network.exe
+            .\cuda_11.8.0_windows_network.exe -s cudart_11.8 nvcc_11.8 cublas_11.8 cublas_dev_11.8
+      - run:
+          name: "Install Dotnet 8"
+          command: |
+            mkdir dotnet
+            cd dotnet
+            $dotnet_url="https://download.visualstudio.microsoft.com/download/pr/5af098e1-e433-4fda-84af-3f54fd27c108/6bd1c6e48e64e64871957289023ca590/dotnet-sdk-8.0.302-win-x64.zip"
+            Invoke-WebRequest -Uri $dotnet_url -Outfile dotnet-sdk-8.0.302-win-x64.zip
+            Expand-Archive -LiteralPath .\dotnet-sdk-8.0.302-win-x64.zip
+            $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet-sdk-8.0.302-win-x64"
+            $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
+            $Env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=$true
+            dotnet tool install --global AzureSignTool
       - run:
           name: Build
+          no_output_timeout: 30m
           command: |
             $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
             $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
@@ -190,17 +482,17 @@ jobs:
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
+            $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64"
+            $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
             mkdir build
             cd build
             & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
-              "-DCMAKE_GENERATOR:STRING=Ninja" `
+              -S ..\gpt4all-chat -B . -G Ninja `
               "-DCMAKE_BUILD_TYPE=Release" `
               "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
               "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
               "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
-              "-DGPT4ALL_OFFLINE_INSTALLER=ON" `
-              "-S ..\gpt4all-chat" `
-              "-B ."
+              "-DGPT4ALL_OFFLINE_INSTALLER=ON"
             & "C:\Qt\Tools\Ninja\ninja.exe"
             & "C:\Qt\Tools\Ninja\ninja.exe" install
             & "C:\Qt\Tools\Ninja\ninja.exe" package
@@ -208,6 +500,178 @@ jobs:
             copy gpt4all-installer-win64.exe upload
       - store_artifacts:
           path: build/upload
+            # add workspace so signing jobs can connect & obtain dmg
+      - persist_to_workspace:
+          root: build
+          # specify path to only include components we want to persist
+          # accross builds
+          paths:
+            - upload
+  sign-offline-chat-installer-windows:
+    machine:
+      image: 'windows-server-2019-vs2019:2022.08.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - attach_workspace:
+          at: build
+      - run:
+          name: "Install Dotnet 8 && Azure Sign Tool"
+          command: |
+            mkdir dotnet
+            cd dotnet
+            $dotnet_url="https://download.visualstudio.microsoft.com/download/pr/5af098e1-e433-4fda-84af-3f54fd27c108/6bd1c6e48e64e64871957289023ca590/dotnet-sdk-8.0.302-win-x64.zip"
+            Invoke-WebRequest -Uri $dotnet_url -Outfile dotnet-sdk-8.0.302-win-x64.zip
+            Expand-Archive -LiteralPath .\dotnet-sdk-8.0.302-win-x64.zip
+            $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet-sdk-8.0.302-win-x64"
+            $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
+            $Env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=$true
+            dotnet tool install --global AzureSignTool
+      - run:
+          name: "Sign Windows Installer With AST"
+          command: |
+            $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64"
+            $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
+            AzureSignTool.exe sign -du "https://gpt4all.io/index.html" -kvu https://gpt4all.vault.azure.net -kvi "$Env:AZSignGUID" -kvs "$Env:AZSignPWD" -kvc "$Env:AZSignCertName" -kvt "$Env:AZSignTID" -tr http://timestamp.digicert.com -v "$($(Get-Location).Path)\build\upload\gpt4all-installer-win64.exe"
+      - store_artifacts:
+          path: build/upload
+  build-online-chat-installer-windows:
+    machine:
+      image: 'windows-server-2019-vs2019:2022.08.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - windows-qt-cache-v2
+      - run:
+          name: Installing Qt
+          command: |
+            if (-not (Test-Path C:\Qt)) {
+              Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+            }
+      - save_cache:  # this is the new step to save cache
+          key: windows-qt-cache-v2
+          paths:
+            - C:\Qt
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe -OutFile cuda_11.8.0_windows_network.exe
+            .\cuda_11.8.0_windows_network.exe -s cudart_11.8 nvcc_11.8 cublas_11.8 cublas_dev_11.8
+      - run:
+          name: "Install Dotnet 8"
+          command: |
+            mkdir dotnet
+            cd dotnet
+            $dotnet_url="https://download.visualstudio.microsoft.com/download/pr/5af098e1-e433-4fda-84af-3f54fd27c108/6bd1c6e48e64e64871957289023ca590/dotnet-sdk-8.0.302-win-x64.zip"
+            Invoke-WebRequest -Uri $dotnet_url -Outfile dotnet-sdk-8.0.302-win-x64.zip
+            Expand-Archive -LiteralPath .\dotnet-sdk-8.0.302-win-x64.zip
+            $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet-sdk-8.0.302-win-x64"
+            $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
+      - run:
+          name: "Setup Azure SignTool"
+          command: |
+            $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64"
+            $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
+            $Env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=$true
+            dotnet tool install --global AzureSignTool
+      - run:
+          name: Build
+          no_output_timeout: 30m
+          command: |
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
+            $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
+            $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.7\bin"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
+            $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64"
+            $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
+            mkdir build
+            cd build
+            & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
+              -S ..\gpt4all-chat -B . -G Ninja `
+              "-DCMAKE_BUILD_TYPE=Release" `
+              "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
+              "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
+              "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
+              "-DGPT4ALL_OFFLINE_INSTALLER=OFF"
+            & "C:\Qt\Tools\Ninja\ninja.exe"
+            & "C:\Qt\Tools\Ninja\ninja.exe" install
+            & "C:\Qt\Tools\Ninja\ninja.exe" package
+            mkdir upload
+            copy gpt4all-installer-win64.exe upload
+            Set-Location -Path "_CPack_Packages/win64/IFW/gpt4all-installer-win64"
+            Compress-Archive -Path 'repository' -DestinationPath '..\..\..\..\upload\repository.zip'
+      - store_artifacts:
+          path: build/upload
+      # add workspace so signing jobs can connect & obtain dmg
+      - persist_to_workspace:
+          root: build
+          # specify path to only include components we want to persist
+          # accross builds
+          paths:
+            - upload
+  sign-online-chat-installer-windows:
+    machine:
+      image: 'windows-server-2019-vs2019:2022.08.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - attach_workspace:
+          at: build
+      - run:
+          name: "Install Dotnet 8"
+          command: |
+            mkdir dotnet
+            cd dotnet
+            $dotnet_url="https://download.visualstudio.microsoft.com/download/pr/5af098e1-e433-4fda-84af-3f54fd27c108/6bd1c6e48e64e64871957289023ca590/dotnet-sdk-8.0.302-win-x64.zip"
+            Invoke-WebRequest -Uri $dotnet_url -Outfile dotnet-sdk-8.0.302-win-x64.zip
+            Expand-Archive -LiteralPath .\dotnet-sdk-8.0.302-win-x64.zip
+            $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet-sdk-8.0.302-win-x64"
+            $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
+      - run:
+          name: "Setup Azure SignTool"
+          command: |
+            $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64"
+            $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
+            $Env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=$true
+            dotnet tool install --global AzureSignTool
+      - run:
+          name: "Sign Windows Installer With AST"
+          command: |
+            $Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64"
+            $Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
+            AzureSignTool.exe sign -du "https://gpt4all.io/index.html" -kvu https://gpt4all.vault.azure.net -kvi "$Env:AZSignGUID" -kvs "$Env:AZSignPWD" -kvc "$Env:AZSignCertName" -kvt "$Env:AZSignTID" -tr http://timestamp.digicert.com -v "$($(Get-Location).Path)/build/upload/gpt4all-installer-win64.exe"
+      - store_artifacts:
+          path: build/upload
+
   build-gpt4all-chat-linux:
     machine:
       image: ubuntu-2204:2023.04.2
@@ -228,7 +692,7 @@ jobs:
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
             wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
             sudo dpkg -i cuda-keyring_1.1-1_all.deb
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Installing Qt
           command: |
@@ -246,8 +710,10 @@ jobs:
           command: |
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
             export PATH=$PATH:/usr/local/cuda/bin
-            ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S gpt4all-chat -B build
-            ~/Qt/Tools/CMake/bin/cmake --build build --target all
+            ~/Qt/Tools/CMake/bin/cmake \
+              -S gpt4all-chat -B build \
+              -DCMAKE_BUILD_TYPE=Release
+            ~/Qt/Tools/CMake/bin/cmake --build build -j$(nproc) --target all
 
   build-gpt4all-chat-windows:
     machine:
@@ -283,8 +749,8 @@ jobs:
       - run:
           name: Install CUDA Toolkit
           command: |
-            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
-            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe -OutFile cuda_11.8.0_windows_network.exe
+            .\cuda_11.8.0_windows_network.exe -s cudart_11.8 nvcc_11.8 cublas_11.8 cublas_dev_11.8
       - run:
           name: Build
           command: |
@@ -306,18 +772,16 @@ jobs:
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
             $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
             & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
-              "-DCMAKE_GENERATOR:STRING=Ninja" `
+              -S gpt4all-chat -B build -G Ninja `
               "-DCMAKE_BUILD_TYPE=Release" `
               "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
               "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
-              "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
-              "-S gpt4all-chat" `
-              "-B build"
+              "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON"
             & "C:\Qt\Tools\Ninja\ninja.exe" -C build
 
   build-gpt4all-chat-macos:
     macos:
-      xcode: 14.0.0
+      xcode: 15.4.0
     steps:
       - checkout
       - run:
@@ -348,13 +812,13 @@ jobs:
           name: Build
           command: |
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
-              -DCMAKE_GENERATOR:STRING=Ninja \
-              -DBUILD_UNIVERSAL=ON \
+              -S gpt4all-chat -B build -G Ninja \
               -DCMAKE_BUILD_TYPE=Release \
               -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
               -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-              -S gpt4all-chat \
-              -B build
+              -DBUILD_UNIVERSAL=ON \
+              -DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
+              -DGGML_METAL_MACOSX_VERSION_MIN=12.6
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build build --target all
   build-ts-docs: 
     docker: 
@@ -385,7 +849,7 @@ jobs:
             sudo apt-get update
             sudo apt-get -y install python3 python3-pip
             sudo pip3 install awscli --upgrade
-            sudo pip3 install mkdocs mkdocs-material mkautodoc 'mkdocstrings[python]'
+            sudo pip3 install mkdocs mkdocs-material mkautodoc 'mkdocstrings[python]' markdown-captions pillow cairosvg
       - run:
           name: Make Documentation
           command: |
@@ -416,7 +880,7 @@ jobs:
             wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
             sudo dpkg -i cuda-keyring_1.1-1_all.deb
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
+            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server
             pip install setuptools wheel cmake
       - run:
           name: Build C library
@@ -424,8 +888,11 @@ jobs:
             export PATH=$PATH:/usr/local/cuda/bin
             git submodule update --init --recursive
             cd gpt4all-backend
-            cmake -B build -DCMAKE_BUILD_TYPE=Release
-            cmake --build build --parallel
+            cmake -B build \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON \
+              -DCMAKE_CUDA_ARCHITECTURES='52-virtual;61-virtual;70-virtual;75-virtual'
+            cmake --build build -j$(nproc)
       - run:
           name: Build wheel
           command: |
@@ -440,7 +907,7 @@ jobs:
 
   build-py-macos:
     macos:
-      xcode: "14.2.0"
+      xcode: 15.4.0
     resource_class: macos.m1.large.gen1
     steps:
       - checkout
@@ -454,7 +921,11 @@ jobs:
           command: |
             git submodule update --init  # don't use --recursive because macOS doesn't use Kompute
             cd gpt4all-backend
-            cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake -B build \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DBUILD_UNIVERSAL=ON \
+              -DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
+              -DGGML_METAL_MACOSX_VERSION_MIN=12.6
             cmake --build build --parallel
       - run:
           name: Build wheel
@@ -488,8 +959,8 @@ jobs:
       - run:
           name: Install CUDA Toolkit
           command: |
-            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
-            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe -OutFile cuda_11.8.0_windows_network.exe
+            .\cuda_11.8.0_windows_network.exe -s cudart_11.8 nvcc_11.8 cublas_11.8 cublas_dev_11.8
       - run:
           name: Install dependencies
           command:
@@ -521,7 +992,10 @@ jobs:
             $Env:PATH += ";C:\VulkanSDK\1.3.261.1\bin"
             $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
             cd gpt4all-backend
-            cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=Release -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON
+            cmake -B build -G Ninja `
+              -DCMAKE_BUILD_TYPE=Release `
+              -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON `
+              -DCMAKE_CUDA_ARCHITECTURES='52-virtual;61-virtual;70-virtual;75-virtual'
             cmake --build build --parallel
       - run:
           name: Build wheel
@@ -573,7 +1047,7 @@ jobs:
             wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
             sudo dpkg -i cuda-keyring_1.1-1_all.deb
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
+            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Build Libraries
           command: |
@@ -581,8 +1055,9 @@ jobs:
             cd gpt4all-backend
             mkdir -p runtimes/build
             cd runtimes/build
-            cmake ../..
-            cmake --build . --parallel --config Release
+            cmake ../.. \
+              -DCMAKE_BUILD_TYPE=Release
+            cmake --build . -j$(nproc)
             mkdir ../linux-x64
             cp -L *.so ../linux-x64 # otherwise persist_to_workspace seems to mess symlinks
       - persist_to_workspace:
@@ -592,7 +1067,7 @@ jobs:
 
   build-bindings-backend-macos:
     macos:
-      xcode: "14.0.0"
+      xcode: 15.4.0
     steps:
       - checkout
       - run:
@@ -610,8 +1085,12 @@ jobs:
             cd gpt4all-backend
             mkdir -p runtimes/build
             cd runtimes/build
-            cmake ../.. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
-            cmake --build . --parallel --config Release
+            cmake ../.. \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DBUILD_UNIVERSAL=ON \
+              -DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
+              -DGGML_METAL_MACOSX_VERSION_MIN=12.6
+            cmake --build . --parallel
             mkdir ../osx-x64
             cp -L *.dylib ../osx-x64
             cp ../../llama.cpp-mainline/*.metal ../osx-x64
@@ -642,8 +1121,8 @@ jobs:
       - run:
           name: Install CUDA Toolkit
           command: |
-            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
-            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/11.8.0/network_installers/cuda_11.8.0_windows_network.exe -OutFile cuda_11.8.0_windows_network.exe
+            .\cuda_11.8.0_windows_network.exe -s cudart_11.8 nvcc_11.8 cublas_11.8 cublas_dev_11.8
       - run:
           name: Install dependencies
           command: |
@@ -698,7 +1177,7 @@ jobs:
             - runtimes/linux-x64/*-*.so
   build-nodejs-macos: 
     macos:
-      xcode: "14.0.0"
+      xcode: 15.4.0
     steps:
       - checkout
       - attach_workspace:
@@ -848,12 +1327,47 @@ workflows:
       - build-offline-chat-installer-macos:
           requires:
             - hold
+      - sign-offline-chat-installer-macos:
+          requires:
+            - build-offline-chat-installer-macos
+      - notarize-offline-chat-installer-macos:
+          requires:
+            - sign-offline-chat-installer-macos
       - build-offline-chat-installer-windows:
           requires:
             - hold
+      - sign-offline-chat-installer-windows:
+          requires:
+            - build-offline-chat-installer-windows
       - build-offline-chat-installer-linux:
           requires:
             - hold
+  build-chat-online-installers:
+    when:
+      or:
+        - << pipeline.parameters.run-all-workflows >>
+        - << pipeline.parameters.run-chat-workflow >>
+    jobs:
+      - hold:
+          type: approval
+      - build-online-chat-installer-macos:
+          requires:
+            - hold
+      - sign-online-chat-installer-macos:
+          requires:
+            - build-online-chat-installer-macos
+      - notarize-online-chat-installer-macos:
+          requires:
+            - sign-online-chat-installer-macos
+      - build-online-chat-installer-windows:
+          requires:
+            - hold
+      - sign-online-chat-installer-windows:
+          requires:
+            - build-online-chat-installer-windows
+      - build-online-chat-installer-linux:
+          requires:
+            - hold
   build-and-test-gpt4all-chat:
     when:
       or:

.circleci/grab_notary_id.py

@@ -0,0 +1,17 @@
+import re
+import sys
+
+ID_REG = r"id: (.*)"
+
+def main() -> None:
+    notary_log = sys.argv[1]
+    with open(notary_log, "r") as f:
+        notary_output = f.read()
+        id_m = re.search(ID_REG, notary_output)
+        if id_m:
+            print(id_m.group(1))
+        else:
+            raise RuntimeError("Unable to parse ID from notarization logs")
+
+if __name__ == "__main__":
+    main()

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
 ignore-words-list = blong, afterall, som, assistent, crasher
-skip = .git,*.pdf,*.svg,*.lock
+skip = .git,*.pdf,*.svg,*.lock,*.ts

.gitignore

@@ -181,6 +181,7 @@ CMakeLists.txt.user
 gpt4all-chat/models/*
 build_*
 build-*
+cmake-build-*
 
 # IntelliJ
 .idea/

MAINTAINERS.md

@@ -0,0 +1,82 @@
+# MAINTAINERS
+
+## Rules
+
+* All content inside GPT4All shall have a documented maintainer
+* If a maintainer decides to retire or resign a call for volunteers will go
+  out
+* If no further maintainer can be found in a reasonable time frame, then the
+  content will be marked deprecated and removed in time
+
+## Job
+
+Maintainers will be...
+
+1. Responsible for overseeing content under their stewardship
+2. Responsible for triaging new issues, reviewing PRs, assigning priority
+   to tasks
+3. Responsible for keeping content in sufficient quality in a timely fashion
+
+## List
+
+Adam Treat ([@manyoso](https://github.com/manyoso))<br/>
+E-mail: adam@nomic.ai<br/>
+Discord: `@gonzochess75`
+- Overall project maintainer
+- Chat UI
+
+Jared Van Bortel ([@cebtenzzre](https://github.com/cebtenzzre))<br/>
+E-mail: jared@nomic.ai<br/>
+Discord: `@cebtenzzre`
+- gpt4all-backend
+- Python binding
+- Python CLI app
+
+Jacob Nguyen ([@jacoobes](https://github.com/jacoobes))<br/>
+Discord: `@jacoobes`<br/>
+E-mail: `jacoobes@sern.dev`
+- TypeScript binding
+
+Dominik ([@cosmic-snow](https://github.com/cosmic-snow))<br/>
+E-mail: cosmic-snow@mailfence.com<br/>
+Discord: `@cosmic__snow`
+- Community documentation (GitHub Wiki)
+
+Max Cembalest ([@mcembalest](https://github.com/mcembalest))<br/>
+E-mail: max@nomic.ai<br/>
+Discord: `@maxcembalest.`
+- Official documentation (gpt4all-bindings/python/docs -> https://docs.gpt4all.io/)
+
+Thiago Ramos ([@thiagojramos](https://github.com/thiagojramos))<br/>
+E-mail: thiagojramos@outlook.com<br/>
+- pt\_BR translation
+
+Victor Emanuel ([@SINAPSA-IC](https://github.com/SINAPSA-IC))<br/>
+E-mail: contact@sinapsaro.ro<br/>
+Discord: `@sinapsa_ic_56124_99632`
+- ro\_RO translation
+
+不知火 Shiranui ([@supersonictw](https://github.com/supersonictw))<br/>
+E-mail: supersonic@livemail.tw<br/>
+Discord: `@supersonictw`
+- zh\_TW translation
+
+Jeremy Tayco ([@jstayco](https://github.com/jstayco))<br/>
+E-mail: jstayco@protonmail.ch<br/>
+Discord: `@vertana`
+- es\_MX translation
+
+Riccardo Giovanetti ([@Harvester62](https://github.com/Harvester62))<br/>
+E-mail: riccardo.giovanetti@gmail.com<br/>
+Discord: `@harvester62`
+- it\_IT translation
+
+Tim ([@Tim453](https://github.com/Tim453))<br/>
+E-mail: tim453@mailbox.org<br/>
+Discord: `@Tim453`
+- Flatpak
+
+Jack ([@wuodoo](https://github.com/wuodoo))<br/>
+E-mail: 2296103047@qq.com><br/>
+Discord: `@mikage`
+- zh\_CN translation

README.md

@@ -1,13 +1,42 @@
 <h1 align="center">GPT4All</h1>
-<p align="center">Privacy-oriented software for chatting with large language models that run on your own computer.</p>
+
+<p align="center">GPT4All runs large language models (LLMs) privately on everyday desktops & laptops. <br> <br> No API calls or GPUs required - you can just download the application and <a href="https://docs.gpt4all.io/gpt4all_desktop/quickstart.html#quickstart">get started</a>
+
+https://github.com/nomic-ai/gpt4all/assets/70534565/513a0f15-4964-4109-89e4-4f9a9011f311
+
 <p align="center">
-  <a href="https://gpt4all.io">Official Website</a> &bull; <a href="https://docs.gpt4all.io">Documentation</a> &bull; <a href="https://discord.gg/mGZE39AS3e">Discord</a>
+  <a href="https://gpt4all.io/installers/gpt4all-installer-win64.exe">
+    <img src="gpt4all-bindings/python/docs/assets/windows.png" width="80" height="80"><br>
+    Download for Windows
+  </a>
+</p>
+
+<p align="center">
+  <a href="https://gpt4all.io/installers/gpt4all-installer-darwin.dmg">
+    <img src="gpt4all-bindings/python/docs/assets/mac.png" width="85" height="100"><br>
+    Download for MacOS
+  </a>
+</p>
+
+<p align="center">
+  <a href="https://gpt4all.io/installers/gpt4all-installer-linux.run">
+    <img src="gpt4all-bindings/python/docs/assets/ubuntu.svg" width="120" height="120"><br>
+    Download for Ubuntu
+  </a>
+</p>
+
+<p align="center">
+  <a href='https://flathub.org/apps/io.gpt4all.gpt4all'>
+    <img width='240' alt='Get it on Flathub' src='https://flathub.org/api/badge?locale=en'><br>
+    Get it on Flathub (community maintained)
+  </a>
+</p>
+
+<p align="center">
+  <a href="https://gpt4all.io">Website</a> &bull; <a href="https://docs.gpt4all.io">Documentation</a> &bull; <a href="https://discord.gg/mGZE39AS3e">Discord</a>
 </p>
 <p align="center">
-  Official Download Links: <a href="https://gpt4all.io/installers/gpt4all-installer-win64.exe">Windows</a> &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-darwin.dmg">macOS</a> &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-linux.run">Ubuntu</a>
-</p>
-<p align="center">
-  <b>NEW:</b> <a href="https://forms.nomic.ai/gpt4all-release-notes-signup">Subscribe to our mailing list</a> for updates and news!
+  <a href="https://forms.nomic.ai/gpt4all-release-notes-signup">Subscribe to the newsletter</a>
 </p>
 <p align="center">
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
@@ -16,31 +45,35 @@ GPT4All is made possible by our compute partner <a href="https://www.paperspace.
  <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg" alt="phorm.ai"></a>
 </p>
 
-<p align="center">
-  <img width="auto" height="400" src="https://github.com/nomic-ai/gpt4all/assets/14168726/495fce3e-769b-4e5a-a394-99f072ac4d29">
-</p>
-<p align="center">
-Run on an M2 MacBook Pro (not sped up!)
-</p>
+## Install GPT4All Python
+
+`gpt4all` gives you access to LLMs with our Python client around [`llama.cpp`](https://github.com/ggerganov/llama.cpp) implementations. 
+
+Nomic contributes to open source software like [`llama.cpp`](https://github.com/ggerganov/llama.cpp) to make LLMs accessible and efficient **for all**.
+
+```bash
+pip install gpt4all
+```
+
+```python
+from gpt4all import GPT4All
+model = GPT4All("Meta-Llama-3-8B-Instruct.Q4_0.gguf") # downloads / loads a 4.66GB LLM
+with model.chat_session():
+    print(model.generate("How can I run LLMs efficiently on my laptop?", max_tokens=1024))
+```
 
 
-## About GPT4All
+## Integrations
 
-GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and NVIDIA and AMD GPUs. Note that your CPU needs to support [AVX instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
+:parrot::link: [Langchain](https://python.langchain.com/v0.2/docs/integrations/providers/gpt4all/)
+:card_file_box: [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
+:telescope: [OpenLIT (OTel-native Monitoring)](https://github.com/openlit/openlit) - [Docs](https://docs.openlit.io/latest/integrations/gpt4all)
 
-Learn more in the [documentation](https://docs.gpt4all.io).
-
-A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily deploy their own on-edge large language models.
-
-
-### Installation
-
-The recommended way to install GPT4All is to use one of the online installers linked above in this README, which are also available at the [GPT4All website](https://gpt4all.io/). These require an internet connection at install time, are slightly easier to use on macOS due to code signing, and provide a version of GPT4All that can check for updates.
-
-An alternative way to install GPT4All is to use one of the offline installers available on the [Releases page](https://github.com/nomic-ai/gpt4all/releases). These do not require an internet connection at install time, and can be used to install an older version of GPT4All if so desired. But using these requires acknowledging a security warning on macOS, and they provide a version of GPT4All that is unable to notify you of updates, so you should enable notifications for Releases on this repository (Watch > Custom > Releases) or sign up for announcements in our [Discord server](https://discord.gg/mGZE39AS3e).
-
-
-### What's New
+## Release History
+- **July 2nd, 2024**: V3.0.0 Release
+    - Fresh redesign of the chat application UI
+    - Improved user workflow for LocalDocs
+    - Expanded access to more model architectures
 - **October 19th, 2023**: GGUF Support Launches with Support for:
     - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
     - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4\_0 and Q4\_1 quantizations in GGUF.
@@ -51,25 +84,6 @@ An alternative way to install GPT4All is to use one of the offline installers av
 
 [Docker-based API server]: https://github.com/nomic-ai/gpt4all/tree/cef74c2be20f5b697055d5b8b506861c7b997fab/gpt4all-api
 
-
-### Building From Source
-
-* Follow the instructions [here](gpt4all-chat/build_and_run.md) to build the GPT4All Chat UI from source.
-
-
-### Bindings
-
-* :snake: <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python">Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
-* :computer: <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">Typescript Bindings</a>
-
-
-### Integrations
-
-* :parrot::link: [Langchain](https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html)
-* :card_file_box: [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
-* :telescope: [OpenLIT (OTel-native Monitoring)](https://github.com/openlit/openlit) - [Docs](https://docs.openlit.io/latest/integrations/gpt4all)
-
-
 ## Contributing
 GPT4All welcomes contributions, involvement, and discussion from the open source community!
 Please see CONTRIBUTING.md and follow the issues, bug reports, and PR markdown templates.
@@ -78,74 +92,6 @@ Check project discord, with project owners, or through existing issues/PRs to av
 Please make sure to tag all of the above with relevant project identifiers or your contribution could potentially get lost.
 Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 
-
-## GPT4All 2024 Roadmap
-To contribute to the development of any of the below roadmap items, make or find the corresponding issue and cross-reference the [in-progress task](https://github.com/orgs/nomic-ai/projects/2/views/1).
-
-Each item should have an issue link below.
-
-- Chat UI Language Localization (localize UI into the native languages of users)
-    - [ ] Chinese
-    - [ ] German
-    - [ ] French
-    - [ ] Portuguese
-    - [ ] Your native language here. 
-- UI Redesign: an internal effort at Nomic to improve the UI/UX of gpt4all for all users.
-    - [ ] Design new user interface and gather community feedback
-    - [ ] Implement the new user interface and experience.
-- Installer and Update Improvements
-    - [ ] Seamless native installation and update process on OSX
-    - [ ] Seamless native installation and update process on Windows
-    - [ ] Seamless native installation and update process on Linux
-- Model discoverability improvements:
-    - [x] Support huggingface model discoverability
-    - [ ] Support Nomic hosted model discoverability
-- LocalDocs (towards a local perplexity)
-    - Multilingual LocalDocs Support
-        - [ ] Create a multilingual experience
-        - [ ] Incorporate a multilingual embedding model
-        - [ ] Specify a preferred multilingual LLM for localdocs
-    - Improved RAG techniques
-        - [ ] Query augmentation and re-writing
-        - [ ] Improved chunking and text extraction from arbitrary modalities
-            - [ ] Custom PDF extractor past the QT default (charts, tables, text)
-        - [ ] Faster indexing and local exact search with v1.5 hamming embeddings and reranking (skip ANN index construction!)
-    - Support queries like 'summarize X document'
-    - Multimodal LocalDocs support with Nomic Embed
-    - Nomic Dataset Integration with real-time LocalDocs
-        - [ ] Include an option to allow the export of private LocalDocs collections to Nomic Atlas for debugging data/chat quality
-        - [ ] Allow optional sharing of LocalDocs collections between users.
-        - [ ] Allow the import of a LocalDocs collection from an Atlas Datasets
-            - Chat with live version of Wikipedia, Chat with Pubmed, chat with the latest snapshot of world news.
-- First class Multilingual LLM Support
-    - [ ] Recommend and set a default LLM for German
-    - [ ] Recommend and set a default LLM for English
-    - [ ] Recommend and set a default LLM for Chinese
-    - [ ] Recommend and set a default LLM for Spanish
-
-- Server Mode improvements
-    - Improved UI and new requested features:
-        - [ ] Fix outstanding bugs and feature requests around networking configurations.
-        - [ ] Support Nomic Embed inferencing
-        - [ ] First class documentation
-        - [ ] Improving developer use and quality of server mode (e.g. support larger batches)
-
-
-## Technical Reports
-
-<p align="center">
-<a href="https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf">:green_book: Technical Report 3: GPT4All Snoozy and Groovy </a>
-</p>
-
-<p align="center">
-<a href="https://static.nomic.ai/gpt4all/2023_GPT4All-J_Technical_Report_2.pdf">:green_book: Technical Report 2: GPT4All-J </a>
-</p>
-
-<p align="center">
-<a href="https://s3.amazonaws.com/static.nomic.ai/gpt4all/2023_GPT4All_Technical_Report.pdf">:green_book: Technical Report 1: GPT4All</a>
-</p>
-
-
 ## Citation
 
 If you utilize this repository, models or data in a downstream project, please consider citing it with:

gpt4all-backend/CMakeLists.txt

@@ -51,13 +51,11 @@ set(DIRECTORY llama.cpp-mainline)
 include(llama.cpp.cmake)
 
 set(BUILD_VARIANTS)
-set(GPTJ_BUILD_VARIANT cpu)
 if (APPLE)
     list(APPEND BUILD_VARIANTS metal)
 endif()
 if (LLMODEL_KOMPUTE)
     list(APPEND BUILD_VARIANTS kompute kompute-avxonly)
-    set(GPTJ_BUILD_VARIANT kompute)
 else()
     list(PREPEND BUILD_VARIANTS cpu cpu-avxonly)
 endif()
@@ -65,6 +63,24 @@ if (LLMODEL_VULKAN)
     list(APPEND BUILD_VARIANTS vulkan vulkan-avxonly)
 endif()
 if (LLMODEL_CUDA)
+    cmake_minimum_required(VERSION 3.18)  # for CMAKE_CUDA_ARCHITECTURES
+
+    # Defaults must be set before enable_language(CUDA).
+    # Keep this in sync with the arch list in ggml/src/CMakeLists.txt.
+    if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+        # 52 == lowest CUDA 12 standard
+        # 60 == f16 CUDA intrinsics
+        # 61 == integer CUDA intrinsics
+        # 70 == compute capability at which unrolling a loop in mul_mat_q kernels is faster
+        if (GGML_CUDA_F16 OR GGML_CUDA_DMMV_F16)
+            set(CMAKE_CUDA_ARCHITECTURES "60;61;70;75") # needed for f16 CUDA intrinsics
+        else()
+            set(CMAKE_CUDA_ARCHITECTURES "52;61;70;75") # lowest CUDA 12 standard + lowest for integer intrinsics
+            #set(CMAKE_CUDA_ARCHITECTURES "OFF") # use this to compile much faster, but only F16 models work
+        endif()
+    endif()
+    message(STATUS "Using CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}")
+
     include(CheckLanguage)
     check_language(CUDA)
     if (NOT CMAKE_CUDA_COMPILER)
@@ -88,25 +104,25 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     else()
         set(GPT4ALL_ALLOW_NON_AVX ON)
     endif()
-    set(LLAMA_AVX2 ${GPT4ALL_ALLOW_NON_AVX})
-    set(LLAMA_F16C ${GPT4ALL_ALLOW_NON_AVX})
-    set(LLAMA_FMA  ${GPT4ALL_ALLOW_NON_AVX})
+    set(GGML_AVX2 ${GPT4ALL_ALLOW_NON_AVX})
+    set(GGML_F16C ${GPT4ALL_ALLOW_NON_AVX})
+    set(GGML_FMA  ${GPT4ALL_ALLOW_NON_AVX})
 
-    set(LLAMA_METAL   OFF)
-    set(LLAMA_KOMPUTE OFF)
-    set(LLAMA_VULKAN  OFF)
-    set(LLAMA_CUDA    OFF)
-    set(LLAMA_ROCM    OFF)
+    set(GGML_METAL   OFF)
+    set(GGML_KOMPUTE OFF)
+    set(GGML_VULKAN  OFF)
+    set(GGML_CUDA    OFF)
+    set(GGML_ROCM    OFF)
     if (BUILD_VARIANT MATCHES metal)
-        set(LLAMA_METAL   ON)
+        set(GGML_METAL   ON)
     elseif (BUILD_VARIANT MATCHES kompute)
-        set(LLAMA_KOMPUTE ON)
+        set(GGML_KOMPUTE ON)
     elseif (BUILD_VARIANT MATCHES vulkan)
-        set(LLAMA_VULKAN  ON)
+        set(GGML_VULKAN  ON)
     elseif (BUILD_VARIANT MATCHES cuda)
-        set(LLAMA_CUDA    ON)
+        set(GGML_CUDA    ON)
     elseif (BUILD_VARIANT MATCHES rocm)
-        set(LLAMA_HIPBLAS ON)
+        set(GGML_HIPBLAS ON)
     endif()
 
     # Include GGML
@@ -135,12 +151,6 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    if (BUILD_VARIANT MATCHES ${GPTJ_BUILD_VARIANT})
-        add_library(gptj-${BUILD_VARIANT} SHARED
-            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        prepare_target(gptj llama-mainline)
-    endif()
-
     if (NOT PROJECT_IS_TOP_LEVEL AND BUILD_VARIANT STREQUAL cuda)
         set(CUDAToolkit_BIN_DIR ${CUDAToolkit_BIN_DIR} PARENT_SCOPE)
     endif()

gpt4all-backend/gptj.cpp

@@ -1,853 +0,0 @@
-#define GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "gptj_impl.h"
-
-#include "llmodel.h"
-#include "llmodel_shared.h"
-#include "utils.h"
-
-#include <ggml.h>
-
-#include <algorithm>
-#include <cassert>
-#include <cinttypes>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <ctime>
-#include <iostream>
-#include <map>
-#include <memory>
-#include <random>
-#include <sstream>
-#include <stdexcept>
-#include <string>
-#include <thread>
-#include <vector>
-
-namespace {
-const char *modelType_ = "GPT-J";
-}
-
-// default hparams (GPT-J 6B)
-struct gptj_hparams {
-    int32_t n_vocab = 50400;
-    int32_t n_ctx   = 2048;
-    int32_t n_embd  = 4096;
-    int32_t n_head  = 16;
-    int32_t n_layer = 28;
-    int32_t n_rot   = 64;
-    float norm_eps  = 1e-5;
-};
-
-struct gptj_layer {
-    // normalization
-    struct ggml_tensor * ln_1_g;
-    struct ggml_tensor * ln_1_b;
-
-    // attention
-    struct ggml_tensor * c_attn_q_proj_w;
-    struct ggml_tensor * c_attn_k_proj_w;
-    struct ggml_tensor * c_attn_v_proj_w;
-
-    struct ggml_tensor * c_attn_proj_w;
-
-    // ff
-    struct ggml_tensor * c_mlp_fc_w;
-    struct ggml_tensor * c_mlp_fc_b;
-
-    struct ggml_tensor * c_mlp_proj_w;
-    struct ggml_tensor * c_mlp_proj_b;
-};
-
-struct gptj_model {
-    gptj_hparams hparams;
-
-    // normalization
-    struct ggml_tensor * ln_f_g;
-    struct ggml_tensor * ln_f_b;
-
-    struct ggml_tensor * wte; // position embedding
-
-    struct ggml_tensor * lmh_g; // language model head
-    struct ggml_tensor * lmh_b; // language model bias
-
-    std::vector<gptj_layer> layers;
-
-    // key + value memory
-    struct llm_kv_cache kv_self;
-
-    //
-    struct ggml_context * ctx;
-    std::map<std::string, struct ggml_tensor *> tensors;
-
-    llm_buffer eval_buf;
-    llm_buffer scr0_buf;
-    llm_buffer scr1_buf;
-
-    ~gptj_model() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
-static bool kv_cache_init(
-        const struct gptj_hparams & hparams,
-              struct llm_kv_cache & cache,
-                         ggml_type   wtype,
-                               int   n_ctx) {
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-
-    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
-    const int64_t n_elements = n_embd*n_mem;
-
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
-
-    struct ggml_init_params params;
-    params.mem_size   = cache.buf.size;
-    params.mem_buffer = cache.buf.addr;
-    params.no_alloc   = false;
-
-    cache.ctx = ggml_init(params);
-
-    if (!cache.ctx) {
-        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
-        return false;
-    }
-
-    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-
-    return true;
-}
-
-// load the model's weights from a file path
-bool gptj_model_load(const std::string &fname, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr)
-{
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
-    if(mem_req != nullptr) {
-        *mem_req = 0;
-    }
-
-    // create the ggml context
-    struct gguf_init_params params = {
-        /*.no_alloc = */ false,
-        /*.ctx      = */ &model.ctx,
-    };
-
-    gguf_context *ggufctx = gguf_init_from_file(fname.c_str(), params);
-    if (!ggufctx) {
-        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
-        return false;
-    }
-
-    // load hparams
-    {
-        auto & hparams = model.hparams;
-
-        bool ok = false;
-        int keyidx;
-
-        do {
-            keyidx = gguf_find_key(ggufctx, "gptj.context_length");
-            if (keyidx == -1) { break; }
-            hparams.n_ctx = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.embedding_length");
-            if (keyidx == -1) { break; }
-            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.attention.head_count");
-            if (keyidx == -1) { break; }
-            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.block_count");
-            if (keyidx == -1) { break; }
-            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.rope.dimension_count");
-            if (keyidx == -1) { break; }
-            hparams.n_rot = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.attention.layer_norm_epsilon");
-            if (keyidx == -1) { break; }
-            hparams.norm_eps = gguf_get_val_f32(ggufctx, keyidx);
-
-            ok = true;
-        } while (false);
-
-        if (!ok) {
-            fprintf(stderr, "%s: required hparam missing!\n", __func__);
-            return false;
-        }
-
-        printf("%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
-        printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_head  = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
-        printf("%s: n_rot   = %d\n", __func__, hparams.n_rot);
-    }
-
-    // load vocab
-    {
-        auto & hparams = model.hparams;
-
-        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
-            return false;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "gpt2") != 0) {
-            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
-            return false;
-        }
-
-        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
-        if (tokens_keyidx == -1) {
-            fprintf(stderr, "%s: gpt2 tokenizer vocab not found!\n", __func__);
-            return false;
-        }
-
-        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
-        printf("%s: gpt2 tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
-
-        for (int i = 0; i < hparams.n_vocab; i++) {
-            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
-            vocab.token_to_id[word] = i;
-            vocab.id_to_token[i] = word;
-        }
-    }
-
-    auto & ctx = model.ctx;
-
-    size_t ctx_size = ggml_get_mem_size(ctx);
-    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size / (1024.0 * 1024.0));
-
-    if (mem_req != nullptr) {
-        *mem_req = ctx_size;
-        gguf_free(ggufctx);
-        return false;
-    }
-
-    // prepare memory for the weights
-    {
-        const auto & hparams = model.hparams;
-        model.layers.resize(hparams.n_layer);
-
-        model.wte    = ggml_get_tensor(ctx, "token_embd.weight");
-
-        model.ln_f_g = ggml_get_tensor(ctx, "output_norm.weight");
-        model.ln_f_b = ggml_get_tensor(ctx, "output_norm.bias");
-
-        model.lmh_g  = ggml_get_tensor(ctx, "output.weight");
-        model.lmh_b  = ggml_get_tensor(ctx, "output.bias");
-
-        auto name = [](int i, std::string n) {
-            static std::string key;
-            key = "blk." + std::to_string(i) + "." + n;
-            return key.c_str();
-        };
-
-        for (int i = 0; i < hparams.n_layer; ++i) {
-            auto & layer = model.layers[i];
-
-            layer.ln_1_g          = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
-            layer.ln_1_b          = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
-
-            layer.c_attn_q_proj_w = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
-            layer.c_attn_k_proj_w = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
-            layer.c_attn_v_proj_w = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
-
-            layer.c_attn_proj_w   = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
-
-            layer.c_mlp_fc_w      = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
-            layer.c_mlp_fc_b      = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
-
-            layer.c_mlp_proj_w    = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
-            layer.c_mlp_proj_b    = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
-        }
-    }
-
-    // key + value memory
-    {
-        const auto & hparams = model.hparams;
-        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
-            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
-            ggml_free(ctx);
-            return false;
-        }
-
-        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
-        printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
-    }
-
-    model.scr0_buf.resize(256u * 1024 * 1024);
-    model.scr1_buf.resize(256u * 1024 * 1024);
-
-    return true;
-}
-
-// evaluate the transformer
-//
-//   - model:     the model
-//   - n_threads: number of threads to use
-//   - n_past:    the context size so far
-//   - embd_inp:  the embeddings of the tokens in the context
-//   - embd_w:    the predicted logits for the next token
-//
-// The GPT-J model requires about 16MB of memory per input token.
-//
-bool gptj_eval(
-        gptj_model & model,
-        const int n_threads,
-        const int n_past,
-        const std::vector<gpt_vocab::id> & embd_inp,
-              std::vector<float>         & embd_w,
-              size_t                     & mem_per_token) {
-    const int N = embd_inp.size();
-
-    const auto & hparams = model.hparams;
-
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_ctx   = hparams.n_ctx;
-    const int n_head  = hparams.n_head;
-    const int n_vocab = hparams.n_vocab;
-    const int n_rot   = hparams.n_rot;
-
-    const size_t init_buf_size = 1024_MiB;
-    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
-        model.eval_buf.resize(init_buf_size);
-
-    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
-        const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
-        printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.eval_buf.size, buf_size_new);
-
-        // reallocate
-        model.eval_buf.resize(buf_size_new);
-        if (model.eval_buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
-            return false;
-        }
-    }
-
-    struct ggml_init_params params = {
-        .mem_size   = model.eval_buf.size,
-        .mem_buffer = model.eval_buf.addr,
-        .no_alloc = false
-    };
-
-    struct ggml_context * ctx0 = ggml_init(params);
-    struct ggml_cgraph * gf = ggml_new_graph(ctx0);
-
-    // KQ_pos - contains the positions
-    struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    int * data = (int *) KQ_pos->data;
-    for (int i = 0; i < N; ++i) {
-        data[i] = n_past + i;
-    }
-
-    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
-
-    // wte
-    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
-
-    for (int il = 0; il < n_layer; ++il) {
-        struct ggml_tensor * cur;
-        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-        // norm
-        {
-            cur = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
-
-            // cur = ln_1_g*cur + ln_1_b
-            cur = ggml_add(ctx0,
-                    ggml_mul(ctx0,
-                        ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),
-                        cur),
-                    ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));
-        }
-
-        struct ggml_tensor * inpSA = cur;
-
-        // self-attention
-        {
-            struct ggml_tensor * Qcur = ggml_rope(
-                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N),
-                KQ_pos, n_rot, 0, 0
-            );
-            struct ggml_tensor * Kcur = ggml_rope(
-                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N),
-                KQ_pos, n_rot, 0, 0
-            );
-
-            // store key and value to memory
-            {
-                struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur));
-
-                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
-                        (   n_ctx)*ggml_element_size(model.kv_self.v),
-                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
-
-                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
-            }
-
-            // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
-
-            // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
-            struct ggml_tensor * K =
-                ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0,
-                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                            n_embd/n_head, n_head, n_past + N),
-                        0, 2, 1, 3);
-
-            // K * Q
-            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
-
-            // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled = ggml_scale(ctx0, KQ, 1.0f/sqrt(float(n_embd)/n_head));
-
-            // KQ_masked = mask_past(KQ_scaled)
-            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past);
-
-            // KQ = soft_max(KQ_masked)
-            struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
-
-            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V =
-                ggml_view_3d(ctx0, model.kv_self.v,
-                        n_past + N, n_embd/n_head, n_head,
-                        n_ctx*ggml_element_size(model.kv_self.v),
-                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
-                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
-
-            // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
-
-            // KQV_merged = KQV.permute(0, 2, 1, 3)
-            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            // cur = KQV_merged.contiguous().view(n_embd, N)
-            cur = ggml_cpy(ctx0,
-                    KQV_merged,
-                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-            // projection (no bias)
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].c_attn_proj_w,
-                    cur);
-        }
-
-        struct ggml_tensor * inpFF = cur;
-
-        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
-        // feed-forward network
-        // this is independent of the self-attention result, so it could be done in parallel to the self-attention
-        {
-            // note here we pass inpSA instead of cur
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].c_mlp_fc_w,
-                    inpSA);
-
-            cur = ggml_add(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),
-                    cur);
-
-            // GELU activation
-            cur = ggml_gelu(ctx0, cur);
-
-            // projection
-            // cur = proj_w*cur + proj_b
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].c_mlp_proj_w,
-                    cur);
-
-            cur = ggml_add(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),
-                    cur);
-        }
-
-        // self-attention + FF
-        cur  = ggml_add(ctx0, cur, inpFF);
-
-        // input for next layer
-        inpL = ggml_add(ctx0, cur, inpL);
-    }
-
-    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-    // norm
-    {
-        inpL = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
-
-        // inpL = ln_f_g*inpL + ln_f_b
-        inpL = ggml_add(ctx0,
-                ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.ln_f_g, inpL),
-                    inpL),
-                ggml_repeat(ctx0, model.ln_f_b, inpL));
-    }
-
-    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
-
-    // lm_head
-    {
-        inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL);
-
-        inpL = ggml_add(ctx0,
-                ggml_repeat(ctx0, model.lmh_b, inpL),
-                inpL);
-    }
-
-    // logits -> probs
-    //inpL = ggml_soft_max(ctx0, inpL);
-
-    ggml_build_forward_expand(gf, inpL);
-
-    // run the computation
-    {
-        std::unique_ptr<uint8_t []> data;
-        auto plan = ggml_graph_plan(gf, n_threads);
-        if (plan.work_size > 0) {
-            data.reset(new uint8_t[plan.work_size]);
-            plan.work_data = data.get();
-        }
-        ggml_graph_compute(gf, &plan);
-    }
-
-    //if (n_past%100 == 0) {
-    //    ggml_graph_print   (gf);
-    //    ggml_graph_dump_dot(gf, NULL, "gpt-2.dot");
-    //}
-
-    //embd_w.resize(n_vocab*N);
-    //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
-
-    // return result for just the last token
-    embd_w.resize(n_vocab);
-    memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
-
-    if (mem_per_token == 0) {
-        mem_per_token = ggml_used_mem(ctx0)/N;
-    }
-    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
-
-    ggml_free(ctx0);
-
-    return true;
-}
-
-#define GPTJ_MAX_RNG_STATE 64*1024
-
-size_t gptj_get_state_size(const gptj_model &model)
-{
-    // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state.
-    // for reference, std::mt19937(1337) serializes to 6701 bytes.
-    const size_t s_rng_size        = sizeof(size_t);
-    const size_t s_rng             = GPTJ_MAX_RNG_STATE;
-    const size_t s_kv_size         = sizeof(size_t);
-    const size_t s_kv_ntok         = sizeof(int);
-    const size_t s_kv              = model.kv_self.buf.size;
-    const size_t s_total = (
-        + s_rng_size
-        + s_rng
-        + s_kv_size
-        + s_kv_ntok
-        + s_kv
-    );
-    fflush(stdout);
-    return s_total;
-}
-
-size_t gptj_copy_state_data(const gptj_model &model, const std::mt19937 &rng, uint8_t *dest)
-{
-    uint8_t * out = dest;
-    fflush(stdout);
-    // copy rng
-    {
-        std::stringstream rng_ss;
-        rng_ss << rng;
-
-        const size_t rng_size = rng_ss.str().size();
-        char rng_buf[GPTJ_MAX_RNG_STATE];
-
-        memset(&rng_buf[0], 0, GPTJ_MAX_RNG_STATE);
-        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
-
-        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
-        memcpy(out, &rng_buf[0], GPTJ_MAX_RNG_STATE); out += GPTJ_MAX_RNG_STATE;
-    }
-
-    // copy kv cache
-    {
-        const size_t kv_size = model.kv_self.buf.size;
-        const int    kv_ntok = model.kv_self.n;
-
-        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
-        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
-
-        if (kv_size) {
-            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
-        }
-    }
-
-    const size_t written  = out - dest;
-    assert(written == gptj_get_state_size(model));
-    fflush(stdout);
-    return written;
-}
-
-size_t gptj_set_state_data(gptj_model *model, std::mt19937 *rng, const uint8_t *src)
-{
-    const uint8_t * in = src;
-
-    // set rng
-    {
-        size_t rng_size;
-        char   rng_buf[GPTJ_MAX_RNG_STATE];
-
-        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
-        memcpy(&rng_buf[0], in, GPTJ_MAX_RNG_STATE); in += GPTJ_MAX_RNG_STATE;
-
-        std::stringstream rng_ss;
-        rng_ss.str(std::string(&rng_buf[0], rng_size));
-        rng_ss >> *rng;
-
-        assert(rng_ss.fail() == false);
-    }
-
-    // set kv cache
-    {
-        size_t kv_size;
-        int kv_ntok;
-
-        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
-        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
-
-        if (kv_size) {
-            assert(model->kv_self.buf.size == kv_size);
-
-            void * k_data = model->kv_self.k->data; // remember data pointers
-            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
-
-            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
-
-            model->kv_self.k->data = k_data; // restore correct data pointers
-            model->kv_self.v->data = v_data;
-
-        }
-
-        model->kv_self.n = kv_ntok;
-    }
-
-    const size_t nread    = in - src;
-    assert(nread == gptj_get_state_size(*model));
-    fflush(stdout);
-    return nread;
-}
-
-struct GPTJPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    gpt_vocab vocab;
-    gptj_model *model = nullptr;
-    int64_t n_threads = 0;
-    size_t mem_per_token = 0;
-    std::mt19937 rng;
-};
-
-GPTJ::GPTJ()
-    : d_ptr(new GPTJPrivate) {
-    d_ptr->model = new gptj_model;
-    d_ptr->model->ctx = nullptr;
-    d_ptr->modelLoaded = false;
-}
-
-size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
-{
-    (void)n_ctx;
-    (void)ngl;
-    gptj_model dummy_model;
-    gpt_vocab dummy_vocab;
-    size_t mem_req;
-    gptj_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
-    return mem_req;
-}
-
-bool GPTJ::loadModel(const std::string &modelPath, int n_ctx, int ngl)
-{
-    (void)n_ctx;
-    (void)ngl;
-    d_ptr->modelLoaded = false;
-
-    std::mt19937 rng(time(NULL));
-    d_ptr->rng = rng;
-
-    // load the model
-    bool ok = gptj_model_load(modelPath, *d_ptr->model, d_ptr->vocab);
-    fflush(stdout);
-    if (!ok) {
-        std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
-        return false;
-    }
-
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    return true;
-}
-
-void GPTJ::setThreadCount(int32_t n_threads)
-{
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t GPTJ::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-GPTJ::~GPTJ()
-{
-    delete d_ptr->model;
-}
-
-bool GPTJ::isModelLoaded() const
-{
-    return d_ptr->modelLoaded;
-}
-
-size_t GPTJ::stateSize() const
-{
-    return gptj_get_state_size(*d_ptr->model);
-}
-
-size_t GPTJ::saveState(uint8_t *dest) const
-{
-    return gptj_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
-}
-
-size_t GPTJ::restoreState(const uint8_t *src)
-{
-    return gptj_set_state_data(d_ptr->model, &d_ptr->rng, src);
-}
-
-std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &ctx, const std::string &str, bool special) const
-{
-    (void)ctx;
-    (void)special;
-    return ::gpt_tokenize(d_ptr->vocab, str);
-}
-
-LLModel::Token GPTJ::sampleToken(PromptContext &promptCtx) const
-{
-    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
-    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
-        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks,
-        promptCtx.logits,
-        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty,
-        d_ptr->rng);
-}
-
-std::string GPTJ::tokenToString(Token id) const
-{
-    return d_ptr->vocab.id_to_token[id];
-}
-
-bool GPTJ::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    // determine the required inference memory per token:
-    static bool initialized = false;
-    if (!initialized) {
-        gptj_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
-            d_ptr->mem_per_token);
-        initialized = true;
-    }
-
-    return gptj_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
-}
-
-int32_t GPTJ::contextLength() const
-{
-    return d_ptr->model->hparams.n_ctx;
-}
-
-const std::vector<LLModel::Token> &GPTJ::endTokens() const
-{
-    static const std::vector<LLModel::Token> fres = {50256};
-    return fres;
-}
-
-const char *get_arch_name(gguf_context *ctx_gguf)
-{
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    if (kid == -1)
-        throw std::runtime_error("key not found in model: general.architecture");
-
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING)
-        throw std::runtime_error("key general.architecture has wrong type");
-
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation()
-{
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type()
-{
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant()
-{
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT char *get_file_arch(const char *fname)
-{
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-
-    char *arch = nullptr;
-    if (ctx_gguf && gguf_get_version(ctx_gguf) <= 3) {
-        try {
-            arch = strdup(get_arch_name(ctx_gguf));
-        } catch (const std::runtime_error &) {
-            // cannot read key -> return null
-        }
-    }
-
-    gguf_free(ctx_gguf);
-    return arch;
-}
-
-DLL_EXPORT bool is_arch_supported(const char *arch)
-{
-    return !strcmp(arch, "gptj");
-}
-
-DLL_EXPORT LLModel *construct()
-{
-    return new GPTJ;
-}
-}

gpt4all-backend/gptj_impl.h

@@ -1,43 +0,0 @@
-#ifndef GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of gptj.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef GPTJ_H
-#define GPTJ_H
-
-#include "llmodel.h"
-
-#include <functional>
-#include <string>
-#include <vector>
-
-struct GPTJPrivate;
-class GPTJ : public LLModel {
-public:
-    GPTJ();
-    ~GPTJ();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    GPTJPrivate *d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token id) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token> &endTokens() const override;
-    bool shouldAddBOS() const override { return false; }
-};
-
-#endif // GPTJ_H

gpt4all-backend/llama.cpp.cmake

@@ -7,7 +7,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 #
 # some of the options here are commented out so they can be set "dynamically" before calling include_ggml()
 
-set(LLAMA_LLAMAFILE_DEFAULT ON)
+set(GGML_LLAMAFILE_DEFAULT ON)
 
 # general
 option(LLAMA_STATIC                     "llama: static link libraries"                          OFF)
@@ -22,15 +22,15 @@ option(LLAMA_GPROF                      "llama: enable gprof"
 option(LLAMA_FATAL_WARNINGS             "llama: enable -Werror flag"                            OFF)
 
 # instruction set specific
-#option(LLAMA_AVX                    "llama: enable AVX"                                     ON)
-#option(LLAMA_AVX2                   "llama: enable AVX2"                                    ON)
-#option(LLAMA_AVX512                 "llama: enable AVX512"                                  OFF)
-#option(LLAMA_AVX512_VBMI            "llama: enable AVX512-VBMI"                             OFF)
-#option(LLAMA_AVX512_VNNI            "llama: enable AVX512-VNNI"                             OFF)
-#option(LLAMA_FMA                    "llama: enable FMA"                                     ON)
+#option(GGML_AVX                     "ggml: enable AVX"                                     ON)
+#option(GGML_AVX2                    "ggml: enable AVX2"                                    ON)
+#option(GGML_AVX512                  "ggml: enable AVX512"                                  OFF)
+#option(GGML_AVX512_VBMI             "ggml: enable AVX512-VBMI"                             OFF)
+#option(GGML_AVX512_VNNI             "ggml: enable AVX512-VNNI"                             OFF)
+#option(GGML_FMA                     "ggml: enable FMA"                                     ON)
 # in MSVC F16C is implied with AVX2/AVX512
 #if (NOT MSVC)
-#    option(LLAMA_F16C               "llama: enable F16C"                                    ON)
+#    option(GGML_F16C                "ggml: enable F16C"                                    ON)
 #endif()
 
 if (WIN32)
@@ -38,40 +38,46 @@ if (WIN32)
 endif()
 
 # 3rd party libs
-option(LLAMA_ACCELERATE                      "llama: enable Accelerate framework"               ON)
-option(LLAMA_BLAS                            "llama: use BLAS"                                  OFF)
-option(LLAMA_LLAMAFILE                       "llama: use llamafile SGEMM"                       ${LLAMA_LLAMAFILE_DEFAULT})
-set(LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor")
-#option(LLAMA_CUDA                            "llama: use CUDA"                                  OFF)
-option(LLAMA_CUDA_FORCE_DMMV                 "llama: use dmmv instead of mmvq CUDA kernels"     OFF)
-option(LLAMA_CUDA_FORCE_MMQ                  "llama: use mmq kernels instead of cuBLAS"         OFF)
-set(LLAMA_CUDA_DMMV_X      "32" CACHE STRING "llama: x stride for dmmv CUDA kernels")
-set(LLAMA_CUDA_MMV_Y        "1" CACHE STRING "llama: y block size for mmv CUDA kernels")
-option(LLAMA_CUDA_F16                        "llama: use 16 bit floats for some calculations"   OFF)
-set(LLAMA_CUDA_KQUANTS_ITER "2" CACHE STRING "llama: iters./thread per block for Q2_K/Q6_K")
-set(LLAMA_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
-                                             "llama: max. batch size for using peer access")
-option(LLAMA_CUDA_NO_PEER_COPY               "llama: do not use peer to peer copies"            OFF)
-#option(LLAMA_HIPBLAS                         "llama: use hipBLAS"                               OFF)
-option(LLAMA_HIP_UMA                         "llama: use HIP unified memory architecture"       OFF)
-#option(LLAMA_CLBLAST                         "llama: use CLBlast"                               OFF)
-#option(LLAMA_VULKAN                          "llama: use Vulkan"                                OFF)
-option(LLAMA_VULKAN_CHECK_RESULTS            "llama: run Vulkan op checks"                      OFF)
-option(LLAMA_VULKAN_DEBUG                    "llama: enable Vulkan debug output"                OFF)
-option(LLAMA_VULKAN_VALIDATE                 "llama: enable Vulkan validation"                  OFF)
-option(LLAMA_VULKAN_RUN_TESTS                "llama: run Vulkan tests"                          OFF)
-#option(LLAMA_METAL                           "llama: use Metal"                                 ${LLAMA_METAL_DEFAULT})
-option(LLAMA_METAL_NDEBUG                    "llama: disable Metal debugging"                   OFF)
-option(LLAMA_METAL_SHADER_DEBUG              "llama: compile Metal with -fno-fast-math"         OFF)
-set(LLAMA_METAL_MACOSX_VERSION_MIN "" CACHE STRING
-                                             "llama: metal minimum macOS version")
-set(LLAMA_METAL_STD "" CACHE STRING          "llama: metal standard version (-std flag)")
-#option(LLAMA_KOMPUTE                         "llama: use Kompute"                               OFF)
-option(LLAMA_QKK_64                          "llama: use super-block size of 64 for k-quants"   OFF)
-set(LLAMA_SCHED_MAX_COPIES  "4" CACHE STRING "llama: max input copies for pipeline parallelism")
+option(GGML_ACCELERATE                      "ggml: enable Accelerate framework"               ON)
+option(GGML_BLAS                            "ggml: use BLAS"                                  OFF)
+option(GGML_LLAMAFILE                       "ggml: use llamafile SGEMM"                       ${GGML_LLAMAFILE_DEFAULT})
+set(GGML_BLAS_VENDOR "Generic" CACHE STRING "ggml: BLAS library vendor")
+
+#option(GGML_CUDA                            "ggml: use CUDA"                                  OFF)
+option(GGML_CUDA_FORCE_DMMV                 "ggml: use dmmv instead of mmvq CUDA kernels"     OFF)
+option(GGML_CUDA_FORCE_MMQ                  "ggml: use mmq kernels instead of cuBLAS"         OFF)
+option(GGML_CUDA_FORCE_CUBLAS               "ggml: always use cuBLAS instead of mmq kernels"  OFF)
+set   (GGML_CUDA_DMMV_X   "32" CACHE STRING "ggml: x stride for dmmv CUDA kernels")
+set   (GGML_CUDA_MMV_Y     "1" CACHE STRING "ggml: y block size for mmv CUDA kernels")
+option(GGML_CUDA_F16                        "ggml: use 16 bit floats for some calculations"   OFF)
+set   (GGML_CUDA_KQUANTS_ITER "2" CACHE STRING
+                                            "ggml: iters./thread per block for Q2_K/Q6_K")
+set   (GGML_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
+                                            "ggml: max. batch size for using peer access")
+option(GGML_CUDA_NO_PEER_COPY               "ggml: do not use peer to peer copies"            OFF)
+option(GGML_CUDA_NO_VMM                     "ggml: do not try to use CUDA VMM"                OFF)
+option(GGML_CUDA_FA_ALL_QUANTS              "ggml: compile all quants for FlashAttention"     OFF)
+option(GGML_CUDA_USE_GRAPHS                 "ggml: use CUDA graphs (llama.cpp only)"          OFF)
+
+#option(GGML_HIPBLAS                         "ggml: use hipBLAS"                               OFF)
+option(GGML_HIP_UMA                         "ggml: use HIP unified memory architecture"       OFF)
+#option(GGML_VULKAN                          "ggml: use Vulkan"                                OFF)
+option(GGML_VULKAN_CHECK_RESULTS            "ggml: run Vulkan op checks"                      OFF)
+option(GGML_VULKAN_DEBUG                    "ggml: enable Vulkan debug output"                OFF)
+option(GGML_VULKAN_VALIDATE                 "ggml: enable Vulkan validation"                  OFF)
+option(GGML_VULKAN_RUN_TESTS                "ggml: run Vulkan tests"                          OFF)
+#option(GGML_METAL                           "ggml: use Metal"                                 ${GGML_METAL_DEFAULT})
+option(GGML_METAL_NDEBUG                    "ggml: disable Metal debugging"                   OFF)
+option(GGML_METAL_SHADER_DEBUG              "ggml: compile Metal with -fno-fast-math"         OFF)
+set(GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING
+                                            "ggml: metal minimum macOS version")
+set(GGML_METAL_STD "" CACHE STRING          "ggml: metal standard version (-std flag)")
+#option(GGML_KOMPUTE                        "ggml: use Kompute"                               OFF)
+option(GGML_QKK_64                          "ggml: use super-block size of 64 for k-quants"   OFF)
+set(GGML_SCHED_MAX_COPIES  "4" CACHE STRING "ggml: max input copies for pipeline parallelism")
 
 # add perf arguments
-option(LLAMA_PERF                            "llama: enable perf"                               OFF)
+option(LLAMA_PERF                           "llama: enable perf"                               OFF)
 
 #
 # Compile flags
@@ -80,14 +86,14 @@ option(LLAMA_PERF                            "llama: enable perf"
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-list(APPEND GGML_COMPILE_DEFS GGML_SCHED_MAX_COPIES=${LLAMA_SCHED_MAX_COPIES})
+list(APPEND GGML_COMPILE_DEFS GGML_SCHED_MAX_COPIES=${GGML_SCHED_MAX_COPIES})
 
 # enable libstdc++ assertions for debug builds
 if (CMAKE_SYSTEM_NAME MATCHES "Linux")
     list(APPEND GGML_COMPILE_DEFS $<$<CONFIG:Debug>:_GLIBCXX_ASSERTIONS>)
 endif()
 
-if (APPLE AND LLAMA_ACCELERATE)
+if (APPLE AND GGML_ACCELERATE)
     find_library(ACCELERATE_FRAMEWORK Accelerate)
     if (ACCELERATE_FRAMEWORK)
         message(STATUS "Accelerate framework found")
@@ -101,7 +107,7 @@ if (APPLE AND LLAMA_ACCELERATE)
     endif()
 endif()
 
-if (LLAMA_BLAS)
+if (GGML_BLAS)
     if (LLAMA_STATIC)
         set(BLA_STATIC ON)
     endif()
@@ -109,7 +115,7 @@ if (LLAMA_BLAS)
         set(BLA_SIZEOF_INTEGER 8)
     endif()
 
-    set(BLA_VENDOR ${LLAMA_BLAS_VENDOR})
+    set(BLA_VENDOR ${GGML_BLAS_VENDOR})
     find_package(BLAS)
 
     if (BLAS_FOUND)
@@ -119,24 +125,24 @@ if (LLAMA_BLAS)
             # BLAS_INCLUDE_DIRS is missing in FindBLAS.cmake.
             # see https://gitlab.kitware.com/cmake/cmake/-/issues/20268
             find_package(PkgConfig REQUIRED)
-            if (${LLAMA_BLAS_VENDOR} MATCHES "Generic")
+            if (${GGML_BLAS_VENDOR} MATCHES "Generic")
                 pkg_check_modules(DepBLAS REQUIRED blas)
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "OpenBLAS")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "OpenBLAS")
                 # As of openblas v0.3.22, the 64-bit is named openblas64.pc
                 pkg_check_modules(DepBLAS openblas64)
                 if (NOT DepBLAS_FOUND)
                     pkg_check_modules(DepBLAS REQUIRED openblas)
                 endif()
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "FLAME")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "FLAME")
                 pkg_check_modules(DepBLAS REQUIRED blis)
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "ATLAS")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "ATLAS")
                 pkg_check_modules(DepBLAS REQUIRED blas-atlas)
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "FlexiBLAS")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "FlexiBLAS")
                 pkg_check_modules(DepBLAS REQUIRED flexiblas_api)
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "Intel")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "Intel")
                 # all Intel* libraries share the same include path
                 pkg_check_modules(DepBLAS REQUIRED mkl-sdl)
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "NVHPC")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "NVHPC")
                 # this doesn't provide pkg-config
                 # suggest to assign BLAS_INCLUDE_DIRS on your own
                 if ("${NVHPC_VERSION}" STREQUAL "")
@@ -170,7 +176,7 @@ if (LLAMA_BLAS)
 
         list(APPEND GGML_COMPILE_DEFS GGML_USE_OPENBLAS)
 
-        if (${BLAS_INCLUDE_DIRS} MATCHES "mkl" AND (${LLAMA_BLAS_VENDOR} MATCHES "Generic" OR ${LLAMA_BLAS_VENDOR} MATCHES "Intel"))
+        if (${BLAS_INCLUDE_DIRS} MATCHES "mkl" AND (${GGML_BLAS_VENDOR} MATCHES "Generic" OR ${GGML_BLAS_VENDOR} MATCHES "Intel"))
             list(APPEND GGML_COMPILE_DEFS GGML_BLAS_USE_MKL)
         endif()
 
@@ -179,18 +185,18 @@ if (LLAMA_BLAS)
     else()
         message(WARNING "BLAS not found, please refer to "
         "https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors"
-        " to set correct LLAMA_BLAS_VENDOR")
+        " to set correct GGML_BLAS_VENDOR")
     endif()
 endif()
 
-if (LLAMA_LLAMAFILE)
+if (GGML_LLAMAFILE)
     list(APPEND GGML_COMPILE_DEFS GGML_USE_LLAMAFILE)
 
-    set(GGML_HEADERS_LLAMAFILE ${DIRECTORY}/sgemm.h)
-    set(GGML_SOURCES_LLAMAFILE ${DIRECTORY}/sgemm.cpp)
+    set(GGML_HEADERS_LLAMAFILE ${DIRECTORY}/ggml/src/llamafile/sgemm.h)
+    set(GGML_SOURCES_LLAMAFILE ${DIRECTORY}/ggml/src/llamafile/sgemm.cpp)
 endif()
 
-if (LLAMA_QKK_64)
+if (GGML_QKK_64)
     list(APPEND GGML_COMPILE_DEFS GGML_QKK_64)
 endif()
 
@@ -361,8 +367,9 @@ function(include_ggml SUFFIX)
     # libraries
     #
 
-    if (LLAMA_CUDA)
-        cmake_minimum_required(VERSION 3.17)
+    if (GGML_CUDA)
+        cmake_minimum_required(VERSION 3.18)  # for CMAKE_CUDA_ARCHITECTURES
+
         get_property(LANGS GLOBAL PROPERTY ENABLED_LANGUAGES)
         if (NOT CUDA IN_LIST LANGS)
             message(FATAL_ERROR "The CUDA language must be enabled.")
@@ -371,26 +378,64 @@ function(include_ggml SUFFIX)
         find_package(CUDAToolkit REQUIRED)
         set(CUDAToolkit_BIN_DIR ${CUDAToolkit_BIN_DIR} PARENT_SCOPE)
 
-        set(GGML_HEADERS_CUDA ${DIRECTORY}/ggml-cuda.h)
+        # architectures are set in gpt4all-backend/CMakeLists.txt
 
-        file(GLOB GGML_SOURCES_CUDA "${DIRECTORY}/ggml-cuda/*.cu")
-        list(APPEND GGML_SOURCES_CUDA "${DIRECTORY}/ggml-cuda.cu")
+        set(GGML_HEADERS_CUDA ${DIRECTORY}/ggml/include/ggml-cuda.h)
+        file(GLOB   GGML_HEADERS_CUDA "${DIRECTORY}/ggml/src/ggml-cuda/*.cuh")
+        list(APPEND GGML_HEADERS_CUDA "${DIRECTORY}/ggml/include/ggml-cuda.h")
+
+        file(GLOB   GGML_SOURCES_CUDA "${DIRECTORY}/ggml/src/ggml-cuda/*.cu")
+        list(APPEND GGML_SOURCES_CUDA "${DIRECTORY}/ggml/src/ggml-cuda.cu")
+        file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/fattn-wmma*.cu")
+        list(APPEND GGML_SOURCES_CUDA ${SRCS})
+        file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/mmq*.cu")
+        list(APPEND GGML_SOURCES_CUDA ${SRCS})
+
+        if (GGML_CUDA_FA_ALL_QUANTS)
+            file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/fattn-vec*.cu")
+            list(APPEND GGML_SOURCES_CUDA ${SRCS})
+            add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS)
+        else()
+            file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/fattn-vec*q4_0-q4_0.cu")
+            list(APPEND GGML_SOURCES_CUDA ${SRCS})
+            file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/fattn-vec*q8_0-q8_0.cu")
+            list(APPEND GGML_SOURCES_CUDA ${SRCS})
+            file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/fattn-vec*f16-f16.cu")
+            list(APPEND GGML_SOURCES_CUDA ${SRCS})
+        endif()
 
         list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_CUDA)
-        if (LLAMA_CUDA_FORCE_DMMV)
+
+        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_DMMV_X=${GGML_CUDA_DMMV_X})
+        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_MMV_Y=${GGML_CUDA_MMV_Y})
+        list(APPEND GGML_COMPILE_DEFS K_QUANTS_PER_ITERATION=${GGML_CUDA_KQUANTS_ITER})
+        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE})
+
+        if (GGML_CUDA_USE_GRAPHS)
+            list(APPEND GGML_COMPILE_DEFS GGML_CUDA_USE_GRAPHS)
+        endif()
+
+        if (GGML_CUDA_FORCE_DMMV)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_FORCE_DMMV)
         endif()
-        if (LLAMA_CUDA_FORCE_MMQ)
+
+        if (GGML_CUDA_FORCE_MMQ)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_FORCE_MMQ)
         endif()
-        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X})
-        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_MMV_Y=${LLAMA_CUDA_MMV_Y})
-        if (LLAMA_CUDA_F16)
+
+        if (GGML_CUDA_FORCE_CUBLAS)
+            list(APPEND GGML_COMPILE_DEFS GGML_CUDA_FORCE_CUBLAS)
+        endif()
+
+        if (GGML_CUDA_NO_VMM)
+            list(APPEND GGML_COMPILE_DEFS GGML_CUDA_NO_VMM)
+        endif()
+
+        if (GGML_CUDA_F16)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_F16)
         endif()
-        list(APPEND GGML_COMPILE_DEFS K_QUANTS_PER_ITERATION=${LLAMA_CUDA_KQUANTS_ITER})
-        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_PEER_MAX_BATCH_SIZE=${LLAMA_CUDA_PEER_MAX_BATCH_SIZE})
-        if (LLAMA_CUDA_NO_PEER_COPY)
+
+        if (GGML_CUDA_NO_PEER_COPY)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_NO_PEER_COPY)
         endif()
 
@@ -406,63 +451,36 @@ function(include_ggml SUFFIX)
         endif()
 
         set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cuda_driver)
-
-        if (DEFINED CMAKE_CUDA_ARCHITECTURES)
-            set(GGML_CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}")
-        else()
-            # 52 == lowest CUDA 12 standard
-            # 60 == f16 CUDA intrinsics
-            # 61 == integer CUDA intrinsics
-            # 70 == compute capability at which unrolling a loop in mul_mat_q kernels is faster
-            if (LLAMA_CUDA_F16 OR LLAMA_CUDA_DMMV_F16)
-                set(GGML_CUDA_ARCHITECTURES "60;61;70") # needed for f16 CUDA intrinsics
-            else()
-                set(GGML_CUDA_ARCHITECTURES "52;61;70") # lowest CUDA 12 standard + lowest for integer intrinsics
-                #set(GGML_CUDA_ARCHITECTURES "") # use this to compile much faster, but only F16 models work
-            endif()
-        endif()
-        message(STATUS "Using CUDA architectures: ${GGML_CUDA_ARCHITECTURES}")
     endif()
 
-    if (LLAMA_CLBLAST)
-        find_package(CLBlast REQUIRED)
-
-        set(GGML_HEADERS_OPENCL ${DIRECTORY}/ggml-opencl.h)
-        set(GGML_SOURCES_OPENCL ${DIRECTORY}/ggml-opencl.cpp)
-
-        list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_CLBLAST)
-
-        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} clblast)
-    endif()
-
-    if (LLAMA_VULKAN)
+    if (GGML_VULKAN)
         find_package(Vulkan REQUIRED)
 
-        set(GGML_HEADERS_VULKAN ${DIRECTORY}/ggml-vulkan.h)
-        set(GGML_SOURCES_VULKAN ${DIRECTORY}/ggml-vulkan.cpp)
+        set(GGML_HEADERS_VULKAN ${DIRECTORY}/ggml/include/ggml-vulkan.h)
+        set(GGML_SOURCES_VULKAN ${DIRECTORY}/ggml/src/ggml-vulkan.cpp)
 
         list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_VULKAN)
 
-        if (LLAMA_VULKAN_CHECK_RESULTS)
+        if (GGML_VULKAN_CHECK_RESULTS)
             list(APPEND GGML_COMPILE_DEFS GGML_VULKAN_CHECK_RESULTS)
         endif()
 
-        if (LLAMA_VULKAN_DEBUG)
+        if (GGML_VULKAN_DEBUG)
             list(APPEND GGML_COMPILE_DEFS GGML_VULKAN_DEBUG)
         endif()
 
-        if (LLAMA_VULKAN_VALIDATE)
+        if (GGML_VULKAN_VALIDATE)
             list(APPEND GGML_COMPILE_DEFS GGML_VULKAN_VALIDATE)
         endif()
 
-        if (LLAMA_VULKAN_RUN_TESTS)
+        if (GGML_VULKAN_RUN_TESTS)
             list(APPEND GGML_COMPILE_DEFS GGML_VULKAN_RUN_TESTS)
         endif()
 
         set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} Vulkan::Vulkan)
     endif()
 
-    if (LLAMA_HIPBLAS)
+    if (GGML_HIPBLAS)
         if ($ENV{ROCM_PATH})
             set(ROCM_PATH $ENV{ROCM_PATH})
         else()
@@ -492,32 +510,32 @@ function(include_ggml SUFFIX)
 
         message(STATUS "HIP and hipBLAS found")
 
-        set(GGML_HEADERS_ROCM ${DIRECTORY}/ggml-cuda.h)
+        set(GGML_HEADERS_ROCM ${DIRECTORY}/ggml/include/ggml-cuda.h)
 
-        file(GLOB GGML_SOURCES_ROCM "${DIRECTORY}/ggml-rocm/*.cu")
-        list(APPEND GGML_SOURCES_ROCM "${DIRECTORY}/ggml-rocm.cu")
+        file(GLOB GGML_SOURCES_ROCM "${DIRECTORY}/ggml/src/ggml-rocm/*.cu")
+        list(APPEND GGML_SOURCES_ROCM "${DIRECTORY}/ggml/src/ggml-rocm.cu")
 
         list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_HIPBLAS GGML_USE_CUDA)
 
-        if (LLAMA_HIP_UMA)
+        if (GGML_HIP_UMA)
             list(APPEND GGML_COMPILE_DEFS GGML_HIP_UMA)
         endif()
 
-        if (LLAMA_CUDA_FORCE_DMMV)
+        if (GGML_CUDA_FORCE_DMMV)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_FORCE_DMMV)
         endif()
 
-        if (LLAMA_CUDA_FORCE_MMQ)
+        if (GGML_CUDA_FORCE_MMQ)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_FORCE_MMQ)
         endif()
 
-        if (LLAMA_CUDA_NO_PEER_COPY)
+        if (GGML_CUDA_NO_PEER_COPY)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_NO_PEER_COPY)
         endif()
 
-        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X})
-        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_MMV_Y=${LLAMA_CUDA_MMV_Y})
-        list(APPEND GGML_COMPILE_DEFS K_QUANTS_PER_ITERATION=${LLAMA_CUDA_KQUANTS_ITER})
+        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_DMMV_X=${GGML_CUDA_DMMV_X})
+        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_MMV_Y=${GGML_CUDA_MMV_Y})
+        list(APPEND GGML_COMPILE_DEFS K_QUANTS_PER_ITERATION=${GGML_CUDA_KQUANTS_ITER})
 
         if (CXX_IS_HIPCC)
             set_source_files_properties(${GGML_SOURCES_ROCM} PROPERTIES LANGUAGE CXX)
@@ -535,9 +553,9 @@ function(include_ggml SUFFIX)
 
     set(LLAMA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY})
 
-    if (LLAMA_KOMPUTE AND NOT GGML_KOMPUTE_ONCE)
+    if (GGML_KOMPUTE AND NOT GGML_KOMPUTE_ONCE)
         set(GGML_KOMPUTE_ONCE ON PARENT_SCOPE)
-        if (NOT EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
+        if (NOT EXISTS "${LLAMA_DIR}/ggml/src/kompute/CMakeLists.txt")
             message(FATAL_ERROR "Kompute not found")
         endif()
         message(STATUS "Kompute found")
@@ -561,12 +579,12 @@ function(include_ggml SUFFIX)
                 set(spv_file ${CMAKE_CURRENT_BINARY_DIR}/${OP_FILE}.spv)
                 add_custom_command(
                     OUTPUT ${spv_file}
-                    DEPENDS ${LLAMA_DIR}/${source}
-                        ${LLAMA_DIR}/kompute-shaders/common.comp
-                        ${LLAMA_DIR}/kompute-shaders/op_getrows.comp
-                        ${LLAMA_DIR}/kompute-shaders/op_mul_mv_q_n_pre.comp
-                        ${LLAMA_DIR}/kompute-shaders/op_mul_mv_q_n.comp
-                    COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
+                    DEPENDS ${LLAMA_DIR}/ggml/src/kompute-shaders/${source}
+                        ${LLAMA_DIR}/ggml/src/kompute-shaders/common.comp
+                        ${LLAMA_DIR}/ggml/src/kompute-shaders/op_getrows.comp
+                        ${LLAMA_DIR}/ggml/src/kompute-shaders/op_mul_mv_q_n_pre.comp
+                        ${LLAMA_DIR}/ggml/src/kompute-shaders/op_mul_mv_q_n.comp
+                    COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/ggml/src/kompute-shaders/${source}
                     COMMENT "Compiling ${source} to ${source}.spv"
                     )
 
@@ -612,39 +630,39 @@ function(include_ggml SUFFIX)
         set(KOMPUTE_OPT_BUILT_IN_VULKAN_HEADER_TAG "v1.3.239" CACHE STRING "Kompute Vulkan headers tag")
         set(KOMPUTE_OPT_LOG_LEVEL Critical CACHE STRING "Kompute log level")
         set(FMT_INSTALL OFF)
-        add_subdirectory(${LLAMA_DIR}/kompute)
+        add_subdirectory(${LLAMA_DIR}/ggml/src/kompute)
 
         # Compile our shaders
         compile_shader(SOURCES
-            kompute-shaders/op_scale.comp
-            kompute-shaders/op_scale_8.comp
-            kompute-shaders/op_add.comp
-            kompute-shaders/op_addrow.comp
-            kompute-shaders/op_mul.comp
-            kompute-shaders/op_silu.comp
-            kompute-shaders/op_relu.comp
-            kompute-shaders/op_gelu.comp
-            kompute-shaders/op_softmax.comp
-            kompute-shaders/op_norm.comp
-            kompute-shaders/op_rmsnorm.comp
-            kompute-shaders/op_diagmask.comp
-            kompute-shaders/op_mul_mat_mat_f32.comp
-            kompute-shaders/op_mul_mat_f16.comp
-            kompute-shaders/op_mul_mat_q8_0.comp
-            kompute-shaders/op_mul_mat_q4_0.comp
-            kompute-shaders/op_mul_mat_q4_1.comp
-            kompute-shaders/op_mul_mat_q6_k.comp
-            kompute-shaders/op_getrows_f32.comp
-            kompute-shaders/op_getrows_f16.comp
-            kompute-shaders/op_getrows_q4_0.comp
-            kompute-shaders/op_getrows_q4_1.comp
-            kompute-shaders/op_getrows_q6_k.comp
-            kompute-shaders/op_rope_f16.comp
-            kompute-shaders/op_rope_f32.comp
-            kompute-shaders/op_cpy_f16_f16.comp
-            kompute-shaders/op_cpy_f16_f32.comp
-            kompute-shaders/op_cpy_f32_f16.comp
-            kompute-shaders/op_cpy_f32_f32.comp
+            op_scale.comp
+            op_scale_8.comp
+            op_add.comp
+            op_addrow.comp
+            op_mul.comp
+            op_silu.comp
+            op_relu.comp
+            op_gelu.comp
+            op_softmax.comp
+            op_norm.comp
+            op_rmsnorm.comp
+            op_diagmask.comp
+            op_mul_mat_mat_f32.comp
+            op_mul_mat_f16.comp
+            op_mul_mat_q8_0.comp
+            op_mul_mat_q4_0.comp
+            op_mul_mat_q4_1.comp
+            op_mul_mat_q6_k.comp
+            op_getrows_f32.comp
+            op_getrows_f16.comp
+            op_getrows_q4_0.comp
+            op_getrows_q4_1.comp
+            op_getrows_q6_k.comp
+            op_rope_f16.comp
+            op_rope_f32.comp
+            op_cpy_f16_f16.comp
+            op_cpy_f16_f32.comp
+            op_cpy_f32_f16.comp
+            op_cpy_f32_f32.comp
         )
 
         # Create a custom target for our generated shaders
@@ -689,12 +707,12 @@ function(include_ggml SUFFIX)
         )
     endif()
 
-    if (LLAMA_KOMPUTE)
+    if (GGML_KOMPUTE)
         list(APPEND GGML_COMPILE_DEFS VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
 
         # Add the stamp to the main sources to ensure dependency tracking
-        set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml-kompute.cpp ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp)
-        set(GGML_HEADERS_KOMPUTE ${LLAMA_DIR}/ggml-kompute.h)
+        set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml/src/ggml-kompute.cpp ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp)
+        set(GGML_HEADERS_KOMPUTE ${LLAMA_DIR}/ggml/include/ggml-kompute.h)
 
         list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_KOMPUTE)
 
@@ -703,7 +721,7 @@ function(include_ggml SUFFIX)
 
     set(CUDA_CXX_FLAGS "")
 
-    if (LLAMA_CUDA)
+    if (GGML_CUDA)
         set(CUDA_FLAGS -use_fast_math)
 
         if (LLAMA_FATAL_WARNINGS)
@@ -750,25 +768,25 @@ function(include_ggml SUFFIX)
         endif()
     endif()
 
-    if (LLAMA_METAL)
+    if (GGML_METAL)
         find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
         find_library(METAL_FRAMEWORK    Metal      REQUIRED)
         find_library(METALKIT_FRAMEWORK MetalKit   REQUIRED)
 
         message(STATUS "Metal framework found")
-        set(GGML_HEADERS_METAL ${DIRECTORY}/ggml-metal.h)
-        set(GGML_SOURCES_METAL ${DIRECTORY}/ggml-metal.m)
+        set(GGML_HEADERS_METAL ${DIRECTORY}/ggml/include/ggml-metal.h)
+        set(GGML_SOURCES_METAL ${DIRECTORY}/ggml/src/ggml-metal.m)
 
         list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_METAL)
-        if (LLAMA_METAL_NDEBUG)
+        if (GGML_METAL_NDEBUG)
             list(APPEND GGML_COMPILE_DEFS GGML_METAL_NDEBUG)
         endif()
 
         # copy ggml-common.h and ggml-metal.metal to bin directory
-        configure_file(${DIRECTORY}/ggml-common.h    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h    COPYONLY)
-        configure_file(${DIRECTORY}/ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)
+        configure_file(${DIRECTORY}/ggml/src/ggml-common.h    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h    COPYONLY)
+        configure_file(${DIRECTORY}/ggml/src/ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)
 
-        if (LLAMA_METAL_SHADER_DEBUG)
+        if (GGML_METAL_SHADER_DEBUG)
             # custom command to do the following:
             #   xcrun -sdk macosx metal    -fno-fast-math -c ggml-metal.metal -o ggml-metal.air
             #   xcrun -sdk macosx metallib                   ggml-metal.air   -o default.metallib
@@ -784,13 +802,13 @@ function(include_ggml SUFFIX)
         endif()
 
         # Append macOS metal versioning flags
-        if (LLAMA_METAL_MACOSX_VERSION_MIN)
-            message(STATUS "Adding -mmacosx-version-min=${LLAMA_METAL_MACOSX_VERSION_MIN} flag to metal compilation")
-            list(APPEND XC_FLAGS -mmacosx-version-min=${LLAMA_METAL_MACOSX_VERSION_MIN})
+        if (GGML_METAL_MACOSX_VERSION_MIN)
+            message(STATUS "Adding -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN} flag to metal compilation")
+            list(APPEND XC_FLAGS -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN})
         endif()
-        if (LLAMA_METAL_STD)
-            message(STATUS "Adding -std=${LLAMA_METAL_STD} flag to metal compilation")
-            list(APPEND XC_FLAGS -std=${LLAMA_METAL_STD})
+        if (GGML_METAL_STD)
+            message(STATUS "Adding -std=${GGML_METAL_STD} flag to metal compilation")
+            list(APPEND XC_FLAGS -std=${GGML_METAL_STD})
         endif()
 
         set(GGML_METALLIB ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib)
@@ -801,7 +819,7 @@ function(include_ggml SUFFIX)
             COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air
             COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h
             COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal
-            DEPENDS ${DIRECTORY}/ggml-metal.metal ${DIRECTORY}/ggml-common.h
+            DEPENDS ${DIRECTORY}/ggml/src/ggml-metal.metal ${DIRECTORY}/ggml/src/ggml-common.h
             COMMENT "Compiling Metal kernels"
             )
         set_source_files_properties(${GGML_METALLIB} DIRECTORY ${CMAKE_SOURCE_DIR} PROPERTIES GENERATED ON)
@@ -855,49 +873,49 @@ function(include_ggml SUFFIX)
              CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64)$"))
         message(STATUS "x86 detected")
         if (MSVC)
-            if (LLAMA_AVX512)
+            if (GGML_AVX512)
                 list(APPEND ARCH_FLAGS /arch:AVX512)
                 # MSVC has no compile-time flags enabling specific
                 # AVX512 extensions, neither it defines the
                 # macros corresponding to the extensions.
                 # Do it manually.
-                if (LLAMA_AVX512_VBMI)
+                if (GGML_AVX512_VBMI)
                     list(APPEND GGML_COMPILE_DEFS $<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
                     list(APPEND GGML_COMPILE_DEFS $<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
                 endif()
-                if (LLAMA_AVX512_VNNI)
+                if (GGML_AVX512_VNNI)
                     list(APPEND GGML_COMPILE_DEFS $<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
                     list(APPEND GGML_COMPILE_DEFS $<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
                 endif()
-            elseif (LLAMA_AVX2)
+            elseif (GGML_AVX2)
                 list(APPEND ARCH_FLAGS /arch:AVX2)
-            elseif (LLAMA_AVX)
+            elseif (GGML_AVX)
                 list(APPEND ARCH_FLAGS /arch:AVX)
             endif()
         else()
-            if (LLAMA_NATIVE)
+            if (GGML_NATIVE)
                 list(APPEND ARCH_FLAGS -march=native)
             endif()
-            if (LLAMA_F16C)
+            if (GGML_F16C)
                 list(APPEND ARCH_FLAGS -mf16c)
             endif()
-            if (LLAMA_FMA)
+            if (GGML_FMA)
                 list(APPEND ARCH_FLAGS -mfma)
             endif()
-            if (LLAMA_AVX)
+            if (GGML_AVX)
                 list(APPEND ARCH_FLAGS -mavx)
             endif()
-            if (LLAMA_AVX2)
+            if (GGML_AVX2)
                 list(APPEND ARCH_FLAGS -mavx2)
             endif()
-            if (LLAMA_AVX512)
+            if (GGML_AVX512)
                 list(APPEND ARCH_FLAGS -mavx512f)
                 list(APPEND ARCH_FLAGS -mavx512bw)
             endif()
-            if (LLAMA_AVX512_VBMI)
+            if (GGML_AVX512_VBMI)
                 list(APPEND ARCH_FLAGS -mavx512vbmi)
             endif()
-            if (LLAMA_AVX512_VNNI)
+            if (GGML_AVX512_VNNI)
                 list(APPEND ARCH_FLAGS -mavx512vnni)
             endif()
         endif()
@@ -916,7 +934,7 @@ function(include_ggml SUFFIX)
     list(APPEND GGML_COMPILE_OPTS "$<$<COMPILE_LANGUAGE:CXX>:${ARCH_FLAGS}>")
     list(APPEND GGML_COMPILE_OPTS "$<$<COMPILE_LANGUAGE:C>:${ARCH_FLAGS}>")
 
-    if (LLAMA_CUDA)
+    if (GGML_CUDA)
         list(APPEND CUDA_CXX_FLAGS ${ARCH_FLAGS})
         list(JOIN CUDA_CXX_FLAGS " " CUDA_CXX_FLAGS_JOINED)  # pass host compiler flags as a single argument
         if (NOT CUDA_CXX_FLAGS_JOINED STREQUAL "")
@@ -928,24 +946,26 @@ function(include_ggml SUFFIX)
     # ggml
 
     add_library(ggml${SUFFIX} OBJECT
-                ${DIRECTORY}/ggml.c
-                ${DIRECTORY}/ggml.h
-                ${DIRECTORY}/ggml-alloc.c
-                ${DIRECTORY}/ggml-alloc.h
-                ${DIRECTORY}/ggml-backend.c
-                ${DIRECTORY}/ggml-backend.h
-                ${DIRECTORY}/ggml-quants.c
-                ${DIRECTORY}/ggml-quants.h
+                ${DIRECTORY}/ggml/include/ggml.h
+                ${DIRECTORY}/ggml/include/ggml-alloc.h
+                ${DIRECTORY}/ggml/include/ggml-backend.h
+                ${DIRECTORY}/ggml/src/ggml.c
+                ${DIRECTORY}/ggml/src/ggml-alloc.c
+                ${DIRECTORY}/ggml/src/ggml-backend.c
+                ${DIRECTORY}/ggml/src/ggml-quants.c
+                ${DIRECTORY}/ggml/src/ggml-quants.h
                 ${GGML_SOURCES_CUDA}      ${GGML_HEADERS_CUDA}
-                ${GGML_SOURCES_OPENCL}    ${GGML_HEADERS_OPENCL}
                 ${GGML_SOURCES_METAL}     ${GGML_HEADERS_METAL}
                 ${GGML_SOURCES_KOMPUTE}   ${GGML_HEADERS_KOMPUTE}
                 ${GGML_SOURCES_VULKAN}    ${GGML_HEADERS_VULKAN}
                 ${GGML_SOURCES_ROCM}      ${GGML_HEADERS_ROCM}
                 ${GGML_SOURCES_LLAMAFILE} ${GGML_HEADERS_LLAMAFILE}
+                ${DIRECTORY}/ggml/src/ggml-aarch64.c
+                ${DIRECTORY}/ggml/src/ggml-aarch64.h
                 )
 
-    target_include_directories(ggml${SUFFIX} PUBLIC ${DIRECTORY} ${LLAMA_EXTRA_INCLUDES})
+    target_include_directories(ggml${SUFFIX} PUBLIC ${DIRECTORY}/ggml/include ${LLAMA_EXTRA_INCLUDES})
+    target_include_directories(ggml${SUFFIX} PRIVATE ${DIRECTORY}/ggml/src)
     target_compile_features(ggml${SUFFIX} PUBLIC c_std_11) # don't bump
 
     target_link_libraries(ggml${SUFFIX} PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
@@ -957,14 +977,15 @@ function(include_ggml SUFFIX)
     # llama
 
     add_library(llama${SUFFIX} STATIC
-                ${DIRECTORY}/llama.cpp
-                ${DIRECTORY}/llama.h
-                ${DIRECTORY}/unicode.h
-                ${DIRECTORY}/unicode.cpp
-                ${DIRECTORY}/unicode-data.cpp
+                ${DIRECTORY}/include/llama.h
+                ${DIRECTORY}/src/llama.cpp
+                ${DIRECTORY}/src/unicode.h
+                ${DIRECTORY}/src/unicode.cpp
+                ${DIRECTORY}/src/unicode-data.cpp
                 )
 
-    target_include_directories(llama${SUFFIX} PUBLIC ${DIRECTORY})
+    target_include_directories(llama${SUFFIX} PUBLIC  ${DIRECTORY}/include ${DIRECTORY}/ggml/include)
+    target_include_directories(llama${SUFFIX} PRIVATE ${DIRECTORY}/src)
     target_compile_features   (llama${SUFFIX} PUBLIC cxx_std_11) # don't bump
 
     target_link_libraries(llama${SUFFIX} PRIVATE
@@ -985,9 +1006,6 @@ function(include_ggml SUFFIX)
         C_STANDARD 11
         C_STANDARD_REQUIRED true
         )
-    if (GGML_CUDA_ARCHITECTURES)
-        set_property(TARGET ggml${SUFFIX} llama${SUFFIX} PROPERTY CUDA_ARCHITECTURES "${GGML_CUDA_ARCHITECTURES}")
-    endif()
 
     target_compile_options(ggml${SUFFIX} PRIVATE "${GGML_COMPILE_OPTS}")
     target_compile_options(llama${SUFFIX} PRIVATE "${GGML_COMPILE_OPTS}")

gpt4all-backend/llamamodel.cpp

@@ -30,9 +30,9 @@
 
 #ifdef GGML_USE_KOMPUTE
 #   include <ggml-kompute.h>
-#elif GGML_USE_VULKAN
+#elif defined(GGML_USE_VULKAN)
 #   include <ggml-vulkan.h>
-#elif GGML_USE_CUDA
+#elif defined(GGML_USE_CUDA)
 #   include <ggml-cuda.h>
 #endif
 
@@ -51,14 +51,14 @@ static const std::vector<const char *> KNOWN_ARCHES {
     // "grok", -- 314B parameters
     "gpt2",
     // "gptj", -- no inference code
-    // "gptneox", -- no inference code
+    "gptneox",
     "mpt",
     "baichuan",
     "starcoder",
-    // "persimmon", -- CUDA generates garbage
     "refact",
     "bert",
     "nomic-bert",
+    // "jina-bert-v2", -- Assertion `i01 >= 0 && i01 < ne01' failed.
     "bloom",
     "stablelm",
     "qwen",
@@ -72,12 +72,20 @@ static const std::vector<const char *> KNOWN_ARCHES {
     "internlm2",
     // "minicpm", -- CUDA generates garbage
     "gemma",
+    "gemma2",
     "starcoder2",
     // "mamba", -- CUDA missing SSM_CONV
     "xverse",
     "command-r",
     // "dbrx", -- 16x12B parameters
     "olmo",
+    "openelm",
+    // "arctic", -- 10B+128x3.66B parameters
+    "deepseek2",
+    "chatglm",
+    // "bitnet", -- tensor not within file bounds?
+    // "t5", -- seq2seq model
+    "jais",
 };
 
 static const std::vector<const char *> EMBEDDING_ARCHES {
@@ -103,6 +111,16 @@ static void llama_log_callback(enum ggml_log_level level, const char *text, void
     }
 }
 
+#ifdef GGML_USE_CUDA
+static void cuda_log_callback(enum ggml_log_level level, const char *text, void *userdata)
+{
+    (void)userdata;
+    if (llama_verbose() || level <= GGML_LOG_LEVEL_WARN) {
+        fputs(text, stderr);
+    }
+}
+#endif
+
 struct gpt_params {
     int32_t seed          = -1;   // RNG seed
     int32_t n_keep        = 0;    // number of tokens to keep from initial prompt
@@ -119,7 +137,7 @@ struct gpt_params {
     bool use_mlock         = false; // use mlock to keep model in memory
 };
 
-static int llama_sample_top_p_top_k(
+static llama_token llama_sample_top_p_top_k(
         llama_context *ctx,
         const llama_token *last_n_tokens_data,
         int last_n_tokens_size,
@@ -127,9 +145,8 @@ static int llama_sample_top_p_top_k(
         float top_p,
         float min_p,
         float temp,
-        float repeat_penalty,
-        int32_t pos) {
-    auto logits = llama_get_logits_ith(ctx, pos);
+        float repeat_penalty) {
+    auto logits = llama_get_logits_ith(ctx, -1);
     auto n_vocab = llama_n_vocab(llama_get_model(ctx));
     // Populate initial list of all candidates
     std::vector<llama_token_data> candidates;
@@ -140,14 +157,22 @@ static int llama_sample_top_p_top_k(
     llama_token_data_array candidates_p = {candidates.data(), candidates.size(), false};
     // Sample repeat penalty
     llama_sample_repetition_penalties(nullptr, &candidates_p, last_n_tokens_data, last_n_tokens_size, repeat_penalty, 0.0f, 0.0f);
-    // Temperature sampling
-    llama_sample_top_k(ctx, &candidates_p, top_k, 1);
-    llama_sample_tail_free(ctx, &candidates_p, 1.0f, 1);
-    llama_sample_typical(ctx, &candidates_p, 1.0f, 1);
-    llama_sample_top_p(ctx, &candidates_p, top_p, 1);
-    llama_sample_min_p(ctx, &candidates_p, min_p, 1);
-    llama_sample_temp(ctx, &candidates_p, temp);
-    return llama_sample_token(ctx, &candidates_p);
+
+    llama_token id;
+    if (temp == 0.0) {
+        // greedy sampling, no probs
+        id = llama_sample_token_greedy(ctx, &candidates_p);
+    } else {
+        // temperature sampling
+        llama_sample_top_k(ctx, &candidates_p, top_k, 1);
+        llama_sample_tail_free(ctx, &candidates_p, 1.0f, 1);
+        llama_sample_typical(ctx, &candidates_p, 1.0f, 1);
+        llama_sample_top_p(ctx, &candidates_p, top_p, 1);
+        llama_sample_min_p(ctx, &candidates_p, min_p, 1);
+        llama_sample_temp(ctx, &candidates_p, temp);
+        id = llama_sample_token(ctx, &candidates_p);
+    }
+    return id;
 }
 
 const char *get_arch_name(gguf_context *ctx_gguf)
@@ -371,6 +396,11 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
         d_ptr->model_params.main_gpu = d_ptr->device;
         d_ptr->model_params.n_gpu_layers = ngl;
         d_ptr->model_params.split_mode = LLAMA_SPLIT_MODE_NONE;
+    } else {
+#ifdef GGML_USE_CUDA
+        std::cerr << "Llama ERROR: CUDA loadModel was called without a device\n";
+        return false;
+#endif // GGML_USE_CUDA
     }
 #elif defined(GGML_USE_METAL)
     (void)ngl;
@@ -383,15 +413,17 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     // always fully offload on Metal
     // TODO(cebtenzzre): use this parameter to allow using more than 53% of system RAM to load a model
     d_ptr->model_params.n_gpu_layers = 100;
-#else
+#else // !KOMPUTE && !VULKAN && !CUDA && !METAL
     (void)ngl;
 #endif
 
-    d_ptr->model = llama_load_model_from_file_gpt4all(modelPath.c_str(), &d_ptr->model_params);
+    d_ptr->model = llama_load_model_from_file(modelPath.c_str(), d_ptr->model_params);
     if (!d_ptr->model) {
         fflush(stdout);
+#ifndef GGML_USE_CUDA
         d_ptr->device = -1;
         d_ptr->deviceName.clear();
+#endif
         std::cerr << "LLAMA ERROR: failed to load model from " << modelPath << std::endl;
         return false;
     }
@@ -434,8 +466,10 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
         std::cerr << "LLAMA ERROR: failed to init context for model " <<  modelPath << std::endl;
         llama_free_model(d_ptr->model);
         d_ptr->model = nullptr;
+#ifndef GGML_USE_CUDA
         d_ptr->device = -1;
         d_ptr->deviceName.clear();
+#endif
         return false;
     }
 
@@ -502,24 +536,34 @@ size_t LLamaModel::restoreState(const uint8_t *src)
     return llama_set_state_data(d_ptr->ctx, const_cast<uint8_t*>(src));
 }
 
-std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str, bool special) const
+std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str, bool special)
 {
-    const bool wantBOS = ctx.n_past == 0 && ctx.tokens.empty();
-    const bool useBOS = wantBOS && shouldAddBOS();
-    auto strCat = wantBOS && !special ? " " + str : str; // insert leading space ourselves, llama.cpp fork doesn't anymore
-    std::vector<LLModel::Token> fres(strCat.size()+4);
-    auto fres_len = llama_tokenize(d_ptr->model, strCat.c_str(), strCat.length(), fres.data(), fres.size(), useBOS, special);
+    bool atStart = m_tokenize_last_token == -1;
+    bool insertSpace = atStart || isSpecialToken(m_tokenize_last_token);
+    std::vector<LLModel::Token> fres(str.length() + 4);
+    int32_t fres_len = llama_tokenize_gpt4all(
+        d_ptr->model, str.c_str(), str.length(), fres.data(), fres.size(), /*add_special*/ atStart,
+        /*parse_special*/ special, /*insert_space*/ insertSpace
+    );
     fres.resize(fres_len);
+    if (fres_len)
+        m_tokenize_last_token = fres.back();
     return fres;
 }
 
+bool LLamaModel::isSpecialToken(Token id) const
+{
+    return llama_token_get_attr(d_ptr->model, id)
+        & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED | LLAMA_TOKEN_ATTR_UNKNOWN);
+}
+
 std::string LLamaModel::tokenToString(Token id) const
 {
     std::vector<char> result(8, 0);
-    const int n_tokens = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), false);
+    const int n_tokens = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), 0, true);
     if (n_tokens < 0) {
         result.resize(-n_tokens);
-        int check = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), false);
+        int check = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), 0, true);
         GGML_ASSERT(check == -n_tokens);
     }
     else {
@@ -535,7 +579,7 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
     return llama_sample_top_p_top_k(d_ptr->ctx,
         promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
         n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.min_p, promptCtx.temp,
-        promptCtx.repeat_penalty, promptCtx.n_last_batch_tokens - 1);
+        promptCtx.repeat_penalty);
 }
 
 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
@@ -545,7 +589,6 @@ bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &toke
     llama_batch batch = llama_batch_init(tokens.size(), 0, 1);
 
     batch.n_tokens = tokens.size();
-    ctx.n_last_batch_tokens = tokens.size();
 
     for (int32_t i = 0; i < batch.n_tokens; i++) {
         batch.token   [i] = tokens[i];
@@ -563,6 +606,30 @@ bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &toke
     return res == 0;
 }
 
+void LLamaModel::shiftContext(PromptContext &promptCtx)
+{
+    // infinite text generation via context shifting
+
+    // erase up to n_ctx*contextErase tokens
+    int n_keep = shouldAddBOS();
+    int n_past = promptCtx.n_past;
+    int n_discard = std::min(n_past - n_keep, int(promptCtx.n_ctx * promptCtx.contextErase));
+
+    assert(n_discard > 0);
+    if (n_discard <= 0)
+        return;
+
+    std::cerr << "Llama: context full, swapping: n_past = " << n_past << ", n_keep = " << n_keep
+              << ", n_discard = " << n_discard << "\n";
+
+    // erase the first n_discard tokens from the context
+    llama_kv_cache_seq_rm (d_ptr->ctx, 0, n_keep,             n_keep + n_discard);
+    llama_kv_cache_seq_add(d_ptr->ctx, 0, n_keep + n_discard, n_past,             -n_discard);
+
+    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
+    promptCtx.n_past = promptCtx.tokens.size();
+}
+
 int32_t LLamaModel::contextLength() const
 {
     return llama_n_ctx(d_ptr->ctx);
@@ -575,10 +642,7 @@ const std::vector<LLModel::Token> &LLamaModel::endTokens() const
 
 bool LLamaModel::shouldAddBOS() const
 {
-    int add_bos = llama_add_bos_token(d_ptr->model);
-    if (add_bos != -1) { return add_bos; }
-    auto vocab_type = llama_vocab_type(d_ptr->model);
-    return vocab_type == LLAMA_VOCAB_TYPE_SPM || vocab_type == LLAMA_VOCAB_TYPE_WPM;
+    return llama_add_bos_token(d_ptr->model);
 }
 
 int32_t LLamaModel::maxContextLength(std::string const &modelPath) const
@@ -723,31 +787,16 @@ bool LLamaModel::initializeGPUDevice(int device, std::string *unavail_reason) co
 #endif
 }
 
-bool LLamaModel::hasGPUDevice() const
-{
-#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
-    return d_ptr->device != -1;
-#else
-    return false;
-#endif
-}
-
 bool LLamaModel::usingGPUDevice() const
 {
-    bool hasDevice;
+    if (!d_ptr->model)
+        return false;
 
+    bool usingGPU = llama_model_using_gpu(d_ptr->model);
 #ifdef GGML_USE_KOMPUTE
-    hasDevice = hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
-    assert(!hasDevice || ggml_vk_has_device());
-#elif defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
-    hasDevice = hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
-#elif defined(GGML_USE_METAL)
-    hasDevice = true;
-#else
-    hasDevice = false;
+    assert(!usingGPU || ggml_vk_has_device());
 #endif
-
-    return hasDevice;
+    return usingGPU;
 }
 
 const char *LLamaModel::backendName() const
@@ -760,6 +809,8 @@ const char *LLamaModel::gpuDeviceName() const
     if (usingGPUDevice()) {
 #if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CUDA)
         return d_ptr->deviceName.c_str();
+#elif defined(GGML_USE_METAL)
+        return "Metal";
 #endif
     }
     return nullptr;
@@ -933,7 +984,7 @@ void LLamaModel::embedInternal(
     const llama_token bos_token = llama_token_bos(d_ptr->model);
     const llama_token eos_token = llama_token_eos(d_ptr->model);
 
-    bool useBOS = shouldAddBOS();
+    bool useBOS = llama_add_bos_token(d_ptr->model);
     bool useEOS = llama_vocab_type(d_ptr->model) == LLAMA_VOCAB_TYPE_WPM;
 
     // no EOS, optional BOS
@@ -941,13 +992,16 @@ void LLamaModel::embedInternal(
         if (!text.empty() && text[0] != ' ') {
             text = ' ' + text; // normalize for SPM - our fork of llama.cpp doesn't add a space prefix
         }
-        wantBOS &= useBOS;
 
         tokens.resize(text.length()+4);
-        int32_t n_tokens = llama_tokenize(d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), wantBOS, false);
+        int32_t n_tokens = llama_tokenize_gpt4all(
+            d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), /*add_special*/ wantBOS,
+            /*parse_special*/ false, /*insert_space*/ false
+        );
         if (n_tokens) {
             (void)eos_token;
-            assert((useEOS && wantBOS) == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
+            (void)useBOS;
+            assert((useEOS && wantBOS && useBOS) == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
             if (useEOS && wantBOS)
                 n_tokens--; // erase EOS/SEP
         }
@@ -1174,6 +1228,9 @@ DLL_EXPORT bool is_arch_supported(const char *arch)
 DLL_EXPORT LLModel *construct()
 {
     llama_log_set(llama_log_callback, nullptr);
+#ifdef GGML_USE_CUDA
+    ggml_backend_cuda_log_set_callback(cuda_log_callback, nullptr);
+#endif
     return new LLamaModel;
 }
 }

gpt4all-backend/llamamodel_impl.h

@@ -6,7 +6,6 @@
 
 #include "llmodel.h"
 
-#include <functional>
 #include <memory>
 #include <string>
 #include <vector>
@@ -34,7 +33,6 @@ public:
     std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0) const override;
     bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const override;
     bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const override;
-    bool hasGPUDevice() const override;
     bool usingGPUDevice() const override;
     const char *backendName() const override;
     const char *gpuDeviceName() const override;
@@ -54,10 +52,12 @@ private:
     bool m_supportsCompletion = false;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) override;
+    bool isSpecialToken(Token id) const override;
     std::string tokenToString(Token id) const override;
     Token sampleToken(PromptContext &ctx) const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    void shiftContext(PromptContext &promptCtx) override;
     int32_t contextLength() const override;
     const std::vector<Token> &endTokens() const override;
     bool shouldAddBOS() const override;

gpt4all-backend/llmodel.cpp

@@ -130,7 +130,7 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
 
         addCudaSearchPath();
 
-        std::string impl_name_re = "(gptj|llamamodel-mainline)-(cpu|metal|kompute|vulkan|cuda)";
+        std::string impl_name_re = "llamamodel-mainline-(cpu|metal|kompute|vulkan|cuda)";
         if (cpu_supports_avx2() == 0) {
             impl_name_re += "-avxonly";
         }

gpt4all-backend/llmodel.h

@@ -2,6 +2,7 @@
 #define LLMODEL_H
 
 #include <algorithm>
+#include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <functional>
@@ -13,11 +14,12 @@
 #include <utility>
 #include <vector>
 
+class Dlhandle;
+
 using namespace std::string_literals;
 
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
-class Dlhandle;
 class LLModel {
 public:
     using Token = int32_t;
@@ -57,23 +59,30 @@ public:
             backend(backend), index(index), type(type), heapSize(heapSize), name(std::move(name)),
             vendor(std::move(vendor)) {}
 
-        std::string selectionName() const { return m_backendNames.at(backend) + ": " + name; }
-        std::string reportedName()  const { return name + " (" + m_backendNames.at(backend) + ")"; }
+        std::string selectionName() const
+        {
+            assert(backend == "cuda"s || backend == "kompute"s);
+            return backendName() + ": " + name;
+        }
+
+        std::string backendName() const { return backendIdToName(backend); }
+
+        static std::string backendIdToName(const std::string &backend) { return s_backendNames.at(backend); }
 
         static std::string updateSelectionName(const std::string &name) {
             if (name == "Auto" || name == "CPU" || name == "Metal")
                 return name;
-            auto it = std::find_if(m_backendNames.begin(), m_backendNames.end(), [&name](const auto &entry) {
+            auto it = std::find_if(s_backendNames.begin(), s_backendNames.end(), [&name](const auto &entry) {
                 return name.starts_with(entry.second + ": ");
             });
-            if (it != m_backendNames.end())
+            if (it != s_backendNames.end())
                 return name;
             return "Vulkan: " + name; // previously, there were only Vulkan devices
         }
 
     private:
-        static inline const std::unordered_map<std::string, std::string> m_backendNames {
-            {"cuda", "CUDA"}, {"kompute", "Vulkan"},
+        static inline const std::unordered_map<std::string, std::string> s_backendNames {
+            {"cpu", "CPU"}, {"metal", "Metal"}, {"cuda", "CUDA"}, {"kompute", "Vulkan"},
         };
     };
 
@@ -114,7 +123,6 @@ public:
     };
 
     struct PromptContext {
-        std::vector<float> logits;      // logits of current context
         std::vector<int32_t> tokens;    // current tokens in the context window
         int32_t n_past = 0;             // number of tokens in past conversation
         int32_t n_ctx = 0;              // number of tokens possible in context window
@@ -126,8 +134,7 @@ public:
         int32_t n_batch = 9;
         float   repeat_penalty = 1.10f;
         int32_t repeat_last_n = 64;     // last n tokens to penalize
-        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context window
-        int32_t n_last_batch_tokens = 0;
+        float   contextErase = 0.5f;    // percent of context to erase if we exceed the context window
     };
 
     using ProgressCallback = std::function<bool(float progress)>;
@@ -152,7 +159,7 @@ public:
                         const std::string &promptTemplate,
                         std::function<bool(int32_t)> promptCallback,
                         std::function<bool(int32_t, const std::string&)> responseCallback,
-                        std::function<bool(bool)> recalculateCallback,
+                        bool allowContextShift,
                         PromptContext &ctx,
                         bool special = false,
                         std::string *fakeReply = nullptr);
@@ -196,7 +203,6 @@ public:
         return false;
     }
 
-    virtual bool hasGPUDevice() const { return false; }
     virtual bool usingGPUDevice() const { return false; }
     virtual const char *backendName() const { return "cpu"; }
     virtual const char *gpuDeviceName() const { return nullptr; }
@@ -206,10 +212,12 @@ public:
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
-    virtual std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special = false) const = 0;
+    virtual std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special = false) = 0;
+    virtual bool isSpecialToken(Token id) const = 0;
     virtual std::string tokenToString(Token id) const = 0;
     virtual Token sampleToken(PromptContext &ctx) const = 0;
     virtual bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const = 0;
+    virtual void shiftContext(PromptContext &promptCtx) = 0;
     virtual int32_t contextLength() const = 0;
     virtual const std::vector<Token> &endTokens() const = 0;
     virtual bool shouldAddBOS() const = 0;
@@ -226,10 +234,6 @@ protected:
         return -1;
     }
 
-    // This is a helper function called from the default implementation of 'prompt' but it can be
-    // shared by all base classes so it isn't virtual
-    void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate);
-
     const Implementation *m_implementation = nullptr;
 
     ProgressCallback m_progressCallback;
@@ -241,16 +245,17 @@ protected:
         return true;
     }
 
-    void decodePrompt(std::function<bool(int32_t)> promptCallback,
+    bool decodePrompt(std::function<bool(int32_t)> promptCallback,
                       std::function<bool(int32_t, const std::string&)> responseCallback,
-                      std::function<bool(bool)> recalculateCallback,
+                      bool allowContextShift,
                       PromptContext &promptCtx,
                       std::vector<Token> embd_inp);
     void generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
-                          std::function<bool(bool)> recalculateCallback,
+                          bool allowContextShift,
                           PromptContext &promptCtx);
 
-private:
+    Token m_tokenize_last_token = -1; // not serialized
+
     friend class LLMImplementation;
 };
 

gpt4all-backend/llmodel_c.cpp

@@ -106,7 +106,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
                     const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
-                    llmodel_recalculate_callback recalculate_callback,
+                    bool allow_context_shift,
                     llmodel_prompt_context *ctx,
                     bool special,
                     const char *fake_reply)
@@ -117,9 +117,6 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
         return response_callback(token_id, response.c_str());
     };
 
-    if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
-        wrapper->promptContext.tokens.resize(ctx->n_past);
-
     // Copy the C prompt context
     wrapper->promptContext.n_past = ctx->n_past;
     wrapper->promptContext.n_ctx = ctx->n_ctx;
@@ -138,13 +135,11 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     auto *fake_reply_p = fake_reply ? &fake_reply_str : nullptr;
 
     // Call the C++ prompt method
-    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, recalculate_callback,
+    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, allow_context_shift,
                              wrapper->promptContext, special, fake_reply_p);
 
     // Update the C context by giving access to the wrappers raw pointers to std::vector data
     // which involves no copies
-    ctx->logits = wrapper->promptContext.logits.data();
-    ctx->logits_size = wrapper->promptContext.logits.size();
     ctx->tokens = wrapper->promptContext.tokens.data();
     ctx->tokens_size = wrapper->promptContext.tokens.size();
 
@@ -287,12 +282,6 @@ bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
     return wrapper->llModel->initializeGPUDevice(device);
 }
 
-bool llmodel_has_gpu_device(llmodel_model model)
-{
-    const auto *wrapper = static_cast<LLModelWrapper *>(model);
-    return wrapper->llModel->hasGPUDevice();
-}
-
 const char *llmodel_model_backend_name(llmodel_model model)
 {
     const auto *wrapper = static_cast<LLModelWrapper *>(model);

gpt4all-backend/llmodel_c.h

@@ -30,8 +30,6 @@ typedef void *llmodel_model;
  * behavior.
  */
 struct llmodel_prompt_context {
-    float *logits;          // logits of current context
-    size_t logits_size;     // the size of the raw logits vector
     int32_t *tokens;        // current tokens in the context window
     size_t tokens_size;     // the size of the raw tokens vector
     int32_t n_past;         // number of tokens in past conversation
@@ -76,13 +74,6 @@ typedef bool (*llmodel_prompt_callback)(int32_t token_id);
  */
 typedef bool (*llmodel_response_callback)(int32_t token_id, const char *response);
 
-/**
- * Callback type for recalculation of context.
- * @param whether the model is recalculating the context.
- * @return a bool indicating whether the model should keep generating.
- */
-typedef bool (*llmodel_recalculate_callback)(bool is_recalculating);
-
 /**
  * Embedding cancellation callback for use with llmodel_embed.
  * @param batch_sizes The number of tokens in each batch that will be embedded.
@@ -177,7 +168,7 @@ uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src);
  * @param prompt_template A string representing the input prompt template.
  * @param prompt_callback A callback function for handling the processing of prompt.
  * @param response_callback A callback function for handling the generated response.
- * @param recalculate_callback A callback function for handling recalculation requests.
+ * @param allow_context_shift Whether to allow shifting of context to make room for more input.
  * @param special True if special tokens in the prompt should be processed, false otherwise.
  * @param fake_reply A string to insert into context as the model's reply, or NULL to generate one.
  * @param ctx A pointer to the llmodel_prompt_context structure.
@@ -186,7 +177,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
                     const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
-                    llmodel_recalculate_callback recalculate_callback,
+                    bool allow_context_shift,
                     llmodel_prompt_context *ctx,
                     bool special,
                     const char *fake_reply);
@@ -291,11 +282,6 @@ bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gp
  */
 bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
 
-/**
- * @return True if a GPU device is successfully initialized, false otherwise.
- */
-bool llmodel_has_gpu_device(llmodel_model model);
-
 /**
  * @return The name of the llama.cpp backend currently in use. One of "cpu", "kompute", or "metal".
  */

gpt4all-backend/llmodel_shared.cpp

@@ -8,41 +8,12 @@
 #include <iostream>
 #include <optional>
 #include <regex>
+#include <sstream>
 #include <stdexcept>
 #include <string>
-#include <unordered_set>
 #include <vector>
 
-// TODO(cebtenzzre): replace this with llama_kv_cache_seq_shift for llamamodel (GPT-J needs this as-is)
-void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate)
-{
-    int n_keep = shouldAddBOS();
-    const int32_t n_discard = (promptCtx.n_ctx - n_keep) * promptCtx.contextErase;
-
-    // Erase the first percentage of context from the tokens
-    std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
-    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
-
-    size_t i = n_keep;
-    promptCtx.n_past = n_keep;
-    while (i < promptCtx.tokens.size()) {
-        size_t batch_end = std::min(i + promptCtx.n_batch, promptCtx.tokens.size());
-        std::vector<int32_t> batch(promptCtx.tokens.begin() + i, promptCtx.tokens.begin() + batch_end);
-        assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
-        if (!evalTokens(promptCtx, batch)) {
-            std::cerr << "LLModel ERROR: Failed to process prompt\n";
-            goto stop_generating;
-        }
-        promptCtx.n_past += batch.size();
-        if (!recalculate(true))
-            goto stop_generating;
-        i = batch_end;
-    }
-    assert(promptCtx.n_past == int32_t(promptCtx.tokens.size()));
-
-stop_generating:
-    recalculate(false);
-}
+namespace ranges = std::ranges;
 
 static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err)
 {
@@ -71,7 +42,7 @@ void LLModel::prompt(const std::string &prompt,
                      const std::string &promptTemplate,
                      std::function<bool(int32_t)> promptCallback,
                      std::function<bool(int32_t, const std::string&)> responseCallback,
-                     std::function<bool(bool)> recalculateCallback,
+                     bool allowContextShift,
                      PromptContext &promptCtx,
                      bool special,
                      std::string *fakeReply)
@@ -88,6 +59,25 @@ void LLModel::prompt(const std::string &prompt,
         return;
     }
 
+    // sanity checks
+    if (promptCtx.n_past > contextLength()) {
+        std::ostringstream ss;
+        ss << "n_past=" << promptCtx.n_past << " is past end of context length=" << contextLength();
+        throw std::out_of_range(ss.str());
+    }
+    if (promptCtx.n_past > promptCtx.tokens.size()) {
+        std::ostringstream ss;
+        ss << "n_past=" << promptCtx.n_past << " is past end of token cache length=" << promptCtx.tokens.size();
+        throw std::out_of_range(ss.str());
+    }
+
+    promptCtx.n_ctx = contextLength();
+    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
+
+    if (promptCtx.n_past < promptCtx.tokens.size())
+        promptCtx.tokens.resize(promptCtx.n_past);
+    m_tokenize_last_token = promptCtx.tokens.empty() ? -1 : promptCtx.tokens.back(); // not serialized
+
     // parse the prompt template
     std::vector<std::smatch> placeholders;
     {
@@ -135,14 +125,16 @@ void LLModel::prompt(const std::string &prompt,
     promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
 
     // decode the user prompt
-    decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+    if (!decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp))
+        return; // error
 
     // decode the assistant's reply, either generated or spoofed
     if (fakeReply == nullptr) {
-        generateResponse(responseCallback, recalculateCallback, promptCtx);
+        generateResponse(responseCallback, allowContextShift, promptCtx);
     } else {
         embd_inp = tokenize(promptCtx, *fakeReply, false);
-        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+        if (!decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp))
+            return; // error
     }
 
     // decode the rest of the prompt template
@@ -156,28 +148,31 @@ void LLModel::prompt(const std::string &prompt,
     }
     if (!asstSuffix.empty()) {
         embd_inp = tokenize(promptCtx, asstSuffix, true);
-        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+        decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp);
     }
 }
 
-void LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
+// returns false on error
+bool LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
                            std::function<bool(int32_t, const std::string&)> responseCallback,
-                           std::function<bool(bool)> recalculateCallback,
+                           bool allowContextShift,
                            PromptContext &promptCtx,
                            std::vector<Token> embd_inp) {
-    // save the context size
-    promptCtx.n_ctx = contextLength();
-
     if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
         responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed.");
         std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
             " tokens and the context window is " << promptCtx.n_ctx << "!\n";
-        return;
+        return false;
     }
 
-    promptCtx.n_predict = std::min(promptCtx.n_predict, promptCtx.n_ctx - (int) embd_inp.size());
-    promptCtx.n_past = std::min(promptCtx.n_past, promptCtx.n_ctx);
-    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
+    // FIXME(jared): There are mitigations for this situation, such as making room before
+    // copying the prompt context, or restoring the KV cache when we restore the prompt
+    // context.
+    if (!allowContextShift && promptCtx.n_past + embd_inp.size() > promptCtx.n_ctx) {
+        std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_eval=" << embd_inp.size()
+                  << ", n_ctx=" << promptCtx.n_ctx << "\n";
+        return false;
+    }
 
     // process the prompt in batches
     size_t i = 0;
@@ -187,94 +182,193 @@ void LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
 
         // Check if the context has run out...
         if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
-            recalculateContext(promptCtx, recalculateCallback);
+            assert(allowContextShift);
+            shiftContext(promptCtx);
             assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
         }
 
         if (!evalTokens(promptCtx, batch)) {
             std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
-            return;
+            return false;
         }
 
         size_t tokens = batch_end - i;
         for (size_t t = 0; t < tokens; ++t) {
-            if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
-                promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(batch.at(t));
             promptCtx.n_past += 1;
             if (!promptCallback(batch.at(t)))
-                return;
+                return false;
         }
         i = batch_end;
     }
+
+    return true;
+}
+
+/*
+ * If string s overlaps with the string key such that some prefix of the key is at the end
+ * of the string, return the position in s where the first match starts. Otherwise, return
+ * std::string::npos. Examples:
+ * s = "bfo",  key = "foo" -> 1
+ * s = "fooa", key = "foo" -> npos
+ */
+static std::string::size_type stringsOverlap(const std::string &s, const std::string &key)
+{
+    if (s.empty() || key.empty())
+        throw std::invalid_argument("arguments to stringsOverlap must not be empty");
+
+    for (int start = std::max(0, int(s.size()) - int(key.size())); start < s.size(); start++) {
+        if (s.compare(start, s.size(), key, 0, s.size() - start) == 0)
+            return start;
+    }
+    return std::string::npos;
 }
 
 void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
-                               std::function<bool(bool)> recalculateCallback,
+                               bool allowContextShift,
                                PromptContext &promptCtx) {
+    static const char *stopSequences[] {
+        "### Instruction", "### Prompt", "### Response", "### Human", "### Assistant", "### Context",
+    };
+
+    // Don't even start if there is no room
+    if (!promptCtx.n_predict)
+        return;
+    if (!allowContextShift && promptCtx.n_past >= promptCtx.n_ctx) {
+        std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_ctx=" << promptCtx.n_ctx
+                  << "\n";
+        return;
+    }
+
     std::string cachedResponse;
     std::vector<Token> cachedTokens;
-    std::unordered_set<std::string> reversePrompts
-        = { "### Instruction", "### Prompt", "### Response", "### Human", "### Assistant", "### Context" };
+    int n_predicted = 0;
 
-    // predict next tokens
-    for (int i = 0; i < promptCtx.n_predict; i++) {
+    // Predict next tokens
+    for (bool stop = false; !stop;) {
+        // Sample next token
+        std::optional<Token> new_tok = sampleToken(promptCtx);
+        std::string new_piece = tokenToString(new_tok.value());
+        cachedTokens.push_back(new_tok.value());
+        cachedResponse += new_piece;
 
-        // sample next token
-        auto id = sampleToken(promptCtx);
+        auto accept = [this, &promptCtx, &cachedTokens, &new_tok, allowContextShift]() -> bool {
+            // Shift context if out of space
+            if (promptCtx.n_past >= promptCtx.n_ctx) {
+                (void)allowContextShift;
+                assert(allowContextShift);
+                shiftContext(promptCtx);
+                assert(promptCtx.n_past < promptCtx.n_ctx);
+            }
 
-        // Check if the context has run out...
-        if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
-            recalculateContext(promptCtx, recalculateCallback);
-            assert(promptCtx.n_past + 1 <= promptCtx.n_ctx);
-        }
+            // Accept the token
+            Token tok = std::exchange(new_tok, std::nullopt).value();
+            if (!evalTokens(promptCtx, { tok })) {
+                // TODO(jared): raise an exception
+                std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
+                return false;
+            }
 
-        if (!evalTokens(promptCtx, { id })) {
-            std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
-            return;
-        }
+            promptCtx.tokens.push_back(tok);
+            promptCtx.n_past += 1;
+            return true;
+        };
 
-        // display text
+        // Check for EOS
+        auto lengthLimit = std::string::npos;
         for (const auto token : endTokens()) {
-            if (id == token) return;
-        }
-
-        const std::string str = tokenToString(id);
-
-        // Check if the provided str is part of our reverse prompts
-        bool foundPartialReversePrompt = false;
-        const std::string completed = cachedResponse + std::string(str);
-        if (reversePrompts.find(completed) != reversePrompts.end())
-            return;
-
-        // Check if it partially matches our reverse prompts and if so, cache
-        for (const auto& s : reversePrompts) {
-            if (s.compare(0, completed.size(), completed) == 0) {
-                foundPartialReversePrompt = true;
-                cachedResponse = completed;
-                break;
+            if (new_tok == token) {
+                stop = true;
+                lengthLimit = cachedResponse.size() - new_piece.size();
             }
         }
 
-        // Regardless the token gets added to our cache
-        cachedTokens.push_back(id);
+        if (lengthLimit != std::string::npos) {
+            // EOS matched
+        } else if (!isSpecialToken(new_tok.value())) {
+            // Check if the response contains a stop sequence
+            for (const auto &p : stopSequences) {
+                auto match = cachedResponse.find(p);
+                if (match != std::string::npos) stop = true;
+                lengthLimit = std::min(lengthLimit, match);
+                if (match == 0) break;
+            }
 
-        // Continue if we have found a partial match
-        if (foundPartialReversePrompt)
-            continue;
-
-        // Empty the cache
-        for (auto t : cachedTokens) {
-            if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
-                promptCtx.tokens.erase(promptCtx.tokens.begin());
-            promptCtx.tokens.push_back(t);
-            promptCtx.n_past += 1;
-            //TODO: Conversion to std::string can be avoided here...
-            if (!responseCallback(t, std::string(tokenToString(t))))
-                return;
+            // Check if the response matches the start of a stop sequence
+            if (lengthLimit == std::string::npos) {
+                for (const auto &p : stopSequences) {
+                    auto match = stringsOverlap(cachedResponse, p);
+                    lengthLimit = std::min(lengthLimit, match);
+                    if (match == 0) break;
+                }
+            }
+        } else if (ranges::find(stopSequences, new_piece) < std::end(stopSequences)) {
+            // Special tokens must exactly match a stop sequence
+            stop = true;
+            lengthLimit = cachedResponse.size() - new_piece.size();
+        }
+
+        // Optionally stop if the context will run out
+        if (!allowContextShift && promptCtx.n_past + cachedTokens.size() >= promptCtx.n_ctx) {
+            std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_ctx="
+                      << promptCtx.n_ctx << "\n";
+            stop = true;
+        }
+
+        // Empty the cache, up to the length limit
+        std::string::size_type responseLength = 0;
+        while (!cachedTokens.empty()) {
+            Token tok = cachedTokens.front();
+            std::string piece = tokenToString(tok);
+
+            // Stop if the piece (or part of it) does not fit within the length limit
+            if (responseLength + (stop ? 1 : piece.size()) > lengthLimit)
+                break;
+
+            // Remove token from cache
+            assert(cachedResponse.starts_with(piece));
+            cachedTokens.erase(cachedTokens.begin(), cachedTokens.begin() + 1);
+            cachedResponse.erase(cachedResponse.begin(), cachedResponse.begin() + piece.size());
+
+            // Accept the token, if needed (not cached)
+            if (cachedTokens.empty() && new_tok && !accept())
+                return;
+
+            // Send the token
+            if (!responseCallback(tok, piece) || ++n_predicted >= promptCtx.n_predict) {
+                stop = true;
+                break;
+            }
+
+            // FIXME(jared): we could avoid printing partial stop sequences if we didn't have to
+            // output token IDs and could cache a partial token for the next prompt call
+            responseLength += piece.size();
+        }
+        assert(cachedTokens.empty() == cachedResponse.empty());
+
+        // Accept the token, if needed (in cache)
+        if (new_tok) {
+            assert(!cachedTokens.empty() && cachedTokens.back() == new_tok);
+            if (stop) {
+                cachedTokens.pop_back();
+            } else if (!accept()) {
+                return;
+            }
         }
-        cachedTokens.clear();
     }
+
+    auto &tokens = promptCtx.tokens;
+    if (tokens.size() < cachedTokens.size()) {
+        /* This is theoretically possible if the longest stop sequence is greater than
+         * n_ctx * contextErase tokens. */
+        throw std::runtime_error("shifted too much context, can't go back");
+    }
+
+    auto discard_start = tokens.end() - cachedTokens.size();
+    assert(std::equal(discard_start, tokens.end(), cachedTokens.begin()));
+    tokens.erase(discard_start, tokens.end());
+
+    promptCtx.n_past -= cachedTokens.size();
 }
 
 void LLModel::embed(

gpt4all-backend/scripts/convert_bert_hf_to_gguf.py

@@ -1,140 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import json
-import struct
-import sys
-from pathlib import Path
-
-import gguf
-import numpy as np
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-
-
-if not 2 <= len(sys.argv) < 4:
-    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = Path(sys.argv[1])
-
-with open(dir_model / "vocab.txt", encoding="utf-8") as f:
-    vocab = f.readlines()
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
-
-
-ARCH = gguf.MODEL_ARCH.BERT
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(dir_model)
-
-block_count = config.num_hidden_layers
-gguf_writer.add_name("BERT")
-gguf_writer.add_context_length(config.max_position_embeddings)
-gguf_writer.add_embedding_length(config.hidden_size)
-gguf_writer.add_feed_forward_length(config.intermediate_size)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_head_count(config.num_attention_heads)
-gguf_writer.add_file_type(ftype)
-
-print("gguf: get tokenizer metadata")
-
-try:
-    with open(dir_model / "tokenizer.json", encoding="utf-8") as f:
-        tokenizer_json = json.load(f)
-except FileNotFoundError as e:
-    print(f'Error: Missing {e.filename!r}', file=sys.stderr)
-    sys.exit(1)
-
-print("gguf: get wordpiece tokenizer vocab")
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-print(tokenizer.encode('I believe the meaning of life is'))
-
-tokens: list[bytearray] = []
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-
-# The number of tokens in tokenizer.json can differ from the expected vocab size.
-# This causes downstream issues with mismatched tensor sizes when running the inference
-for i in range(config.vocab_size):
-    try:
-        text = reverse_vocab[i]
-    except KeyError:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-
-    tokens.append(text)
-
-gguf_writer.add_tokenizer_model("bert")  # wordpiece
-gguf_writer.add_token_list(tokens)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-model = AutoModel.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
-print(model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
-        continue
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        l_type = 1
-    else:
-        l_type = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-backend/scripts/convert_gptj_to_gguf.py

@@ -1,165 +0,0 @@
-#!/usr/bin/env python3
-# Convert GPT-J-6B h5 transformer model to ggml format
-#
-# Load the model using GPTJForCausalLM.
-# Iterate over all variables and write them to a binary file.
-#
-# For each variable, write the following:
-#   - Number of dimensions (int)
-#   - Name length (int)
-#   - Dimensions (int[n_dims])
-#   - Name (char[name_length])
-#   - Data (float[n_dims])
-#
-# By default, the bigger matrices are converted to 16-bit floats.
-# This can be disabled by adding the "ftype" CLI argument.
-#
-# At the start of the ggml file we write the model parameters
-# and vocabulary.
-#
-
-from __future__ import annotations
-
-import sys
-import struct
-import json
-from pathlib import Path
-
-import gguf
-import numpy as np
-from transformers import AutoConfig, AutoTokenizer, GPTJForCausalLM
-from transformers.models.gpt2 import tokenization_gpt2
-
-
-if not 2 <= len(sys.argv) < 4:
-    print("Usage: python {} dir-model [ftype]\n".format(Path(__file__).name))
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = Path(sys.argv[1])
-fname_out = dir_model / "ggml-model.gguf"
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
-
-
-ARCH = gguf.MODEL_ARCH.GPTJ
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(dir_model)
-
-block_count = config.n_layer
-gguf_writer.add_name("GPT-J")
-gguf_writer.add_context_length(config.n_positions)
-gguf_writer.add_embedding_length(config.n_embd)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_feed_forward_length(4 * config.n_embd)
-gguf_writer.add_head_count(config.n_head)
-gguf_writer.add_rope_dimension_count(config.rotary_dim)
-gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
-gguf_writer.add_file_type(ftype)
-
-print("gguf: get gpt2 tokenizer vocab")
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-byte_encoder = tokenization_gpt2.bytes_to_unicode()
-byte_decoder = {v: k for k, v in byte_encoder.items()}
-
-tokens: list[bytearray] = []
-
-for i in range(config.vocab_size):
-    if i in reverse_vocab:
-        try:
-            text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
-        except KeyError:
-            text = bytearray()
-            for c in reverse_vocab[i]:
-                if ord(c) < 256:  # single byte character
-                    text.append(byte_decoder[c])
-                else:  # multibyte special token character
-                    text.extend(c.encode('utf-8'))
-    else:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-
-    tokens.append(text)
-
-
-gguf_writer.add_tokenizer_model("gpt2")
-gguf_writer.add_token_list(tokens)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-model = GPTJForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
-#print (model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-#print (list_vars)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    # we don't need these
-    if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
-        print("  Skipping variable:", name)
-        continue
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        ftype_cur = 1
-    elif ftype == 1 or data.dtype != np.float32:
-        print("  Converting to float32")
-        data = data.astype(np.float32)
-        ftype_cur = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-bindings/cli/README.md

@@ -2,8 +2,7 @@
 
 GPT4All on the command-line.
 
-## Documentation
-<https://docs.gpt4all.io/gpt4all_cli.html>
+More details on the [wiki](https://github.com/nomic-ai/gpt4all/wiki/Python-CLI).
 
 ## Quickstart
 
@@ -34,11 +33,11 @@ python -m pip install --user --upgrade gpt4all typer
 # run the CLI
 python app.py repl
 ```
-By default, it will automatically download the `groovy` model to `.cache/gpt4all/` in your user
-directory, if necessary.
+By default, it will automatically download the `Mistral Instruct` model to `.cache/gpt4all/` in your
+user directory, if necessary.
 
 If you have already saved a model beforehand, specify its path with the `-m`/`--model` argument,
 for example:
 ```shell
-python app.py repl --model /home/user/my-gpt4all-models/gpt4all-13b-snoozy-q4_0.gguf
+python app.py repl --model /home/user/my-gpt4all-models/mistral-7b-instruct-v0.1.Q4_0.gguf
 ```

gpt4all-bindings/python/CHANGELOG.md

@@ -0,0 +1,55 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+
+## [2.8.1] - 2024-08-13
+
+### Added
+- Use greedy sampling when temperature is set to zero ([#2854](https://github.com/nomic-ai/gpt4all/pull/2854))
+
+### Changed
+- Search for pip-installed CUDA 11 as well as CUDA 12 ([#2802](https://github.com/nomic-ai/gpt4all/pull/2802))
+- Stop shipping CUBINs to reduce wheel size ([#2802](https://github.com/nomic-ai/gpt4all/pull/2802))
+- Use llama\_kv\_cache ops to shift context faster ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Don't stop generating at end of context ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+
+### Fixed
+- Make reverse prompt detection work more reliably and prevent it from breaking output ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Explicitly target macOS 12.6 in CI to fix Metal compatibility on older macOS ([#2849](https://github.com/nomic-ai/gpt4all/pull/2849))
+- Do not initialize Vulkan driver when only using CPU ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+- Fix a segfault on exit when using CPU mode on Linux with NVIDIA and EGL ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+
+## [2.8.0] - 2024-08-05
+
+### Added
+- Support GPT-NeoX, Gemma 2, OpenELM, ChatGLM, and Jais architectures (all with Vulkan support) ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
+- Enable Vulkan support for StarCoder2, XVERSE, Command R, and OLMo ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
+- Support DeepSeek-V2 architecture (no Vulkan support) ([#2702](https://github.com/nomic-ai/gpt4all/pull/2702))
+- Add Llama 3.1 8B Instruct to models3.json (by [@3Simplex](https://github.com/3Simplex) in [#2731](https://github.com/nomic-ai/gpt4all/pull/2731) and [#2732](https://github.com/nomic-ai/gpt4all/pull/2732))
+- Support Llama 3.1 RoPE scaling ([#2758](https://github.com/nomic-ai/gpt4all/pull/2758))
+- Add Qwen2-1.5B-Instruct to models3.json (by [@ThiloteE](https://github.com/ThiloteE) in [#2759](https://github.com/nomic-ai/gpt4all/pull/2759))
+- Detect use of a Python interpreter under Rosetta for a clearer error message ([#2793](https://github.com/nomic-ai/gpt4all/pull/2793))
+
+### Changed
+- Build against CUDA 11.8 instead of CUDA 12 for better compatibility with older drivers ([#2639](https://github.com/nomic-ai/gpt4all/pull/2639))
+- Update llama.cpp to commit 87e397d00 from July 19th ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
+
+### Removed
+- Remove unused internal llmodel\_has\_gpu\_device ([#2409](https://github.com/nomic-ai/gpt4all/pull/2409))
+- Remove support for GPT-J models ([#2676](https://github.com/nomic-ai/gpt4all/pull/2676), [#2693](https://github.com/nomic-ai/gpt4all/pull/2693))
+
+### Fixed
+- Fix debug mode crash on Windows and undefined behavior in LLamaModel::embedInternal ([#2467](https://github.com/nomic-ai/gpt4all/pull/2467))
+- Fix CUDA PTX errors with some GPT4All builds ([#2421](https://github.com/nomic-ai/gpt4all/pull/2421))
+- Fix mishandling of inputs greater than n\_ctx tokens after [#1970](https://github.com/nomic-ai/gpt4all/pull/1970) ([#2498](https://github.com/nomic-ai/gpt4all/pull/2498))
+- Fix crash when Kompute falls back to CPU ([#2640](https://github.com/nomic-ai/gpt4all/pull/2640))
+- Fix several Kompute resource management issues ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
+- Fix crash/hang when some models stop generating, by showing special tokens ([#2701](https://github.com/nomic-ai/gpt4all/pull/2701))
+- Fix several backend issues ([#2778](https://github.com/nomic-ai/gpt4all/pull/2778))
+  - Restore leading space removal logic that was incorrectly removed in [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
+  - CUDA: Cherry-pick llama.cpp DMMV cols requirement fix that caused a crash with long conversations since [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
+
+[2.8.1]: https://github.com/nomic-ai/gpt4all/compare/python-v2.8.0...python-v2.8.1
+[2.8.0]: https://github.com/nomic-ai/gpt4all/compare/python-v2.7.0...python-v2.8.0

gpt4all-bindings/python/docs/assets/ubuntu.svg

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="285" height="285" viewBox="-142.5 -142.5 285 285" xmlns:xlink="http://www.w3.org/1999/xlink">
+<circle fill="#FFFFFF" r="141.732"/><g id="U" fill="#DD4814"><circle cx="-96.3772" r="18.9215"/>
+<path d="M-45.6059,68.395C-62.1655,57.3316-74.4844,40.4175-79.6011,20.6065-73.623,15.7354-69.8047,8.3164-69.8047,0-69.8047-8.3164-73.623-15.7354-79.6011-20.6065-74.4844-40.4175-62.1655-57.3316-45.6059-68.395L-31.7715-45.2212C-45.9824-35.2197-55.2754-18.7026-55.2754,0-55.2754,18.7026-45.9824,35.2197-31.7715,45.2212Z"/></g>
+<use xlink:href="#U" transform="rotate(120)"/><use xlink:href="#U" transform="rotate(240)"/></svg>

gpt4all-bindings/python/docs/css/custom.css

@@ -1,5 +1,5 @@
-/* Remove the `In` and `Out` block in rendered Jupyter notebooks */
-.md-container .jp-Cell-outputWrapper .jp-OutputPrompt.jp-OutputArea-prompt,
-.md-container .jp-Cell-inputWrapper .jp-InputPrompt.jp-InputArea-prompt {
-  display: none !important;
-}
+.md-content h1,
+.md-content h2 {
+  margin-top: 0.5em;
+  margin-bottom: 0.5em;
+}

gpt4all-bindings/python/docs/gpt4all_desktop/chats.md

@@ -0,0 +1,124 @@
+# Chats
+
+Chats are conversations with [language models](models.md) that run locally on your device.
+
+!!! note "Example Chats"
+
+    === "Llama 3"
+        Download `Llama 3` and prompt:
+        ```
+        explain why the sky is blue in a way that is correct and makes sense to a child
+        ```
+
+        With the default sampling settings, you should see text resembling the following:
+
+        ```
+        What a great question!
+        So, you know how we can see different colors like red, yellow, green, and orange? Well, when sunlight enters Earth's atmosphere, it starts to interact with tiny particles called molecules of gases like nitrogen (N2) and oxygen (02). These molecules are really small, but they're all around us!
+        Now, here's the cool part: these molecules scatter, or bounce off, shorter wavelengths of light more than longer wavelengths. Think of it like a game of pool where the cue ball hits other balls with different sizes and speeds.
+        The shortest wavelength of visible light is violet (about 400 nanometers), followed by blue (around 450-495 nanometers). These shorter wavelengths are scattered in all directions, kind of like how a basketball player might bounce the ball around on the court. This scattering effect makes the sky appear blue to our eyes!
+        Here's why it doesn't look as bright or intense as a blue crayon: there are many more molecules in the atmosphere that scatter longer wavelengths (like red and orange) than shorter ones, so those colors get spread out and become less noticeable.
+        To sum it up:
+        1. Sunlight enters Earth's atmosphere.
+        2. Tiny gas molecule particles interact with light, scattering shorter wavelengths like blue and violet.
+        3. This scattered blue light reaches our eyes from all directions, making the sky appear blue!
+        Now, go outside and enjoy that beautiful blue sky!
+        ```
+
+    === "Nous Hermes 2 Mistral DPO"
+        Download `Nous Hermes 2 Mistral DPO` and prompt:
+        ```
+        write me a react app i can run from the command line to play a quick game
+        ```
+
+        With the default sampling settings, you should see text and code blocks resembling the following:
+        
+        ```
+        I'll provide you with a simple React app that allows you to play a guessing game. This app will generate a random number between 1 and 10, and ask you to guess it. You can run this app locally using Create React App. Follow the steps below:
+        Open your terminal or command line interface.
+        Navigate to the directory where you want to create the project (e.g., cd Documents/Projects).
+        Run the following commands:
+        ```
+
+        ```bash
+        npx create-react-app guessing-game
+        cd guessing-game
+        ```
+
+        ```
+        Open the project in your preferred code editor by running code . (for VS Code) or any other command to open the folder with your text editor.
+        Replace the content of the src/App.js file with the following code:
+        ```
+
+        ```javascript
+        import React, { useState } from 'react';
+
+        function App() {
+        const [guess, setGuess] = useState('');
+        const [message, setMessage] = useState('');
+        const [numberToGuess, setNumberToGuess] = useState(Math.floor(Math.random() * 10) + 1);
+
+        function handleChange(e) {
+            setGuess(e.target.value);
+        }
+
+        function handleSubmit(e) {
+            e.preventDefault();
+            if (parseInt(guess) === numberToGuess) {
+            setMessage('You guessed it right!');
+            } else {
+            setMessage(`Wrong guess, the correct number was ${numberToGuess}. Try again next time.`);
+            }
+        }
+
+        return (
+            <div>
+            <h1>Guess a Number</h1>
+            <p>{message}</p>
+            <form onSubmit={handleSubmit}>
+                <input type="number" value={guess} onChange={handleChange} />
+                <button type="submit">Submit</button>
+            </form>
+            </div>
+        );
+        }
+
+        export default App;
+        ```
+
+        ```
+        Save the file and go back to your terminal or command line interface. Run npm start. This will start a local development server, and you can access the app in your browser at http://localhost:3000.
+        Play the guessing game by entering a number between 1 and 10 into the input field and clicking "Submit". The app will tell you if your guess is correct or not.
+        Remember that this is just a simple example, and you can expand upon it to make the game more interesting with additional features like high scores, multiple difficulty levels, etc.
+        ```
+
+## New Chat
+
+Choose a model with the dropdown at the top of the Chats page
+
+If you don't have any models, [download one](models.md#download-models). Once you have models, you can start chats by loading your default model, which you can configure in [settings](settings.md#application-settings)
+
+![Choose a model](../assets/three_model_options.png)
+
+## LocalDocs
+
+Open the [LocalDocs](localdocs.md) panel with the button in the top-right corner to bring your files into the chat. With LocalDocs, your chats are enhanced with semantically related snippets from your files included in the model's context.
+
+![Open LocalDocs](../assets/open_local_docs.png)
+
+## Chat History
+
+View your chat history with the button in the top-left corner of the Chats page.
+
+<table>
+<tr>
+    <td>
+    <img src="../assets/closed_chat_panel.png" alt="Close chats" style="width:100%">
+    </td>
+    <td>
+    <img src="../assets/open_chat_panel.png" alt="Open chats" style="width:100%">
+    </td>
+</tr>
+</table>
+
+You can change a chat name or delete it from your chat history at any time.

gpt4all-bindings/python/docs/gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-Obsidian.md

@@ -0,0 +1,109 @@
+# Using GPT4All to Privately Chat with your Obsidian Vault
+
+Obsidian for Desktop is a powerful management and note-taking software designed to create and organize markdown notes. This tutorial allows you to sync and access your Obsidian note files directly on your computer. By connecting it to LocalDocs, you can integrate these files into your LLM chats for private access and enhanced context.
+
+## Download Obsidian for Desktop
+
+!!! note "Download Obsidian for Desktop"
+
+      1. **Download Obsidian for Desktop**:
+         - Visit the [Obsidian website](https://obsidian.md) and create an account account.
+         - Click the Download button in the center of the homepage
+         - For more help with installing Obsidian see [Getting Started with Obsidian](https://help.obsidian.md/Getting+started/Download+and+install+Obsidian)
+      
+      2. **Set Up Obsidian**:
+         - Launch Obsidian from your Applications folder (macOS), Start menu (Windows), or equivalent location (Linux).
+         - On the welcome screen, you can either create a new vault (a collection of notes) or open an existing one.
+         - To create a new vault, click Create a new vault, name your vault, choose a location on your computer, and click Create.
+   
+   
+      3. **Sign in and Sync**:
+            - Once installed, you can start adding and organizing notes.
+            - Choose the folders you want to sync to your computer.
+   
+
+
+## Connect Obsidian to LocalDocs
+
+!!! note "Connect Obsidian to LocalDocs"
+
+      1. **Open LocalDocs**:
+         - Navigate to the LocalDocs feature within GPT4All.
+
+         <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of LocalDocs interface -->
+                  <img width="1348" alt="LocalDocs interface" src="https://github.com/nomic-ai/gpt4all/assets/132290469/d8fb2d79-2063-45d4-bcce-7299fb75b144">
+               </td>
+            </tr>
+         </table>
+   
+      2. **Add Collection**:
+         - Click on **+ Add Collection** to begin linking your Obsidian Vault.
+      
+         <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of adding collection in LocalDocs -->
+                  <img width="1348" alt="Screenshot of adding collection" src="https://raw.githubusercontent.com/nomic-ai/gpt4all/124ef867a9d9afd9e14d3858cd77bce858f79773/gpt4all-bindings/python/docs/assets/obsidian_adding_collection.png">
+               </td>
+            </tr>
+         </table>
+   
+         - Name your collection
+   
+   
+      3. **Create Collection**:
+         - Click **Create Collection** to initiate the embedding process. Progress will be displayed within the LocalDocs interface.
+   
+      4. **Access Files in Chats**:
+         - Load a model to chat with your files (Llama 3 Instruct is the fastest)
+         - In your chat, open 'LocalDocs' with the button in the top-right corner to provide context from your synced Obsidian notes.
+      
+         <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of accessing LocalDocs in chats -->
+                  <img width="1447" alt="Accessing LocalDocs in chats" src="https://raw.githubusercontent.com/nomic-ai/gpt4all/124ef867a9d9afd9e14d3858cd77bce858f79773/gpt4all-bindings/python/docs/assets/obsidian_docs.png">
+               </td>
+            </tr>
+         </table>
+   
+      5. **Interact With Your Notes:**
+         - Use the model to interact with your files
+         <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of interacting sources -->
+                  <img width="662" alt="osbsidian user interaction" src="https://raw.githubusercontent.com/nomic-ai/gpt4all/124ef867a9d9afd9e14d3858cd77bce858f79773/gpt4all-bindings/python/docs/assets/osbsidian_user_interaction.png">
+               </td>
+            </tr>
+         </table>
+         <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of viewing sources -->
+                  <img width="662" alt="osbsidian GPT4ALL response" src="https://raw.githubusercontent.com/nomic-ai/gpt4all/124ef867a9d9afd9e14d3858cd77bce858f79773/gpt4all-bindings/python/docs/assets/obsidian_response.png">
+               </td>
+            </tr>
+         </table>
+   
+      6. **View Referenced Files**:
+         - Click on **Sources** below LLM responses to see which Obsidian Notes were referenced.
+      
+         <table>
+            <tr>
+               <td>
+                  <!-- Referenced Files  -->
+                  <img width="643" alt="Referenced Files" src="https://raw.githubusercontent.com/nomic-ai/gpt4all/124ef867a9d9afd9e14d3858cd77bce858f79773/gpt4all-bindings/python/docs/assets/obsidian_sources.png">
+               </td>
+            </tr>
+         </table>
+
+## How It Works
+
+Obsidian for Desktop syncs your Obsidian notes to your computer, while LocalDocs integrates these files into your LLM chats using embedding models. These models find semantically similar snippets from your files to enhance the context of your interactions.
+
+To learn more about embedding models and explore further, refer to the [Nomic Python SDK documentation](https://docs.nomic.ai/atlas/capabilities/embeddings).
+

gpt4all-bindings/python/docs/gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-One-Drive.md

@@ -0,0 +1,112 @@
+# Using GPT4All to Privately Chat with your OneDrive Data
+
+Local and Private AI Chat with your OneDrive Data
+
+OneDrive for Desktop allows you to sync and access your OneDrive files directly on your computer. By connecting your synced directory to LocalDocs, you can start using GPT4All to privately chat with data stored in your OneDrive.
+
+## Download OneDrive for Desktop
+
+!!! note "Download OneDrive for Desktop"
+
+    1. **Download OneDrive for Desktop**:
+    - Visit [Microsoft OneDrive](https://www.microsoft.com/en-us/microsoft-365/onedrive/download).
+    - Press 'download' for your respective device type.
+    - Download the OneDrive for Desktop application.
+    
+    2. **Install OneDrive for Desktop**
+    - Run the installer file you downloaded.
+    - Follow the prompts to complete the installation process.
+    
+    3. **Sign in and Sync**
+    - Once installed, sign in to OneDrive for Desktop with your Microsoft account credentials.
+    - Choose the folders you want to sync to your computer.
+
+## Connect OneDrive to LocalDocs
+
+!!! note "Connect OneDrive to LocalDocs"
+
+    1. **Install GPT4All and Open LocalDocs**:
+    
+        - Go to [nomic.ai/gpt4all](https://nomic.ai/gpt4all) to install GPT4All for your operating system.
+        
+        - Navigate to the LocalDocs feature within GPT4All to configure it to use your synced OneDrive directory.
+
+        <table>
+        <tr>
+            <td>
+                <!-- Placeholder for screenshot of LocalDocs interface -->
+                <img width="1348" alt="Screenshot 2024-07-10 at 10 55 41 AM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/54254bc0-d9a0-40c4-9fd1-5059abaad583">
+            </td>
+        </tr>
+        </table>
+
+    2. **Add Collection**:
+    
+        - Click on **+ Add Collection** to begin linking your OneDrive folders.
+
+        <table>
+        <tr>
+            <td>
+                <!-- Placeholder for screenshot of adding collection in LocalDocs -->
+               <img width="1348" alt="Screenshot 2024-07-10 at 10 56 29 AM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/7f12969a-753a-4757-bb9e-9b607cf315ca">
+            </td>
+        </tr>
+        </table>
+
+        - Name the Collection and specify the OneDrive folder path.
+
+    3. **Create Collection**:
+    
+        - Click **Create Collection** to initiate the embedding process. Progress will be displayed within the LocalDocs interface.
+
+    4. **Access Files in Chats**:
+    
+        - Load a model within GPT4All to chat with your files.
+        
+        - In your chat, open 'LocalDocs' using the button in the top-right corner to provide context from your synced OneDrive files.
+
+        <table>
+        <tr>
+            <td>
+                <!-- Placeholder for screenshot of accessing LocalDocs in chats -->
+                <img width="1447" alt="Screenshot 2024-07-10 at 10 58 55 AM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/b5a67fe6-0d6a-42ae-b3b8-cc0f91cbf5b1">
+            </td>
+        </tr>
+        </table>
+
+    5. **Interact With Your OneDrive**:
+    
+        - Use the model to interact with your files directly from OneDrive.
+        
+        <table>
+        <tr>
+            <td>
+                <!-- Placeholder for screenshot of interacting with sources -->
+                <img width="662" alt="Screenshot 2024-07-10 at 11 04 55 AM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/2c9815b8-3d1c-4179-bf76-3ddbafb193bf">
+            </td>
+        </tr>
+        </table>
+        
+        <table>
+        <tr>
+            <td>
+                <img width="662" alt="Screenshot 2024-07-11 at 11 21 46 AM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/ce8be292-b025-415a-bd54-f11868e0cd0a">
+            </td>
+        </tr>
+        </table>
+
+    6. **View Referenced Files**:
+    
+        - Click on **Sources** below responses to see which OneDrive files were referenced.
+
+        <table>
+        <tr>
+            <td>
+              <img width="643" alt="Screenshot 2024-07-11 at 11 22 49 AM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/6fe3f10d-2791-4153-88a7-2198ab3ac945">
+            </td>
+        </tr>
+        </table>
+
+## How It Works
+
+OneDrive for Desktop syncs your OneDrive files to your computer, while LocalDocs maintains a database of these synced files for use by your local GPT4All model. As your OneDrive updates, LocalDocs will automatically detect file changes and stay up to date. LocalDocs leverages [Nomic Embedding](https://docs.nomic.ai/atlas/capabilities/embeddings) models to find semantically similar snippets from your files, enhancing the context of your interactions.

gpt4all-bindings/python/docs/gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-google-drive.md

@@ -0,0 +1,113 @@
+# Using GPT4All to Privately Chat with your Google Drive Data
+Local and Private AI Chat with your Google Drive Data
+
+Google Drive for Desktop allows you to sync and access your Google Drive files directly on your computer. By connecting your synced directory to LocalDocs, you can start using GPT4All to privately chat with data stored in your Google Drive.
+
+## Download Google Drive for Desktop
+
+!!! note "Download Google Drive for Desktop"
+
+    1. **Download Google Drive for Desktop**:
+    - Visit [drive.google.com](https://drive.google.com) and sign in with your Google account.
+    - Navigate to the **Settings** (gear icon) and select **Settings** from the dropdown menu.
+    - Scroll down to **Google Drive for desktop** and click **Download**.
+
+    2. **Install Google Drive for Desktop**
+    - Run the installer file you downloaded.
+    - Follow the prompts to complete the installation process.
+
+    3. **Sign in and Sync**
+    - Once installed, sign in to Google Drive for Desktop with your Google account credentials.
+    - Choose the folders you want to sync to your computer.
+
+For advanced help, see [Setting up Google Drive for Desktop](https://support.google.com/drive/answer/10838124?hl=en)
+## Connect Google Drive to LocalDocs
+
+!!! note "Connect Google Drive to LocalDocs"
+
+    1. **Install GPT4All and Open LocalDocs**:
+    
+        - Go to [nomic.ai/gpt4all](https://nomic.ai/gpt4all) to install GPT4All for your operating system.
+        
+        - Navigate to the LocalDocs feature within GPT4All to configure it to use your synced directory.
+
+        <table>
+        <tr>
+            <td>
+                <!-- Screenshot of LocalDocs interface -->
+                <img width="1348" alt="Screenshot 2024-07-09 at 3 15 35 PM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/d8fb2d79-2063-45d4-bcce-7299fb75b144">
+            </td>
+        </tr>
+        </table>
+
+    2. **Add Collection**:
+    
+        - Click on **+ Add Collection** to begin linking your Google Drive folders.
+
+        <table>
+        <tr>
+            <td>
+                <!-- Screenshot of adding collection in LocalDocs -->
+                <img width="1348" alt="Screenshot 2024-07-09 at 3 17 24 PM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/39063615-9eb6-4c47-bde7-c9f04f9b168b">
+            </td>
+        </tr>
+        </table>
+
+        - Name Collection
+
+
+    3. **Create Collection**:
+    
+        - Click **Create Collection** to initiate the embedding process. Progress will be displayed within the LocalDocs interface.
+
+    4. **Access Files in Chats**:
+    
+        - Load a model to chat with your files (Llama 3 Instruct performs best)
+        
+        - In your chat, open 'LocalDocs' with the button in the top-right corner to provide context from your synced Google Drive files.
+
+        <table>
+        <tr>
+            <td>
+                <!-- Screenshot of accessing LocalDocs in chats -->
+                <img width="1447" alt="Screenshot 2024-07-09 at 3 20 53 PM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/ce68811f-9abd-451b-ac0a-fb941e185d7a">
+            </td>
+        </tr>
+        </table>
+
+    5. **Interact With Your Drive:**
+    
+        - Use the model to interact with your files
+        
+        <table>
+        <tr>
+            <td>
+                <!-- Screenshot of interacting sources -->
+                <img width="662" alt="Screenshot 2024-07-09 at 3 36 51 PM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/bc55bc36-e613-419d-a568-adb1cd993854">
+            </td>
+        </tr>
+        </table>
+
+        <table>
+        <tr>
+            <td>
+              <img width="662" alt="Screenshot 2024-07-11 at 11 34 00 AM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/1c0fd19a-5a22-4726-a841-d26c1bea81fc">
+            </td>
+        </tr>
+        </table>
+    
+    6. **View Referenced Files**:
+    
+        - Click on **Sources** below LLM responses to see which Google Drive files were referenced.
+
+        <table>
+        <tr>
+            <td>  
+           <img width="643" alt="Screenshot 2024-07-11 at 11 34 37 AM" src="https://github.com/nomic-ai/gpt4all/assets/132290469/78527d30-8d24-4b4c-8311-b611a2d66fcd">
+            </td>
+        </tr>
+        </table>
+
+## How It Works
+
+Google Drive for Desktop syncs your Google Drive files to your computer, while LocalDocs maintains a database of these synced files for use by your local LLM. As your Google Drive updates, LocalDocs will automatically detect file changes and get up to date. LocalDocs is powered by [Nomic Embedding](https://docs.nomic.ai/atlas/capabilities/embeddings) models which find semantically similar snippets from your files to enhance the context of your interactions.

gpt4all-bindings/python/docs/gpt4all_desktop/localdocs.md

@@ -0,0 +1,48 @@
+# LocalDocs
+
+LocalDocs brings the information you have from files on-device into your LLM chats - **privately**.
+
+## Create LocalDocs
+
+!!! note "Create LocalDocs"
+
+    1. Click `+ Add Collection`.
+    
+    2. Name your collection and link it to a folder.
+
+        <table>
+        <tr>
+            <td>
+            <img src="../assets/new_docs_annotated.png" alt="new GOT Docs" style="width:100%">
+            </td>
+            <td>
+            <img src="../assets/new_docs_annotated_filled.png" alt="new GOT Docs filled out" style="width:100%">
+            </td>
+        </tr>
+        </table>
+
+    3. Click `Create Collection`. Progress for the collection is displayed on the LocalDocs page. 
+
+        ![Embedding in progress](../assets/baelor.png)
+
+        You will see a green `Ready` indicator when the entire collection is ready. 
+
+        Note: you can still chat with the files that are ready before the entire collection is ready.
+
+        ![Embedding complete](../assets/got_done.png)
+
+        Later on if you modify your LocalDocs settings you can rebuild your collections with your new settings.
+
+    4. In your chats, open `LocalDocs` with button in top-right corner to give your LLM context from those files.
+
+        ![LocalDocs result](../assets/syrio_snippets.png)
+
+    5. See which files were referenced by clicking `Sources` below the LLM responses.
+
+        ![Sources](../assets/open_sources.png)
+
+## How It Works
+
+A LocalDocs collection uses Nomic AI's free and fast on-device embedding models to index your folder into text snippets that each get an **embedding vector**. These vectors allow us to find snippets from your files that are semantically similar to the questions and prompts you enter in your chats. We then include those semantically similar snippets in the prompt to the LLM.
+
+To try the embedding models yourself, we recommend using the [Nomic Python SDK](https://docs.nomic.ai/atlas/capabilities/embeddings)

gpt4all-bindings/python/docs/gpt4all_desktop/models.md

@@ -0,0 +1,79 @@
+# Models
+
+GPT4All is optimized to run LLMs in the 3-13B parameter range on consumer-grade hardware.
+
+LLMs are downloaded to your device so you can run them locally and privately. With our backend anyone can interact with LLMs efficiently and securely on their own hardware.
+
+## Download Models
+
+!!! note "Download Models"
+
+    <div style="text-align: center; margin-top: 20px;">
+        <table style="margin-left: auto; margin-right: auto;">
+            <tr>
+                <td style="text-align: right; padding-right: 10px;">1.</td>
+                <td style="text-align: left;">Click `Models` in the menu on the left (below `Chats` and above `LocalDocs`)</td>
+                <td><img src="../assets/models_page_icon.png" alt="Models Page Icon" style="width: 80px; height: auto;"></td>
+            </tr>
+            <tr>
+                <td style="text-align: right; padding-right: 10px;">2.</td>
+                <td style="text-align: left;">Click `+ Add Model` to navigate to the `Explore Models` page</td>
+                <td><img src="../assets/add.png" alt="Add Model button" style="width: 100px; height: auto;"></td>
+            </tr>
+            <tr>
+                <td style="text-align: right; padding-right: 10px;">3.</td>
+                <td style="text-align: left;">Search for models available online</td>
+                <td><img src="../assets/explore.png" alt="Explore Models search" style="width: 120px; height: auto;"></td>
+            </tr>
+            <tr>
+                <td style="text-align: right; padding-right: 10px;">4.</td>
+                <td style="text-align: left;">Hit `Download` to save a model to your device</td>
+                <td><img src="../assets/download.png" alt="Download Models button" style="width: 120px; height: auto;"></td>
+            </tr>
+            <tr>
+                <td style="text-align: right; padding-right: 10px;">5.</td>
+                <td style="text-align: left;">Once the model is downloaded you will see it in `Models`.</td>
+                <td><img src="../assets/installed_models.png" alt="Download Models button" style="width: 120px; height: auto;"></td>
+            </tr>
+        </table>
+    </div>
+
+## Explore Models
+
+GPT4All connects you with LLMs from HuggingFace with a [`llama.cpp`](https://github.com/ggerganov/llama.cpp) backend so that they will run efficiently on your hardware. Many of these models can be identified by the file type `.gguf`.
+
+![Explore models](../assets/search_mistral.png)
+
+## Example Models
+
+Many LLMs are available at various sizes, quantizations, and licenses. 
+
+- LLMs with more parameters tend to be better at coherently responding to instructions
+
+- LLMs with a smaller quantization (e.g. 4bit instead of 16bit) are much faster and less memory intensive, and tend to have slightly worse performance
+
+- Licenses vary in their terms for personal and commercial use
+
+Here are a few examples:
+
+| Model| Filesize| RAM Required| Parameters| Quantization| Developer| License| MD5 Sum (Unique Hash)|
+|------|---------|-------------|-----------|-------------|----------|--------|----------------------|
+| Llama 3 Instruct  | 4.66 GB| 8 GB| 8 Billion| q4_0| Meta| [Llama 3 License](https://llama.meta.com/llama3/license/)| c87ad09e1e4c8f9c35a5fcef52b6f1c9|
+| Nous Hermes 2 Mistral DPO| 4.11 GB| 8 GB| 7 Billion| q4_0| Mistral & Nous Research | [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0)| Coa5f6b4eabd3992da4d7fb7f020f921eb|
+| Phi-3 Mini Instruct | 2.18 GB| 4 GB| 4 billion| q4_0| Microsoft| [MIT](https://opensource.org/license/mit)| f8347badde9bfc2efbe89124d78ddaf5|
+| Mini Orca (Small)| 1.98 GB| 4 GB| 3 billion| q4_0| Microsoft | [CC-BY-NC-SA-4.0](https://spdx.org/licenses/CC-BY-NC-SA-4.0)| 0e769317b90ac30d6e09486d61fefa26|
+| GPT4All Snoozy| 7.37 GB| 16 GB| 13 billion| q4_0| Nomic AI| [GPL](https://www.gnu.org/licenses/gpl-3.0.en.html)| 40388eb2f8d16bb5d08c96fdfaac6b2c|
+
+### Search Results
+
+You can click the gear icon in the search bar to sort search results by their # of likes, # of downloads, or date of upload (all from HuggingFace).
+
+![Sort search results](../assets/search_settings.png)
+
+## Connect Model APIs
+
+You can add your API key for remote model providers.
+
+**Note**: this does not download a model file to your computer to use securely. Instead, this way of interacting with models has your prompts leave your computer to the API provider and returns the response to your computer.
+
+![Connect APIs](../assets/add_model_gpt4.png)

gpt4all-bindings/python/docs/gpt4all_desktop/quickstart.md

@@ -0,0 +1,40 @@
+# GPT4All Desktop
+
+The GPT4All Desktop Application allows you to download and run large language models (LLMs) locally & privately on your device.
+
+With GPT4All, you can chat with models, turn your local files into information sources for models [(LocalDocs)](localdocs.md), or browse models available online to download onto your device.
+
+## Quickstart
+
+!!! note "Quickstart"
+
+    1. Install GPT4All for your operating system and open the application.
+
+        <div style="text-align: center; margin-top: 20px;">
+            [Download for Windows](https://gpt4all.io/installers/gpt4all-installer-win64.exe) &nbsp;&nbsp;&nbsp;&nbsp;
+            [Download for Mac](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg) &nbsp;&nbsp;&nbsp;&nbsp;
+            [Download for Linux](https://gpt4all.io/installers/gpt4all-installer-linux.run)
+        </div>
+
+    2. Hit `Start Chatting`. ![GPT4All home page](../assets/gpt4all_home.png)
+
+    3. Click `+ Add Model`.
+
+    4. Download a model. We recommend starting with Llama 3, but you can [browse more models](models.md). ![Download a model](../assets/download_llama.png)
+
+    5. Once downloaded, go to Chats (below Home and above Models in the menu on the left).  
+
+    6. Click "Load Default Model" (will be Llama 3 or whichever model you downloaded). 
+
+        <table>
+        <tr>
+            <td>
+            <img src="../assets/before_first_chat.png" alt="Before first chat" style="width:100%">
+            </td>
+            <td>
+            <img src="../assets/new_first_chat.png" alt="New first chat" style="width:100%">
+            </td>
+        </tr>
+        </table>
+
+    7. Try the [example chats](chats.md) or your own prompts!

gpt4all-bindings/python/docs/gpt4all_desktop/settings.md

@@ -0,0 +1,75 @@
+# Settings
+
+## Application Settings
+
+!!! note "General Application Settings"
+
+    | Setting | Description | Default Value |
+    | --- | --- | --- |
+    | **Theme** | Color theme for the application. Options are `Light`, `Dark`, and `LegacyDark` | `Light` |
+    | **Font Size** | Font size setting for text throughout the application. Options are Small, Medium, and Large | Small |
+    | **Device** | Device that will run your models. Options are `Auto` (GPT4All chooses), `Metal` (Apple Silicon M1+), `CPU`, and `GPU` | `Auto` |
+    | **Default Model** | Choose your preferred LLM to load by default on startup| Auto |
+    | **Download Path** | Select a destination on your device to save downloaded models | Windows: `C:\Users\{username}\AppData\Local\nomic.ai\GPT4All`<br><br>Mac: `/Users/{username}/Library/Application Support/nomic.ai/GPT4All/`<br><br>Linux: `/home/{username}/.local/share/nomic.ai/GPT4All` |
+    
+    | **Enable Datalake** | Opt-in to sharing interactions with GPT4All community (**anonymous** and **optional**) | Off |
+
+!!! note "Advanced Application Settings"
+
+    | Setting | Description | Default Value |
+    | --- | --- | --- |
+    | **CPU Threads** | Number of concurrently running CPU threads (more can speed up responses) | 4 |
+    | **Save Chat Context** | Save chat context to disk to pick up exactly where a model left off. | Off |
+    | **Enable Local Server** | Allow any application on your device to use GPT4All via an OpenAI-compatible GPT4All API | Off |
+    | **API Server Port** | Local HTTP port for the local API server | 4891 |
+
+## Model Settings
+
+!!! note "Model / Character Settings"
+
+    | Setting | Description | Default Value |
+    | --- | --- | --- |
+    | **Name** | Unique name of this model / character| set by model uploader |
+    | **System Prompt** | General instructions for the chats this model will be used for | set by model uploader |
+    | **Prompt Template** | Format of user <-> assistant interactions for the chats this model will be used for | set by model uploader |
+
+### Clone
+
+You can **clone** an existing model, which allows you to save a configuration of a model file with different prompt templates and sampling settings.
+
+### Sampling Settings
+
+!!! note "Model Sampling Settings"
+
+    | Setting             | Description                          | Default Value |
+    |----------------------------|------------------------------------------|-----------|
+    | **Context Length**         | Maximum length of input sequence in tokens        | 2048      |
+    | **Max Length**             | Maximum length of response in tokens     | 4096      |
+    | **Prompt Batch Size**      | Token batch size for parallel processing | 128      |
+    | **Temperature**            | Lower temperature gives more likely generations | 0.7       |
+    | **Top P**                  | Prevents choosing highly unlikely tokens  | 0.4       |
+    | **Top K**                  | Size of selection pool for tokens         | 40        |
+    | **Min P**                  | Minimum relative probability              | 0         |
+    | **Repeat Penalty Tokens**  | Length to apply penalty                   | 64        |
+    | **Repeat Penalty**         | Penalize repetitiveness                   | 1.18      |
+    | **GPU Layers**             | How many model layers to load into VRAM     | 32        |
+
+## LocalDocs Settings
+
+!!! note "General LocalDocs Settings"
+
+    | Setting | Description | Default Value |
+    | --- | --- | --- |
+    | **Allowed File Extensions** | Choose which file types will be indexed into LocalDocs collections as text snippets with embedding vectors | `.txt`, `.pdf`, `.md`, `.rst` |
+    | **Use Nomic Embed API** | Use Nomic API to create LocalDocs collections fast and off-device; [Nomic API Key](https://atlas.nomic.ai/) required | Off |
+    | **Embeddings Device** | Device that will run embedding models. Options are `Auto` (GPT4All chooses), `Metal` (Apple Silicon M1+), `CPU`, and `GPU` | `Auto` |
+    | **Show Sources** | Titles of source files retrieved by LocalDocs will be displayed directly in your chats.| On |
+
+!!! note "Advanced LocalDocs Settings"
+
+    Note that increasing these settings can increase the likelihood of factual responses, but may result in slower generation times.
+
+    | Setting | Description | Default Value |
+    | --- | --- | --- |
+    | **Document Snippet Size** | Number of string characters per document snippet | 512 |
+    | **Maximum Document Snippets Per Prompt** | Upper limit for the number of snippets from your files LocalDocs can retrieve for LLM context | 3 |

gpt4all-bindings/python/docs/gpt4all_help/faq.md

@@ -0,0 +1,43 @@
+# Frequently Asked Questions
+
+## Models
+
+### Which language models are supported?
+
+We support models with a `llama.cpp` implementation which have been uploaded to [HuggingFace](https://huggingface.co/).
+
+### Which embedding models are supported?
+
+We support SBert and Nomic Embed Text v1 & v1.5.
+
+## Software
+
+### What software do I need?
+
+All you need is to [install GPT4all](../index.md) onto you Windows, Mac, or Linux computer.
+
+### Which SDK languages are supported?
+
+Our SDK is in Python for usability, but these are light bindings around [`llama.cpp`](https://github.com/ggerganov/llama.cpp) implementations that we contribute to for efficiency and accessibility on everyday computers.
+
+### Is there an API?
+
+Yes, you can run your model in server-mode with our [OpenAI-compatible API](https://platform.openai.com/docs/api-reference/completions), which you can configure in [settings](../gpt4all_desktop/settings.md#application-settings)
+
+### Can I monitor a GPT4All deployment?
+
+Yes, GPT4All [integrates](../gpt4all_python/monitoring.md) with [OpenLIT](https://github.com/openlit/openlit) so you can deploy LLMs with user interactions and hardware usage automatically monitored for full observability.
+
+### Is there a command line interface (CLI)?
+
+[Yes](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/cli), we have a lightweight use of the Python client as a CLI. We welcome further contributions!
+
+## Hardware
+
+### What hardware do I need?
+
+GPT4All can run on CPU, Metal (Apple Silicon M1+), and GPU.
+
+### What are the system requirements?
+
+Your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) and you need enough RAM to load a model into memory.

gpt4all-bindings/python/docs/gpt4all_help/troubleshooting.md

@@ -0,0 +1,27 @@
+# Troubleshooting
+
+## Error Loading Models
+
+It is possible you are trying to load a model from HuggingFace whose weights are not compatible with our [backend](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings).
+
+Try downloading one of the officially supported models mentioned our [website](https://gpt4all.io/). If the problem persists, please share your experience on our [Discord](https://discord.com/channels/1076964370942267462).
+
+## Bad Responses 
+
+Try the [example chats](../gpt4all_desktop/chats.md) to double check that your system is implementing models correctly.
+
+### Responses Incoherent
+
+If you are seeing something **not at all** resembling the [example chats](../gpt4all_desktop/chats.md) - for example, if the responses you are seeing look nonsensical - try [downloading a different model](../gpt4all_desktop/models.md), and please share your experience on our [Discord](https://discord.com/channels/1076964370942267462).
+
+### Responses Incorrect
+
+LLMs can be unreliable. It's helpful to know what their training data was - they are less likely to be correct when asking about data they were not trained on unless you give the necessary information in the prompt as **context**.
+
+Giving LLMs additional context, like chatting using [LocalDocs](../gpt4all_desktop/localdocs.md), can help merge the language model's ability to understand text with the files that you trust to contain the information you need. 
+
+Including information in a prompt is not a guarantee that it will be used correctly, but the more clear and concise your prompts, and the more relevant your prompts are to your files, the better.
+
+### LocalDocs Issues
+
+Occasionally a model - particularly a smaller or overall weaker LLM - may not use the relevant text snippets from the files that were referenced via LocalDocs. If you are seeing this, it can help to use phrases like "in the docs" or "from the provided files" when prompting your model.

gpt4all-bindings/python/docs/gpt4all_python/home.md

@@ -0,0 +1,159 @@
+# GPT4All Python SDK
+
+## Installation
+
+To get started, pip-install the `gpt4all` package into your python environment.
+
+```bash
+pip install gpt4all
+```
+
+We recommend installing `gpt4all` into its own virtual environment using `venv` or `conda`
+
+## Load LLM
+
+Models are loaded by name via the `GPT4All` class. If it's your first time loading a model, it will be downloaded to your device and saved so it can be quickly reloaded next time you create a `GPT4All` model with the same name.
+
+!!! note "Load LLM"
+
+    ```python
+    from gpt4all import GPT4All
+    model = GPT4All("Meta-Llama-3-8B-Instruct.Q4_0.gguf") # downloads / loads a 4.66GB LLM
+    with model.chat_session():
+        print(model.generate("How can I run LLMs efficiently on my laptop?", max_tokens=1024))
+    ```
+
+| `GPT4All` model name| Filesize| RAM Required| Parameters| Quantization| Developer| License| MD5 Sum (Unique Hash)|
+|------|---------|-------|-------|-----------|----------|--------|----------------------|
+|  `Meta-Llama-3-8B-Instruct.Q4_0.gguf`| 4.66 GB| 8 GB| 8 Billion| q4_0| Meta| [Llama 3 License](https://llama.meta.com/llama3/license/)| c87ad09e1e4c8f9c35a5fcef52b6f1c9|
+| `Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf`| 4.11 GB| 8 GB| 7 Billion| q4_0| Mistral & Nous Research | [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0)| Coa5f6b4eabd3992da4d7fb7f020f921eb|
+| `Phi-3-mini-4k-instruct.Q4_0.gguf` | 2.18 GB| 4 GB| 3.8 billion| q4_0| Microsoft| [MIT](https://opensource.org/license/mit)| f8347badde9bfc2efbe89124d78ddaf5|
+| `orca-mini-3b-gguf2-q4_0.gguf`| 1.98 GB| 4 GB| 3 billion| q4_0| Microsoft | [CC-BY-NC-SA-4.0](https://spdx.org/licenses/CC-BY-NC-SA-4.0)| 0e769317b90ac30d6e09486d61fefa26|
+| `gpt4all-13b-snoozy-q4_0.gguf`| 7.37 GB| 16 GB| 13 billion| q4_0| Nomic AI| [GPL](https://www.gnu.org/licenses/gpl-3.0.en.html)| 40388eb2f8d16bb5d08c96fdfaac6b2c|
+
+
+## Chat Session Generation
+
+Most of the language models you will be able to access from HuggingFace have been trained as assistants. This guides language models to not just answer with relevant text, but *helpful* text.
+
+If you want your LLM's responses to be helpful in the typical sense, we recommend you apply the chat templates the models were finetuned with. Information about specific prompt templates is typically available on the official HuggingFace page for the model.
+
+!!! note "Example LLM Chat Session Generation"
+
+    === "Code"
+
+        Load `Llama 3` and enter the following prompt in a chat session:
+
+        ```python
+        from gpt4all import GPT4All
+        model = GPT4All("Meta-Llama-3-8B-Instruct.Q4_0.gguf")
+        with model.chat_session():
+            print(model.generate("quadratic formula"))
+        ```
+
+    === "Output"
+   
+        With the default sampling settings, you should see something resembling the following:
+        ```
+        The quadratic formula!
+
+        The quadratic formula is a mathematical formula that provides the solutions to a quadratic equation of the form:
+
+        ax^2 + bx + c = 0
+
+        where a, b, and c are constants. The formula is:
+
+        x = (-b ± √(b^2 - 4ac)) / 2a
+
+        Let's break it down:
+
+        * x is the variable we're trying to solve for.
+        * a, b, and c are the coefficients of the quadratic equation.
+        * ± means "plus or minus".
+        * √ denotes the square root.
+
+        To use the formula, simply plug in the values of a, b, and c into the expression above. The resulting value(s) will be the solutions to the original quadratic equation!
+
+        For example, let's say we have the quadratic equation:
+
+        x^2 + 5x + 6 = 0
+
+        We can plug these values into the formula as follows:
+
+        a = 1
+        b = 5
+        ```
+
+## Direct Generation
+
+Directly calling `model.generate()` prompts the model without applying any templates. 
+
+Note: this can result in responses that are less like helpful responses and more like mirroring the tone of your prompt. In general, a language model outside of a chat session is less of a helpful assistant and more of a lens into the distribution of the model's training data.
+
+As an example, see how the model's response changes when we give the same prompt as above without applying a chat session:
+
+!!! note "Example LLM Direct Generation"
+
+    === "Code"
+    
+        Load `Llama 3` and enter the following prompt:
+
+        ```python
+        from gpt4all import GPT4All
+        model = GPT4All("Meta-Llama-3-8B-Instruct.Q4_0.gguf")
+        print(model.generate("quadratic formula"))
+        ```
+    === "Output"
+
+        With the default sampling settings, you should see something resembling the following:
+        ```
+        . The equation is in the form of a + bx = c, where a and b are constants.
+        The solution to this problem involves using the quadratic formula which states that for any quadratic equation ax^2+bx+c=0, its solutions can be found by:
+        x = (-b ± √(b^2-4ac)) / 2a
+        In your case, since you have a + bx = c, we need to rewrite it in the form of ax^2+bx+c=0. To do this, subtract both sides from c, so that:
+        c - (a + bx) = 0
+        Now, combine like terms on the left side and simplify:
+        ax^2 + (-b)x + (c-a) = 0\n\nSo now we have a quadratic equation in standard form: ax^2+bx+c=0. We can use this to find its solutions using the quadratic formula:
+        
+        x = ((-b ± √((-b)^2
+        ```
+
+Why did it respond differently? Because language models, before being fine-tuned as assistants, are trained to be more like a data mimic than a helpful assistant. Therefore our responses ends up more like a typical continuation of math-style text rather than a helpful answer in dialog. 
+
+## Embeddings
+
+Nomic trains and open-sources free embedding models that will run very fast on your hardware.
+
+The easiest way to run the text embedding model locally uses the [`nomic`](https://github.com/nomic-ai/nomic) python library to interface with our fast [C/C++ implementations](ref.md#gpt4all.gpt4all.Embed4All).
+
+!!! note "Example Embeddings Generation"
+
+    === "Code"
+
+        Importing `embed` from the [`nomic`](https://github.com/nomic-ai/nomic) library, you can call `embed.text()` with `inference_mode="local"`. This downloads an embedding model and saves it for later.
+
+        ```python
+        from nomic import embed
+        embeddings = embed.text(["String 1", "String 2"], inference_mode="local")['embeddings']
+        print("Number of embeddings created:", len(embeddings))
+        print("Number of dimensions per embedding:", len(embeddings[0]))
+        ```
+    
+    === "Output"
+
+        ```
+        Number of embeddings created: 2
+        Number of dimensions per embedding: 768
+        ```
+
+![Nomic embed text local inference](../assets/local_embed.gif)
+
+To learn more about making embeddings locally with `nomic`, visit our [embeddings guide](https://docs.nomic.ai/atlas/guides/embeddings#local-inference).
+
+The following embedding models can be used within the application and with the `Embed4All` class from the `gpt4all` Python library. The default context length as GGUF files is 2048 but can be [extended](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF#description).
+
+| Name| Using with `nomic`| `Embed4All` model name| Context Length| # Embedding Dimensions| File Size|
+|--------------------|-|------------------------------------------------------|---------------:|-----------------:|----------:|
+| [Nomic Embed v1](https://huggingface.co/nomic-ai/nomic-embed-text-v1-GGUF)   | ```embed.text(strings, model="nomic-embed-text-v1", inference_mode="local")```| ```Embed4All("nomic-embed-text-v1.f16.gguf")```|           2048 |              768 |   262 MiB |
+| [Nomic Embed v1.5](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF) | ```embed.text(strings, model="nomic-embed-text-v1.5", inference_mode="local")```| ```Embed4All("nomic-embed-text-v1.5.f16.gguf")``` |           2048| 64-768 |   262 MiB |
+| [SBert](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2)| n/a| ```Embed4All("all-MiniLM-L6-v2.gguf2.f16.gguf")```|            512 |              384 |    44 MiB |

gpt4all-bindings/python/docs/gpt4all_python/monitoring.md

@@ -0,0 +1,49 @@
+# GPT4All Monitoring
+
+GPT4All integrates with [OpenLIT](https://github.com/openlit/openlit) OpenTelemetry auto-instrumentation to perform real-time monitoring of your LLM application and GPU hardware.
+
+Monitoring can enhance your GPT4All deployment with auto-generated traces and metrics for
+
+- **Performance Optimization:** Analyze latency, cost and token usage to ensure your LLM application runs efficiently, identifying and resolving performance bottlenecks swiftly.
+  
+- **User Interaction Insights:** Capture each prompt and response to understand user behavior and usage patterns better, improving user experience and engagement.
+  
+- **Detailed GPU Metrics:** Monitor essential GPU parameters such as utilization, memory consumption, temperature, and power usage to maintain optimal hardware performance and avert potential issues.
+
+## Setup Monitoring
+
+!!! note "Setup Monitoring"
+
+    With [OpenLIT](https://github.com/openlit/openlit), you can automatically monitor traces and metrics for your LLM deployment:
+
+    ```shell
+    pip install openlit
+    ```
+
+    ```python
+    from gpt4all import GPT4All
+    import openlit
+
+    openlit.init()  # start
+    # openlit.init(collect_gpu_stats=True)  # Optional: To configure GPU monitoring
+
+    model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf')
+
+    # Start a chat session and send queries
+    with model.chat_session():
+        response1 = model.generate(prompt='hello', temp=0)
+        response2 = model.generate(prompt='write me a short poem', temp=0)
+        response3 = model.generate(prompt='thank you', temp=0)
+
+        print(model.current_chat_session)
+    ```
+
+## Visualization
+
+### OpenLIT UI
+
+Connect to OpenLIT's UI to start exploring the collected LLM performance metrics and traces. Visit the OpenLIT [Quickstart Guide](https://docs.openlit.io/latest/quickstart) for step-by-step details.
+
+### Grafana, DataDog, & Other Integrations
+
+You can also send the data collected by OpenLIT to popular monitoring tools like Grafana and DataDog. For detailed instructions on setting up these connections, please refer to the OpenLIT [Connections Guide](https://docs.openlit.io/latest/connections/intro).

gpt4all-bindings/python/docs/gpt4all_python/ref.md

@@ -0,0 +1,4 @@
+# GPT4All Python SDK Reference
+::: gpt4all.gpt4all.GPT4All
+
+::: gpt4all.gpt4all.Embed4All

gpt4all-bindings/python/docs/index.md

@@ -1,66 +1,28 @@
-# GPT4All
-Welcome to the GPT4All technical documentation.
+# GPT4All Documentation
 
-GPT4All is an open-source software ecosystem that allows anyone to train and deploy **powerful** and **customized** large language models (LLMs) on **everyday hardware**.
-Nomic AI oversees contributions to the open-source ecosystem ensuring quality, security and maintainability.
+GPT4All runs large language models (LLMs) privately on everyday desktops & laptops. 
 
-GPT4All software is optimized to run inference of 3-13 billion parameter large language models on the CPUs of laptops, desktops and servers.
+No API calls or GPUs required - you can just download the application and [get started](gpt4all_desktop/quickstart.md#quickstart).
 
-=== "GPT4All Example"
-    ``` py
+!!! note "Desktop Application"
+    GPT4All runs LLMs as an application on your computer. Nomic's embedding models can bring information from your local documents and files into your chats. It's fast, on-device, and completely **private**.
+
+    <div style="text-align: center; margin-top: 20px;">
+        [Download for Windows](https://gpt4all.io/installers/gpt4all-installer-win64.exe) &nbsp;&nbsp;&nbsp;&nbsp;
+        [Download for Mac](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg) &nbsp;&nbsp;&nbsp;&nbsp;
+        [Download for Linux](https://gpt4all.io/installers/gpt4all-installer-linux.run)
+    </div>
+
+!!! note "Python SDK"
+    Use GPT4All in Python to program with LLMs implemented with the [`llama.cpp`](https://github.com/ggerganov/llama.cpp) backend and [Nomic's C backend](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-backend). Nomic contributes to open source software like [`llama.cpp`](https://github.com/ggerganov/llama.cpp) to make LLMs accessible and efficient **for all**.
+
+    ```bash
+    pip install gpt4all
+    ```
+
+    ```python
     from gpt4all import GPT4All
-    model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
-    output = model.generate("The capital of France is ", max_tokens=3)
-    print(output)
+    model = GPT4All("Meta-Llama-3-8B-Instruct.Q4_0.gguf") # downloads / loads a 4.66GB LLM
+    with model.chat_session():
+        print(model.generate("How can I run LLMs efficiently on my laptop?", max_tokens=1024))
     ```
-=== "Output"
-    ```
-    1. Paris
-    ```
-See [Python Bindings](gpt4all_python.md) to use GPT4All.
-
-### Navigating the Documentation
-In an effort to ensure cross-operating-system and cross-language compatibility, the [GPT4All software ecosystem](https://github.com/nomic-ai/gpt4all)
-is organized as a monorepo with the following structure:
-
-- **gpt4all-backend**: The GPT4All backend maintains and exposes a universal, performance optimized C API for running inference with multi-billion parameter Transformer Decoders.
-This C API is then bound to any higher level programming language such as C++, Python, Go, etc.
-- **gpt4all-bindings**: GPT4All bindings contain a variety of high-level programming languages that implement the C API. Each directory is a bound programming language. The [CLI](gpt4all_cli.md) is included here, as well.
-- **gpt4all-chat**: GPT4All Chat is an OS native chat application that runs on macOS, Windows and Linux. It is the easiest way to run local, privacy aware chat assistants on everyday hardware. You can download it on the [GPT4All Website](https://gpt4all.io) and read its source code in the monorepo.
-
-Explore detailed documentation for the backend, bindings and chat client in the sidebar.
-## Models
-The GPT4All software ecosystem is compatible with the following Transformer architectures:
-
-- `Falcon`
-- `LLaMA` (including `OpenLLaMA`)
-- `MPT` (including `Replit`)
-- `GPT-J`
-
-You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models3.json)
-
-
-GPT4All models are artifacts produced through a process known as neural network quantization.
-A multi-billion parameter Transformer Decoder usually takes 30+ GB of VRAM to execute a forward pass.
-Most people do not have such a powerful computer or access to GPU hardware. By running trained LLMs through quantization algorithms, 
-some GPT4All models can run on your laptop using only 4-8GB of RAM enabling their wide-spread usage.
-Bigger models might still require more RAM, however.
-
-Any model trained with one of these architectures can be quantized and run locally with all GPT4All bindings and in the
-chat client. You can add new variants by contributing to the gpt4all-backend.
-
-## Frequently Asked Questions
-Find answers to frequently asked questions by searching the [Github issues](https://github.com/nomic-ai/gpt4all/issues) or in the [documentation FAQ](gpt4all_faq.md).
-
-## Getting the most of your local LLM
-
-**Inference Speed**
-of a local LLM depends on two factors: model size and the number of tokens given as input. 
-It is not advised to prompt local LLMs with large chunks of context as their inference speed will heavily degrade.
-You will likely want to run GPT4All models on GPU if you would like to utilize context windows larger than 750 tokens. Native GPU support for GPT4All models is planned.
-
-**Inference Performance:**
-Which model is best? That question depends on your use-case. The ability of an LLM to faithfully follow instructions is conditioned
-on the quantity and diversity of the pre-training data it trained on and the diversity, quality and factuality of the data the LLM
-was fine-tuned on. A goal of GPT4All is to bring the most powerful local assistant model to your desktop and Nomic AI is actively
-working on efforts to improve their performance and quality.

gpt4all-bindings/python/docs/old/index.md

@@ -0,0 +1,71 @@
+# GPT4All
+Welcome to the GPT4All documentation LOCAL EDIT
+
+GPT4All is an open-source software ecosystem for anyone to run large language models (LLMs) **privately** on **everyday laptop & desktop computers**. No API calls or GPUs required.
+
+The GPT4All Desktop Application is a touchpoint to interact with LLMs and integrate them with your local docs & local data for RAG (retrieval-augmented generation). No coding is required, just install the application, download the models of your choice, and you are ready to use your LLM.
+
+Your local data is **yours**. GPT4All handles the retrieval privately and on-device to fetch relevant data to support your queries to your LLM.
+
+Nomic AI oversees contributions to GPT4All to ensure quality, security, and maintainability. Additionally, Nomic AI has open-sourced code for training and deploying your own customized LLMs internally.
+
+GPT4All software is optimized to run inference of 3-13 billion parameter large language models on the CPUs of laptops, desktops and servers.
+
+=== "GPT4All Example"
+    ``` py
+    from gpt4all import GPT4All
+    model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
+    output = model.generate("The capital of France is ", max_tokens=3)
+    print(output)
+    ```
+=== "Output"
+    ```
+    1. Paris
+    ```
+See [Python Bindings](gpt4all_python.md) to use GPT4All.
+
+### Navigating the Documentation
+In an effort to ensure cross-operating-system and cross-language compatibility, the [GPT4All software ecosystem](https://github.com/nomic-ai/gpt4all)
+is organized as a monorepo with the following structure:
+
+- **gpt4all-backend**: The GPT4All backend maintains and exposes a universal, performance optimized C API for running inference with multi-billion parameter Transformer Decoders.
+This C API is then bound to any higher level programming language such as C++, Python, Go, etc.
+- **gpt4all-bindings**: GPT4All bindings contain a variety of high-level programming languages that implement the C API. Each directory is a bound programming language. The [CLI](gpt4all_cli.md) is included here, as well.
+- **gpt4all-chat**: GPT4All Chat is an OS native chat application that runs on macOS, Windows and Linux. It is the easiest way to run local, privacy aware chat assistants on everyday hardware. You can download it on the [GPT4All Website](https://gpt4all.io) and read its source code in the monorepo.
+
+Explore detailed documentation for the backend, bindings and chat client in the sidebar.
+## Models
+The GPT4All software ecosystem is compatible with the following Transformer architectures:
+
+- `Falcon`
+- `LLaMA` (including `OpenLLaMA`)
+- `MPT` (including `Replit`)
+- `GPT-J`
+
+You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models3.json)
+
+
+GPT4All models are artifacts produced through a process known as neural network quantization.
+A multi-billion parameter Transformer Decoder usually takes 30+ GB of VRAM to execute a forward pass.
+Most people do not have such a powerful computer or access to GPU hardware. By running trained LLMs through quantization algorithms, 
+some GPT4All models can run on your laptop using only 4-8GB of RAM enabling their wide-spread usage.
+Bigger models might still require more RAM, however.
+
+Any model trained with one of these architectures can be quantized and run locally with all GPT4All bindings and in the
+chat client. You can add new variants by contributing to the gpt4all-backend.
+
+## Frequently Asked Questions
+Find answers to frequently asked questions by searching the [Github issues](https://github.com/nomic-ai/gpt4all/issues) or in the [documentation FAQ](gpt4all_faq.md).
+
+## Getting the most of your local LLM
+
+**Inference Speed**
+of a local LLM depends on two factors: model size and the number of tokens given as input. 
+It is not advised to prompt local LLMs with large chunks of context as their inference speed will heavily degrade.
+You will likely want to run GPT4All models on GPU if you would like to utilize context windows larger than 750 tokens. Native GPU support for GPT4All models is planned.
+
+**Inference Performance:**
+Which model is best? That question depends on your use-case. The ability of an LLM to faithfully follow instructions is conditioned
+on the quantity and diversity of the pre-training data it trained on and the diversity, quality and factuality of the data the LLM
+was fine-tuned on. A goal of GPT4All is to bring the most powerful local assistant model to your desktop and Nomic AI is actively
+working on efforts to improve their performance and quality.

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -6,6 +6,7 @@ import platform
 import re
 import subprocess
 import sys
+import textwrap
 import threading
 from enum import Enum
 from queue import Queue
@@ -28,25 +29,44 @@ if TYPE_CHECKING:
 EmbeddingsType = TypeVar('EmbeddingsType', bound='list[Any]')
 
 
+# Detect Rosetta 2
+if platform.system() == "Darwin" and platform.processor() == "i386":
+    if subprocess.run(
+        "sysctl -n sysctl.proc_translated".split(), check=True, capture_output=True, text=True,
+    ).stdout.strip() == "1":
+        raise RuntimeError(textwrap.dedent("""\
+            Running GPT4All under Rosetta is not supported due to CPU feature requirements.
+            Please install GPT4All in an environment that uses a native ARM64 Python interpreter.
+        """))
+
+
+def _load_cuda(rtver: str, blasver: str) -> None:
+    if platform.system() == "Linux":
+        cudalib   = f"lib/libcudart.so.{rtver}"
+        cublaslib = f"lib/libcublas.so.{blasver}"
+    else:  # Windows
+        cudalib   = fr"bin\cudart64_{rtver.replace(".", "")}.dll"
+        cublaslib = fr"bin\cublas64_{blasver}.dll"
+
+    # preload the CUDA libs so the backend can find them
+    ctypes.CDLL(os.path.join(cuda_runtime.__path__[0], cudalib), mode=ctypes.RTLD_GLOBAL)
+    ctypes.CDLL(os.path.join(cublas.__path__[0], cublaslib), mode=ctypes.RTLD_GLOBAL)
+
+
 # Find CUDA libraries from the official packages
 cuda_found = False
-if platform.system() in ('Linux', 'Windows'):
+if platform.system() in ("Linux", "Windows"):
     try:
         from nvidia import cuda_runtime, cublas
     except ImportError:
         pass  # CUDA is optional
     else:
-        if platform.system() == 'Linux':
-            cudalib   = 'lib/libcudart.so.12'
-            cublaslib = 'lib/libcublas.so.12'
-        else:  # Windows
-            cudalib   = r'bin\cudart64_12.dll'
-            cublaslib = r'bin\cublas64_12.dll'
-
-        # preload the CUDA libs so the backend can find them
-        ctypes.CDLL(os.path.join(cuda_runtime.__path__[0], cudalib), mode=ctypes.RTLD_GLOBAL)
-        ctypes.CDLL(os.path.join(cublas.__path__[0], cublaslib), mode=ctypes.RTLD_GLOBAL)
-        cuda_found = True
+        for rtver, blasver in [("12", "12"), ("11.0", "11")]:
+            try:
+                _load_cuda(rtver, blasver)
+                cuda_found = True
+            except OSError:  # dlopen() does not give specific error codes
+                pass  # try the next one
 
 
 # TODO: provide a config file to make this more robust
@@ -73,8 +93,6 @@ llmodel = load_llmodel_library()
 
 class LLModelPromptContext(ctypes.Structure):
     _fields_ = [
-        ("logits", ctypes.POINTER(ctypes.c_float)),
-        ("logits_size", ctypes.c_size_t),
         ("tokens", ctypes.POINTER(ctypes.c_int32)),
         ("tokens_size", ctypes.c_size_t),
         ("n_past", ctypes.c_int32),
@@ -119,7 +137,6 @@ llmodel.llmodel_isModelLoaded.restype = ctypes.c_bool
 
 PromptCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32)
 ResponseCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32, ctypes.c_char_p)
-RecalculateCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_bool)
 EmbCancelCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.POINTER(ctypes.c_uint), ctypes.c_uint, ctypes.c_char_p)
 
 llmodel.llmodel_prompt.argtypes = [
@@ -128,7 +145,7 @@ llmodel.llmodel_prompt.argtypes = [
     ctypes.c_char_p,
     PromptCallback,
     ResponseCallback,
-    RecalculateCallback,
+    ctypes.c_bool,
     ctypes.POINTER(LLModelPromptContext),
     ctypes.c_bool,
     ctypes.c_char_p,
@@ -177,9 +194,6 @@ llmodel.llmodel_gpu_init_gpu_device_by_struct.restype = ctypes.c_bool
 llmodel.llmodel_gpu_init_gpu_device_by_int.argtypes = [ctypes.c_void_p, ctypes.c_int32]
 llmodel.llmodel_gpu_init_gpu_device_by_int.restype = ctypes.c_bool
 
-llmodel.llmodel_has_gpu_device.argtypes = [ctypes.c_void_p]
-llmodel.llmodel_has_gpu_device.restype = ctypes.c_bool
-
 llmodel.llmodel_model_backend_name.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_model_backend_name.restype = ctypes.c_char_p
 
@@ -354,7 +368,6 @@ class LLModel:
     ):
         if self.context is None:
             context = LLModelPromptContext(
-                logits_size=0,
                 tokens_size=0,
                 n_past=0,
                 n_ctx=0,
@@ -508,7 +521,7 @@ class LLModel:
             ctypes.c_char_p(prompt_template.encode()),
             PromptCallback(self._prompt_callback),
             ResponseCallback(self._callback_decoder(callback)),
-            RecalculateCallback(self._recalculate_callback),
+            True,
             self.context,
             special,
             ctypes.c_char_p(),
@@ -601,8 +614,3 @@ class LLModel:
     @staticmethod
     def _prompt_callback(token_id: int) -> bool:
         return True
-
-    # Empty recalculate callback
-    @staticmethod
-    def _recalculate_callback(is_recalculating: bool) -> bool:
-        return is_recalculating

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -502,7 +502,7 @@ class GPT4All:
         Generate outputs from any GPT4All model.
 
         Args:
-            prompt: The prompt for the model the complete.
+            prompt: The prompt for the model to complete.
             max_tokens: The maximum number of tokens to generate.
             temp: The model temperature. Larger values increase creativity but decrease factuality.
             top_k: Randomly sample from the top_k most likely tokens at each generation step. Set this to 1 for greedy decoding.

gpt4all-bindings/python/mkdocs.yml

@@ -1,24 +1,31 @@
-site_name: GPT4All Documentation
+site_name: GPT4All
 repo_url: https://github.com/nomic-ai/gpt4all
 repo_name: nomic-ai/gpt4all
 site_url: https://docs.gpt4all.io
 edit_uri: edit/main/docs/
-site_description: Documentation for running GPT4All anywhere.
-copyright: Copyright © 2023 Nomic, Inc
+site_description: GPT4All Docs - run LLMs efficiently on your hardware
+copyright: Copyright © 2024 Nomic, Inc
 use_directory_urls: false
 
 nav:
-    - 'index.md'
-    - 'GPT4All Chat Client': 'gpt4all_chat.md'
-    - 'Bindings':
-      - 'GPT4All in Python':
-        - 'Generation': 'gpt4all_python.md'
-        - 'Embedding': 'gpt4all_python_embedding.md'
-        - 'Monitoring with OpenLIT': 'gpt4all_monitoring.md'
-      - 'GPT4ALL in NodeJs': 'gpt4all_nodejs.md'
-      - 'gpt4all_cli.md'
-    - 'Wiki':
-      - 'gpt4all_faq.md'
+    - 'index.md' 
+    - 'Quickstart' : 'gpt4all_desktop/quickstart.md'
+    - 'Chats' : 'gpt4all_desktop/chats.md'
+    - 'Models' : 'gpt4all_desktop/models.md'
+    - 'LocalDocs' : 'gpt4all_desktop/localdocs.md'
+    - 'Settings' : 'gpt4all_desktop/settings.md'
+    - 'Cookbook':
+      - 'Local AI Chat with your Google Drive': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-google-drive.md'
+      - 'Local AI Chat with your Obsidian Vault': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-Obsidian.md'
+      - 'Local AI Chat with your OneDrive': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-One-Drive.md'
+    - 'Python SDK':
+      - 'gpt4all_python/home.md'
+      - 'Monitoring': 'gpt4all_python/monitoring.md'
+      - 'SDK Reference': 'gpt4all_python/ref.md'
+    - 'Help':
+      - 'FAQ': 'gpt4all_help/faq.md'
+      - 'Troubleshooting': 'gpt4all_help/troubleshooting.md'
+
 
 theme:
   name: material
@@ -27,6 +34,7 @@ theme:
   logo: assets/nomic.png
   favicon: assets/favicon.ico
   features:
+    - content.code.copy
     - navigation.instant
     - navigation.tracking
     - navigation.sections
@@ -52,11 +60,15 @@ markdown_extensions:
   - admonition
   - codehilite:
       css_class: highlight
+  - markdown_captions
 
 extra_css:
     - css/custom.css
 
+
+
 plugins:
+  - search
   - mkdocstrings:
       handlers:
         python:
@@ -65,9 +77,13 @@ plugins:
             heading_level: 4
             show_root_full_path: false
             docstring_section_style: list
-  #- material/social:
-  #    cards_font: Roboto
+  - material/social:
+      cards_layout_options:
+          font_family: Roboto
+          description: GPT4All runs LLMs efficiently on your hardware
 
-  #- mkdocs-jupyter:
-  #    ignore_h1_titles: True
-  #    show_input: True
+extra:
+  generator: false
+  analytics:
+    provider: google
+    property: G-NPXC8BYHJV

gpt4all-bindings/python/setup.py

@@ -68,7 +68,7 @@ def get_long_description():
 
 setup(
     name=package_name,
-    version="2.7.0",
+    version="2.8.1",
     description="Python bindings for GPT4All",
     long_description=get_long_description(),
     long_description_content_type="text/markdown",
@@ -94,8 +94,8 @@ setup(
     ],
     extras_require={
         'cuda': [
-            'nvidia-cuda-runtime-cu12',
-            'nvidia-cublas-cu12',
+            'nvidia-cuda-runtime-cu11',
+            'nvidia-cublas-cu11',
         ],
         'all': [
             'gpt4all[cuda]; platform_system == "Windows" or platform_system == "Linux"',
@@ -107,6 +107,7 @@ setup(
             'wheel',
             'setuptools',
             'mkdocs-material',
+            'mkdocs-material[imaging]',
             'mkautodoc',
             'mkdocstrings[python]',
             'mkdocs-jupyter',

gpt4all-chat/CHANGELOG.md

@@ -0,0 +1,102 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+
+## [Unreleased]
+
+### Added
+- Use greedy sampling when temperature is set to zero ([#2854](https://github.com/nomic-ai/gpt4all/pull/2854))
+
+## [3.2.1] - 2024-08-13
+
+### Fixed
+- Do not initialize Vulkan driver when only using CPU ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+- Fix a potential crash on exit when using only CPU on Linux with NVIDIA (does not affect X11) ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+- Fix default CUDA architecture list after [#2802](https://github.com/nomic-ai/gpt4all/pull/2802) ([#2855](https://github.com/nomic-ai/gpt4all/pull/2855))
+
+## [3.2.0] - 2024-08-12
+
+### Added
+- Add Qwen2-1.5B-Instruct to models3.json (by [@ThiloteE](https://github.com/ThiloteE) in [#2759](https://github.com/nomic-ai/gpt4all/pull/2759))
+- Enable translation feature for seven languages: English, Spanish, Italian, Portuguese, Chinese Simplified, Chinese Traditional, Romanian ([#2830](https://github.com/nomic-ai/gpt4all/pull/2830))
+
+### Changed
+- Add missing entries to Italian transltation (by [@Harvester62](https://github.com/Harvester62) in [#2783](https://github.com/nomic-ai/gpt4all/pull/2783))
+- Use llama\_kv\_cache ops to shift context faster ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Don't stop generating at end of context ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+
+### Fixed
+- Case-insensitive LocalDocs source icon detection (by [@cosmic-snow](https://github.com/cosmic-snow) in [#2761](https://github.com/nomic-ai/gpt4all/pull/2761))
+- Fix comparison of pre- and post-release versions for update check and models3.json ([#2762](https://github.com/nomic-ai/gpt4all/pull/2762), [#2772](https://github.com/nomic-ai/gpt4all/pull/2772))
+- Fix several backend issues ([#2778](https://github.com/nomic-ai/gpt4all/pull/2778))
+  - Restore leading space removal logic that was incorrectly removed in [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
+  - CUDA: Cherry-pick llama.cpp DMMV cols requirement fix that caused a crash with long conversations since [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
+- Make reverse prompt detection work more reliably and prevent it from breaking output ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Disallow context shift for chat name and follow-up generation to prevent bugs ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Explicitly target macOS 12.6 in CI to fix Metal compatibility on older macOS ([#2846](https://github.com/nomic-ai/gpt4all/pull/2846))
+
+## [3.1.1] - 2024-07-27
+
+### Added
+- Add Llama 3.1 8B Instruct to models3.json (by [@3Simplex](https://github.com/3Simplex) in [#2731](https://github.com/nomic-ai/gpt4all/pull/2731) and [#2732](https://github.com/nomic-ai/gpt4all/pull/2732))
+- Portuguese (BR) translation (by [thiagojramos](https://github.com/thiagojramos) in [#2733](https://github.com/nomic-ai/gpt4all/pull/2733))
+- Support adding arbitrary OpenAI-compatible models by URL (by [@supersonictw](https://github.com/supersonictw) in [#2683](https://github.com/nomic-ai/gpt4all/pull/2683))
+- Support Llama 3.1 RoPE scaling ([#2758](https://github.com/nomic-ai/gpt4all/pull/2758))
+
+### Changed
+- Add missing entries to Chinese (Simplified) translation (by [wuodoo](https://github.com/wuodoo) in [#2716](https://github.com/nomic-ai/gpt4all/pull/2716) and [#2749](https://github.com/nomic-ai/gpt4all/pull/2749))
+- Update translation files and add missing paths to CMakeLists.txt ([#2735](https://github.com/nomic-ai/gpt4all/2735))
+
+## [3.1.0] - 2024-07-24
+
+### Added
+- Generate suggested follow-up questions ([#2634](https://github.com/nomic-ai/gpt4all/pull/2634), [#2723](https://github.com/nomic-ai/gpt4all/pull/2723))
+  - Also add options for the chat name and follow-up question prompt templates
+- Scaffolding for translations ([#2612](https://github.com/nomic-ai/gpt4all/pull/2612))
+- Spanish (MX) translation (by [@jstayco](https://github.com/jstayco) in [#2654](https://github.com/nomic-ai/gpt4all/pull/2654))
+- Chinese (Simplified) translation by mikage ([#2657](https://github.com/nomic-ai/gpt4all/pull/2657))
+- Dynamic changes of language and locale at runtime ([#2659](https://github.com/nomic-ai/gpt4all/pull/2659), [#2677](https://github.com/nomic-ai/gpt4all/pull/2677))
+- Romanian translation by [@SINAPSA\_IC](https://github.com/SINAPSA_IC) ([#2662](https://github.com/nomic-ai/gpt4all/pull/2662))
+- Chinese (Traditional) translation (by [@supersonictw](https://github.com/supersonictw) in [#2661](https://github.com/nomic-ai/gpt4all/pull/2661))
+- Italian translation (by [@Harvester62](https://github.com/Harvester62) in [#2700](https://github.com/nomic-ai/gpt4all/pull/2700))
+
+### Changed
+- Customize combo boxes and context menus to fit the new style ([#2535](https://github.com/nomic-ai/gpt4all/pull/2535))
+- Improve view bar scaling and Model Settings layout ([#2520](https://github.com/nomic-ai/gpt4all/pull/2520)
+- Make the logo spin while the model is generating ([#2557](https://github.com/nomic-ai/gpt4all/pull/2557))
+- Server: Reply to wrong GET/POST method with HTTP 405 instead of 404 (by [@cosmic-snow](https://github.com/cosmic-snow) in [#2615](https://github.com/nomic-ai/gpt4all/pull/2615))
+- Update theme for menus (by [@3Simplex](https://github.com/3Simplex) in [#2578](https://github.com/nomic-ai/gpt4all/pull/2578))
+- Move the "stop" button to the message box ([#2561](https://github.com/nomic-ai/gpt4all/pull/2561))
+- Build with CUDA 11.8 for better compatibility ([#2639](https://github.com/nomic-ai/gpt4all/pull/2639))
+- Make links in latest news section clickable ([#2643](https://github.com/nomic-ai/gpt4all/pull/2643))
+- Support translation of settings choices ([#2667](https://github.com/nomic-ai/gpt4all/pull/2667), [#2690](https://github.com/nomic-ai/gpt4all/pull/2690))
+- Improve LocalDocs view's error message (by @cosmic-snow in [#2679](https://github.com/nomic-ai/gpt4all/pull/2679))
+- Ignore case of LocalDocs file extensions ([#2642](https://github.com/nomic-ai/gpt4all/pull/2642), [#2684](https://github.com/nomic-ai/gpt4all/pull/2684))
+- Update llama.cpp to commit 87e397d00 from July 19th ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694), [#2702](https://github.com/nomic-ai/gpt4all/pull/2702))
+  - Add support for GPT-NeoX, Gemma 2, OpenELM, ChatGLM, and Jais architectures (all with Vulkan support)
+  - Add support for DeepSeek-V2 architecture (no Vulkan support)
+  - Enable Vulkan support for StarCoder2, XVERSE, Command R, and OLMo
+- Show scrollbar in chat collections list as needed (by [@cosmic-snow](https://github.com/cosmic-snow) in [#2691](https://github.com/nomic-ai/gpt4all/pull/2691))
+
+### Removed
+- Remove support for GPT-J models ([#2676](https://github.com/nomic-ai/gpt4all/pull/2676), [#2693](https://github.com/nomic-ai/gpt4all/pull/2693))
+
+### Fixed
+- Fix placement of thumbs-down and datalake opt-in dialogs ([#2540](https://github.com/nomic-ai/gpt4all/pull/2540))
+- Select the correct folder with the Linux fallback folder dialog ([#2541](https://github.com/nomic-ai/gpt4all/pull/2541))
+- Fix clone button sometimes producing blank model info ([#2545](https://github.com/nomic-ai/gpt4all/pull/2545))
+- Fix jerky chat view scrolling ([#2555](https://github.com/nomic-ai/gpt4all/pull/2555))
+- Fix "reload" showing for chats with missing models ([#2520](https://github.com/nomic-ai/gpt4all/pull/2520)
+- Fix property binding loop warning ([#2601](https://github.com/nomic-ai/gpt4all/pull/2601))
+- Fix UI hang with certain chat view content ([#2543](https://github.com/nomic-ai/gpt4all/pull/2543))
+- Fix crash when Kompute falls back to CPU ([#2640](https://github.com/nomic-ai/gpt4all/pull/2640))
+- Fix several Vulkan resource management issues ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
+- Fix crash/hang when some models stop generating, by showing special tokens ([#2701](https://github.com/nomic-ai/gpt4all/pull/2701))
+
+[Unreleased]: https://github.com/nomic-ai/gpt4all/compare/v3.2.1...HEAD
+[3.2.1]: https://github.com/nomic-ai/gpt4all/compare/v3.2.0...v3.2.1
+[3.2.0]: https://github.com/nomic-ai/gpt4all/compare/v3.1.1...v3.2.0
+[3.1.1]: https://github.com/nomic-ai/gpt4all/compare/v3.1.0...v3.1.1
+[3.1.0]: https://github.com/nomic-ai/gpt4all/compare/v3.0.0...v3.1.0

gpt4all-chat/CMakeLists.txt

@@ -17,10 +17,12 @@ if(APPLE)
 endif()
 
 set(APP_VERSION_MAJOR 3)
-set(APP_VERSION_MINOR 0)
-set(APP_VERSION_PATCH 0)
+set(APP_VERSION_MINOR 2)
+set(APP_VERSION_PATCH 1)
 set(APP_VERSION_BASE "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
-set(APP_VERSION "${APP_VERSION_BASE}-rc1")
+set(APP_VERSION "${APP_VERSION_BASE}")
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/Modules")
 
 # Include the binary directory for the generated header file
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")
@@ -29,10 +31,10 @@ project(gpt4all VERSION ${APP_VERSION_BASE} LANGUAGES CXX C)
 
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTORCC ON)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-option(GPT4ALL_LOCALHOST OFF "Build installer for localhost repo")
+option(GPT4ALL_LOCALHOST "Build installer for localhost repo" OFF)
 option(GPT4ALL_OFFLINE_INSTALLER "Build an offline installer" OFF)
+option(GPT4ALL_SIGN_INSTALL "Sign installed binaries and installers (requires signing identities)" OFF)
 
 # Generate a header file with the version number
 configure_file(
@@ -41,9 +43,9 @@ configure_file(
 )
 
 if(LINUX)
-  find_package(Qt6 6.4 COMPONENTS Core Quick WaylandCompositor QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
+  find_package(Qt6 6.4 COMPONENTS Core Quick WaylandCompositor QuickDialogs2 Svg HttpServer Sql Pdf LinguistTools REQUIRED)
 else()
-  find_package(Qt6 6.4 COMPONENTS Core Quick QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
+  find_package(Qt6 6.4 COMPONENTS Core Quick QuickDialogs2 Svg HttpServer Sql Pdf LinguistTools REQUIRED)
 endif()
 
 # Get the Qt6Core target properties
@@ -109,6 +111,7 @@ qt_add_executable(chat
     chatllm.h chatllm.cpp
     chatmodel.h chatlistmodel.h chatlistmodel.cpp
     chatapi.h chatapi.cpp
+    chatviewtextprocessor.h chatviewtextprocessor.cpp
     database.h database.cpp
     download.h download.cpp
     embllm.cpp embllm.h
@@ -119,7 +122,6 @@ qt_add_executable(chat
     network.h network.cpp
     server.h server.cpp
     logger.h logger.cpp
-    responsetext.h responsetext.cpp
     ${APP_ICON_RESOURCE}
     ${CHAT_EXE_RESOURCES}
 )
@@ -132,91 +134,121 @@ qt_add_qml_module(chat
       main.qml
       qml/AddCollectionView.qml
       qml/AddModelView.qml
+      qml/ApplicationSettings.qml
       qml/ChatDrawer.qml
       qml/ChatView.qml
       qml/CollectionsDrawer.qml
       qml/HomeView.qml
+      qml/LocalDocsSettings.qml
+      qml/LocalDocsView.qml
+      qml/ModelSettings.qml
       qml/ModelsView.qml
       qml/NetworkDialog.qml
       qml/NewVersionDialog.qml
-      qml/ThumbsDownDialog.qml
+      qml/PopupDialog.qml
       qml/SettingsView.qml
       qml/StartupDialog.qml
-      qml/PopupDialog.qml
-      qml/Theme.qml
-      qml/ModelSettings.qml
-      qml/ApplicationSettings.qml
-      qml/LocalDocsSettings.qml
-      qml/LocalDocsView.qml
       qml/SwitchModelDialog.qml
-      qml/MySettingsTab.qml
-      qml/MySettingsStack.qml
-      qml/MySettingsDestructiveButton.qml
-      qml/MySettingsButton.qml
-      qml/MySettingsLabel.qml
-      qml/MySlug.qml
+      qml/Theme.qml
+      qml/ThumbsDownDialog.qml
+      qml/Toast.qml
+      qml/ToastManager.qml
+      qml/MyBusyIndicator.qml
       qml/MyButton.qml
+      qml/MyCheckBox.qml
       qml/MyComboBox.qml
       qml/MyDialog.qml
       qml/MyDirectoryField.qml
       qml/MyFancyLink.qml
-      qml/MyTextArea.qml
-      qml/MyTextField.qml
-      qml/MyCheckBox.qml
-      qml/MyBusyIndicator.qml
+      qml/MyMenu.qml
+      qml/MyMenuItem.qml
       qml/MyMiniButton.qml
+      qml/MySettingsButton.qml
+      qml/MySettingsDestructiveButton.qml
+      qml/MySettingsLabel.qml
+      qml/MySettingsStack.qml
+      qml/MySettingsTab.qml
+      qml/MySlug.qml
+      qml/MyTextArea.qml
+      qml/MyTextButton.qml
+      qml/MyTextField.qml
       qml/MyToolButton.qml
       qml/MyWelcomeButton.qml
     RESOURCES
       icons/antenna_1.svg
       icons/antenna_2.svg
       icons/antenna_3.svg
-      icons/send_message.svg
-      icons/stop_generating.svg
-      icons/regenerate.svg
-      icons/chat.svg
+      icons/caret_down.svg
+      icons/caret_right.svg
       icons/changelog.svg
+      icons/chat.svg
+      icons/check.svg
       icons/close.svg
       icons/copy.svg
       icons/db.svg
       icons/discord.svg
       icons/download.svg
-      icons/settings.svg
-      icons/eject.svg
       icons/edit.svg
+      icons/eject.svg
       icons/email.svg
-      icons/file.svg
       icons/file-md.svg
       icons/file-pdf.svg
       icons/file-txt.svg
+      icons/file.svg
       icons/github.svg
       icons/globe.svg
+      icons/gpt4all-32.png
+      icons/gpt4all-48.png
+      icons/gpt4all.svg
+      icons/gpt4all_transparent.svg
       icons/home.svg
       icons/image.svg
       icons/info.svg
-      icons/local-docs.svg
-      icons/models.svg
-      icons/nomic_logo.svg
-      icons/notes.svg
-      icons/search.svg
-      icons/trash.svg
-      icons/network.svg
-      icons/thumbs_up.svg
-      icons/thumbs_down.svg
-      icons/twitter.svg
       icons/left_panel_closed.svg
       icons/left_panel_open.svg
-      icons/logo.svg
-      icons/logo-32.png
-      icons/logo-48.png
+      icons/local-docs.svg
+      icons/models.svg
+      icons/network.svg
+      icons/nomic_logo.svg
+      icons/notes.svg
+      icons/plus.svg
+      icons/recycle.svg
+      icons/regenerate.svg
+      icons/search.svg
+      icons/send_message.svg
+      icons/settings.svg
+      icons/stack.svg
+      icons/stop_generating.svg
+      icons/thumbs_down.svg
+      icons/thumbs_up.svg
+      icons/trash.svg
+      icons/twitter.svg
+      icons/up_down.svg
       icons/you.svg
-      icons/alt_logo.svg
+)
+
+qt_add_translations(chat
+    TS_FILES
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_en_US.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_es_MX.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_zh_CN.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_zh_TW.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_ro_RO.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_it_IT.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_pt_BR.ts
 )
 
 set_target_properties(chat PROPERTIES
     WIN32_EXECUTABLE TRUE
 )
 
+macro(REPORT_MISSING_SIGNING_CONTEXT)
+    message(FATAL_ERROR [=[
+        Signing requested but no identity configured.
+        Please set the correct env variable or provide the MAC_SIGNING_IDENTITY argument on the command line
+        ]=])
+endmacro()
+
 if (APPLE)
     set_target_properties(chat PROPERTIES
         MACOSX_BUNDLE TRUE
@@ -227,6 +259,28 @@ if (APPLE)
         OUTPUT_NAME gpt4all
     )
     add_dependencies(chat ggml-metal)
+
+    if(NOT MAC_SIGNING_IDENTITY)
+        if(NOT DEFINED ENV{MAC_SIGNING_CERT_NAME} AND GPT4ALL_SIGN_INSTALL)
+            REPORT_MISSING_SIGNING_CONTEXT()
+        endif()
+        set(MAC_SIGNING_IDENTITY $ENV{MAC_SIGNING_CERT_NAME})
+    endif()
+    if(NOT MAC_SIGNING_TID)
+        if(NOT DEFINED ENV{MAC_NOTARIZATION_TID} AND GPT4ALL_SIGN_INSTALL)
+            REPORT_MISSING_SIGNING_CONTEXT()
+        endif()
+        set(MAC_SIGNING_TID $ENV{MAC_NOTARIZATION_TID})
+    endif()
+
+    # Setup MacOS signing for individual binaries
+    set_target_properties(chat PROPERTIES
+        XCODE_ATTRIBUTE_CODE_SIGN_STYLE "Manual"
+        XCODE_ATTRIBUTE_DEVELOPMENT_TEAM ${MAC_SIGNING_TID}
+        XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY ${MAC_SIGNING_IDENTITY}
+        XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED True
+        XCODE_ATTRIBUTE_OTHER_CODE_SIGN_FLAGS "--timestamp=http://timestamp.apple.com/ts01  --options=runtime,library"
+    )
 endif()
 
 target_compile_definitions(chat
@@ -271,15 +325,11 @@ if (LLMODEL_KOMPUTE)
     set(MODEL_IMPL_TARGETS
         llamamodel-mainline-kompute
         llamamodel-mainline-kompute-avxonly
-        gptj-kompute
-        gptj-kompute-avxonly
     )
 else()
     set(MODEL_IMPL_TARGETS
         llamamodel-mainline-cpu
         llamamodel-mainline-cpu-avxonly
-        gptj-cpu
-        gptj-cpu-avxonly
     )
 endif()
 
@@ -293,6 +343,24 @@ install(
     RUNTIME DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN}  # .dll
 )
 
+if(APPLE AND GPT4ALL_SIGN_INSTALL)
+    include(SignMacOSBinaries)
+    install_sign_osx(chat)
+    install_sign_osx(llmodel)
+    foreach(tgt ${MODEL_IMPL_TARGETS})
+        install_sign_osx(${tgt})
+    endforeach()
+endif()
+
+if(WIN32 AND GPT4ALL_SIGN_INSTALL)
+    include(SignWindowsBinaries)
+    sign_target_windows(chat)
+    sign_target_windows(llmodel)
+    foreach(tgt ${MODEL_IMPL_TARGETS})
+        sign_target_windows(${tgt})
+    endforeach()
+endif()
+
 if (LLMODEL_CUDA)
     set_property(TARGET llamamodel-mainline-cuda llamamodel-mainline-cuda-avxonly
                  APPEND PROPERTY INSTALL_RPATH "$ORIGIN")
@@ -360,7 +428,7 @@ set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR})
 set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR})
 SET(CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH})
 set(CPACK_PACKAGE_HOMEPAGE_URL "https://gpt4all.io")
-set(CPACK_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-48.png")
+set(CPACK_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/gpt4all-48.png")
 set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE)
 set(CPACK_RESOURCE_FILE_README ${CMAKE_CURRENT_SOURCE_DIR}/README.md)
 set(CPACK_PACKAGE_EXECUTABLES "GPT4All")
@@ -370,8 +438,8 @@ set(CPACK_IFW_PACKAGE_TITLE "GPT4All Installer")
 set(CPACK_IFW_PACKAGE_PUBLISHER "Nomic, Inc.")
 set(CPACK_IFW_PRODUCT_URL "https://gpt4all.io")
 set(CPACK_IFW_PACKAGE_WIZARD_STYLE "Aero")
-set(CPACK_IFW_PACKAGE_LOGO "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-48.png")
-set(CPACK_IFW_PACKAGE_WINDOW_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-32.png")
+set(CPACK_IFW_PACKAGE_LOGO "${CMAKE_CURRENT_SOURCE_DIR}/icons/gpt4all-48.png")
+set(CPACK_IFW_PACKAGE_WINDOW_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/gpt4all-32.png")
 set(CPACK_IFW_PACKAGE_WIZARD_SHOW_PAGE_LIST OFF)
 
 include(InstallRequiredSystemLibraries)

gpt4all-chat/LICENSE

@@ -1,4 +1,4 @@
-Copyright 2023 Nomic, Inc., Aaron Miller
+Copyright 2023-2024 Nomic, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

gpt4all-chat/README.md

@@ -16,27 +16,17 @@ One click installers for macOS, Linux, and Windows at https://gpt4all.io
 ## Features
 
 * Cross-platform (Linux, Windows, MacOSX)
-* Fast CPU based inference using ggml for GPT-J based models
 * The UI is made to look and feel like you've come to expect from a chatty gpt
 * Check for updates so you can always stay fresh with latest models
 * Easy to install with precompiled binaries available for all three major desktop platforms
 * Multi-modal - Ability to load more than one model and switch between them
-* Supports both llama.cpp and gptj.cpp style models
-* Model downloader in GUI featuring many popular open source models
-* Settings dialog to change temp, top_p, top_k, threads, etc
-* Copy your conversation to clipboard
-* Check for updates to get the very latest GUI
-
-## Feature wishlist
-
 * Multi-chat - a list of current and past chats and the ability to save/delete/export and switch between
-* Text to speech - have the AI response with voice
-* Speech to text - give the prompt with your voice
-* Plugin support for langchain other developer tools
-* chat gui headless operation mode
-* Advanced settings for changing temperature, topk, etc. (DONE)
-* * Improve the accessibility of the installer for screen reader users
-* YOUR IDEA HERE
+* Supports models that are supported by llama.cpp
+* Model downloader in GUI featuring many popular open source models
+* Settings dialog to change temp, top_p, min_p, top_k, threads, etc
+* Copy your conversation to clipboard
+* RAG via LocalDocs feature
+* Check for updates to get the very latest GUI
 
 ## Building and running
 
@@ -44,14 +34,7 @@ One click installers for macOS, Linux, and Windows at https://gpt4all.io
 
 ## Getting the latest
 
-If you've already checked out the source code and/or built the program make sure when you do a git fetch to get the latest changes and that you also do ```git submodule update --init --recursive``` to update the submodules.
-
-## Manual download of models
-* You can find a 'Model Explorer' on the official website where you can manually download models that we support: https://gpt4all.io/index.html
-
-## Terminal Only Interface with no Qt dependency
-
-Check out https://github.com/kuvaus/LlamaGPTJ-chat which is using the llmodel backend so it is compliant with our ecosystem and all models downloaded above should work with it.
+If you've already checked out the source code and/or built the program make sure when you do a git fetch to get the latest changes and that you also do `git submodule update --init --recursive` to update the submodules. (If you ever run into trouble, deinitializing via `git submodule deinit -f .` and then initializing again via `git submodule update --init --recursive` fixes most issues)
 
 ## Contributing
 
@@ -59,6 +42,4 @@ Check out https://github.com/kuvaus/LlamaGPTJ-chat which is using the llmodel ba
 
 
 ## License
-The source code of this chat interface is currently under a MIT license. The underlying GPT4All-j model is released under non-restrictive open-source Apache 2 License.
-
-The GPT4All-J license allows for users to use generated outputs as they see fit. Users take responsibility for ensuring their content meets applicable requirements for publication in a given context or region.
+The source code of this chat interface is currently under a MIT license.

gpt4all-chat/chat.cpp

@@ -58,14 +58,15 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::modelLoadingPercentageChanged, this, &Chat::handleModelLoadingPercentageChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::responseChanged, this, &Chat::handleResponseChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::promptProcessing, this, &Chat::promptProcessing, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::generatingQuestions, this, &Chat::generatingQuestions, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::responseStopped, this, &Chat::responseStopped, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelLoadingError, this, &Chat::handleModelLoadingError, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelLoadingWarning, this, &Chat::modelLoadingWarning, Qt::QueuedConnection);
-    connect(m_llmodel, &ChatLLM::recalcChanged, this, &Chat::handleRecalculating, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::restoringFromTextChanged, this, &Chat::handleRestoringFromText, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::generatedNameChanged, this, &Chat::generatedNameChanged, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::generatedQuestionFinished, this, &Chat::generatedQuestionFinished, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportSpeed, this, &Chat::handleTokenSpeedChanged, Qt::QueuedConnection);
-    connect(m_llmodel, &ChatLLM::reportDevice, this, &Chat::handleDeviceChanged, Qt::QueuedConnection);
-    connect(m_llmodel, &ChatLLM::reportFallbackReason, this, &Chat::handleFallbackReasonChanged, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::loadedModelInfoChanged, this, &Chat::loadedModelInfoChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::databaseResultsChanged, this, &Chat::handleDatabaseResultsChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelInfoChanged, this, &Chat::handleModelInfoChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::trySwitchContextOfLoadedModelCompleted, this, &Chat::handleTrySwitchContextOfLoadedModelCompleted, Qt::QueuedConnection);
@@ -91,10 +92,10 @@ void Chat::reset()
     emit resetContextRequested();
     m_id = Network::globalInstance()->generateUniqueId();
     emit idChanged(m_id);
-    // NOTE: We deliberately do no reset the name or creation date to indictate that this was originally
+    // NOTE: We deliberately do no reset the name or creation date to indicate that this was originally
     // an older chat that was reset for another purpose. Resetting this data will lead to the chat
     // name label changing back to 'New Chat' and showing up in the chat model list as a 'New Chat'
-    // further down in the list. This might surprise the user. In the future, we me might get rid of
+    // further down in the list. This might surprise the user. In the future, we might get rid of
     // the "reset context" button in the UI. Right now, by changing the model in the combobox dropdown
     // we effectively do a reset context. We *have* to do this right now when switching between different
     // types of models. The only way to get rid of that would be a very long recalculate where we rebuild
@@ -114,6 +115,8 @@ void Chat::resetResponseState()
     if (m_responseInProgress && m_responseState == Chat::LocalDocsRetrieval)
         return;
 
+    m_generatedQuestions = QList<QString>();
+    emit generatedQuestionsChanged();
     m_tokenSpeed = QString();
     emit tokenSpeedChanged();
     m_responseInProgress = true;
@@ -187,6 +190,12 @@ void Chat::handleModelLoadingPercentageChanged(float loadingPercentage)
 void Chat::promptProcessing()
 {
     m_responseState = !databaseResults().isEmpty() ? Chat::LocalDocsProcessing : Chat::PromptProcessing;
+     emit responseStateChanged();
+}
+
+void Chat::generatingQuestions()
+{
+    m_responseState = Chat::GeneratingQuestions;
     emit responseStateChanged();
 }
 
@@ -230,8 +239,8 @@ void Chat::newPromptResponsePair(const QString &prompt)
 {
     resetResponseState();
     m_chatModel->updateCurrentResponse(m_chatModel->count() - 1, false);
-    m_chatModel->appendPrompt(tr("Prompt: "), prompt);
-    m_chatModel->appendResponse(tr("Response: "), prompt);
+    m_chatModel->appendPrompt("Prompt: ", prompt);
+    m_chatModel->appendResponse("Response: ", prompt);
     emit resetResponseRequested();
 }
 
@@ -239,13 +248,13 @@ void Chat::serverNewPromptResponsePair(const QString &prompt)
 {
     resetResponseState();
     m_chatModel->updateCurrentResponse(m_chatModel->count() - 1, false);
-    m_chatModel->appendPrompt(tr("Prompt: "), prompt);
-    m_chatModel->appendResponse(tr("Response: "), prompt);
+    m_chatModel->appendPrompt("Prompt: ", prompt);
+    m_chatModel->appendResponse("Response: ", prompt);
 }
 
-bool Chat::isRecalc() const
+bool Chat::restoringFromText() const
 {
-    return m_llmodel->isRecalc();
+    return m_llmodel->restoringFromText();
 }
 
 void Chat::unloadAndDeleteLater()
@@ -305,10 +314,16 @@ void Chat::generatedNameChanged(const QString &name)
     emit nameChanged();
 }
 
-void Chat::handleRecalculating()
+void Chat::generatedQuestionFinished(const QString &question)
+{
+    m_generatedQuestions << question;
+    emit generatedQuestionsChanged();
+}
+
+void Chat::handleRestoringFromText()
 {
     Network::globalInstance()->trackChatEvent("recalc_context", { {"length", m_chatModel->count()} });
-    emit recalcChanged();
+    emit restoringFromTextChanged();
 }
 
 void Chat::handleModelLoadingError(const QString &error)
@@ -327,16 +342,19 @@ void Chat::handleTokenSpeedChanged(const QString &tokenSpeed)
     emit tokenSpeedChanged();
 }
 
-void Chat::handleDeviceChanged(const QString &device)
+QString Chat::deviceBackend() const
 {
-    m_device = device;
-    emit deviceChanged();
+    return m_llmodel->deviceBackend();
 }
 
-void Chat::handleFallbackReasonChanged(const QString &fallbackReason)
+QString Chat::device() const
 {
-    m_fallbackReason = fallbackReason;
-    emit fallbackReasonChanged();
+    return m_llmodel->device();
+}
+
+QString Chat::fallbackReason() const
+{
+    return m_llmodel->fallbackReason();
 }
 
 void Chat::handleDatabaseResultsChanged(const QList<ResultInfo> &results)
@@ -408,11 +426,6 @@ bool Chat::deserialize(QDataStream &stream, int version)
 
     bool discardKV = m_modelInfo.id().isEmpty();
 
-    // Prior to version 2 gptj models had a bug that fixed the kv_cache to F32 instead of F16 so
-    // unfortunately, we cannot deserialize these
-    if (version < 2 && m_modelInfo.filename().contains("gpt4all-j"))
-        discardKV = true;
-
     if (version > 2) {
         stream >> m_collections;
         emit collectionListChanged(m_collections);

gpt4all-chat/chat.h

@@ -27,17 +27,19 @@ class Chat : public QObject
     Q_PROPERTY(QString response READ response NOTIFY responseChanged)
     Q_PROPERTY(ModelInfo modelInfo READ modelInfo WRITE setModelInfo NOTIFY modelInfoChanged)
     Q_PROPERTY(bool responseInProgress READ responseInProgress NOTIFY responseInProgressChanged)
-    Q_PROPERTY(bool isRecalc READ isRecalc NOTIFY recalcChanged)
+    Q_PROPERTY(bool restoringFromText READ restoringFromText NOTIFY restoringFromTextChanged)
     Q_PROPERTY(bool isServer READ isServer NOTIFY isServerChanged)
     Q_PROPERTY(ResponseState responseState READ responseState NOTIFY responseStateChanged)
     Q_PROPERTY(QList<QString> collectionList READ collectionList NOTIFY collectionListChanged)
     Q_PROPERTY(QString modelLoadingError READ modelLoadingError NOTIFY modelLoadingErrorChanged)
     Q_PROPERTY(QString tokenSpeed READ tokenSpeed NOTIFY tokenSpeedChanged);
-    Q_PROPERTY(QString device READ device NOTIFY deviceChanged);
-    Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY fallbackReasonChanged);
+    Q_PROPERTY(QString deviceBackend READ deviceBackend NOTIFY loadedModelInfoChanged)
+    Q_PROPERTY(QString device READ device NOTIFY loadedModelInfoChanged)
+    Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY loadedModelInfoChanged)
     Q_PROPERTY(LocalDocsCollectionsModel *collectionModel READ collectionModel NOTIFY collectionModelChanged)
     // 0=no, 1=waiting, 2=working
     Q_PROPERTY(int trySwitchContextInProgress READ trySwitchContextInProgress NOTIFY trySwitchContextInProgressChanged)
+    Q_PROPERTY(QList<QString> generatedQuestions READ generatedQuestions NOTIFY generatedQuestionsChanged)
     QML_ELEMENT
     QML_UNCREATABLE("Only creatable from c++!")
 
@@ -47,6 +49,7 @@ public:
         LocalDocsRetrieval,
         LocalDocsProcessing,
         PromptProcessing,
+        GeneratingQuestions,
         ResponseGeneration
     };
     Q_ENUM(ResponseState)
@@ -85,7 +88,7 @@ public:
     ResponseState responseState() const;
     ModelInfo modelInfo() const;
     void setModelInfo(const ModelInfo &modelInfo);
-    bool isRecalc() const;
+    bool restoringFromText() const;
 
     Q_INVOKABLE void unloadModel();
     Q_INVOKABLE void reloadModel();
@@ -111,11 +114,15 @@ public:
     QString modelLoadingError() const { return m_modelLoadingError; }
 
     QString tokenSpeed() const { return m_tokenSpeed; }
-    QString device() const { return m_device; }
-    QString fallbackReason() const { return m_fallbackReason; }
+    QString deviceBackend() const;
+    QString device() const;
+    // not loaded -> QString(), no fallback -> QString("")
+    QString fallbackReason() const;
 
     int trySwitchContextInProgress() const { return m_trySwitchContextInProgress; }
 
+    QList<QString> generatedQuestions() const { return m_generatedQuestions; }
+
 public Q_SLOTS:
     void serverNewPromptResponsePair(const QString &prompt);
 
@@ -137,7 +144,7 @@ Q_SIGNALS:
     void processSystemPromptRequested();
     void modelChangeRequested(const ModelInfo &modelInfo);
     void modelInfoChanged();
-    void recalcChanged();
+    void restoringFromTextChanged();
     void loadDefaultModelRequested();
     void loadModelRequested(const ModelInfo &modelInfo);
     void generateNameRequested();
@@ -149,18 +156,20 @@ Q_SIGNALS:
     void fallbackReasonChanged();
     void collectionModelChanged();
     void trySwitchContextInProgressChanged();
+    void loadedModelInfoChanged();
+    void generatedQuestionsChanged();
 
 private Q_SLOTS:
     void handleResponseChanged(const QString &response);
     void handleModelLoadingPercentageChanged(float);
     void promptProcessing();
+    void generatingQuestions();
     void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
-    void handleRecalculating();
+    void generatedQuestionFinished(const QString &question);
+    void handleRestoringFromText();
     void handleModelLoadingError(const QString &error);
     void handleTokenSpeedChanged(const QString &tokenSpeed);
-    void handleDeviceChanged(const QString &device);
-    void handleFallbackReasonChanged(const QString &device);
     void handleDatabaseResultsChanged(const QList<ResultInfo> &results);
     void handleModelInfoChanged(const ModelInfo &modelInfo);
     void handleTrySwitchContextOfLoadedModelCompleted(int value);
@@ -177,6 +186,7 @@ private:
     QString m_fallbackReason;
     QString m_response;
     QList<QString> m_collections;
+    QList<QString> m_generatedQuestions;
     ChatModel *m_chatModel;
     bool m_responseInProgress = false;
     ResponseState m_responseState;

gpt4all-chat/chatapi.cpp

@@ -90,13 +90,13 @@ void ChatAPI::prompt(const std::string &prompt,
                      const std::string &promptTemplate,
                      std::function<bool(int32_t)> promptCallback,
                      std::function<bool(int32_t, const std::string&)> responseCallback,
-                     std::function<bool(bool)> recalculateCallback,
+                     bool allowContextShift,
                      PromptContext &promptCtx,
                      bool special,
                      std::string *fakeReply) {
 
     Q_UNUSED(promptCallback);
-    Q_UNUSED(recalculateCallback);
+    Q_UNUSED(allowContextShift);
     Q_UNUSED(special);
 
     if (!isModelLoaded()) {
@@ -201,6 +201,11 @@ void ChatAPIWorker::request(const QString &apiKey,
     QNetworkRequest request(apiUrl);
     request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
     request.setRawHeader("Authorization", authorization.toUtf8());
+#if defined(DEBUG)
+    qDebug() << "ChatAPI::request"
+             << "API URL: " << apiUrl.toString()
+             << "Authorization: " << authorization.toUtf8();
+#endif
     m_networkManager = new QNetworkAccessManager(this);
     QNetworkReply *reply = m_networkManager->post(request, array);
     connect(qGuiApp, &QCoreApplication::aboutToQuit, reply, &QNetworkReply::abort);
@@ -218,10 +223,28 @@ void ChatAPIWorker::handleFinished()
     }
 
     QVariant response = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
-    Q_ASSERT(response.isValid());
+
+    if (!response.isValid()) {
+        m_chat->callResponse(
+            -1,
+            tr("ERROR: Network error occurred while connecting to the API server")
+                .toStdString()
+        );
+        return;
+    }
+
     bool ok;
     int code = response.toInt(&ok);
     if (!ok || code != 200) {
+        bool isReplyEmpty(reply->readAll().isEmpty());
+        if (isReplyEmpty)
+            m_chat->callResponse(
+                -1,
+                tr("ChatAPIWorker::handleFinished got HTTP Error %1 %2")
+                    .arg(code)
+                    .arg(reply->errorString())
+                    .toStdString()
+            );
         qWarning().noquote() << "ERROR: ChatAPIWorker::handleFinished got HTTP Error" << code << "response:"
                              << reply->errorString();
     }
@@ -238,7 +261,10 @@ void ChatAPIWorker::handleReadyRead()
     }
 
     QVariant response = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
-    Q_ASSERT(response.isValid());
+
+    if (!response.isValid())
+        return;
+
     bool ok;
     int code = response.toInt(&ok);
     if (!ok || code != 200) {

gpt4all-chat/chatapi.h

@@ -69,7 +69,7 @@ public:
                 const std::string &promptTemplate,
                 std::function<bool(int32_t)> promptCallback,
                 std::function<bool(int32_t, const std::string&)> responseCallback,
-                std::function<bool(bool)> recalculateCallback,
+                bool allowContextShift,
                 PromptContext &ctx,
                 bool special,
                 std::string *fakeReply) override;
@@ -97,38 +97,57 @@ protected:
     // them as they are only called from the default implementation of 'prompt' which we override and
     // completely replace
 
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override {
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) override
+    {
         (void)ctx;
         (void)str;
         (void)special;
         throw std::logic_error("not implemented");
     }
 
-    std::string tokenToString(Token id) const override {
+    bool isSpecialToken(Token id) const override
+    {
         (void)id;
         throw std::logic_error("not implemented");
     }
 
-    Token sampleToken(PromptContext &ctx) const override {
+    std::string tokenToString(Token id) const override
+    {
+        (void)id;
+        throw std::logic_error("not implemented");
+    }
+
+    Token sampleToken(PromptContext &ctx) const override
+    {
         (void)ctx;
         throw std::logic_error("not implemented");
     }
 
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override {
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override
+    {
         (void)ctx;
         (void)tokens;
         throw std::logic_error("not implemented");
     }
 
-    int32_t contextLength() const override {
+    void shiftContext(PromptContext &promptCtx) override
+    {
+        (void)promptCtx;
         throw std::logic_error("not implemented");
     }
 
-    const std::vector<Token> &endTokens() const override {
+    int32_t contextLength() const override
+    {
         throw std::logic_error("not implemented");
     }
 
-    bool shouldAddBOS() const override {
+    const std::vector<Token> &endTokens() const override
+    {
+        throw std::logic_error("not implemented");
+    }
+
+    bool shouldAddBOS() const override
+    {
         throw std::logic_error("not implemented");
     }
 

gpt4all-chat/chatlistmodel.cpp

@@ -19,7 +19,7 @@
 #include <algorithm>
 
 #define CHAT_FORMAT_MAGIC 0xF5D553CC
-#define CHAT_FORMAT_VERSION 8
+#define CHAT_FORMAT_VERSION 9
 
 class MyChatListModel: public ChatListModel { };
 Q_GLOBAL_STATIC(MyChatListModel, chatListModelInstance)
@@ -29,7 +29,17 @@ ChatListModel *ChatListModel::globalInstance()
 }
 
 ChatListModel::ChatListModel()
-    : QAbstractListModel(nullptr) {}
+    : QAbstractListModel(nullptr) {
+
+        QCoreApplication::instance()->installEventFilter(this);
+}
+
+bool ChatListModel::eventFilter(QObject *obj, QEvent *ev)
+{
+    if (obj == QCoreApplication::instance() && ev->type() == QEvent::LanguageChange)
+        emit dataChanged(index(0, 0), index(m_chats.size() - 1, 0));
+    return false;
+}
 
 void ChatListModel::loadChats()
 {

gpt4all-chat/chatlistmodel.h

@@ -242,6 +242,9 @@ Q_SIGNALS:
     void requestSaveChats(const QVector<Chat*> &);
     void saveChatsFinished();
 
+protected:
+    bool eventFilter(QObject *obj, QEvent *ev) override;
+
 private Q_SLOTS:
     void newChatCountChanged()
     {

gpt4all-chat/chatllm.cpp

@@ -17,7 +17,6 @@
 #include <QMutexLocker>
 #include <QSet>
 #include <QStringList>
-#include <QVariantMap>
 #include <QWaitCondition>
 #include <Qt>
 #include <QtLogging>
@@ -38,7 +37,7 @@ using namespace Qt::Literals::StringLiterals;
 //#define DEBUG
 //#define DEBUG_MODEL_LOADING
 
-#define GPTJ_INTERNAL_STATE_VERSION 0
+#define GPTJ_INTERNAL_STATE_VERSION  0 // GPT-J is gone but old chats still use this
 #define LLAMA_INTERNAL_STATE_VERSION 0
 
 class LLModelStore {
@@ -93,11 +92,17 @@ void LLModelStore::destroy()
     m_availableModel.reset();
 }
 
+void LLModelInfo::resetModel(ChatLLM *cllm, LLModel *model) {
+    this->model.reset(model);
+    fallbackReason.reset();
+    emit cllm->loadedModelInfoChanged();
+}
+
 ChatLLM::ChatLLM(Chat *parent, bool isServer)
     : QObject{nullptr}
     , m_promptResponseTokens(0)
     , m_promptTokens(0)
-    , m_isRecalc(false)
+    , m_restoringFromText(false)
     , m_shouldBeLoaded(false)
     , m_forceUnloadModel(false)
     , m_markedForDeletion(false)
@@ -141,7 +146,7 @@ void ChatLLM::destroy()
     // The only time we should have a model loaded here is on shutdown
     // as we explicitly unload the model in all other circumstances
     if (isModelLoaded()) {
-        m_llModelInfo.model.reset();
+        m_llModelInfo.resetModel(this);
     }
 }
 
@@ -208,7 +213,7 @@ void ChatLLM::trySwitchContextOfLoadedModel(const ModelInfo &modelInfo)
     QString filePath = modelInfo.dirpath + modelInfo.filename();
     QFileInfo fileInfo(filePath);
 
-    m_llModelInfo = LLModelStore::globalInstance()->acquireModel();
+    acquireModel();
 #if defined(DEBUG_MODEL_LOADING)
         qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
@@ -251,8 +256,6 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     // reset status
     emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
     emit modelLoadingError("");
-    emit reportFallbackReason("");
-    emit reportDevice("");
     m_pristineLoadedState = false;
 
     QString filePath = modelInfo.dirpath + modelInfo.filename();
@@ -265,12 +268,12 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 #if defined(DEBUG_MODEL_LOADING)
         qDebug() << "already acquired model deleted" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
-        m_llModelInfo.model.reset();
+        m_llModelInfo.resetModel(this);
     } else if (!m_isServer) {
         // This is a blocking call that tries to retrieve the model we need from the model store.
         // If it succeeds, then we just have to restore state. If the store has never had a model
         // returned to it, then the modelInfo.model pointer should be null which will happen on startup
-        m_llModelInfo = LLModelStore::globalInstance()->acquireModel();
+        acquireModel();
 #if defined(DEBUG_MODEL_LOADING)
         qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
@@ -305,7 +308,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 #if defined(DEBUG_MODEL_LOADING)
             qDebug() << "deleting model" << m_llmThread.objectName() << m_llModelInfo.model.get();
 #endif
-            m_llModelInfo.model.reset();
+            m_llModelInfo.resetModel(this);
         }
     }
 
@@ -319,6 +322,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         QVariantMap modelLoadProps;
         if (modelInfo.isOnline) {
             QString apiKey;
+            QString requestUrl;
             QString modelName;
             {
                 QFile file(filePath);
@@ -329,190 +333,28 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                 QJsonObject obj = doc.object();
                 apiKey = obj["apiKey"].toString();
                 modelName = obj["modelName"].toString();
+                if (modelInfo.isCompatibleApi) {
+                    QString baseUrl(obj["baseUrl"].toString());
+                    QUrl apiUrl(QUrl::fromUserInput(baseUrl));
+                    if (!Network::isHttpUrlValid(apiUrl)) {
+                        return false;
+                    }
+                    QString currentPath(apiUrl.path());
+                    QString suffixPath("%1/chat/completions");
+                    apiUrl.setPath(suffixPath.arg(currentPath));
+                    requestUrl = apiUrl.toString();
+                } else {
+                    requestUrl = modelInfo.url();
+                }
             }
             m_llModelType = LLModelType::API_;
             ChatAPI *model = new ChatAPI();
             model->setModelName(modelName);
-            model->setRequestURL(modelInfo.url());
+            model->setRequestURL(requestUrl);
             model->setAPIKey(apiKey);
-            m_llModelInfo.model.reset(model);
-        } else {
-            QElapsedTimer modelLoadTimer;
-            modelLoadTimer.start();
-
-            auto requestedDevice = MySettings::globalInstance()->device();
-            auto n_ctx = MySettings::globalInstance()->modelContextLength(modelInfo);
-            m_ctx.n_ctx = n_ctx;
-            auto ngl = MySettings::globalInstance()->modelGpuLayers(modelInfo);
-
-            std::string backend = "auto";
-#ifdef Q_OS_MAC
-            if (requestedDevice == "CPU") {
-                backend = "cpu";
-            } else if (m_forceMetal) {
-#ifdef __aarch64__
-                backend = "metal";
-#endif
-            }
-#else // !defined(Q_OS_MAC)
-            if (requestedDevice.startsWith("CUDA: "))
-                backend = "cuda";
-#endif
-
-            QString constructError;
-            m_llModelInfo.model.reset();
-            try {
-                auto *model = LLModel::Implementation::construct(filePath.toStdString(), backend, n_ctx);
-                m_llModelInfo.model.reset(model);
-            } catch (const LLModel::MissingImplementationError &e) {
-                modelLoadProps.insert("error", "missing_model_impl");
-                constructError = e.what();
-            } catch (const LLModel::UnsupportedModelError &e) {
-                modelLoadProps.insert("error", "unsupported_model_file");
-                constructError = e.what();
-            } catch (const LLModel::BadArchError &e) {
-                constructError = e.what();
-                modelLoadProps.insert("error", "unsupported_model_arch");
-                modelLoadProps.insert("model_arch", QString::fromStdString(e.arch()));
-            }
-
-            if (m_llModelInfo.model) {
-                if (m_llModelInfo.model->isModelBlacklisted(filePath.toStdString())) {
-                    static QSet<QString> warned;
-                    auto fname = modelInfo.filename();
-                    if (!warned.contains(fname)) {
-                        emit modelLoadingWarning(
-                            u"%1 is known to be broken. Please get a replacement via the download dialog."_s.arg(fname)
-                        );
-                        warned.insert(fname); // don't warn again until restart
-                    }
-                }
-
-                m_llModelInfo.model->setProgressCallback([this](float progress) -> bool {
-                    progress = std::max(progress, std::numeric_limits<float>::min()); // keep progress above zero
-                    emit modelLoadingPercentageChanged(progress);
-                    return m_shouldBeLoaded;
-                });
-
-                auto approxDeviceMemGB = [](const LLModel::GPUDevice *dev) {
-                    float memGB = dev->heapSize / float(1024 * 1024 * 1024);
-                    return std::floor(memGB * 10.f) / 10.f; // truncate to 1 decimal place
-                };
-
-                std::vector<LLModel::GPUDevice> availableDevices;
-                const LLModel::GPUDevice *defaultDevice = nullptr;
-                {
-                    const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString(), n_ctx, ngl);
-                    availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
-                    // Pick the best device
-                    // NB: relies on the fact that Kompute devices are listed first
-                    if (!availableDevices.empty() && availableDevices.front().type == 2 /*a discrete gpu*/) {
-                        defaultDevice = &availableDevices.front();
-                        float memGB = defaultDevice->heapSize / float(1024 * 1024 * 1024);
-                        memGB = std::floor(memGB * 10.f) / 10.f; // truncate to 1 decimal place
-                        modelLoadProps.insert("default_device", QString::fromStdString(defaultDevice->name));
-                        modelLoadProps.insert("default_device_mem", approxDeviceMemGB(defaultDevice));
-                    }
-                }
-
-                QString actualDevice("CPU");
-
-#if defined(Q_OS_MAC) && defined(__aarch64__)
-                if (m_llModelInfo.model->implementation().buildVariant() == "metal")
-                    actualDevice = "Metal";
-#else
-                if (requestedDevice != "CPU") {
-                    const auto *device = defaultDevice;
-                    if (requestedDevice != "Auto") {
-                        // Use the selected device
-                        for (const LLModel::GPUDevice &d : availableDevices) {
-                            if (QString::fromStdString(d.selectionName()) == requestedDevice) {
-                                device = &d;
-                                break;
-                            }
-                        }
-                    }
-
-                    std::string unavail_reason;
-                    if (!device) {
-                        // GPU not available
-                    } else if (!m_llModelInfo.model->initializeGPUDevice(device->index, &unavail_reason)) {
-                        emit reportFallbackReason(QString::fromStdString("<br>" + unavail_reason));
-                    } else {
-                        actualDevice = QString::fromStdString(device->reportedName());
-                        modelLoadProps.insert("requested_device_mem", approxDeviceMemGB(device));
-                    }
-                }
-#endif
-
-                // Report which device we're actually using
-                emit reportDevice(actualDevice);
-                bool success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, ngl);
-
-                if (!m_shouldBeLoaded) {
-                    m_llModelInfo.model.reset();
-                    if (!m_isServer)
-                        LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
-                    m_llModelInfo = LLModelInfo();
-                    emit modelLoadingPercentageChanged(0.0f);
-                    return false;
-                }
-
-                if (actualDevice == "CPU") {
-                    // we asked llama.cpp to use the CPU
-                } else if (!success) {
-                    // llama_init_from_file returned nullptr
-                    emit reportDevice("CPU");
-                    emit reportFallbackReason("<br>GPU loading failed (out of VRAM?)");
-                    modelLoadProps.insert("cpu_fallback_reason", "gpu_load_failed");
-                    success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, 0);
-
-                    if (!m_shouldBeLoaded) {
-                        m_llModelInfo.model.reset();
-                        if (!m_isServer)
-                            LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
-                        m_llModelInfo = LLModelInfo();
-                        emit modelLoadingPercentageChanged(0.0f);
-                        return false;
-                    }
-                } else if (!m_llModelInfo.model->usingGPUDevice()) {
-                    // ggml_vk_init was not called in llama.cpp
-                    // We might have had to fallback to CPU after load if the model is not possible to accelerate
-                    // for instance if the quantization method is not supported on Vulkan yet
-                    emit reportDevice("CPU");
-                    emit reportFallbackReason("<br>model or quant has no GPU support");
-                    modelLoadProps.insert("cpu_fallback_reason", "gpu_unsupported_model");
-                }
-
-                if (!success) {
-                    m_llModelInfo.model.reset();
-                    if (!m_isServer)
-                        LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
-                    m_llModelInfo = LLModelInfo();
-                    emit modelLoadingError(u"Could not load model due to invalid model file for %1"_s.arg(modelInfo.filename()));
-                    modelLoadProps.insert("error", "loadmodel_failed");
-                } else {
-                    switch (m_llModelInfo.model->implementation().modelType()[0]) {
-                    case 'L': m_llModelType = LLModelType::LLAMA_; break;
-                    case 'G': m_llModelType = LLModelType::GPTJ_; break;
-                    default:
-                        {
-                            m_llModelInfo.model.reset();
-                            if (!m_isServer)
-                                LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
-                            m_llModelInfo = LLModelInfo();
-                            emit modelLoadingError(u"Could not determine model type for %1"_s.arg(modelInfo.filename()));
-                        }
-                    }
-
-                    modelLoadProps.insert("$duration", modelLoadTimer.elapsed() / 1000.);
-                }
-            } else {
-                if (!m_isServer)
-                    LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
-                m_llModelInfo = LLModelInfo();
-                emit modelLoadingError(u"Error loading %1: %2"_s.arg(modelInfo.filename(), constructError));
-            }
+            m_llModelInfo.resetModel(this, model);
+        } else if (!loadNewModel(modelInfo, modelLoadProps)) {
+            return false; // m_shouldBeLoaded became false
         }
 #if defined(DEBUG_MODEL_LOADING)
         qDebug() << "new model" << m_llmThread.objectName() << m_llModelInfo.model.get();
@@ -523,6 +365,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         fflush(stdout);
 #endif
         emit modelLoadingPercentageChanged(isModelLoaded() ? 1.0f : 0.0f);
+        emit loadedModelInfoChanged();
 
         modelLoadProps.insert("requestedDevice", MySettings::globalInstance()->device());
         modelLoadProps.insert("model", modelInfo.filename());
@@ -530,7 +373,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     } else {
         if (!m_isServer)
             LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo)); // release back into the store
-        m_llModelInfo = LLModelInfo();
+        resetModel();
         emit modelLoadingError(u"Could not find file for model %1"_s.arg(modelInfo.filename()));
     }
 
@@ -541,6 +384,200 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     return bool(m_llModelInfo.model);
 }
 
+/* Returns false if the model should no longer be loaded (!m_shouldBeLoaded).
+ * Otherwise returns true, even on error. */
+bool ChatLLM::loadNewModel(const ModelInfo &modelInfo, QVariantMap &modelLoadProps)
+{
+    QElapsedTimer modelLoadTimer;
+    modelLoadTimer.start();
+
+    QString requestedDevice = MySettings::globalInstance()->device();
+    int n_ctx = MySettings::globalInstance()->modelContextLength(modelInfo);
+    m_ctx.n_ctx = n_ctx;
+    int ngl = MySettings::globalInstance()->modelGpuLayers(modelInfo);
+
+    std::string backend = "auto";
+#ifdef Q_OS_MAC
+    if (requestedDevice == "CPU") {
+        backend = "cpu";
+    } else if (m_forceMetal) {
+#ifdef __aarch64__
+        backend = "metal";
+#endif
+    }
+#else // !defined(Q_OS_MAC)
+    if (requestedDevice.startsWith("CUDA: "))
+        backend = "cuda";
+#endif
+
+    QString filePath = modelInfo.dirpath + modelInfo.filename();
+
+    auto construct = [this, &filePath, &modelInfo, &modelLoadProps, n_ctx](std::string const &backend) {
+        QString constructError;
+        m_llModelInfo.resetModel(this);
+        try {
+            auto *model = LLModel::Implementation::construct(filePath.toStdString(), backend, n_ctx);
+            m_llModelInfo.resetModel(this, model);
+        } catch (const LLModel::MissingImplementationError &e) {
+            modelLoadProps.insert("error", "missing_model_impl");
+            constructError = e.what();
+        } catch (const LLModel::UnsupportedModelError &e) {
+            modelLoadProps.insert("error", "unsupported_model_file");
+            constructError = e.what();
+        } catch (const LLModel::BadArchError &e) {
+            constructError = e.what();
+            modelLoadProps.insert("error", "unsupported_model_arch");
+            modelLoadProps.insert("model_arch", QString::fromStdString(e.arch()));
+        }
+
+        if (!m_llModelInfo.model) {
+            if (!m_isServer)
+                LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+            resetModel();
+            emit modelLoadingError(u"Error loading %1: %2"_s.arg(modelInfo.filename(), constructError));
+            return false;
+        }
+
+        m_llModelInfo.model->setProgressCallback([this](float progress) -> bool {
+            progress = std::max(progress, std::numeric_limits<float>::min()); // keep progress above zero
+            emit modelLoadingPercentageChanged(progress);
+            return m_shouldBeLoaded;
+        });
+        return true;
+    };
+
+    if (!construct(backend))
+        return true;
+
+    if (m_llModelInfo.model->isModelBlacklisted(filePath.toStdString())) {
+        static QSet<QString> warned;
+        auto fname = modelInfo.filename();
+        if (!warned.contains(fname)) {
+            emit modelLoadingWarning(
+                u"%1 is known to be broken. Please get a replacement via the download dialog."_s.arg(fname)
+            );
+            warned.insert(fname); // don't warn again until restart
+        }
+    }
+
+    auto approxDeviceMemGB = [](const LLModel::GPUDevice *dev) {
+        float memGB = dev->heapSize / float(1024 * 1024 * 1024);
+        return std::floor(memGB * 10.f) / 10.f; // truncate to 1 decimal place
+    };
+
+    std::vector<LLModel::GPUDevice> availableDevices;
+    const LLModel::GPUDevice *defaultDevice = nullptr;
+    {
+        const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString(), n_ctx, ngl);
+        availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
+        // Pick the best device
+        // NB: relies on the fact that Kompute devices are listed first
+        if (!availableDevices.empty() && availableDevices.front().type == 2 /*a discrete gpu*/) {
+            defaultDevice = &availableDevices.front();
+            float memGB = defaultDevice->heapSize / float(1024 * 1024 * 1024);
+            memGB = std::floor(memGB * 10.f) / 10.f; // truncate to 1 decimal place
+            modelLoadProps.insert("default_device", QString::fromStdString(defaultDevice->name));
+            modelLoadProps.insert("default_device_mem", approxDeviceMemGB(defaultDevice));
+            modelLoadProps.insert("default_device_backend", QString::fromStdString(defaultDevice->backendName()));
+        }
+    }
+
+    bool actualDeviceIsCPU = true;
+
+#if defined(Q_OS_MAC) && defined(__aarch64__)
+    if (m_llModelInfo.model->implementation().buildVariant() == "metal")
+        actualDeviceIsCPU = false;
+#else
+    if (requestedDevice != "CPU") {
+        const auto *device = defaultDevice;
+        if (requestedDevice != "Auto") {
+            // Use the selected device
+            for (const LLModel::GPUDevice &d : availableDevices) {
+                if (QString::fromStdString(d.selectionName()) == requestedDevice) {
+                    device = &d;
+                    break;
+                }
+            }
+        }
+
+        std::string unavail_reason;
+        if (!device) {
+            // GPU not available
+        } else if (!m_llModelInfo.model->initializeGPUDevice(device->index, &unavail_reason)) {
+            m_llModelInfo.fallbackReason = QString::fromStdString(unavail_reason);
+        } else {
+            actualDeviceIsCPU = false;
+            modelLoadProps.insert("requested_device_mem", approxDeviceMemGB(device));
+        }
+    }
+#endif
+
+    bool success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, ngl);
+
+    if (!m_shouldBeLoaded) {
+        m_llModelInfo.resetModel(this);
+        if (!m_isServer)
+            LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+        resetModel();
+        emit modelLoadingPercentageChanged(0.0f);
+        return false;
+    }
+
+    if (actualDeviceIsCPU) {
+        // we asked llama.cpp to use the CPU
+    } else if (!success) {
+        // llama_init_from_file returned nullptr
+        m_llModelInfo.fallbackReason = "GPU loading failed (out of VRAM?)";
+        modelLoadProps.insert("cpu_fallback_reason", "gpu_load_failed");
+
+        // For CUDA, make sure we don't use the GPU at all - ngl=0 still offloads matmuls
+        if (backend == "cuda" && !construct("auto"))
+            return true;
+
+        success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, 0);
+
+        if (!m_shouldBeLoaded) {
+            m_llModelInfo.resetModel(this);
+            if (!m_isServer)
+                LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+            resetModel();
+            emit modelLoadingPercentageChanged(0.0f);
+            return false;
+        }
+    } else if (!m_llModelInfo.model->usingGPUDevice()) {
+        // ggml_vk_init was not called in llama.cpp
+        // We might have had to fallback to CPU after load if the model is not possible to accelerate
+        // for instance if the quantization method is not supported on Vulkan yet
+        m_llModelInfo.fallbackReason = "model or quant has no GPU support";
+        modelLoadProps.insert("cpu_fallback_reason", "gpu_unsupported_model");
+    }
+
+    if (!success) {
+        m_llModelInfo.resetModel(this);
+        if (!m_isServer)
+            LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+        resetModel();
+        emit modelLoadingError(u"Could not load model due to invalid model file for %1"_s.arg(modelInfo.filename()));
+        modelLoadProps.insert("error", "loadmodel_failed");
+        return true;
+    }
+
+    switch (m_llModelInfo.model->implementation().modelType()[0]) {
+    case 'L': m_llModelType = LLModelType::LLAMA_; break;
+    default:
+        {
+            m_llModelInfo.resetModel(this);
+            if (!m_isServer)
+                LLModelStore::globalInstance()->releaseModel(std::move(m_llModelInfo));
+            resetModel();
+            emit modelLoadingError(u"Could not determine model type for %1"_s.arg(modelInfo.filename()));
+        }
+    }
+
+    modelLoadProps.insert("$duration", modelLoadTimer.elapsed() / 1000.);
+    return true;
+};
+
 bool ChatLLM::isModelLoaded() const
 {
     return m_llModelInfo.model && m_llModelInfo.model->isModelLoaded();
@@ -574,6 +611,7 @@ std::string trim_whitespace(const std::string& input)
     return std::string(first_non_whitespace, last_non_whitespace);
 }
 
+// FIXME(jared): we don't actually have to re-decode the prompt to generate a new response
 void ChatLLM::regenerateResponse()
 {
     // ChatGPT uses a different semantic meaning for n_past than local models. For ChatGPT, the meaning
@@ -621,6 +659,16 @@ void ChatLLM::setModelInfo(const ModelInfo &modelInfo)
     emit modelInfoChanged(modelInfo);
 }
 
+void ChatLLM::acquireModel() {
+    m_llModelInfo = LLModelStore::globalInstance()->acquireModel();
+    emit loadedModelInfoChanged();
+}
+
+void ChatLLM::resetModel() {
+    m_llModelInfo = {};
+    emit loadedModelInfoChanged();
+}
+
 void ChatLLM::modelChangeRequested(const ModelInfo &modelInfo)
 {
     m_shouldBeLoaded = true;
@@ -664,17 +712,6 @@ bool ChatLLM::handleResponse(int32_t token, const std::string &response)
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "recalculate" << m_llmThread.objectName() << isRecalc;
-#endif
-    if (m_isRecalc != isRecalc) {
-        m_isRecalc = isRecalc;
-        emit recalcChanged();
-    }
-    return !m_stopGenerating;
-}
 bool ChatLLM::prompt(const QList<QString> &collectionList, const QString &prompt)
 {
     if (m_restoreStateFromText) {
@@ -716,7 +753,7 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     if (!databaseResults.isEmpty()) {
         QStringList results;
         for (const ResultInfo &info : databaseResults)
-            results << u"Collection: %1\nPath: %2\nSnippet: %3"_s.arg(info.collection, info.path, info.text);
+            results << u"Collection: %1\nPath: %2\nExcerpt: %3"_s.arg(info.collection, info.path, info.text);
 
         // FIXME(jared): use a Jinja prompt template instead of hardcoded Alpaca-style localdocs template
         docsContext = u"### Context:\n%1\n\n"_s.arg(results.join("\n\n"));
@@ -728,7 +765,6 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     auto promptFunc = std::bind(&ChatLLM::handlePrompt, this, std::placeholders::_1);
     auto responseFunc = std::bind(&ChatLLM::handleResponse, this, std::placeholders::_1,
         std::placeholders::_2);
-    auto recalcFunc = std::bind(&ChatLLM::handleRecalculate, this, std::placeholders::_1);
     emit promptProcessing();
     m_ctx.n_predict = n_predict;
     m_ctx.top_k = top_k;
@@ -748,10 +784,12 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     m_timer->start();
     if (!docsContext.isEmpty()) {
         auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode localdocs context without a response
-        m_llModelInfo.model->prompt(docsContext.toStdString(), "%1", promptFunc, responseFunc, recalcFunc, m_ctx);
+        m_llModelInfo.model->prompt(docsContext.toStdString(), "%1", promptFunc, responseFunc,
+                                    /*allowContextShift*/ true, m_ctx);
         m_ctx.n_predict = old_n_predict; // now we are ready for a response
     }
-    m_llModelInfo.model->prompt(prompt.toStdString(), promptTemplate.toStdString(), promptFunc, responseFunc, recalcFunc, m_ctx);
+    m_llModelInfo.model->prompt(prompt.toStdString(), promptTemplate.toStdString(), promptFunc, responseFunc,
+                                /*allowContextShift*/ true, m_ctx);
 #if defined(DEBUG)
     printf("\n");
     fflush(stdout);
@@ -763,7 +801,13 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
         m_response = trimmed;
         emit responseChanged(QString::fromStdString(m_response));
     }
-    emit responseStopped(elapsed);
+
+    SuggestionMode mode = MySettings::globalInstance()->suggestionMode();
+    if (mode == SuggestionMode::On || (!databaseResults.isEmpty() && mode == SuggestionMode::LocalDocsOnly))
+        generateQuestions(elapsed);
+    else
+        emit responseStopped(elapsed);
+
     m_pristineLoadedState = false;
     return true;
 }
@@ -809,7 +853,7 @@ void ChatLLM::unloadModel()
 #endif
 
     if (m_forceUnloadModel) {
-        m_llModelInfo.model.reset();
+        m_llModelInfo.resetModel(this);
         m_forceUnloadModel = false;
     }
 
@@ -841,13 +885,18 @@ void ChatLLM::generateName()
     if (!isModelLoaded())
         return;
 
+    const QString chatNamePrompt = MySettings::globalInstance()->modelChatNamePrompt(m_modelInfo);
+    if (chatNamePrompt.trimmed().isEmpty()) {
+        qWarning() << "ChatLLM: not generating chat name because prompt is empty";
+        return;
+    }
+
     auto promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     auto promptFunc = std::bind(&ChatLLM::handleNamePrompt, this, std::placeholders::_1);
     auto responseFunc = std::bind(&ChatLLM::handleNameResponse, this, std::placeholders::_1, std::placeholders::_2);
-    auto recalcFunc = std::bind(&ChatLLM::handleNameRecalculate, this, std::placeholders::_1);
     LLModel::PromptContext ctx = m_ctx;
-    m_llModelInfo.model->prompt("Describe the above conversation in seven words or less.",
-                                promptTemplate.toStdString(), promptFunc, responseFunc, recalcFunc, ctx);
+    m_llModelInfo.model->prompt(chatNamePrompt.toStdString(), promptTemplate.toStdString(),
+                                promptFunc, responseFunc, /*allowContextShift*/ false, ctx);
     std::string trimmed = trim_whitespace(m_nameResponse);
     if (trimmed != m_nameResponse) {
         m_nameResponse = trimmed;
@@ -867,7 +916,6 @@ bool ChatLLM::handleNamePrompt(int32_t token)
     qDebug() << "name prompt" << m_llmThread.objectName() << token;
 #endif
     Q_UNUSED(token);
-    qt_noop();
     return !m_stopGenerating;
 }
 
@@ -885,16 +933,73 @@ bool ChatLLM::handleNameResponse(int32_t token, const std::string &response)
     return words.size() <= 3;
 }
 
-bool ChatLLM::handleNameRecalculate(bool isRecalc)
+bool ChatLLM::handleQuestionPrompt(int32_t token)
 {
 #if defined(DEBUG)
-    qDebug() << "name recalc" << m_llmThread.objectName() << isRecalc;
+    qDebug() << "question prompt" << m_llmThread.objectName() << token;
 #endif
-    Q_UNUSED(isRecalc);
-    qt_noop();
+    Q_UNUSED(token);
+    return !m_stopGenerating;
+}
+
+bool ChatLLM::handleQuestionResponse(int32_t token, const std::string &response)
+{
+#if defined(DEBUG)
+    qDebug() << "question response" << m_llmThread.objectName() << token << response;
+#endif
+    Q_UNUSED(token);
+
+    // add token to buffer
+    m_questionResponse.append(response);
+
+    // match whole question sentences
+    // FIXME: This only works with response by the model in english which is not ideal for a multi-language
+    // model.
+    static const QRegularExpression reQuestion(R"(\b(What|Where|How|Why|When|Who|Which|Whose|Whom)\b[^?]*\?)");
+
+    // extract all questions from response
+    int lastMatchEnd = -1;
+    for (const auto &match : reQuestion.globalMatch(m_questionResponse)) {
+        lastMatchEnd = match.capturedEnd();
+        emit generatedQuestionFinished(match.captured());
+    }
+
+    // remove processed input from buffer
+    if (lastMatchEnd != -1)
+        m_questionResponse.erase(m_questionResponse.cbegin(), m_questionResponse.cbegin() + lastMatchEnd);
+
     return true;
 }
 
+void ChatLLM::generateQuestions(qint64 elapsed)
+{
+    Q_ASSERT(isModelLoaded());
+    if (!isModelLoaded()) {
+        emit responseStopped(elapsed);
+        return;
+    }
+
+    const std::string suggestedFollowUpPrompt = MySettings::globalInstance()->modelSuggestedFollowUpPrompt(m_modelInfo).toStdString();
+    if (QString::fromStdString(suggestedFollowUpPrompt).trimmed().isEmpty()) {
+        emit responseStopped(elapsed);
+        return;
+    }
+
+    emit generatingQuestions();
+    m_questionResponse.clear();
+    auto promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
+    auto promptFunc = std::bind(&ChatLLM::handleQuestionPrompt, this, std::placeholders::_1);
+    auto responseFunc = std::bind(&ChatLLM::handleQuestionResponse, this, std::placeholders::_1, std::placeholders::_2);
+    LLModel::PromptContext ctx = m_ctx;
+    QElapsedTimer totalTime;
+    totalTime.start();
+    m_llModelInfo.model->prompt(suggestedFollowUpPrompt, promptTemplate.toStdString(), promptFunc, responseFunc,
+                                /*allowContextShift*/ false, ctx);
+    elapsed += totalTime.elapsed();
+    emit responseStopped(elapsed);
+}
+
+
 bool ChatLLM::handleSystemPrompt(int32_t token)
 {
 #if defined(DEBUG)
@@ -904,15 +1009,6 @@ bool ChatLLM::handleSystemPrompt(int32_t token)
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleSystemRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "system recalc" << m_llmThread.objectName() << isRecalc;
-#endif
-    Q_UNUSED(isRecalc);
-    return false;
-}
-
 bool ChatLLM::handleRestoreStateFromTextPrompt(int32_t token)
 {
 #if defined(DEBUG)
@@ -922,15 +1018,6 @@ bool ChatLLM::handleRestoreStateFromTextPrompt(int32_t token)
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleRestoreStateFromTextRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "restore state from text recalc" << m_llmThread.objectName() << isRecalc;
-#endif
-    Q_UNUSED(isRecalc);
-    return false;
-}
-
 // this function serialized the cached model state to disk.
 // we want to also serialize n_ctx, and read it at load time.
 bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
@@ -962,8 +1049,6 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
     if (version >= 7) {
         stream << m_ctx.n_ctx;
     }
-    stream << quint64(m_ctx.logits.size());
-    stream.writeRawData(reinterpret_cast<const char*>(m_ctx.logits.data()), m_ctx.logits.size() * sizeof(float));
     stream << quint64(m_ctx.tokens.size());
     stream.writeRawData(reinterpret_cast<const char*>(m_ctx.tokens.data()), m_ctx.tokens.size() * sizeof(int));
     saveState();
@@ -1020,12 +1105,9 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
         if (!discardKV) m_ctx.n_ctx = n_ctx;
     }
 
-    quint64 logitsSize;
-    stream >> logitsSize;
-    if (!discardKV) {
-        m_ctx.logits.resize(logitsSize);
-        stream.readRawData(reinterpret_cast<char*>(m_ctx.logits.data()), logitsSize * sizeof(float));
-    } else {
+    if (version < 9) {
+        quint64 logitsSize;
+        stream >> logitsSize;
         stream.skipRawData(logitsSize * sizeof(float));
     }
 
@@ -1138,7 +1220,6 @@ void ChatLLM::processSystemPrompt()
     m_ctx = LLModel::PromptContext();
 
     auto promptFunc = std::bind(&ChatLLM::handleSystemPrompt, this, std::placeholders::_1);
-    auto recalcFunc = std::bind(&ChatLLM::handleSystemRecalculate, this, std::placeholders::_1);
 
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
     const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
@@ -1164,7 +1245,7 @@ void ChatLLM::processSystemPrompt()
 #endif
     auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode system prompt without a response
     // use "%1%2" and not "%1" to avoid implicit whitespace
-    m_llModelInfo.model->prompt(systemPrompt, "%1%2", promptFunc, nullptr, recalcFunc, m_ctx, true);
+    m_llModelInfo.model->prompt(systemPrompt, "%1%2", promptFunc, nullptr, /*allowContextShift*/ true, m_ctx, true);
     m_ctx.n_predict = old_n_predict;
 #if defined(DEBUG)
     printf("\n");
@@ -1181,14 +1262,13 @@ void ChatLLM::processRestoreStateFromText()
     if (!isModelLoaded() || !m_restoreStateFromText || m_isServer)
         return;
 
-    m_isRecalc = true;
-    emit recalcChanged();
+    m_restoringFromText = true;
+    emit restoringFromTextChanged();
 
     m_stopGenerating = false;
     m_ctx = LLModel::PromptContext();
 
     auto promptFunc = std::bind(&ChatLLM::handleRestoreStateFromTextPrompt, this, std::placeholders::_1);
-    auto recalcFunc = std::bind(&ChatLLM::handleRestoreStateFromTextRecalculate, this, std::placeholders::_1);
 
     const QString promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
@@ -1221,7 +1301,7 @@ void ChatLLM::processRestoreStateFromText()
         auto responseText = response.second.toStdString();
 
         m_llModelInfo.model->prompt(prompt.second.toStdString(), promptTemplate.toStdString(), promptFunc, nullptr,
-                                    recalcFunc, m_ctx, false, &responseText);
+                                    /*allowContextShift*/ true, m_ctx, false, &responseText);
     }
 
     if (!m_stopGenerating) {
@@ -1229,8 +1309,8 @@ void ChatLLM::processRestoreStateFromText()
         m_stateFromText.clear();
     }
 
-    m_isRecalc = false;
-    emit recalcChanged();
+    m_restoringFromText = false;
+    emit restoringFromTextChanged();
 
     m_pristineLoadedState = false;
 }

gpt4all-chat/chatllm.h

@@ -14,29 +14,39 @@
 #include <QPair>
 #include <QString>
 #include <QThread>
+#include <QVariantMap>
 #include <QVector>
 #include <QtGlobal>
 
 #include <atomic>
 #include <cstdint>
 #include <memory>
+#include <optional>
 #include <string>
 
 using namespace Qt::Literals::StringLiterals;
 
 class QDataStream;
 
+// NOTE: values serialized to disk, do not change or reuse
 enum LLModelType {
-    GPTJ_,
-    LLAMA_,
-    API_,
+    GPTJ_  = 0, // no longer used
+    LLAMA_ = 1,
+    API_   = 2,
+    BERT_  = 3, // no longer used
 };
 
+class ChatLLM;
+
 struct LLModelInfo {
     std::unique_ptr<LLModel> model;
     QFileInfo fileInfo;
+    std::optional<QString> fallbackReason;
+
     // NOTE: This does not store the model type or name on purpose as this is left for ChatLLM which
     // must be able to serialize the information even if it is in the unloaded state
+
+    void resetModel(ChatLLM *cllm, LLModel *model = nullptr);
 };
 
 class TokenTimer : public QObject {
@@ -83,7 +93,10 @@ class Chat;
 class ChatLLM : public QObject
 {
     Q_OBJECT
-    Q_PROPERTY(bool isRecalc READ isRecalc NOTIFY recalcChanged)
+    Q_PROPERTY(bool restoringFromText READ restoringFromText NOTIFY restoringFromTextChanged)
+    Q_PROPERTY(QString deviceBackend READ deviceBackend NOTIFY loadedModelInfoChanged)
+    Q_PROPERTY(QString device READ device NOTIFY loadedModelInfoChanged)
+    Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY loadedModelInfoChanged)
 public:
     ChatLLM(Chat *parent, bool isServer = false);
     virtual ~ChatLLM();
@@ -108,7 +121,31 @@ public:
     ModelInfo modelInfo() const;
     void setModelInfo(const ModelInfo &info);
 
-    bool isRecalc() const { return m_isRecalc; }
+    bool restoringFromText() const { return m_restoringFromText; }
+
+    void acquireModel();
+    void resetModel();
+
+    QString deviceBackend() const
+    {
+        if (!isModelLoaded()) return QString();
+        std::string name = LLModel::GPUDevice::backendIdToName(m_llModelInfo.model->backendName());
+        return QString::fromStdString(name);
+    }
+
+    QString device() const
+    {
+        if (!isModelLoaded()) return QString();
+        const char *name = m_llModelInfo.model->gpuDeviceName();
+        return name ? QString(name) : u"CPU"_s;
+    }
+
+    // not loaded -> QString(), no fallback -> QString("")
+    QString fallbackReason() const
+    {
+        if (!isModelLoaded()) return QString();
+        return m_llModelInfo.fallbackReason.value_or(u""_s);
+    }
 
     QString generatedName() const { return QString::fromStdString(m_nameResponse); }
 
@@ -125,6 +162,7 @@ public Q_SLOTS:
     void unloadModel();
     void reloadModel();
     void generateName();
+    void generateQuestions(qint64 elapsed);
     void handleChatIdChanged(const QString &id);
     void handleShouldBeLoadedChanged();
     void handleThreadStarted();
@@ -134,14 +172,17 @@ public Q_SLOTS:
     void processRestoreStateFromText();
 
 Q_SIGNALS:
-    void recalcChanged();
+    void restoringFromTextChanged();
+    void loadedModelInfoChanged();
     void modelLoadingPercentageChanged(float);
     void modelLoadingError(const QString &error);
     void modelLoadingWarning(const QString &warning);
     void responseChanged(const QString &response);
     void promptProcessing();
+    void generatingQuestions();
     void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
+    void generatedQuestionFinished(const QString &generatedQuestion);
     void stateChanged();
     void threadStarted();
     void shouldBeLoadedChanged();
@@ -160,16 +201,14 @@ protected:
         int32_t repeat_penalty_tokens);
     bool handlePrompt(int32_t token);
     bool handleResponse(int32_t token, const std::string &response);
-    bool handleRecalculate(bool isRecalc);
     bool handleNamePrompt(int32_t token);
     bool handleNameResponse(int32_t token, const std::string &response);
-    bool handleNameRecalculate(bool isRecalc);
     bool handleSystemPrompt(int32_t token);
     bool handleSystemResponse(int32_t token, const std::string &response);
-    bool handleSystemRecalculate(bool isRecalc);
     bool handleRestoreStateFromTextPrompt(int32_t token);
     bool handleRestoreStateFromTextResponse(int32_t token, const std::string &response);
-    bool handleRestoreStateFromTextRecalculate(bool isRecalc);
+    bool handleQuestionPrompt(int32_t token);
+    bool handleQuestionResponse(int32_t token, const std::string &response);
     void saveState();
     void restoreState();
 
@@ -179,8 +218,11 @@ protected:
     quint32 m_promptResponseTokens;
 
 private:
+    bool loadNewModel(const ModelInfo &modelInfo, QVariantMap &modelLoadProps);
+
     std::string m_response;
     std::string m_nameResponse;
+    QString m_questionResponse;
     LLModelInfo m_llModelInfo;
     LLModelType m_llModelType;
     ModelInfo m_modelInfo;
@@ -189,7 +231,7 @@ private:
     QThread m_llmThread;
     std::atomic<bool> m_stopGenerating;
     std::atomic<bool> m_shouldBeLoaded;
-    std::atomic<bool> m_isRecalc;
+    std::atomic<bool> m_restoringFromText; // status indication
     std::atomic<bool> m_forceUnloadModel;
     std::atomic<bool> m_markedForDeletion;
     bool m_isServer;