build_and_run: clarify which additional Qt libs are needed

Signed-off-by: cebtenzzre <cebtenzzre@gmail.com>

.circleci/continue_config.yml

@@ -27,7 +27,176 @@ jobs:
       - image: circleci/python:3.7
     steps:
       - run: echo "CircleCI pipeline triggered"
-
+  build-offline-chat-installer-macos:
+    macos:
+      xcode: 14.0.0
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - macos-qt-cache_v2
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
+              hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: macos-qt-cache_v2
+          paths:
+            - ~/Qt
+      - run:
+          name: Build
+          command: |
+            mkdir build
+            cd build
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
+              -DCMAKE_GENERATOR:STRING=Ninja \
+              -DBUILD_UNIVERSAL=ON \
+              -DMACDEPLOYQT=~/Qt/6.5.1/macos/bin/macdeployqt \
+              -DGPT4ALL_OFFLINE_INSTALLER=ON \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
+              -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
+              -S ../gpt4all-chat \
+              -B .
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package
+            mkdir upload
+            cp gpt4all-installer-* upload
+      - store_artifacts:
+          path: build/upload
+  build-offline-chat-installer-linux:
+    machine:
+      image: ubuntu-2204:2023.04.2
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - linux-qt-cache
+      - run:
+          name: Setup Linux and Dependencies
+          command: |
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
+              chmod +x qt-unified-linux-x64-4.6.0-online.run
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: linux-qt-cache
+          paths:
+            - ~/Qt
+      - run:
+          name: Build linuxdeployqt
+          command: |
+            git clone https://github.com/nomic-ai/linuxdeployqt
+            cd linuxdeployqt && qmake && sudo make install
+      - run:
+          name: Build
+          command: |
+            set -eo pipefail
+            export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
+            mkdir build
+            cd build
+            mkdir upload
+            ~/Qt/Tools/CMake/bin/cmake -DGPT4ALL_OFFLINE_INSTALLER=ON -DCMAKE_BUILD_TYPE=Release -S ../gpt4all-chat -B .
+            ~/Qt/Tools/CMake/bin/cmake --build . --target all
+            ~/Qt/Tools/CMake/bin/cmake --build . --target install
+            ~/Qt/Tools/CMake/bin/cmake --build . --target package
+            cp gpt4all-installer-* upload
+      - store_artifacts:
+          path: build/upload
+  build-offline-chat-installer-windows:
+    machine:
+      image: 'windows-server-2019-vs2019:2022.08.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - windows-qt-cache
+      - run:
+          name: Installing Qt
+          command: |
+            if (-not (Test-Path C:\Qt)) {
+              Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+            }
+      - save_cache:  # this is the new step to save cache
+          key: windows-qt-cache
+          paths:
+            - C:\Qt
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Build
+          command: |
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
+            $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
+            $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.6\bin"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
+            mkdir build
+            cd build
+            & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
+              "-DCMAKE_GENERATOR:STRING=Ninja" `
+              "-DCMAKE_BUILD_TYPE=Release" `
+              "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
+              "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
+              "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
+              "-DGPT4ALL_OFFLINE_INSTALLER=ON" `
+              "-S ..\gpt4all-chat" `
+              "-B ."
+            & "C:\Qt\Tools\Ninja\ninja.exe"
+            & "C:\Qt\Tools\Ninja\ninja.exe" install
+            & "C:\Qt\Tools\Ninja\ninja.exe" package
+            mkdir upload
+            copy gpt4all-installer-win64.exe upload
+      - store_artifacts:
+          path: build/upload
   build-gpt4all-chat-linux:
     machine:
       image: ubuntu-2204:2023.04.2
@@ -163,6 +332,7 @@ jobs:
             cd build
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
               -DCMAKE_GENERATOR:STRING=Ninja \
+              -DBUILD_UNIVERSAL=ON \
               -DCMAKE_BUILD_TYPE=Release \
               -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
               -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
@@ -244,6 +414,8 @@ jobs:
           command: |
             cd gpt4all-bindings/python/
             python setup.py bdist_wheel --plat-name=manylinux1_x86_64
+      - store_artifacts:
+          path: gpt4all-bindings/python/dist
       - persist_to_workspace:
           root: gpt4all-bindings/python/dist
           paths:
@@ -274,7 +446,9 @@ jobs:
           name: Build wheel
           command: |
             cd gpt4all-bindings/python
-            python setup.py bdist_wheel --plat-name=macosx_10_9_universal2
+            python setup.py bdist_wheel --plat-name=macosx_10_15_universal2
+      - store_artifacts:
+          path: gpt4all-bindings/python/dist
       - persist_to_workspace:
           root: gpt4all-bindings/python/dist
           paths:
@@ -288,9 +462,6 @@ jobs:
       - run:
           name: Install MinGW64
           command: choco install -y mingw --force --no-progress
-      - run:
-          name: Add MinGW64 to PATH
-          command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
       - run:
           name: Install VulkanSDK
           command: |
@@ -311,6 +482,7 @@ jobs:
             cd gpt4all-backend
             mkdir build
             cd build
+            $env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
             $env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
             cmake -G "MinGW Makefiles" .. -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
             cmake --build . --parallel
@@ -323,9 +495,11 @@ jobs:
             cd gpt4all
             mkdir llmodel_DO_NOT_MODIFY
             mkdir llmodel_DO_NOT_MODIFY/build/
-            cp 'C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
+            cp 'C:\ProgramData\mingw64\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
             cd ..
             python setup.py bdist_wheel --plat-name=win_amd64
+      - store_artifacts:
+          path: gpt4all-bindings/python/dist
       - persist_to_workspace:
           root: gpt4all-bindings/python/dist
           paths:
@@ -442,7 +616,7 @@ jobs:
       - run:
           name: Build Libraries
           command: |
-            $MinGWBin = "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
+            $MinGWBin = "C:\ProgramData\mingw64\mingw64\bin"
             $Env:Path += ";$MinGwBin"
             $Env:Path += ";C:\Program Files\CMake\bin"
             $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
@@ -828,6 +1002,20 @@ workflows:
     when: << pipeline.parameters.run-default-workflow >>
     jobs:
       - default-job
+  build-chat-offline-installers:
+    when: << pipeline.parameters.run-chat-workflow >>
+    jobs:
+      - hold:
+          type: approval
+      - build-offline-chat-installer-macos:
+          requires:
+            - hold
+      - build-offline-chat-installer-windows:
+          requires:
+            - hold
+      - build-offline-chat-installer-linux:
+          requires:
+            - hold
   build-and-test-gpt4all-chat:
     when: << pipeline.parameters.run-chat-workflow >>
     jobs:

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
-ignore-words-list = blong, belong, afterall, som
+ignore-words-list = blong, afterall, som, assistent, crasher
 skip = .git,*.pdf,*.svg,*.lock

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -27,21 +27,6 @@ body:
         - label: "The official example notebooks/scripts"
         - label: "My own modified scripts"
 
-  - type: checkboxes
-    id: related-components
-    attributes:
-      label: Related Components
-      description: "Select the components related to the issue (if applicable):"
-      options:
-        - label: "backend"
-        - label: "bindings"
-        - label: "python-bindings"
-        - label: "chat-ui"
-        - label: "models"
-        - label: "circleci"
-        - label: "docker"
-        - label: "api"
-
   - type: textarea
     id: reproduction
     validations:
@@ -67,4 +52,4 @@ body:
       required: true
     attributes:
       label: Expected behavior
-      description: "A clear and concise description of what you would expect to happen."
+      description: "A clear and concise description of what you would expect to happen."

.gitmodules

@@ -1,9 +1,4 @@
-[submodule "llama.cpp-230519"]
-	path = gpt4all-backend/llama.cpp-230519
-	url = https://github.com/ggerganov/llama.cpp.git
-[submodule "llama.cpp-230511"]
-	path = gpt4all-backend/llama.cpp-230511
-	url = https://github.com/nomic-ai/llama.cpp
 [submodule "llama.cpp-mainline"]
 	path = gpt4all-backend/llama.cpp-mainline
 	url = https://github.com/nomic-ai/llama.cpp.git
+	branch = gguf

README.md

@@ -1,9 +1,9 @@
 <h1 align="center">GPT4All</h1>
 
-<p align="center">Open-source assistant-style large language models that run locally on your CPU</p>
+<p align="center">Open-source large language models that run locally on your CPU and nearly any GPU</p>
 
 <p align="center">
-<a href="https://gpt4all.io">GPT4All Website</a>
+<a href="https://gpt4all.io">GPT4All Website and Models</a>
 </p>
 
 <p align="center">
@@ -30,13 +30,24 @@ Run on an M1 macOS Device (not sped up!)
 </p>
 
 ## GPT4All: An ecosystem of open-source on-edge large language models.
-GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
+
+> [!IMPORTANT]
+> GPT4All v2.5.0 and newer only supports models in GGUF format (.gguf). Models used with a previous version of GPT4All (.bin extension) will no longer work.
+
+GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and any GPU. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
 
 Learn more in the [documentation](https://docs.gpt4all.io).
 
-The goal is simple - be the best instruction tuned assistant-style language model that any person or enterprise can freely use, distribute and build on.
+A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models.
 
-A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models. 
+### What's New ([Issue Tracker](https://github.com/orgs/nomic-ai/projects/2))
+- **October 19th, 2023**: GGUF Support Launches with Support for:
+    - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
+    - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4_0, Q6 quantizations in GGUF.
+    - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
+- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on AMD, Intel, Samsung, Qualcomm and NVIDIA GPUs.
+- **August 15th, 2023**: GPT4All API launches allowing inference of local LLMs from docker containers.
+- **July 2023**: Stable support for LocalDocs, a GPT4All Plugin that allows you to privately and locally chat with your data.
 
 
 ### Chat Client

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -26,7 +26,7 @@ router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
 async def list_engines():
     '''
     List all available GPT4All models from
-    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json
+    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json
     '''
     raise NotImplementedError()
     return ListEnginesResponse(data=[])

gpt4all-backend/CMakeLists.txt

@@ -20,7 +20,7 @@ endif()
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 
 set(LLMODEL_VERSION_MAJOR 0)
-set(LLMODEL_VERSION_MINOR 4)
+set(LLMODEL_VERSION_MINOR 5)
 set(LLMODEL_VERSION_PATCH 0)
 set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
 project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
@@ -97,35 +97,15 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    add_library(replit-mainline-${BUILD_VARIANT} SHARED
-    replit.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-    target_compile_definitions(replit-mainline-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
-    prepare_target(replit-mainline llama-mainline)
-
     if (NOT LLAMA_METAL)
-# FIXME: These need to be forward ported to latest ggml
-#        add_library(gptj-${BUILD_VARIANT} SHARED
-#            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-#        prepare_target(gptj ggml-230511)
-
-        add_library(falcon-${BUILD_VARIANT} SHARED
-            falcon.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        target_compile_definitions(falcon-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
-        prepare_target(falcon llama-mainline)
-# FIXME: These need to be forward ported to latest ggml
-#        add_library(mpt-${BUILD_VARIANT} SHARED
-#            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-#        prepare_target(mpt ggml-230511)
+        add_library(gptj-${BUILD_VARIANT} SHARED
+            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        prepare_target(gptj llama-mainline)
 
         add_library(bert-${BUILD_VARIANT} SHARED
             bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
         target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
         prepare_target(bert llama-mainline)
-
-        add_library(starcoder-${BUILD_VARIANT} SHARED
-            starcoder.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        target_compile_definitions(starcoder-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
-        prepare_target(starcoder llama-mainline)
     endif()
 endforeach()
 
@@ -134,6 +114,8 @@ add_library(llmodel
     llmodel_c.h llmodel_c.cpp
     dlhandle.h
 )
+target_link_libraries(llmodel PRIVATE ggml-mainline-default)
+target_compile_definitions(llmodel PRIVATE GGML_BUILD_VARIANT="default")
 target_compile_definitions(llmodel PRIVATE LIB_FILE_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}")
 
 set_target_properties(llmodel PROPERTIES

gpt4all-backend/bert.cpp

@@ -4,10 +4,10 @@
 #include "ggml.h"
 
 #include <cassert>
+#include <cinttypes>
 #include <cmath>
 #include <cstdio>
 #include <cstring>
-#include <fstream>
 #include <map>
 #include <string>
 #include <vector>
@@ -34,7 +34,6 @@ struct bert_hparams
     int32_t n_intermediate = 1536;
     int32_t n_head = 12;
     int32_t n_layer = 6;
-    int32_t f16 = 1;
 };
 
 struct bert_layer
@@ -88,7 +87,6 @@ struct bert_model
     std::vector<bert_layer> layers;
 
     struct ggml_context *ctx;
-    std::map<std::string, struct ggml_tensor *> tensors;
 };
 
 // Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
@@ -345,7 +343,7 @@ void bert_eval(
 
     // embd norm
     {
-        inpL = ggml_norm(ctx0, inpL);
+        inpL = ggml_norm(ctx0, inpL, 1e-5f);
 
         inpL = ggml_add(ctx0,
                         ggml_mul(ctx0,
@@ -406,7 +404,7 @@ void bert_eval(
 
         // attention norm
         {
-            cur = ggml_norm(ctx0, cur);
+            cur = ggml_norm(ctx0, cur, 1e-5f);
 
             cur = ggml_add(ctx0,
                            ggml_mul(ctx0,
@@ -432,7 +430,7 @@ void bert_eval(
 
         // output norm
         {
-            cur = ggml_norm(ctx0, cur);
+            cur = ggml_norm(ctx0, cur, 1e-5f);
 
             cur = ggml_add(ctx0,
                            ggml_mul(ctx0,
@@ -482,7 +480,6 @@ void bert_eval(
 //
 
 void bert_free(bert_ctx * ctx) {
-    ggml_free(ctx->model.ctx);
     delete ctx;
 }
 
@@ -492,63 +489,130 @@ struct bert_ctx * bert_load_from_file(const char *fname)
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname);
 #endif
 
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin)
-    {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname);
-        return nullptr;
-    }
-
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *)&magic, sizeof(magic));
-        if (magic != 0x62657274)
-        {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname);
-            return nullptr;
-        }
-    }
-
     bert_ctx * new_bert = new bert_ctx;
     bert_model & model = new_bert->model;
     bert_vocab & vocab = new_bert->vocab;
 
+    struct gguf_init_params params = {
+        /*.no_alloc = */ false,
+        /*.ctx      = */ &model.ctx,
+    };
+    gguf_context *ggufctx = gguf_init_from_file(fname, params);
+    if (!ggufctx) {
+        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
+        return nullptr;
+    }
+
+    printf("%s: gguf version     = %d\n", __func__, gguf_get_version(ggufctx));
+    printf("%s: gguf alignment   = %zu\n", __func__, gguf_get_alignment(ggufctx));
+    printf("%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx));
+
+    // print some standard metadata
+    {
+        int keyidx;
+
+        keyidx = gguf_find_key(ggufctx, "general.name");
+        if (keyidx != -1) { printf("%s: model name           = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.description");
+        if (keyidx != -1) { printf("%s: model description    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.author");
+        if (keyidx != -1) { printf("%s: model author         = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.license");
+        if (keyidx != -1) { printf("%s: model license        = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.architecture");
+        if (keyidx != -1) { printf("%s: model architecture   = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.file_type");
+        if (keyidx != -1) { printf("%s: model file type      = %" PRIu32 "\n", __func__, gguf_get_val_u32(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "gptneox.tensor_data_layout");
+        if (keyidx != -1) { printf("%s: model data layout    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.source.huggingface.repository");
+        if (keyidx != -1) { printf("%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+    }
+
+    // check required metadata
+    {
+        // check model architecture kv
+        int keyidx = gguf_find_key(ggufctx, "general.architecture");
+        if (keyidx == -1) {
+            fprintf(stderr, "%s: gguf model architecture not found!\n", __func__);
+            return nullptr;
+        }
+        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
+            fprintf(stderr, "%s: model architecture not supported!\n", __func__);
+            return nullptr;
+        }
+    }
+
     // load hparams
     {
         auto &hparams = model.hparams;
 
-        fin.read((char *)&hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *)&hparams.n_max_tokens, sizeof(hparams.n_max_tokens));
-        fin.read((char *)&hparams.n_embd, sizeof(hparams.n_embd));
-        fin.read((char *)&hparams.n_intermediate, sizeof(hparams.n_intermediate));
-        fin.read((char *)&hparams.n_head, sizeof(hparams.n_head));
-        fin.read((char *)&hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *)&hparams.f16, sizeof(hparams.f16));
+        bool ok = false;
+        int keyidx;
+
+        do {
+            keyidx = gguf_find_key(ggufctx, "bert.context_length");
+            if (keyidx == -1) { break; }
+            hparams.n_max_tokens = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.embedding_length");
+            if (keyidx == -1) { break; }
+            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.feed_forward_length");
+            if (keyidx == -1) { break; }
+            hparams.n_intermediate = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.attention.head_count");
+            if (keyidx == -1) { break; }
+            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.block_count");
+            if (keyidx == -1) { break; }
+            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
+
+            ok = true;
+        } while (false);
+
+        if (!ok) {
+            fprintf(stderr, "%s: required hparam missing!\n", __func__);
+            return nullptr;
+        }
 
 #if defined(DEBUG_BERT)
-        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
         printf("%s: n_max_tokens   = %d\n", __func__, hparams.n_max_tokens);
-        printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_intermediate  = %d\n", __func__, hparams.n_intermediate);
-        printf("%s: n_head  = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
-        printf("%s: f16     = %d\n", __func__, hparams.f16);
+        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
+        printf("%s: n_intermediate = %d\n", __func__, hparams.n_intermediate);
+        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
+        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
 #endif
     }
 
     // load vocab
     {
-        int32_t n_vocab = model.hparams.n_vocab;
+        auto & hparams = model.hparams;
 
-        std::string word;
-        for (int i = 0; i < n_vocab; i++)
-        {
-            uint32_t len;
-            fin.read((char *)&len, sizeof(len));
+        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
+        if (keyidx == -1) {
+            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
+            return nullptr;
+        }
+        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
+            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
+            return nullptr;
+        }
 
-            word.resize(len);
-            fin.read((char *)word.data(), len);
+        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
+        if (tokens_keyidx == -1) {
+            fprintf(stderr, "%s: bert tokenizer vocab not found!\n", __func__);
+            return nullptr;
+        }
+
+        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
+        printf("%s: bert tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
+
+        for (int i = 0; i < hparams.n_vocab; i++) {
+            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
 
             if (word[0] == '#' && word[1] == '#')
             {
@@ -564,290 +628,52 @@ struct bert_ctx * bert_load_from_file(const char *fname)
         }
     }
 
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = GGML_TYPE_COUNT;
-    switch (model.hparams.f16)
-    {
-    case 0:
-        wtype = GGML_TYPE_F32;
-        break;
-    case 1:
-        wtype = GGML_TYPE_F16;
-        break;
-    case 2:
-        wtype = GGML_TYPE_Q4_0;
-        break;
-    case 3:
-        wtype = GGML_TYPE_Q4_1;
-        break;
-    default:
-    {
-        fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
-                __func__, fname, model.hparams.f16);
-        bert_free(new_bert);
-        return nullptr;
-    }
-    }
-
     auto &ctx = model.ctx;
 
-    size_t model_mem_req = 0;
-
-    {
-        const auto &hparams = model.hparams;
-
-        const int n_embd = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_max_tokens = hparams.n_max_tokens;
-        const int n_intermediate = hparams.n_intermediate;
-        const int n_vocab = hparams.n_vocab;
-
-        // Calculate size requirements
-
-        model_mem_req += n_embd * n_vocab * ggml_type_sizef(wtype); // word_embeddings
-        model_mem_req += n_embd * 2 * ggml_type_sizef(wtype); // token_type_embeddings
-        model_mem_req += n_embd * n_max_tokens * ggml_type_sizef(wtype); // position_embeddings
-
-        model_mem_req += 2 * n_embd * ggml_type_sizef(GGML_TYPE_F32); // ln_e_*
-
-        model_mem_req += 4 * n_layer * (n_embd * ggml_type_sizef(GGML_TYPE_F32)); // ln_*
-
-        model_mem_req += 4 * n_layer * (n_embd * n_embd * ggml_type_sizef(wtype)); // kqvo weights
-        model_mem_req += 4 * n_layer * (n_embd * ggml_type_sizef(GGML_TYPE_F32)); // kqvo bias
-
-        model_mem_req += 2 * n_layer * (n_embd * n_intermediate * ggml_type_sizef(wtype)); // ff_*_w
-        model_mem_req += n_layer * (n_intermediate * ggml_type_sizef(GGML_TYPE_F32)); // ff_i_b
-        model_mem_req += n_layer * (n_embd * ggml_type_sizef(GGML_TYPE_F32)); // ff_o_b
-
-        model_mem_req += (5 + 16 * n_layer) * ggml_tensor_overhead(); // object overhead
-
 #if defined(DEBUG_BERT)
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, model_mem_req / (1024.0 * 1024.0));
+    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ggml_get_mem_size(ctx) / (1024.0 * 1024.0));
 #endif
-    }
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size = model_mem_req,
-            .mem_buffer = NULL,
-            .no_alloc = false,
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx)
-        {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            bert_free(new_bert);
-            return nullptr;
-        }
-    }
 
     // prepare memory for the weights
     {
-        const auto &hparams = model.hparams;
-
-        const int n_embd = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_intermediate = hparams.n_intermediate;
-        const int n_max_tokens = hparams.n_max_tokens;
-        const int n_vocab = hparams.n_vocab;
-
+        const int n_layer = model.hparams.n_layer;
         model.layers.resize(n_layer);
 
-        model.word_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
-        model.token_type_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, 2);
-        model.position_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_max_tokens);
+        model.word_embeddings       = ggml_get_tensor(ctx, "token_embd.weight");
+        model.token_type_embeddings = ggml_get_tensor(ctx, "token_types.weight");
+        model.position_embeddings   = ggml_get_tensor(ctx, "position_embd.weight");
+        model.ln_e_w                = ggml_get_tensor(ctx, "output_norm.weight");
+        model.ln_e_b                = ggml_get_tensor(ctx, "output_norm.bias");
 
-        model.ln_e_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.ln_e_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-        // map by name
-        model.tensors["embeddings.word_embeddings.weight"] = model.word_embeddings;
-        model.tensors["embeddings.token_type_embeddings.weight"] = model.token_type_embeddings;
-        model.tensors["embeddings.position_embeddings.weight"] = model.position_embeddings;
-
-        model.tensors["embeddings.LayerNorm.weight"] = model.ln_e_w;
-        model.tensors["embeddings.LayerNorm.bias"] = model.ln_e_b;
+        auto name = [](int i, std::string n) {
+            static std::string key;
+            key = "blk." + std::to_string(i) + "." + n;
+            return key.c_str();
+        };
 
         for (int i = 0; i < n_layer; ++i)
         {
             auto &layer = model.layers[i];
 
-            layer.ln_att_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.ln_att_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.ln_out_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.ln_out_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            layer.q_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
-            layer.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.k_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
-            layer.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.v_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
-            layer.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.o_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
-            layer.o_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            layer.ff_i_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_intermediate);
-            layer.ff_i_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_intermediate);
-
-            layer.ff_o_w = ggml_new_tensor_2d(ctx, wtype, n_intermediate, n_embd);
-            layer.ff_o_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            // map by name
-
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.query.weight"] = layer.q_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.query.bias"] = layer.q_b;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.key.weight"] = layer.k_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.key.bias"] = layer.k_b;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.value.weight"] = layer.v_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.value.bias"] = layer.v_b;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.LayerNorm.weight"] = layer.ln_att_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.LayerNorm.bias"] = layer.ln_att_b;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.dense.weight"] = layer.o_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.dense.bias"] = layer.o_b;
-
-            model.tensors["encoder.layer." + std::to_string(i) + ".intermediate.dense.weight"] = layer.ff_i_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".intermediate.dense.bias"] = layer.ff_i_b;
-
-            model.tensors["encoder.layer." + std::to_string(i) + ".output.LayerNorm.weight"] = layer.ln_out_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".output.LayerNorm.bias"] = layer.ln_out_b;
-            model.tensors["encoder.layer." + std::to_string(i) + ".output.dense.weight"] = layer.ff_o_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".output.dense.bias"] = layer.ff_o_b;
+            layer.ln_att_w = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
+            layer.ln_att_b = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
+            layer.ln_out_w = ggml_get_tensor(ctx, name(i, "ffn_norm.weight"));
+            layer.ln_out_b = ggml_get_tensor(ctx, name(i, "ffn_norm.bias"));
+            layer.q_w      = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
+            layer.q_b      = ggml_get_tensor(ctx, name(i, "attn_q.bias"));
+            layer.k_w      = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
+            layer.k_b      = ggml_get_tensor(ctx, name(i, "attn_k.bias"));
+            layer.v_w      = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
+            layer.v_b      = ggml_get_tensor(ctx, name(i, "attn_v.bias"));
+            layer.o_w      = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
+            layer.o_b      = ggml_get_tensor(ctx, name(i, "attn_output.bias"));
+            layer.ff_i_w   = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
+            layer.ff_i_b   = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
+            layer.ff_o_w   = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
+            layer.ff_o_b   = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
         }
     }
 
-    // load weights
-    {
-        int n_tensors = 0;
-#if defined(DEBUG_BERT)
-        size_t total_size = 0;
-#endif
-
-#if defined(DEBUG_BERT)
-        printf("%s: ", __func__);
-#endif
-
-        while (true)
-        {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ftype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ftype), sizeof(ftype));
-
-            if (fin.eof())
-            {
-                break;
-            }
-
-            int64_t nelements = 1;
-            int64_t ne[2] = {1, 1};
-            for (int i = 0; i < n_dims; ++i)
-            {
-                int32_t ne_cur;
-                fin.read(reinterpret_cast<char *>(&ne_cur), sizeof(ne_cur));
-                ne[i] = ne_cur;
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-
-            if (model.tensors.find(name.data()) == model.tensors.end())
-            {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                bert_free(new_bert);
-                return nullptr;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements)
-            {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                bert_free(new_bert);
-                return nullptr;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1])
-            {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%ld, %ld], expected [%ld, %ld]\n",
-                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
-                bert_free(new_bert);
-                return nullptr;
-            }
-
-#if defined(DEBUG_BERT)
-            static const char *ftype_str[] = {
-                "f32",
-                "f16",
-                "q4_0",
-                "q4_1",
-            };
-            printf("%24s - [%5ld, %5ld], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ftype_str[ftype], ggml_nbytes(tensor) / 1024.0 / 1024.0, ggml_nbytes(tensor));
-#endif
-
-            size_t bpe = 0;
-
-            switch (ftype)
-            {
-            case 0:
-                bpe = ggml_type_size(GGML_TYPE_F32);
-                break;
-            case 1:
-                bpe = ggml_type_size(GGML_TYPE_F16);
-                break;
-            case 2:
-                bpe = ggml_type_size(GGML_TYPE_Q4_0);
-                assert(ne[0] % 64 == 0);
-                break;
-            case 3:
-                bpe = ggml_type_size(GGML_TYPE_Q4_1);
-                assert(ne[0] % 64 == 0);
-                break;
-            default:
-            {
-                fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype);
-                bert_free(new_bert);
-                return nullptr;
-            }
-            };
-
-            if ((nelements * bpe) / ggml_blck_size(tensor->type) != ggml_nbytes(tensor))
-            {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %lu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements * bpe);
-                bert_free(new_bert);
-                return nullptr;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-#if defined(DEBUG_BERT)
-            // printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
-            total_size += ggml_nbytes(tensor);
-#endif
-
-            if (++n_tensors % 8 == 0)
-            {
-#if defined(DEBUG_BERT)
-                printf(".");
-                fflush(stdout);
-#endif
-            }
-        }
-
-#if defined(DEBUG_BERT)
-        printf(" done\n");
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size / 1024.0 / 1024.0, n_tensors);
-#endif
-    }
-
-    fin.close();
-
     // Calculate space requirements for setting up context buffers later
     {
         bert_vocab_id tokens[] = {0, 1, 2, 3};
@@ -1019,6 +845,16 @@ const std::vector<LLModel::Token> &Bert::endTokens() const
     return out;
 }
 
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != GGUF_TYPE_STRING) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -1038,13 +874,21 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    if (magic != 0x62657274) {
-         return false;
-    }
-    return true;
+DLL_EXPORT bool magic_match(const char * fname) {
+    struct ggml_context * ctx_meta = NULL;
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ &ctx_meta,
+    };
+    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
+    if (!ctx_gguf)
+        return false;
+
+    bool isValid = gguf_get_version(ctx_gguf) <= 3;
+    isValid = isValid && get_arch_name(ctx_gguf) == "bert";
+
+    gguf_free(ctx_gguf);
+    return isValid;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/falcon.cpp

@@ -1,985 +0,0 @@
-#include "ggml.h"
-#define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "falcon_impl.h"
-#include "llama.h"
-#include "llama-util.h"
-#include "utils.h"
-#include "llmodel_shared.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <iostream>
-#include <sstream>
-
-namespace {
-const char *modelType_ = "Falcon";
-}
-
-// commented out 40B support as it presently would require forking ggml/llama.cpp
-// can re-add once mainline ggml supports it
-
-#define FALCON_MAGIC 0x67676a74
-
-// default hparams (Falcon 7B)
-struct falcon_hparams {
-    int32_t n_vocab = 65024;
-    int32_t n_embd  = 4544;
-    int32_t n_head  = 71;
-    int32_t n_head_kv = 1;
-    int32_t n_layer = 32;
-    int32_t falcon_version = 7; // 7 for Falcon-7B, 40 for Falcon-40B
-    int32_t ftype   = 1;
-    int32_t n_ctx   = 2048;
-};
-
-struct falcon_layer {
-    // normalization
-    struct ggml_tensor* input_layernorm;
-    struct ggml_tensor* input_layernorm_b;
-    //struct ggml_tensor* attention_norm;    // Falcon-40B only
-    //struct ggml_tensor* attention_norm_b;  // Falcon-40B only
-
-    // attention
-    struct ggml_tensor* query_key_value;
-    struct ggml_tensor* wo;
-
-    // ff
-    struct ggml_tensor* ffn_up;
-    struct ggml_tensor* ffn_down;
-};
-
-struct falcon_model {
-    falcon_hparams hparams;
-
-    struct ggml_tensor* tok_embeddings;
-    struct ggml_tensor* output_norm;
-    struct ggml_tensor* output_norm_b;
-    struct ggml_tensor* lm_head;
-
-    std::vector<falcon_layer> layers;
-
-    // key + value memory
-    llm_kv_cache kv_self;
-
-    struct ggml_context* ctx;
-    std::map<std::string, struct ggml_tensor*> tensors;
-
-    llm_buffer eval_buf;
-    llm_buffer work_buf;
-    llm_buffer scr0_buf;
-    llm_buffer scr1_buf;
-};
-
-static bool kv_cache_init(
-        const struct falcon_hparams & hparams,
-              struct llm_kv_cache & cache,
-                         ggml_type   wtype,
-                               int   n_ctx) {
-    const int n_embd  = hparams.n_embd;
-    const int dim_head  = n_embd / hparams.n_head;
-    const int dim_kv = dim_head * hparams.n_head_kv;
-    const int n_layer = hparams.n_layer;
-
-    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
-    const int64_t n_elements = dim_kv * n_mem;
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
-    struct ggml_init_params params;
-    params.mem_size   = cache.buf.size;
-    params.mem_buffer = cache.buf.addr;
-    params.no_alloc   = false;
-
-    cache.ctx = ggml_init(params);
-    if (!cache.ctx) {
-        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
-        return false;
-    }
-
-    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    return true;
-}
-
-// load the model's weights from a file
-bool falcon_model_load(const std::string & fname, falcon_model & model, gpt_vocab & vocab, size_t *mem_req) {
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
-    if (mem_req) {
-        *mem_req = 0;
-    }
-
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin) {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
-        return false;
-    }
-
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *) &magic, sizeof(magic));
-        if (magic != FALCON_MAGIC) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
-            return false;
-        }
-    }
-
-    uint32_t format_version;
-    fin.read((char *) &format_version, sizeof(format_version));
-
-    // load hparams
-    {
-        auto & hparams = model.hparams;
-
-        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
-        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
-        fin.read((char *) &hparams.n_head_kv, sizeof(hparams.n_head_kv));
-        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *) &hparams.falcon_version, sizeof(hparams.falcon_version));
-        fin.read((char *) &hparams.ftype,   sizeof(hparams.ftype));
-
-        if (hparams.falcon_version != 7) { // && hparams.falcon_version != 40) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad Falcon version: %d)\n", __func__, fname.c_str(), hparams.falcon_version);
-            return false;
-        }
-
-        const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
-
-        printf("%s: n_vocab   = %d\n", __func__, hparams.n_vocab);
-        printf("%s: n_embd    = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_head    = %d\n", __func__, hparams.n_head);
-        printf("%s: n_head_kv = %d\n", __func__, hparams.n_head_kv);
-        printf("%s: n_layer   = %d\n", __func__, hparams.n_layer);
-        printf("%s: ftype     = %d\n", __func__, hparams.ftype);
-        printf("%s: qntvr     = %d\n", __func__, qntvr);
-
-        hparams.ftype %= GGML_QNT_VERSION_FACTOR;
-    }
-
-    // load vocab
-    {
-        const int32_t n_vocab = model.hparams.n_vocab;
-
-        std::string word;
-        std::vector<char> buf(128);
-
-        for (int i = 0; i < n_vocab; i++) {
-            uint32_t len;
-            fin.read((char *) &len, sizeof(len));
-
-            buf.resize(len);
-            fin.read((char *) buf.data(), len);
-            word.assign(buf.data(), len);
-
-            uint32_t dummy;
-            fin.read((char *) &dummy, sizeof(dummy));
-
-            vocab.token_to_id[word] = i;
-            vocab.id_to_token[i] = word;
-        }
-    }
-
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
-    if (wtype == GGML_TYPE_COUNT) {
-        fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
-                __func__, fname.c_str(), model.hparams.ftype);
-        return false;
-    }
-
-    auto & ctx = model.ctx;
-
-    size_t ctx_size = 0;
-
-    {
-        const auto& hparams = model.hparams;
-
-        const int n_embd = hparams.n_embd;
-        const int n_head = hparams.n_head;
-        const int n_head_kv = hparams.n_head_kv;
-        const int n_layer = hparams.n_layer;
-        const int n_ctx = hparams.n_ctx;
-        const int n_ff = 4 * model.hparams.n_embd;
-        const int n_vocab = hparams.n_vocab;
-        const int head_dim = hparams.n_embd / hparams.n_head;
-
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // tok_embeddings
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm_b
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // lm_head
-
-        // if (hparams.version == 40) { // Falcon-40B
-        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm
-        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm_b
-        // }
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm_b
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * (n_head_kv * 2 + n_head) * head_dim);  // query_key_value
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_embd);  // wo
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_ff);  // ffn_up
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_ff * n_embd);  // ffn_down
- 
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
-    }
-
-    if (mem_req) {
-        const int n_embd  = model.hparams.n_embd;
-        const int dim_head  = n_embd / model.hparams.n_head;
-        const int dim_kv = dim_head * model.hparams.n_head_kv;
-        const int n_layer = model.hparams.n_layer;
-
-        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
-        const int64_t n_elements = dim_kv * n_mem;
-        size_t kv_cache_size = 2u*n_elements*ggml_type_size(wtype) + 2_MiB;
-        *mem_req = ctx_size + kv_cache_size;
-        return false;
-    }
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size   = ctx_size,
-            .mem_buffer = NULL,
-            .no_alloc   = false,
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx) {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            return false;
-        }
-    }
-
-    // prepare memory for the weights
-    {
-        const auto& hparams = model.hparams;
-
-        const int n_embd = hparams.n_embd;
-        const int n_head = hparams.n_head;
-        const int n_head_kv = hparams.n_head_kv;
-        const int n_layer = hparams.n_layer;
-        const int n_ff = 4 * model.hparams.n_embd;
-        const int n_vocab = hparams.n_vocab;
-        const int head_dim = hparams.n_embd / hparams.n_head;
-
-        model.layers.resize(n_layer);
-
-        model.tok_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
-
-        model.output_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.output_norm_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
-
-        // map by name
-        model.tensors["transformer.word_embeddings.weight"] =
-            model.tok_embeddings;
-
-        model.tensors["transformer.ln_f.weight"] = model.output_norm;
-        model.tensors["transformer.ln_f.bias"] = model.output_norm_b;
-        model.tensors["lm_head.weight"] = model.lm_head;
-
-        for (int i = 0; i < n_layer; ++i) {
-            auto& layer = model.layers[i];
-
-            layer.input_layernorm =
-                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.input_layernorm_b =
-                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            // if (hparams.version == 40) { // for Falcon-40B only
-            //     layer.attention_norm =
-            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            //     layer.attention_norm_b =
-            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            // }
-
-            // query_key_value shape for config.multi_query == True:
-            layer.query_key_value = ggml_new_tensor_2d(
-                ctx, wtype, n_embd, (n_head_kv * 2 + n_head) * head_dim);
-            layer.wo = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
-
-            layer.ffn_up = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);
-            layer.ffn_down = ggml_new_tensor_2d(ctx, wtype, n_ff, n_embd);
-
-            // map by name
-            // if (hparams.version == 40) {
-            //     // Falcon-40B:
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_mlp.weight"] = layer.input_layernorm;
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_mlp.bias"] = layer.input_layernorm_b;
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_attn.weight"] = layer.attention_norm;
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_attn.bias"] = layer.attention_norm_b;
-            // } else {
-            // Falcon-7B:
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".input_layernorm.weight"] = layer.input_layernorm;
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".input_layernorm.bias"] = layer.input_layernorm_b;
-            //}
-
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".self_attention.query_key_value.weight"] =
-                layer.query_key_value;
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".self_attention.dense.weight"] = layer.wo;
-
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".mlp.dense_h_to_4h.weight"] = layer.ffn_up;
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".mlp.dense_4h_to_h.weight"] = layer.ffn_down;
-        }
-    }
-
-    // key + value memory
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_layer = hparams.n_layer;
-        const int n_ctx   = hparams.n_ctx;
-        const int n_head_kv = hparams.n_head_kv;
-        const int head_dim = hparams.n_embd / hparams.n_head;
-
-        const int64_t n_mem      = n_layer*n_ctx;
-        const int64_t n_elements = head_dim*n_mem;
-
-        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F32, model.hparams.n_ctx)) {
-            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
-            ggml_free(ctx);
-            return false;
-        }
-        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
-
-        printf("%s: memory_size = %8.2f MB, n_mem = %" PRId64 "\n", __func__, memory_size/1024.0/1024.0, n_mem);
-    }
-
-    // load weights
-    {
-        int n_tensors = 0;
-        size_t total_size = 0;
-
-        printf("%s: ", __func__);
-
-        while (true) {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ttype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
-
-            if (fin.eof()) {
-                break;
-            }
-
-            int32_t nelements = 1;
-            int32_t ne[2] = { 1, 1 };
-            for (int i = 0; i < n_dims; ++i) {
-                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-            fin.seekg(-static_cast<ptrdiff_t>(fin.tellg()) & 31, std::ios_base::cur);
-
-            if (model.tensors.find(name.data()) == model.tensors.end()) {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                return false;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                return false;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%5d, %5d], expected [%5d, %5d]\n",
-                        __func__, name.data(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
-                return false;
-            }
-
-            // for debugging
-            if (0) {
-                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
-            }
-
-            const size_t bpe = ggml_type_size(ggml_type(ttype));
-
-            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
-                return false;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-            total_size += ggml_nbytes(tensor);
-            if (++n_tensors % 8 == 0) {
-                printf(".");
-                fflush(stdout);
-            }
-        }
-
-        printf(" done\n");
-
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
-    }
-
-    fin.close();
-
-    model.eval_buf.resize(1280u * 1024 * 1024);
-    model.scr0_buf.resize(256u * 1024 * 1024);
-    model.scr1_buf.resize(256u * 1024 * 1024);
-    return true;
-}
-
-
-// evaluate the transformer
-//
-//   - model:     the model
-//   - n_threads: number of threads to use
-//   - n_past:    the context size so far
-//   - embd_inp:  the embeddings of the tokens in the context
-//   - embd_w:    the predicted logits for the next token
-//
-bool falcon_eval(
-        falcon_model & model,
-        const int n_threads,
-        const int n_past,
-        const std::vector<gpt_vocab::id> & embd_inp,
-              std::vector<float>         & embd_w,
-              size_t                     & mem_per_token) {
-    const int N = embd_inp.size();
-
-    const auto & hparams = model.hparams;
-
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_ctx   = hparams.n_ctx;
-    const int n_head  = hparams.n_head;
-    const int n_head_kv = hparams.n_head_kv;
-    const int n_vocab = hparams.n_vocab;
-    const int version = hparams.falcon_version;
-    const size_t head_dim = n_embd / n_head;
-
-   struct ggml_init_params eval_ctx_params = {
-        .mem_size = model.eval_buf.size,
-        .mem_buffer = model.eval_buf.addr,
-        .no_alloc = false,
-    };
-
-    struct ggml_context * ctx0 = ggml_init(eval_ctx_params);
-    struct ggml_cgraph gf = {};
-
-    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
-
-    // wte
-    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.tok_embeddings, embd);
-    struct ggml_tensor* repeat_dummy = ggml_new_tensor_3d(ctx0, inpL->type, head_dim, N + n_past, n_head);
-
-    ggml_type wtype = GGML_TYPE_F32;
-    const int sizeof_wtype = ggml_type_sizef(wtype);
-
-    for (int il = 0; il < n_layer; ++il) {
-        struct ggml_tensor * cur;
-        struct ggml_tensor * layernorm_output;
-
-        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-        // self-attention
-        {
-            layernorm_output = ggml_norm(ctx0, inpL);
-
-            layernorm_output = ggml_add(ctx0,
-                    ggml_mul(ctx0,
-                        ggml_repeat(ctx0, model.layers[il].input_layernorm, layernorm_output),
-                        layernorm_output),
-                    ggml_repeat(ctx0, model.layers[il].input_layernorm_b, layernorm_output));
-
-            // if (version == 40) { // Falcon-40B only
-            //     cur = ggml_norm(ctx0, inpL);
-
-            //     cur = ggml_add(ctx0,
-            //             ggml_mul(ctx0,
-            //                 ggml_repeat(ctx0, model.layers[il].attention_norm, cur),
-            //                 cur),
-            //             ggml_repeat(ctx0, model.layers[il].attention_norm_b, cur));
-            // }
-            // else {
-            cur = layernorm_output;
-            // }
-
-            // compute QKV
-
-            cur = ggml_mul_mat(ctx0, model.layers[il].query_key_value, cur);
-
-            // Note that the strides for Kcur, Vcur are set up so that the
-            // resulting views are misaligned with the tensor's storage
-            // (by applying the K/V offset we shift the tensor's original
-            // view to stick out behind the viewed QKV tensor's allocated
-            // memory, so to say). This is ok because no actual accesses
-            // happen to that out-of-range memory, but it can require some
-            // trickery when trying to accurately dump these views for
-            // debugging.
-
-            struct ggml_tensor * Qcur = ggml_view_3d(
-                ctx0, cur, head_dim, n_head, N,
-                head_dim * sizeof_wtype,
-                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
-                0);
-
-            struct ggml_tensor * Kcur = ggml_view_3d(
-                ctx0, cur, head_dim, n_head_kv, N,
-                head_dim * sizeof_wtype,
-                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
-                head_dim * n_head * sizeof_wtype);
-
-            struct ggml_tensor * Vcur = ggml_view_3d(
-                ctx0, cur, head_dim, n_head_kv, N,
-                head_dim * sizeof_wtype,
-                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
-                head_dim * (n_head + n_head_kv) * sizeof_wtype);
-
-            // using mode = 2 for neox mode
-            Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, head_dim, 2, n_ctx);
-            Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, head_dim, 2, n_ctx);
-
-            // store key and value to memory
-            {
-                struct ggml_tensor* k = ggml_view_1d(
-                    ctx0, model.kv_self.k, N * n_head_kv * head_dim,
-                    (ggml_element_size(model.kv_self.k) * n_head_kv * head_dim) *
-                        (il * n_ctx + n_past));
-                struct ggml_tensor* v = ggml_view_1d(
-                    ctx0, model.kv_self.v, N * n_head_kv * head_dim,
-                    (ggml_element_size(model.kv_self.v) * n_head_kv * head_dim) *
-                        (il * n_ctx + n_past));
-
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
-            }
-
-            struct ggml_tensor * K = ggml_permute(
-                ctx0,
-                ggml_view_3d(
-                    ctx0,
-                    model.kv_self.k,
-                    head_dim, n_head_kv, n_past + N,
-                    head_dim * sizeof_wtype,
-                    head_dim * n_head_kv * sizeof_wtype,
-                    il * n_ctx * ggml_element_size(model.kv_self.k) * n_head_kv * head_dim),
-                0, 2, 1, 3);
-
-            // K * Q
-
-            // changed from repeat2 back to repeat, will not support 40B!
-            K = ggml_cont(ctx0, ggml_repeat(ctx0, K, repeat_dummy));
-
-            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
-            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
-
-            // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled =
-                ggml_scale_inplace(ctx0,
-                        KQ,
-                        ggml_new_f32(ctx0, 1.0f/sqrt(float(head_dim)))
-                        );
-
-            // KQ_masked = mask_past(KQ_scaled)
-            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
-
-            // KQ = soft_max(KQ_masked)
-            struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
-
-            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor* V = ggml_permute(
-                ctx0,
-                ggml_view_3d(
-                    ctx0,
-                    model.kv_self.v,
-                    head_dim, n_head_kv, n_past + N,
-                    head_dim * sizeof_wtype,
-                    head_dim * n_head_kv * sizeof_wtype,
-                    il * n_ctx * ggml_element_size(model.kv_self.v) * n_head_kv * head_dim),
-                0, 2, 1, 3);
-
-            // changed from repeat2 back to repeat, will not support 40B!
-            V = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_repeat(ctx0, V, repeat_dummy)));
-
-            // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
-
-            // KQV_merged = KQV.permute(0, 2, 1, 3)
-            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            // cur = KQV_merged.contiguous().view(n_embd, N)
-            cur = ggml_cpy(ctx0,
-                    KQV_merged,
-                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-            // projection
-            {
-                cur = ggml_mul_mat(ctx0,
-                        model.layers[il].wo,
-                        cur);
-            }
-        }
-
-        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
-
-        struct ggml_tensor* inpFF = layernorm_output;
-        struct ggml_tensor* attn_out = ggml_cpy(
-            ctx0, cur, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-        {
-            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_up, inpFF);
-            cur = ggml_gelu(ctx0, cur);
-            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_down, cur);
-        }
-
-        cur = ggml_add(ctx0, cur, attn_out);
-        cur = ggml_add(ctx0, cur, inpL);
-        // input for next layer
-        inpL = cur;
-    }
-
-    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-    // norm
-    {
-        inpL = ggml_norm(ctx0, inpL);
-
-        // inpL = ln_f_g*inpL + ln_f_b
-        inpL = ggml_add(ctx0,
-                ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.output_norm, inpL),
-                    inpL),
-                ggml_repeat(ctx0, model.output_norm_b, inpL));
-    }
-
-    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
-
-    // lm_head
-    {
-        inpL = ggml_mul_mat(ctx0, model.lm_head, inpL);
-
-        //inpL = ggml_add(ctx0,
-        //        ggml_repeat(ctx0, model.lmh_b, inpL),
-        //        inpL);
-    }
-
-    // logits -> probs
-    //inpL = ggml_soft_max_inplace(ctx0, inpL);
-
-    // run the computation
-    ggml_build_forward_expand(&gf, inpL);
-    ggml_graph_compute_g4a(model.work_buf, &gf, n_threads);
-  
-
-    //if (n_past%100 == 0) {
-    //    ggml_graph_print   (&gf);
-    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
-    //}
-
-    //embd_w.resize(n_vocab*N);
-    //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
-
-    // return result for just the last token
-    embd_w.resize(n_vocab);
-    memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
-
-    if (mem_per_token == 0) {
-        mem_per_token = ggml_used_mem(ctx0)/N;
-    }
-    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
-
-    ggml_free(ctx0);
-
-    return true;
-}
-
-
-#define MAX_RNG_STATE 64*1024
-size_t falcon_get_state_size(const falcon_model &model) {
-    const size_t s_rng_size        = sizeof(size_t);
-    const size_t s_rng             = MAX_RNG_STATE;
-    const size_t s_kv_size         = sizeof(size_t);
-    const size_t s_kv_ntok         = sizeof(int);
-    const size_t s_kv              = model.kv_self.buf.size;
-    const size_t s_total = (
-        + s_rng_size
-        + s_rng
-        + s_kv_size
-        + s_kv_ntok
-        + s_kv
-    );
-    return s_total;
-}
-
-size_t falcon_copy_state_data(const falcon_model &model, const std::mt19937 &rng, uint8_t *dest)
-{
-    uint8_t * out = dest;
-    // copy rng
-    {
-        std::stringstream rng_ss;
-        rng_ss << rng;
-
-        const size_t rng_size = rng_ss.str().size();
-        char rng_buf[MAX_RNG_STATE];
-
-        memset(&rng_buf[0], 0, MAX_RNG_STATE);
-        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
-
-        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
-        memcpy(out, &rng_buf[0], MAX_RNG_STATE); out += MAX_RNG_STATE;
-    }
-
-    // copy kv cache
-    {
-        const size_t kv_size = model.kv_self.buf.size;
-        const int    kv_ntok = model.kv_self.n;
-
-        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
-        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
-
-        if (kv_size) {
-            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
-        }
-    }
-
-    const size_t written  = out - dest;
-    assert(written == falcon_get_state_size(model));
-    fflush(stdout);
-    return written;
-}
-
-size_t falcon_set_state_data(falcon_model *model, std::mt19937 *rng, const uint8_t *src)
-{
-    const uint8_t * in = src;
-
-    // set rng
-    {
-        size_t rng_size;
-        char   rng_buf[MAX_RNG_STATE];
-
-        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
-        memcpy(&rng_buf[0], in, MAX_RNG_STATE); in += MAX_RNG_STATE;
-
-        std::stringstream rng_ss;
-        rng_ss.str(std::string(&rng_buf[0], rng_size));
-        rng_ss >> *rng;
-
-        assert(rng_ss.fail() == false);
-    }
-
-    // set kv cache
-    {
-        size_t kv_size;
-        int kv_ntok;
-
-        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
-        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
-
-        if (kv_size) {
-            assert(model->kv_self.buf.size == kv_size);
-
-            void * k_data = model->kv_self.k->data; // remember data pointers
-            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
-
-            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
-
-            model->kv_self.k->data = k_data; // restore correct data pointers
-            model->kv_self.v->data = v_data;
-
-        }
-
-        model->kv_self.n = kv_ntok;
-    }
-
-    const size_t nread    = in - src;
-    assert(nread == falcon_get_state_size(*model));
-    fflush(stdout);
-    return nread;
-}
-
-struct FalconPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    gpt_vocab vocab;
-    falcon_model *model = nullptr;
-    int64_t n_threads = 0;
-    size_t mem_per_token = 0;
-    std::mt19937 rng;
-};
-
-Falcon::Falcon() : d_ptr(new FalconPrivate) {
-    d_ptr->model = new falcon_model;
-    d_ptr->model->ctx = nullptr;
-    d_ptr->modelLoaded = false;
-}
-
-Falcon::~Falcon() {
-    if(d_ptr->model->ctx) {
-        ggml_free(d_ptr->model->ctx);
-        d_ptr->model->ctx = nullptr;
-    }
-    delete d_ptr->model;
-}
-
-bool Falcon::loadModel(const std::string &modelPath)
-{
-    std::mt19937 rng(time(NULL));
-    d_ptr->rng = rng;
-
-    // load the model
-    if (!falcon_model_load(modelPath, *d_ptr->model, d_ptr->vocab, nullptr)) {
-        std::cerr << "FALCON ERROR: failed to load model from " <<  modelPath;
-        return false;
-    }
-
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    fflush(stdout);
-    return true;
-}
-
-bool Falcon::isModelLoaded() const
-{
-    return d_ptr -> modelLoaded;
-}
-
-size_t Falcon::requiredMem(const std::string &modelPath)
-{
-    falcon_model dummy_model;
-    gpt_vocab dummy_vocab;
-    size_t mem_req;
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-    falcon_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
-    return mem_req;
-}
-
-size_t Falcon::stateSize() const
-{
-    return falcon_get_state_size(*d_ptr->model);
-}
-
-size_t Falcon::saveState(uint8_t *dest) const
-{
-    return falcon_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
-}
-
-size_t Falcon::restoreState(const uint8_t *src)
-{
-    return falcon_set_state_data(d_ptr->model, &d_ptr->rng, src);
-}
-
-void Falcon::setThreadCount(int32_t n_threads)
-{
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t Falcon::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-std::vector<LLModel::Token> Falcon::tokenize(PromptContext &, const std::string &str) const
-{
-    return ::gpt_tokenize(d_ptr->vocab, str);
-}
-
-LLModel::Token Falcon::sampleToken(PromptContext &promptCtx) const
-{
-    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
-    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
-        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks,
-        promptCtx.logits,
-        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty,
-        d_ptr->rng);
-}
-
-std::string Falcon::tokenToString(Token id) const
-{
-    return d_ptr->vocab.id_to_token[id];
-}
-
-bool Falcon::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    // determine the required inference memory per token:
-    static bool initialized = false;
-    if (!initialized) {
-        falcon_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
-            d_ptr->mem_per_token);
-        initialized = true;
-    }
-
-    return falcon_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
-}
-
-int32_t Falcon::contextLength() const
-{
-    return d_ptr->model->hparams.n_ctx;
-}
-
-const std::vector<LLModel::Token> &Falcon::endTokens() const
-{
-    static const std::vector<LLModel::Token> out = { 11 };
-    return out;
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    uint32_t version = 0;
-    f.read(reinterpret_cast<char*>(&version), sizeof(version));
-    if (magic != FALCON_MAGIC) {
-         return false; 
-    }
-    falcon_hparams hparams;
-    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
-    // we're matching the file format of existing pre-converted models
-    // compatible with ctransformers llama.cpp based format, which also
-    // unfortunately shares its magic number what llama uses, so we now
-    // differentiate by n_vocab
-    // give some wiggle room over the max to allow for finetunes that expand the
-    // vocabulary
-    if (!(hparams.n_vocab >= 65024 && hparams.n_vocab <= 65100)) {
-        return false;
-    }
-    if (hparams.falcon_version != 7) {
-        return false;
-    }
-    return true;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new Falcon;
-}
-}

gpt4all-backend/falcon_impl.h

@@ -1,42 +0,0 @@
-#ifndef FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of falcon.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef FALCON_H
-#define FALCON_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include <memory>
-#include "llmodel.h"
-
-struct FalconPrivate;
-class Falcon : public LLModel {
-public:
-    Falcon();
-    ~Falcon();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    std::unique_ptr<FalconPrivate> d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // Falcon_H

gpt4all-backend/gptj.cpp

@@ -9,7 +9,6 @@
 #include <cmath>
 #include <cstdio>
 #include <cstring>
-#include <fstream>
 #include <map>
 #include <string>
 #include <vector>
@@ -42,7 +41,7 @@ struct gptj_hparams {
     int32_t n_head  = 16;
     int32_t n_layer = 28;
     int32_t n_rot   = 64;
-    int32_t f16     = 1;
+    float norm_eps  = 1e-5;
 };
 
 struct gptj_layer {
@@ -128,216 +127,149 @@ static bool kv_cache_init(
     return true;
 }
 
-// load the model's weights from a stream
-bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
+// load the model's weights from a file path
+bool gptj_model_load(const std::string &fname, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
     if(mem_req != nullptr) {
         *mem_req = 0;
     }
 
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *) &magic, sizeof(magic));
-        if (magic != 0x67676d6c) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
-            return false;
-        }
+    // create the ggml context
+    struct gguf_init_params params = {
+        /*.no_alloc = */ false,
+        /*.ctx      = */ &model.ctx,
+    };
+
+    gguf_context *ggufctx = gguf_init_from_file(fname.c_str(), params);
+    if (!ggufctx) {
+        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
+        return false;
     }
 
     // load hparams
     {
         auto & hparams = model.hparams;
 
-        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
-        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
-        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
-        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *) &hparams.n_rot,   sizeof(hparams.n_rot));
-        fin.read((char *) &hparams.f16,     sizeof(hparams.f16));
+        bool ok = false;
+        int keyidx;
+
+        do {
+            keyidx = gguf_find_key(ggufctx, "gptj.context_length");
+            if (keyidx == -1) { break; }
+            hparams.n_ctx = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.embedding_length");
+            if (keyidx == -1) { break; }
+            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.attention.head_count");
+            if (keyidx == -1) { break; }
+            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.block_count");
+            if (keyidx == -1) { break; }
+            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.rope.dimension_count");
+            if (keyidx == -1) { break; }
+            hparams.n_rot = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.attention.layer_norm_epsilon");
+            if (keyidx == -1) { break; }
+            hparams.norm_eps = gguf_get_val_f32(ggufctx, keyidx);
+
+            ok = true;
+        } while (false);
+
+        if (!ok) {
+            fprintf(stderr, "%s: required hparam missing!\n", __func__);
+            return false;
+        }
 
-        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
         printf("%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
         printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
         printf("%s: n_head  = %d\n", __func__, hparams.n_head);
         printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
         printf("%s: n_rot   = %d\n", __func__, hparams.n_rot);
-        printf("%s: f16     = %d\n", __func__, hparams.f16);
     }
 
     // load vocab
     {
-        int32_t n_vocab = 0;
-        fin.read((char *) &n_vocab, sizeof(n_vocab));
+        auto & hparams = model.hparams;
 
-        if (n_vocab != model.hparams.n_vocab) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
-                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
+        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
+        if (keyidx == -1) {
+            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
+            return false;
+        }
+        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "gpt2") != 0) {
+            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
             return false;
         }
 
-        std::string word;
-        for (int i = 0; i < n_vocab; i++) {
-            uint32_t len;
-            fin.read((char *) &len, sizeof(len));
+        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
+        if (tokens_keyidx == -1) {
+            fprintf(stderr, "%s: gpt2 tokenizer vocab not found!\n", __func__);
+            return false;
+        }
 
-            word.resize(len);
-            fin.read((char *) word.data(), len);
+        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
+        printf("%s: gpt2 tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
 
+        for (int i = 0; i < hparams.n_vocab; i++) {
+            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
             vocab.token_to_id[word] = i;
             vocab.id_to_token[i] = word;
         }
     }
 
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = GGML_TYPE_COUNT;
-    switch (model.hparams.f16) {
-        case 0: wtype = GGML_TYPE_F32;  break;
-        case 1: wtype = GGML_TYPE_F16;  break;
-        case 2: wtype = GGML_TYPE_Q4_0; break;
-        case 3: wtype = GGML_TYPE_Q4_1; break;
-        case 5: wtype = GGML_TYPE_Q4_2; break;
-        default:
-                {
-                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
-                            __func__, fname.c_str(), model.hparams.f16);
-                    return false;
-                }
-    }
-
     auto & ctx = model.ctx;
 
-    size_t ctx_size = 0;
-
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_ctx   = hparams.n_ctx;
-        const int n_vocab = hparams.n_vocab;
-
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_g
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_b
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype); // wte
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype);         // lmh_g
-        ctx_size +=        n_vocab*ggml_type_sizef(GGML_TYPE_F32); // lmh_b
-
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_g
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_b
-
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_q_proj_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_k_proj_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_v_proj_w
-
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_proj_w
-
-        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_fc_w
-        ctx_size += n_layer*(       4*n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_fc_b
-
-        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_proj_w
-        ctx_size += n_layer*(         n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_proj_b
-
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_k
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_v
-
-        ctx_size += (5 + 10*n_layer)*256; // object overhead
-
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
-    }
+    size_t ctx_size = ggml_get_mem_size(ctx);
+    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size / (1024.0 * 1024.0));
 
     if (mem_req != nullptr) {
-        *mem_req += ctx_size;
-        const int n_embd  = model.hparams.n_embd;
-        const int n_layer = model.hparams.n_layer;
-
-        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
-        const int64_t n_elements = n_embd*n_mem;
-
-        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+        *mem_req = ctx_size;
+        gguf_free(ggufctx);
         return false;
     }
 
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size   = ctx_size,
-            .mem_buffer = NULL,
-            .no_alloc = false
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx) {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            return false;
-        }
-    }
-
     // prepare memory for the weights
     {
         const auto & hparams = model.hparams;
+        model.layers.resize(hparams.n_layer);
 
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_vocab = hparams.n_vocab;
+        model.wte    = ggml_get_tensor(ctx, "token_embd.weight");
 
-        model.layers.resize(n_layer);
+        model.ln_f_g = ggml_get_tensor(ctx, "output_norm.weight");
+        model.ln_f_b = ggml_get_tensor(ctx, "output_norm.bias");
 
-        model.wte    = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
+        model.lmh_g  = ggml_get_tensor(ctx, "output.weight");
+        model.lmh_b  = ggml_get_tensor(ctx, "output.bias");
 
-        model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        auto name = [](int i, std::string n) {
+            static std::string key;
+            key = "blk." + std::to_string(i) + "." + n;
+            return key.c_str();
+        };
 
-        model.lmh_g  = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
-        model.lmh_b  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab);
-
-        // map by name
-        model.tensors["transformer.wte.weight"] = model.wte;
-
-        model.tensors["transformer.ln_f.weight"] = model.ln_f_g;
-        model.tensors["transformer.ln_f.bias"]   = model.ln_f_b;
-
-        model.tensors["lm_head.weight"] = model.lmh_g;
-        model.tensors["lm_head.bias"]   = model.lmh_b;
-
-        for (int i = 0; i < n_layer; ++i) {
+        for (int i = 0; i < hparams.n_layer; ++i) {
             auto & layer = model.layers[i];
 
-            layer.ln_1_g          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
-            layer.ln_1_b          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
+            layer.ln_1_g          = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
+            layer.ln_1_b          = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
 
-            layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
-            layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
-            layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_q_proj_w = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
+            layer.c_attn_k_proj_w = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
+            layer.c_attn_v_proj_w = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
 
-            layer.c_attn_proj_w   = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_proj_w   = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
 
-            layer.c_mlp_fc_w      = ggml_new_tensor_2d(ctx, wtype,           n_embd, 4*n_embd);
-            layer.c_mlp_fc_b      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
+            layer.c_mlp_fc_w      = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
+            layer.c_mlp_fc_b      = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
 
-            layer.c_mlp_proj_w    = ggml_new_tensor_2d(ctx, wtype,         4*n_embd,   n_embd);
-            layer.c_mlp_proj_b    = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
-
-            // map by name
-            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.weight"]          = layer.ln_1_g;
-            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.bias"]            = layer.ln_1_b;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.q_proj.weight"]   = layer.c_attn_q_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.k_proj.weight"]   = layer.c_attn_k_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.v_proj.weight"]   = layer.c_attn_v_proj_w;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.out_proj.weight"] = layer.c_attn_proj_w;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.weight"]     = layer.c_mlp_fc_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.bias"]       = layer.c_mlp_fc_b;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.weight"]    = layer.c_mlp_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.bias"]      = layer.c_mlp_proj_b;
+            layer.c_mlp_proj_w    = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
+            layer.c_mlp_proj_b    = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
         }
     }
 
@@ -354,113 +286,12 @@ bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & m
         printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
     }
 
-    // load weights
-    {
-        int n_tensors = 0;
-        size_t total_size = 0;
-
-        printf("%s: ", __func__);
-
-        while (true) {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ftype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ftype),  sizeof(ftype));
-
-            if (fin.eof()) {
-                break;
-            }
-
-            int32_t nelements = 1;
-            int32_t ne[2] = { 1, 1 };
-            for (int i = 0; i < n_dims; ++i) {
-                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-
-            if (model.tensors.find(name.data()) == model.tensors.end()) {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                return false;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                return false;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
-                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
-                return false;
-            }
-
-            if (0) {
-                static const char * ftype_str[] = { "f32", "f16", "q4_0", "q4_1", };
-                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ftype_str[ftype], ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
-            }
-
-            size_t bpe = 0;
-
-            switch (ftype) {
-                case 0: bpe = ggml_type_size(GGML_TYPE_F32);  break;
-                case 1: bpe = ggml_type_size(GGML_TYPE_F16);  break;
-                case 2: bpe = ggml_type_size(GGML_TYPE_Q4_0); assert(ne[0] % 64 == 0); break;
-                case 3: bpe = ggml_type_size(GGML_TYPE_Q4_1); assert(ne[0] % 64 == 0); break;
-                default:
-                        {
-                            fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype);
-                            return false;
-                        }
-            };
-
-            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
-                return false;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
-            total_size += ggml_nbytes(tensor);
-            if (++n_tensors % 8 == 0) {
-                printf(".");
-                fflush(stdout);
-            }
-        }
-
-        printf(" done\n");
-
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
-    }
-
     model.scr0_buf.resize(256u * 1024 * 1024);
     model.scr1_buf.resize(256u * 1024 * 1024);
 
     return true;
 }
 
-// load the model's weights from a file path
-bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) {
-
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin) {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
-        return false;
-    }
-
-    bool loaded = gptj_model_load(fname, fin, model, vocab);
-    fin.close();
-    return loaded;
-}
-
 // evaluate the transformer
 //
 //   - model:     the model
@@ -513,7 +344,6 @@ bool gptj_eval(
 
     struct ggml_context * ctx0 = ggml_init(params);
     struct ggml_cgraph gf = {};
-    gf.n_threads = n_threads;
 
     struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
     memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -526,7 +356,7 @@ bool gptj_eval(
         ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
         // norm
         {
-            cur = ggml_norm(ctx0, inpL);
+            cur = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
 
             // cur = ln_1_g*cur + ln_1_b
             cur = ggml_add(ctx0,
@@ -540,37 +370,31 @@ bool gptj_eval(
 
         // self-attention
         {
-            struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur);
-            struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur);
-            struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur);
+            struct ggml_tensor * Qcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
+            struct ggml_tensor * Kcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
 
             // store key and value to memory
             {
+                struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur));
+
                 struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_1d(ctx0, model.kv_self.v, N*n_embd, (ggml_element_size(model.kv_self.v)*n_embd)*(il*n_ctx + n_past));
+                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
+                        (   n_ctx)*ggml_element_size(model.kv_self.v),
+                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
 
                 ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
                 ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
             }
 
             // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q =
-                ggml_permute(ctx0,
-                        ggml_rope(ctx0,
-                            ggml_cpy(ctx0,
-                                Qcur,
-                                ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),
-                            n_past, n_rot, 0),
-                        0, 2, 1, 3);
+            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
 
             // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
             struct ggml_tensor * K =
                 ggml_permute(ctx0,
-                        ggml_rope(ctx0,
-                            ggml_reshape_3d(ctx0,
-                                ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                                n_embd/n_head, n_head, n_past + N),
-                            n_past, n_rot, 1),
+                        ggml_reshape_3d(ctx0,
+                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
+                            n_embd/n_head, n_head, n_past + N),
                         0, 2, 1, 3);
 
             // K * Q
@@ -590,17 +414,15 @@ bool gptj_eval(
             struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
 
             // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V_trans =
-                ggml_cpy(ctx0,
-                        ggml_permute(ctx0,
-                            ggml_reshape_3d(ctx0,
-                                ggml_view_1d(ctx0, model.kv_self.v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd),
-                                n_embd/n_head, n_head, n_past + N),
-                            1, 2, 0, 3),
-                        ggml_new_tensor_3d(ctx0, model.kv_self.v->type, n_past + N, n_embd/n_head, n_head));
+            struct ggml_tensor * V =
+                ggml_view_3d(ctx0, model.kv_self.v,
+                        n_past + N, n_embd/n_head, n_head,
+                        n_ctx*ggml_element_size(model.kv_self.v),
+                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
+                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
 
             // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
 
             // KQV_merged = KQV.permute(0, 2, 1, 3)
             struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
@@ -656,7 +478,7 @@ bool gptj_eval(
 
     // norm
     {
-        inpL = ggml_norm(ctx0, inpL);
+        inpL = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
 
         // inpL = ln_f_g*inpL + ln_f_b
         inpL = ggml_add(ctx0,
@@ -680,9 +502,18 @@ bool gptj_eval(
     // logits -> probs
     //inpL = ggml_soft_max(ctx0, inpL);
 
-    // run the computation
     ggml_build_forward_expand(&gf, inpL);
-    ggml_graph_compute       (ctx0, &gf);
+
+    // run the computation
+    {
+        std::unique_ptr<uint8_t []> data;
+        auto plan = ggml_graph_plan(&gf, n_threads);
+        if (plan.work_size > 0) {
+            data.reset(new uint8_t[plan.work_size]);
+            plan.work_data = data.get();
+        }
+        ggml_graph_compute(&gf, &plan);
+    }
 
     //if (n_past%100 == 0) {
     //    ggml_graph_print   (&gf);
@@ -836,8 +667,7 @@ size_t GPTJ::requiredMem(const std::string &modelPath) {
     gptj_model dummy_model;
     gpt_vocab dummy_vocab;
     size_t mem_req;
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-    gptj_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
+    gptj_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
     return mem_req;
 }
 
@@ -845,10 +675,8 @@ bool GPTJ::loadModel(const std::string &modelPath) {
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
 
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-
     // load the model
-    if (!gptj_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab)) {
+    if (!gptj_model_load(modelPath, *d_ptr->model, d_ptr->vocab)) {
         std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
         return false;
     }
@@ -939,6 +767,16 @@ const std::vector<LLModel::Token> &GPTJ::endTokens() const
     return fres;
 }
 
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != GGUF_TYPE_STRING) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -958,15 +796,21 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    gptj_hparams hparams;
-    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
-    if (!(hparams.n_vocab >= 50300 && hparams.n_vocab <= 50400)) {
-        return false; // not a gptj.
-    }
-    return magic == 0x67676d6c;
+DLL_EXPORT bool magic_match(const char * fname) {
+    struct ggml_context * ctx_meta = NULL;
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ &ctx_meta,
+    };
+    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
+    if (!ctx_gguf)
+        return false;
+
+    bool isValid = gguf_get_version(ctx_gguf) <= 3;
+    isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
+
+    gguf_free(ctx_gguf);
+    return isValid;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llama.cpp.cmake

@@ -174,6 +174,9 @@ if (LLAMA_KOMPUTE)
         add_custom_command(
             OUTPUT ${spv_file}
             DEPENDS ${LLAMA_DIR}/${source}
+              ${LLAMA_DIR}/kompute/common.comp
+              ${LLAMA_DIR}/kompute/op_getrows.comp
+              ${LLAMA_DIR}/kompute/op_mul_mv_q_n.comp
             COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
             COMMENT "Compiling ${source} to ${source}.spv"
         )
@@ -185,24 +188,41 @@ if (LLAMA_KOMPUTE)
         string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}")
         set(OUTPUT_HEADER_FILE "${HEADER_FILE}")
         message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}")
-        add_custom_command(
-          OUTPUT ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-          DEPENDS ${spv_file} xxd
-          COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
-        )
+        if(CMAKE_GENERATOR MATCHES "Visual Studio")
+            add_custom_command(
+              OUTPUT ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              DEPENDS ${spv_file} xxd
+              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd"
+            )
+        else()
+            add_custom_command(
+              OUTPUT ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              DEPENDS ${spv_file} xxd
+              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
+            )
+        endif()
       endforeach()
     endfunction()
 
     if (EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
         message(STATUS "Kompute found")
+        set(KOMPUTE_OPT_LOG_LEVEL Critical CACHE STRING "Kompute log level")
         add_subdirectory(${LLAMA_DIR}/kompute)
 
         # Compile our shaders
@@ -219,12 +239,16 @@ if (LLAMA_KOMPUTE)
           kompute/op_norm.comp
           kompute/op_rmsnorm.comp
           kompute/op_diagmask.comp
+          kompute/op_mul_mat_mat_f32.comp
           kompute/op_mul_mat_f16.comp
+          kompute/op_mul_mat_q8_0.comp
           kompute/op_mul_mat_q4_0.comp
           kompute/op_mul_mat_q4_1.comp
+          kompute/op_mul_mat_q6_k.comp
           kompute/op_getrows_f16.comp
           kompute/op_getrows_q4_0.comp
           kompute/op_getrows_q4_1.comp
+          kompute/op_getrows_q6_k.comp
           kompute/op_rope.comp
           kompute/op_cpy_f16_f16.comp
           kompute/op_cpy_f16_f32.comp
@@ -246,12 +270,16 @@ if (LLAMA_KOMPUTE)
           shaderop_norm.h
           shaderop_rmsnorm.h
           shaderop_diagmask.h
+          shaderop_mul_mat_mat_f32.h
           shaderop_mul_mat_f16.h
+          shaderop_mul_mat_q8_0.h
           shaderop_mul_mat_q4_0.h
           shaderop_mul_mat_q4_1.h
+          shaderop_mul_mat_q6_k.h
           shaderop_getrows_f16.h
           shaderop_getrows_q4_0.h
           shaderop_getrows_q4_1.h
+          shaderop_getrows_q6_k.h
           shaderop_rope.h
           shaderop_cpy_f16_f16.h
           shaderop_cpy_f16_f32.h
@@ -330,6 +358,13 @@ endif()
 # TODO: probably these flags need to be tweaked on some architectures
 #       feel free to update the Makefile for your architecture and send a pull request or issue
 message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
+if (MSVC)
+  string(TOLOWER "${CMAKE_GENERATOR_PLATFORM}" CMAKE_GENERATOR_PLATFORM_LWR)
+  message(STATUS "CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}")
+else ()
+  set(CMAKE_GENERATOR_PLATFORM_LWR "")
+endif ()
+
 if (NOT MSVC)
     if (LLAMA_STATIC)
         add_link_options(-static)
@@ -345,6 +380,139 @@ if (NOT MSVC)
     endif()
 endif()
 
+if ((${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm") OR (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") OR ("${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "arm64"))
+    message(STATUS "ARM detected")
+    if (MSVC)
+        add_compile_definitions(__ARM_NEON)
+        add_compile_definitions(__ARM_FEATURE_FMA)
+        add_compile_definitions(__ARM_FEATURE_DOTPROD)
+        # add_compile_definitions(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) # MSVC doesn't support vdupq_n_f16, vld1q_f16, vst1q_f16
+        add_compile_definitions(__aarch64__) # MSVC defines _M_ARM64 instead
+    else()
+        include(CheckCXXCompilerFlag)
+        check_cxx_compiler_flag(-mfp16-format=ieee COMPILER_SUPPORTS_FP16_FORMAT_I3E)
+        if (NOT "${COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
+            add_compile_options(-mfp16-format=ieee)
+        endif()
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv6")
+            # Raspberry Pi 1, Zero
+            add_compile_options(-mfpu=neon-fp-armv8 -mno-unaligned-access)
+        endif()
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7")
+            # Raspberry Pi 2
+            add_compile_options(-mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations)
+        endif()
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv8")
+            # Raspberry Pi 3, 4, Zero 2 (32-bit)
+            add_compile_options(-mno-unaligned-access)
+        endif()
+    endif()
+elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$" OR "${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "^(x86_64|i686|amd64|x64)$" )
+    message(STATUS "x86 detected")
+    if (MSVC)
+        if (LLAMA_AVX512)
+            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX512>)
+            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
+            # MSVC has no compile-time flags enabling specific
+            # AVX512 extensions, neither it defines the
+            # macros corresponding to the extensions.
+            # Do it manually.
+            if (LLAMA_AVX512_VBMI)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
+            endif()
+            if (LLAMA_AVX512_VNNI)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
+            endif()
+        elseif (LLAMA_AVX2)
+            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX2>)
+            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
+        elseif (LLAMA_AVX)
+            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX>)
+            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
+        endif()
+    else()
+        if (LLAMA_F16C)
+            add_compile_options(-mf16c)
+        endif()
+        if (LLAMA_FMA)
+            add_compile_options(-mfma)
+        endif()
+        if (LLAMA_AVX)
+            add_compile_options(-mavx)
+        endif()
+        if (LLAMA_AVX2)
+            add_compile_options(-mavx2)
+        endif()
+        if (LLAMA_AVX512)
+            add_compile_options(-mavx512f)
+            add_compile_options(-mavx512bw)
+        endif()
+        if (LLAMA_AVX512_VBMI)
+            add_compile_options(-mavx512vbmi)
+        endif()
+        if (LLAMA_AVX512_VNNI)
+            add_compile_options(-mavx512vnni)
+        endif()
+    endif()
+elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
+    message(STATUS "PowerPC detected")
+    add_compile_options(-mcpu=native -mtune=native)
+    #TODO: Add  targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be)
+else()
+    message(STATUS "Unknown architecture")
+endif()
+
+#
+# POSIX conformance
+#
+
+# clock_gettime came in POSIX.1b (1993)
+# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional
+# posix_memalign came in POSIX.1-2001 / SUSv3
+# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985)
+add_compile_definitions(_XOPEN_SOURCE=600)
+
+# Somehow in OpenBSD whenever POSIX conformance is specified
+# some string functions rely on locale_t availability,
+# which was introduced in POSIX.1-2008, forcing us to go higher
+if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
+    remove_definitions(-D_XOPEN_SOURCE=600)
+    add_compile_definitions(_XOPEN_SOURCE=700)
+endif()
+
+# Data types, macros and functions related to controlling CPU affinity and
+# some memory allocation are available on Linux through GNU extensions in libc
+if (CMAKE_SYSTEM_NAME MATCHES "Linux")
+    add_compile_definitions(_GNU_SOURCE)
+endif()
+
+# RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1,
+# and on macOS its availability depends on enabling Darwin extensions
+# similarly on DragonFly, enabling BSD extensions is necessary
+if (
+    CMAKE_SYSTEM_NAME MATCHES "Darwin" OR
+    CMAKE_SYSTEM_NAME MATCHES "iOS" OR
+    CMAKE_SYSTEM_NAME MATCHES "tvOS" OR
+    CMAKE_SYSTEM_NAME MATCHES "DragonFly"
+)
+    add_compile_definitions(_DARWIN_C_SOURCE)
+endif()
+
+# alloca is a non-standard interface that is not visible on BSDs when
+# POSIX conformance is specified, but not all of them provide a clean way
+# to enable it in such cases
+if (CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+    add_compile_definitions(__BSD_VISIBLE)
+endif()
+if (CMAKE_SYSTEM_NAME MATCHES "NetBSD")
+    add_compile_definitions(_NETBSD_SOURCE)
+endif()
+if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
+    add_compile_definitions(_BSD_SOURCE)
+endif()
+
 function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
     message(STATUS "Configuring ggml implementation target llama${SUFFIX} in ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}")
 
@@ -452,15 +620,14 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
 
     if (WITH_LLAMA)
         # Backwards compatibility with old llama.cpp versions
-        set(LLAMA_UTIL_SOURCE_FILE llama-util.h)
+#        set(LLAMA_UTIL_SOURCE_FILE llama-util.h)
         if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
             set(LLAMA_UTIL_SOURCE_FILE llama_util.h)
         endif()
 
         add_library(llama${SUFFIX} STATIC
                     ${DIRECTORY}/llama.cpp
-                    ${DIRECTORY}/llama.h
-                    ${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
+                    ${DIRECTORY}/llama.h)
 
         if (LLAMA_METAL AND GGML_METAL_SOURCES)
             target_compile_definitions(llama${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)

gpt4all-backend/llamamodel.cpp

@@ -36,18 +36,25 @@ namespace {
 const char *modelType_ = "LLaMA";
 }
 
+static bool llama_verbose() {
+    const char* var = getenv("GPT4ALL_VERBOSE_LLAMACPP");
+    return var && *var;
+}
+
+static void llama_log_callback(enum ggml_log_level level, const char *text, void *userdata) {
+    (void)userdata;
+    if (llama_verbose() || level <= GGML_LOG_LEVEL_ERROR) {
+        fputs(text, stderr);
+    }
+}
+
 struct gpt_params {
     int32_t seed          = -1;   // RNG seed
     int32_t n_keep        = 0;    // number of tokens to keep from initial prompt
-#if LLAMA_DATE <= 230511
-    int32_t n_parts       = -1;   // amount of model parts (-1 = determine from model dimensions)
-#endif
 
-#if LLAMA_DATE >= 230519
     // sampling parameters
     float   tfs_z         = 1.0f; // 1.0 = disabled
     float   typical_p     = 1.0f; // 1.0 = disabled
-#endif
 
     std::string prompt = "";
 
@@ -57,7 +64,6 @@ struct gpt_params {
     bool use_mlock         = false; // use mlock to keep model in memory
 };
 
-#if LLAMA_DATE >= 230519
 static int llama_sample_top_p_top_k(
         llama_context *ctx,
         const llama_token *last_n_tokens_data,
@@ -85,7 +91,6 @@ static int llama_sample_top_p_top_k(
     llama_sample_temperature(ctx, &candidates_p, temp);
     return llama_sample_token(ctx, &candidates_p);
 }
-#endif
 
 struct LLamaPrivate {
     const std::string modelPath;
@@ -93,6 +98,7 @@ struct LLamaPrivate {
     llama_context *ctx = nullptr;
     llama_context_params params;
     int64_t n_threads = 0;
+    std::vector<LLModel::Token> end_tokens;
 };
 
 LLamaModel::LLamaModel()
@@ -149,11 +155,10 @@ bool LLamaModel::loadModel(const std::string &modelPath)
 #else
     d_ptr->params.use_mlock  = params.use_mlock;
 #endif
-#if LLAMA_DATE <= 230511
-    d_ptr->params.n_parts  = params.n_parts;
-#endif
 #ifdef GGML_USE_METAL
-    std::cerr << "llama.cpp: using Metal" << std::endl;
+    if (llama_verbose()) {
+        std::cerr << "llama.cpp: using Metal" << std::endl;
+    }
     // metal always runs the whole model if n_gpu_layers is not 0, at least
     // currently
     d_ptr->params.n_gpu_layers = 1;
@@ -176,6 +181,8 @@ bool LLamaModel::loadModel(const std::string &modelPath)
         return false;
     }
 
+    d_ptr->end_tokens = {llama_token_eos(d_ptr->ctx)};
+
 #ifdef GGML_USE_KOMPUTE
     if (ggml_vk_has_device()) {
         std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
@@ -226,9 +233,9 @@ size_t LLamaModel::restoreState(const uint8_t *src)
 
 std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str) const
 {
-    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
+    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos(d_ptr->ctx));
     std::vector<LLModel::Token> fres(str.size()+4);
-    auto fres_len = llama_tokenize(d_ptr->ctx, str.c_str(), fres.data(), fres.size(), useBOS);
+    auto fres_len = llama_tokenize(d_ptr->ctx, str.c_str(), str.length(), fres.data(), fres.size(), useBOS);
     fres.resize(fres_len);
     return fres;
 }
@@ -249,16 +256,7 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
 
 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
 {
-    // When we recalculate context we could have erased the original BOS token... we need to replace it
-    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
-    if (useBOS) {
-        std::vector<int32_t> myTokens;
-        myTokens.push_back(llama_token_bos());
-        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
-        ctx.n_past += 1;
-        return llama_eval(d_ptr->ctx, myTokens.data(), myTokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
-    } else
-        return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+    return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
 }
 
 int32_t LLamaModel::contextLength() const
@@ -268,8 +266,7 @@ int32_t LLamaModel::contextLength() const
 
 const std::vector<LLModel::Token> &LLamaModel::endTokens() const
 {
-    static const std::vector<LLModel::Token> fres = {llama_token_eos()};
-    return fres;
+    return d_ptr->end_tokens;
 }
 
 #if defined(GGML_USE_KOMPUTE)
@@ -308,8 +305,9 @@ bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string& d
 #endif
 }
 
-bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device)
+bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device, std::string *unavail_reason)
 {
+    bool result = false;
 #if defined(GGML_USE_KOMPUTE)
     ggml_vk_device vkDevice;
     vkDevice.index = device.index;
@@ -317,10 +315,16 @@ bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device)
     vkDevice.heapSize = device.heapSize;
     vkDevice.name = device.name;
     vkDevice.vendor = device.vendor;
-    return ggml_vk_init_device(vkDevice);
+    result = ggml_vk_init_device(vkDevice);
+    if (!result && unavail_reason) {
+        *unavail_reason = "failed to init GPU";
+    }
 #else
-    return false;
+    if (unavail_reason) {
+        *unavail_reason = "built without Kompute";
+    }
 #endif
+    return result;
 }
 
 bool LLamaModel::initializeGPUDevice(int device)
@@ -351,6 +355,16 @@ bool LLamaModel::usingGPUDevice()
     return false;
 }
 
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != (GGUF_TYPE_STRING)) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -370,42 +384,27 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(std::istream& f) {
-    // Check magic
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    if (magic != 0x67676a74) return false;
-    // Check version
-    uint32_t version = 0;
-    f.read(reinterpret_cast<char*>(&version), sizeof(version));
-    if (!(version LLAMA_VERSIONS)) {
+DLL_EXPORT bool magic_match(const char * fname) {
+
+    struct ggml_context * ctx_meta = NULL;
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ &ctx_meta,
+    };
+    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
+    if (!ctx_gguf)
         return false;
-    }
-    llama_file_hparams hparams;
-    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
-    if (!(hparams.n_vocab >= 32000 && hparams.n_vocab <= 32100)) {
-        return false; // not a llama.
-    }
-#ifdef GGML_USE_METAL
-    // Check quant supported on metal
-    // skip fields
-    switch(hparams.ftype) {
-        // currently supported on Metal https://github.com/ggerganov/llama.cpp/blob/ae9663f1887513e152839e91f61c513075a19422/ggml-metal.m#L51-L55
-        case LLAMA_FTYPE_MOSTLY_F16:
-        case LLAMA_FTYPE_MOSTLY_Q2_K:
-        case LLAMA_FTYPE_MOSTLY_Q4_0:
-        case LLAMA_FTYPE_MOSTLY_Q6_K:
-        case LLAMA_FTYPE_MOSTLY_Q4_K_S:
-        case LLAMA_FTYPE_MOSTLY_Q4_K_M:
-            return true;
-        default: // unsupported quant-type for Metal
-            return false;
-    }
-#endif
-    return true;
+
+    bool isValid = gguf_get_version(ctx_gguf) <= 3;
+    auto arch = get_arch_name(ctx_gguf);
+    isValid = isValid && (arch == "llama" || arch == "starcoder" || arch == "falcon" || arch == "mpt");
+
+    gguf_free(ctx_gguf);
+    return isValid;
 }
 
 DLL_EXPORT LLModel *construct() {
+    llama_log_set(llama_log_callback, nullptr);
     return new LLamaModel;
 }
 }

gpt4all-backend/llamamodel_impl.h

@@ -27,7 +27,7 @@ public:
     int32_t threadCount() const override;
     std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) override;
     bool initializeGPUDevice(size_t memoryRequired, const std::string& device) override;
-    bool initializeGPUDevice(const GPUDevice &device) override;
+    bool initializeGPUDevice(const GPUDevice &device, std::string *unavail_reason) override;
     bool initializeGPUDevice(int device) override;
     bool hasGPUDevice() override;
     bool usingGPUDevice() override;

gpt4all-backend/llmodel.cpp

@@ -10,6 +10,7 @@
 #include <cassert>
 #include <cstdlib>
 #include <sstream>
+#include <regex>
 #ifdef _MSC_VER
 #include <intrin.h>
 #endif
@@ -52,7 +53,7 @@ LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
     m_buildVariant = get_build_variant();
-    m_magicMatch = m_dlhandle->get<bool(std::ifstream&)>("magic_match");
+    m_magicMatch = m_dlhandle->get<bool(const char*)>("magic_match");
     assert(m_magicMatch);
     m_construct = m_dlhandle->get<LLModel *()>("construct");
     assert(m_construct);
@@ -81,6 +82,13 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     static auto* libs = new std::vector<Implementation>([] () {
         std::vector<Implementation> fres;
 
+        std::string impl_name_re = "(bert|llama|gptj|llamamodel-mainline)";
+        if (requires_avxonly()) {
+            impl_name_re += "-avxonly";
+        } else {
+            impl_name_re += "-(default|metal)";
+        }
+        std::regex re(impl_name_re);
         auto search_in_directory = [&](const std::string& paths) {
             std::stringstream ss(paths);
             std::string path;
@@ -90,7 +98,10 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
                 // Iterate over all libraries
                 for (const auto& f : std::filesystem::directory_iterator(fs_path)) {
                     const std::filesystem::path& p = f.path();
+
                     if (p.extension() != LIB_FILE_EXT) continue;
+                    if (!std::regex_search(p.stem().string(), re)) continue;
+
                     // Add to list if model implementation
                     try {
                         Dlhandle dl(p.string());
@@ -111,31 +122,28 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     return *libs;
 }
 
-const LLModel::Implementation* LLModel::Implementation::implementation(std::ifstream& f, const std::string& buildVariant) {
+const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
     for (const auto& i : implementationList()) {
-        f.seekg(0);
-        if (!i.m_magicMatch(f)) continue;
         if (buildVariant != i.m_buildVariant) continue;
+        if (!i.m_magicMatch(fname)) continue;
         return &i;
     }
     return nullptr;
 }
 
 LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant) {
-
-    if (!has_at_least_minimal_hardware())
+    if (!has_at_least_minimal_hardware()) {
+        std::cerr << "LLModel ERROR: CPU does not support AVX\n";
         return nullptr;
+    }
 
-    // Read magic
-    std::ifstream f(modelPath, std::ios::binary);
-    if (!f) return nullptr;
     // Get correct implementation
     const Implementation* impl = nullptr;
 
     #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
         if (buildVariant == "auto") {
             size_t total_mem = getSystemTotalRAMInBytes();
-            impl = implementation(f, "metal");
+            impl = implementation(modelPath.c_str(), "metal");
             if(impl) {
                 LLModel* metalimpl = impl->m_construct();
                 metalimpl->m_implementation = impl;
@@ -161,10 +169,9 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
                 buildVariant = "default";
             }
         }
-        impl = implementation(f, buildVariant);
+        impl = implementation(modelPath.c_str(), buildVariant);
         if (!impl) return nullptr;
     }
-    f.close();
 
     // Construct and return llmodel implementation
     auto fres = impl->m_construct();

gpt4all-backend/llmodel.h

@@ -27,13 +27,13 @@ public:
 
         static bool isImplementation(const Dlhandle&);
         static const std::vector<Implementation>& implementationList();
-        static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
+        static const Implementation *implementation(const char *fname, const std::string& buildVariant);
         static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
         static void setImplementationsSearchPath(const std::string& path);
         static const std::string& implementationsSearchPath();
 
     private:
-        bool (*m_magicMatch)(std::ifstream& f);
+        bool (*m_magicMatch)(const char *fname);
         LLModel *(*m_construct)();
 
     private:
@@ -97,10 +97,16 @@ public:
 
     virtual std::vector<GPUDevice> availableGPUDevices(size_t /*memoryRequired*/) { return std::vector<GPUDevice>(); }
     virtual bool initializeGPUDevice(size_t /*memoryRequired*/, const std::string& /*device*/) { return false; }
-    virtual bool initializeGPUDevice(const GPUDevice &/*device*/) { return false; }
+    virtual bool initializeGPUDevice(const GPUDevice &/*device*/, std::string *unavail_reason = nullptr) {
+        if (unavail_reason) {
+            *unavail_reason = "model has no GPU support";
+        }
+        return false;
+    }
     virtual bool initializeGPUDevice(int /*device*/) { return false; }
     virtual bool hasGPUDevice() { return false; }
     virtual bool usingGPUDevice() { return false; }
+    static std::vector<GPUDevice> availableGPUDevices();
 
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of

gpt4all-backend/llmodel_shared.cpp

@@ -4,6 +4,10 @@
 #include <iostream>
 #include <unordered_set>
 
+#ifdef GGML_USE_KOMPUTE
+#include "ggml-vulkan.h"
+#endif
+
 void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
     size_t i = 0;
     promptCtx.n_past = 0;
@@ -88,10 +92,10 @@ void LLModel::prompt(const std::string &prompt,
             if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
                 promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(batch.at(t));
+            promptCtx.n_past += 1;
             if (!promptCallback(batch.at(t)))
                 return;
         }
-        promptCtx.n_past += batch.size();
         i = batch_end;
     }
 
@@ -122,8 +126,6 @@ void LLModel::prompt(const std::string &prompt,
             return;
         }
 
-        promptCtx.n_past += 1;
-
         // display text
         for (const auto token : endTokens()) {
             if (id == token) return;
@@ -158,6 +160,7 @@ void LLModel::prompt(const std::string &prompt,
             if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
                 promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(t);
+            promptCtx.n_past += 1;
             //TODO: Conversion to std::string can be avoided here...
             if (!responseCallback(t, std::string(tokenToString(t))))
                 return;
@@ -174,3 +177,26 @@ std::vector<float> LLModel::embedding(const std::string &/*text*/)
     }
     return std::vector<float>();
 }
+
+std::vector<LLModel::GPUDevice> LLModel::availableGPUDevices()
+{
+#if defined(GGML_USE_KOMPUTE)
+    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(0);
+
+    std::vector<LLModel::GPUDevice> devices;
+    for(const auto& vkDevice : vkDevices) {
+        LLModel::GPUDevice device;
+        device.index = vkDevice.index;
+        device.type = vkDevice.type;
+        device.heapSize = vkDevice.heapSize;
+        device.name = vkDevice.name;
+        device.vendor = vkDevice.vendor;
+
+        devices.push_back(device);
+    }
+
+    return devices;
+#else
+    return std::vector<LLModel::GPUDevice>();
+#endif
+}

gpt4all-backend/llmodel_shared.h

@@ -80,7 +80,6 @@ struct llm_kv_cache {
     }
 };
 
-#if LLAMA_DATE >= 230519
 inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads) {
     struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
     if (plan.work_size > 0) {
@@ -89,4 +88,3 @@ inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_t
     }
     ggml_graph_compute(graph, &plan);
 }
-#endif

gpt4all-backend/mpt.cpp

@@ -1,893 +0,0 @@
-#define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "mpt_impl.h"
-
-#include "utils.h"
-#include "llmodel_shared.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <fstream>
-#include <map>
-#include <random>
-#include <string>
-#include <vector>
-#include <iostream>
-#if defined(_WIN32) && defined(_MSC_VER)
-    #define WIN32_LEAN_AND_MEAN
-    #ifndef NOMINMAX
-        #define NOMINMAX
-    #endif
-    #include <windows.h>
-    #include <io.h>
-    #include <stdio.h>
-#else
-    #include <unistd.h>
-#endif
-#include <sstream>
-#include <thread>
-#include <unordered_set>
-#include <regex>
-#include <ggml.h>
-
-
-namespace {
-const char *modelType_ = "MPT";
-}
-
-// default hparams (MPT 7B)
-struct mpt_hparams {
-    int32_t n_vocab      = 50432;
-    int32_t n_ctx        = 2048;
-    int32_t n_embd       = 4096;
-    int32_t n_head       = 32;
-    int32_t n_layer      = 32;
-    float alibi_bias_max = 8;
-    float clip_qkv       = 0;
-    int32_t expand       = 4;
-    int32_t f16          = 1;
-};
-
-struct mpt_layer {
-    // normalization
-    struct ggml_tensor * norm_1_w;
-    struct ggml_tensor * norm_2_w;
-
-    // attention
-    struct ggml_tensor * attn_Wqkv_w;
-    struct ggml_tensor * attn_out_proj_w;
-
-    // ff
-    struct ggml_tensor * ffn_up_proj_w;
-    struct ggml_tensor * ffn_down_proj_w;
-};
-
-struct mpt_model {
-    mpt_hparams hparams;
-
-    // normalization
-    struct ggml_tensor * norm_f_w;
-
-    struct ggml_tensor * wte; // position embedding
-
-    // mpt does weight tying
-
-    std::vector<mpt_layer> layers;
-
-    struct llm_kv_cache kv_self;
-    struct ggml_context * ctx;
-    std::map<std::string, struct ggml_tensor *> tensors;
-
-
-    llm_buffer eval_buf;
-    llm_buffer scr0_buf;
-    llm_buffer scr1_buf;
-
-    ~mpt_model() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
-static bool kv_cache_init(
-        const struct mpt_hparams & hparams,
-             struct llm_kv_cache & cache,
-                         ggml_type   wtype,
-                               int   n_ctx) {
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-
-    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
-    const int64_t n_elements = n_embd*n_mem;
-
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
-
-    struct ggml_init_params params;
-    params.mem_size   = cache.buf.size;
-    params.mem_buffer = cache.buf.addr;
-    params.no_alloc   = false;
-
-    cache.ctx = ggml_init(params);
-
-    if (!cache.ctx) {
-        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
-        return false;
-    }
-
-    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-
-    return true;
-}
-
-// load the model's weights from a stream. if mem_req ptr is passed the model is
-// only partially parsed to estimate required memory
-bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & model, gpt_vocab & vocab, size_t * mem_req) {
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
-    if (mem_req != nullptr) {
-        *mem_req = 0;
-    }
-
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *) &magic, sizeof(magic));
-        if (magic != 0x67676d6d) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
-            return false;
-        }
-    }
-
-    // load hparams
-    {
-        auto & hparams = model.hparams;
-
-        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
-        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
-        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
-        fin.read((char *) &hparams.alibi_bias_max,  sizeof(hparams.alibi_bias_max));
-        fin.read((char *) &hparams.clip_qkv,  sizeof(hparams.clip_qkv));
-        fin.read((char *) &hparams.f16,   sizeof(hparams.f16));
-
-        printf("%s: n_vocab        = %d\n", __func__, hparams.n_vocab);
-        printf("%s: n_ctx          = %d\n", __func__, hparams.n_ctx);
-        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
-        printf("%s: alibi_bias_max = %f\n", __func__, hparams.alibi_bias_max);
-        printf("%s: clip_qkv       = %f\n", __func__, hparams.clip_qkv);
-        printf("%s: ftype          = %d\n", __func__, hparams.f16);
-    }
-
-    // load vocab
-    {
-        int32_t n_vocab = model.hparams.n_vocab;
-        fin.read((char *) &n_vocab, sizeof(n_vocab));
-
-        if (n_vocab != model.hparams.n_vocab) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
-                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
-            return false;
-        }
-
-        std::string word;
-        for (int i = 0; i < n_vocab; i++) {
-            uint32_t len;
-            fin.read((char *) &len, sizeof(len));
-            bool special = false;
-            if (len & (1<<31)) {
-                len = len &~ (1<<31);
-                special = true;
-            }
-
-            if (len > 0) {
-                word.resize(len);
-                fin.read((char *) word.data(), len);
-                vocab.token_to_id[word] = i;
-                vocab.id_to_token[i] = word;
-            }
-
-            if(special) {
-                vocab.add_special_token(word);
-            }
-        }
-    }
-
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = GGML_TYPE_COUNT;
-    switch (model.hparams.f16) {
-        case 0: wtype = GGML_TYPE_F32;  break;
-        case 1: wtype = GGML_TYPE_F16;  break;
-        case 2: wtype = GGML_TYPE_Q4_0; break;
-        case 3: wtype = GGML_TYPE_Q4_1; break;
-        case 5: wtype = GGML_TYPE_Q4_2; break;
-        default:
-                {
-                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
-                            __func__, fname.c_str(), model.hparams.f16);
-                    return false;
-                }
-    }
-
-    auto & ctx = model.ctx;
-
-    size_t ctx_size = 0;
-
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_ctx   = hparams.n_ctx;
-        const int n_vocab = hparams.n_vocab;
-        const int expand  = hparams.expand;
-
-
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_w
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(GGML_TYPE_F32); // wte
-
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_1_w
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_2_w
-
-        ctx_size += n_layer*(3*n_embd*n_embd*ggml_type_sizef(wtype)); // attn_Wqkv_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // attn_out_proj_w
-
-        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype));  // ffn_up_proj_w
-        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype)); // ffn_down_proj_w
-
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_k
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_v
-
-        // TODO probably less now?
-        ctx_size += (5 + 10*n_layer)*256; // object overhead
-
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
-    }
-
-    if (mem_req != nullptr) {
-        *mem_req += ctx_size;
-        const int n_embd  = model.hparams.n_embd;
-        const int n_layer = model.hparams.n_layer;
-
-        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
-        const int64_t n_elements = n_embd*n_mem;
-
-        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
-        return false;
-    }
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size   = ctx_size,
-            .mem_buffer = NULL,
-            .no_alloc   = false,
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx) {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            return false;
-        }
-    }
-
-    // prepare memory for the weights
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_vocab = hparams.n_vocab;
-        const int expand  = hparams.expand;
-
-        model.layers.resize(n_layer);
-
-        model.wte    = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab);
-        model.norm_f_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-        // map by name
-        model.tensors["transformer.wte.weight"] = model.wte;
-        model.tensors["transformer.norm_f.weight"] = model.norm_f_w;
-
-        for (int i = 0; i < n_layer; ++i) {
-            auto & layer = model.layers[i];
-
-            layer.norm_1_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.norm_2_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            layer.attn_Wqkv_w     = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd * 3);
-            layer.attn_out_proj_w = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd);
-            layer.ffn_up_proj_w   = ggml_new_tensor_2d(ctx, wtype,        n_embd, expand*n_embd);
-            layer.ffn_down_proj_w = ggml_new_tensor_2d(ctx, wtype, expand*n_embd, n_embd);
-
-            // map by name
-            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_1.weight"]        = layer.norm_1_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_2.weight"]        = layer.norm_2_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.Wqkv.weight"]     = layer.attn_Wqkv_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.out_proj.weight"] = layer.attn_out_proj_w;
-
-            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.up_proj.weight"]   = layer.ffn_up_proj_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.down_proj.weight"] = layer.ffn_down_proj_w;
-        }
-    }
-
-    // key + value memory
-    {
-        const auto & hparams = model.hparams;
-        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
-            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
-            ggml_free(ctx);
-            return false;
-        }
-
-        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
-        printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
-    }
-
-    // load weights
-    {
-        int n_tensors = 0;
-        size_t total_size = 0;
-
-        printf("%s: ", __func__);
-
-        while (true) {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ttype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
-
-            if (fin.eof()) {
-                break;
-            }
-
-            int32_t nelements = 1;
-            int32_t ne[2] = { 1, 1 };
-            for (int i = 0; i < n_dims; ++i) {
-                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-
-            if (model.tensors.find(name.data()) == model.tensors.end()) {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                return false;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                return false;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
-                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
-                return false;
-            }
-
-            // for debugging
-            if (0) {
-                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
-            }
-
-            const size_t bpe = ggml_type_size(ggml_type(ttype));
-
-            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
-                return false;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ttype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
-            total_size += ggml_nbytes(tensor);
-            if (++n_tensors % 8 == 0) {
-                printf(".");
-                fflush(stdout);
-            }
-        }
-
-        printf(" done\n");
-
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
-    }
-
-    model.scr0_buf.resize(256u * 1024 * 1024);
-    model.scr1_buf.resize(256u * 1024 * 1024);
-
-    return true;
-}
-
-// load the model's weights from a file path
-bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vocab) {
-
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin) {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
-        return false;
-    }
-
-    bool loaded = mpt_model_load(fname, fin, model, vocab, nullptr);
-    fin.close();
-    return loaded;
-}
-
-bool mpt_eval(
-        mpt_model & model,
-        const int n_threads,
-        const int n_past,
-        const std::vector<int>           & embd_inp,
-              std::vector<float>         & embd_w,
-              size_t                     & mem_per_token) {
-    const int N = embd_inp.size();
-
-    const auto & hparams = model.hparams;
-
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_ctx   = hparams.n_ctx;
-    const int n_head  = hparams.n_head;
-    const int n_vocab = hparams.n_vocab;
-
-    const size_t init_buf_size = 1024_MiB;
-    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
-        model.eval_buf.resize(init_buf_size);
-
-    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
-        const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
-        // printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
-
-        // reallocate
-        model.eval_buf.resize(buf_size_new);
-        if (model.eval_buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
-            return false;
-        }
-    }
-
-    struct ggml_init_params params = {
-        .mem_size   = model.eval_buf.size,
-        .mem_buffer = model.eval_buf.addr,
-        .no_alloc = false
-    };
-
-    struct ggml_context * ctx0 = ggml_init(params);
-    struct ggml_cgraph gf = {};
-    gf.n_threads = n_threads;
-
-    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
-
-    // wte
-    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
-
-    for (int il = 0; il < n_layer; ++il) {
-        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-        struct ggml_tensor * inpSA = inpL;
-        struct ggml_tensor * cur = inpSA;
-        // self-attention
-        {
-
-            // norm1
-            cur = ggml_norm(ctx0, cur);
-            cur = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].norm_1_w, cur),
-                    cur);
-            // compute QKV
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].attn_Wqkv_w,
-                    cur);
-
-            // TODO: clip_qkv
-            struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 0*ggml_element_size(cur)*n_embd));
-            struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 1*ggml_element_size(cur)*n_embd));
-            struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 2*ggml_element_size(cur)*n_embd));
-
-            // TODO: qk_ln? (seems to be False in MPT-7B configs)
-            {
-                Vcur = ggml_transpose(ctx0, Vcur);
-
-                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
-                                        (   n_ctx)*ggml_element_size(model.kv_self.v),
-                                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
-
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
-            }
-            // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q =
-                ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0, Qcur, n_embd/n_head, n_head, N),
-                        0, 2, 1, 3);
-
-            struct ggml_tensor * K =
-                ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0,
-                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                            n_embd/n_head, n_head, n_past + N),
-                        0, 2, 1, 3);
-
-            // K * Q
-            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
-
-            // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled =
-                ggml_scale(ctx0,
-                        KQ,
-                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
-                        );
-
-
-            // Alibi
-            struct ggml_tensor * KQ_scaled_biased = ggml_alibi(ctx0, ggml_cont(ctx0, KQ_scaled), n_past, n_head);
-
-            // KQ_masked = mask_past(KQ_scaled)
-            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled_biased, n_past);
-
-            // KQ = soft_max(KQ_masked)
-            struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
-
-            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V =
-                ggml_view_3d(ctx0, model.kv_self.v,
-                        n_past + N, n_embd/n_head, n_head,
-                        n_ctx*ggml_element_size(model.kv_self.v),
-                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
-                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
-
-            // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
-
-            // KQV_merged = KQV.permute(0, 2, 1, 3)
-            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            // cur = KQV_merged.contiguous().view(n_embd, N)
-            cur = ggml_cpy(ctx0,
-                    KQV_merged,
-                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-            // projection (no bias)
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].attn_out_proj_w,
-                    cur);
-        }
-
-        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
-        // residual
-        struct ggml_tensor * resSA = ggml_add(ctx0, cur, inpSA);
-        // feed-forward network
-        {
-            cur = resSA;
-            // norm2
-            cur = ggml_norm(ctx0, cur);
-            cur = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].norm_2_w, cur),
-                    cur);
-            // ffn
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].ffn_up_proj_w,
-                    cur);
-            cur = ggml_gelu(ctx0, cur);
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].ffn_down_proj_w,
-                    cur);
-
-        }
-
-        // self-attention + FF
-        inpL = ggml_add(ctx0, cur, resSA);
-    }
-    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-    struct ggml_tensor * out = inpL;
-    // -> logits
-    {
-        out = ggml_norm(ctx0, out);
-        out = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.norm_f_w, out),
-                    out);
-        ggml_set_scratch(ctx0, { 0, 0, nullptr, });
-        out = ggml_mul_mat(ctx0, model.wte, out);
-    }
-
-
-    // run the computation
-    ggml_build_forward_expand(&gf, out);
-    ggml_graph_compute       (ctx0, &gf);
-
-
-    // return result for just the last token
-    embd_w.resize(n_vocab);
-    memcpy(embd_w.data(), (float *) ggml_get_data(out) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
-
-    if (mem_per_token == 0) {
-        mem_per_token = ggml_used_mem(ctx0)/N;
-    }
-    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
-
-    ggml_free(ctx0);
-
-    return true;
-}
-
-
-#define MPT_MAX_RNG_STATE 64*1024
-
-size_t mpt_get_state_size(const mpt_model &model)
-{
-    // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state.
-    // for reference, std::mt19937(1337) serializes to 6701 bytes.
-    const size_t s_rng_size        = sizeof(size_t);
-    const size_t s_rng             = MPT_MAX_RNG_STATE;
-    const size_t s_kv_size         = sizeof(size_t);
-    const size_t s_kv_ntok         = sizeof(int);
-    const size_t s_kv              = model.kv_self.buf.size;
-    const size_t s_total = (
-        + s_rng_size
-        + s_rng
-        + s_kv_size
-        + s_kv_ntok
-        + s_kv
-    );
-    fflush(stdout);
-    return s_total;
-}
-
-size_t mpt_copy_state_data(const mpt_model &model, const std::mt19937 &rng, uint8_t *dest)
-{
-    uint8_t * out = dest;
-    fflush(stdout);
-    // copy rng
-    {
-        std::stringstream rng_ss;
-        rng_ss << rng;
-
-        const size_t rng_size = rng_ss.str().size();
-        char rng_buf[MPT_MAX_RNG_STATE];
-
-        memset(&rng_buf[0], 0, MPT_MAX_RNG_STATE);
-        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
-
-        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
-        memcpy(out, &rng_buf[0], MPT_MAX_RNG_STATE); out += MPT_MAX_RNG_STATE;
-    }
-
-    // copy kv cache
-    {
-        const size_t kv_size = model.kv_self.buf.size;
-        const int    kv_ntok = model.kv_self.n;
-
-        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
-        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
-
-        if (kv_size) {
-            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
-        }
-    }
-
-    const size_t written  = out - dest;
-    assert(written == mpt_get_state_size(model));
-    fflush(stdout);
-    return written;
-}
-
-size_t mpt_set_state_data(mpt_model *model, std::mt19937 *rng, const uint8_t *src)
-{
-    const uint8_t * in = src;
-
-    // set rng
-    {
-        size_t rng_size;
-        char   rng_buf[MPT_MAX_RNG_STATE];
-
-        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
-        memcpy(&rng_buf[0], in, MPT_MAX_RNG_STATE); in += MPT_MAX_RNG_STATE;
-
-        std::stringstream rng_ss;
-        rng_ss.str(std::string(&rng_buf[0], rng_size));
-        rng_ss >> *rng;
-
-        assert(rng_ss.fail() == false);
-    }
-
-    // set kv cache
-    {
-        size_t kv_size;
-        int kv_ntok;
-
-        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
-        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
-
-        if (kv_size) {
-            assert(model->kv_self.buf.size == kv_size);
-
-            void * k_data = model->kv_self.k->data; // remember data pointers
-            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
-
-            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
-
-            model->kv_self.k->data = k_data; // restore correct data pointers
-            model->kv_self.v->data = v_data;
-
-        }
-
-        model->kv_self.n = kv_ntok;
-    }
-
-    const size_t nread    = in - src;
-    assert(nread == mpt_get_state_size(*model));
-    fflush(stdout);
-    return nread;
-}
-
-struct MPTPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    gpt_vocab vocab;
-    mpt_model *model = nullptr;
-    int64_t n_threads = 0;
-    size_t mem_per_token = 0;
-    std::mt19937 rng;
-    bool has_im_end = false;
-};
-
-MPT::MPT()
-    : d_ptr(new MPTPrivate) {
-    d_ptr->model = new mpt_model;
-    d_ptr->model->ctx = nullptr;
-    d_ptr->modelLoaded = false;
-}
-
-size_t MPT::requiredMem(const std::string &modelPath) {
-    mpt_model dummy_model;
-    gpt_vocab dummy_vocab;
-    size_t mem_req;
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-    mpt_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
-    return mem_req;
-}
-
-bool MPT::loadModel(const std::string &modelPath) {
-    std::mt19937 rng(time(NULL));
-    d_ptr->rng = rng;
-
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-
-    // load the model
-    if (!mpt_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab, nullptr)) {
-        std::cerr << "MPT ERROR: failed to load model from " <<  modelPath;
-        return false;
-    }
-
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    d_ptr->has_im_end = d_ptr->vocab.token_to_id.find("<|im_end|>") != d_ptr->vocab.token_to_id.end();
-    fflush(stdout);
-    return true;
-}
-
-void MPT::setThreadCount(int32_t n_threads) {
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t MPT::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-MPT::~MPT()
-{
-    delete d_ptr->model;
-}
-
-bool MPT::isModelLoaded() const
-{
-    return d_ptr->modelLoaded;
-}
-
-size_t MPT::stateSize() const
-{
-    return mpt_get_state_size(*d_ptr->model);
-}
-
-size_t MPT::saveState(uint8_t *dest) const
-{
-    return mpt_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
-}
-
-size_t MPT::restoreState(const uint8_t *src)
-{
-    return mpt_set_state_data(d_ptr->model, &d_ptr->rng, src);
-}
-
-std::vector<LLModel::Token> MPT::tokenize(PromptContext &, const std::string &str) const
-{
-    return ::gpt_tokenize(d_ptr->vocab, str);
-}
-
-std::string MPT::tokenToString(Token id) const
-{
-    return d_ptr->vocab.id_to_token[id];
-}
-
-LLModel::Token MPT::sampleToken(PromptContext &promptCtx) const
-{
-    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
-    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
-        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks,
-        promptCtx.logits,
-        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty,
-        d_ptr->rng);
-}
-
-bool MPT::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    // determine the required inference memory per token:
-    static bool initialized = false;
-    if (!initialized) {
-        mpt_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
-            d_ptr->mem_per_token);
-        initialized = true;
-    }
-
-    return mpt_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
-}
-
-int32_t MPT::contextLength() const
-{
-    return d_ptr->model->hparams.n_ctx;
-}
-
-const std::vector<LLModel::Token> &MPT::endTokens() const
-{
-    static const std::vector<LLModel::Token> fres = {0, d_ptr->vocab.token_to_id["<|im_end|>"]};
-    return fres;
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    return magic == 0x67676d6d;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new MPT;
-}
-}

gpt4all-backend/mpt_impl.h

@@ -1,41 +0,0 @@
-#ifndef MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of mpt.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef MPT_H
-#define MPT_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include "llmodel.h"
-
-struct MPTPrivate;
-class MPT : public LLModel {
-public:
-    MPT();
-    ~MPT();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    MPTPrivate *d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string tokenToString(Token) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // MPT_H

gpt4all-backend/replit_impl.h

@@ -1,44 +0,0 @@
-#ifndef REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of replit.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef REPLIT_H
-#define REPLIT_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include "llmodel.h"
-
-#define GGML_QNT_VERSION_FACTOR 1000 // do not change this
-
-struct ReplitPrivate;
-class Replit : public LLModel {
-public:
-    Replit();
-    ~Replit();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string & modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-    bool usingGPUDevice() override;
-
-private:
-    ReplitPrivate *d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string tokenToString(Token) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // REPLIT_H

gpt4all-backend/scripts/convert_bert_hf_to_ggml.py

@@ -1,102 +0,0 @@
-import sys
-import struct
-import json
-import torch
-import numpy as np
-
-from transformers import AutoModel, AutoTokenizer
-
-if len(sys.argv) < 3:
-    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = sys.argv[1]
-fname_out = sys.argv[1] + "/ggml-model.bin"
-
-with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f:
-    encoder = json.load(f)
-
-with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
-    hparams = json.load(f)
-
-with open(dir_model + "/vocab.txt", "r", encoding="utf-8") as f:
-    vocab = f.readlines()
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-    fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
-
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-model = AutoModel.from_pretrained(dir_model, low_cpu_mem_usage=True)
-print (model)
-
-print(tokenizer.encode('I believe the meaning of life is'))
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-fout = open(fname_out, "wb")
-
-print(hparams)
-
-fout.write(struct.pack("i", 0x62657274)) # magic: ggml in hex
-fout.write(struct.pack("i", hparams["vocab_size"]))
-fout.write(struct.pack("i", hparams["max_position_embeddings"]))
-fout.write(struct.pack("i", hparams["hidden_size"]))
-fout.write(struct.pack("i", hparams["intermediate_size"]))
-fout.write(struct.pack("i", hparams["num_attention_heads"]))
-fout.write(struct.pack("i", hparams["num_hidden_layers"]))
-fout.write(struct.pack("i", ftype))
-
-for i in range(hparams["vocab_size"]):
-    text = vocab[i][:-1] # strips newline at the end
-    #print(f"{i}:{text}")
-    data = bytes(text, 'utf-8')
-    fout.write(struct.pack("i", len(data)))
-    fout.write(data)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
-        continue
-    print("Processing variable: " + name + " with shape: ", data.shape)
-
-    n_dims = len(data.shape);
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-            print("  Converting to float16")
-            data = data.astype(np.float16)
-            l_type = 1
-    else:
-        l_type = 0
-
-    # header
-    str = name.encode('utf-8')
-    fout.write(struct.pack("iii", n_dims, len(str), l_type))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str);
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-backend/scripts/convert_bert_hf_to_gguf.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import struct
+import sys
+from pathlib import Path
+
+import gguf
+import numpy as np
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+
+
+if not 2 <= len(sys.argv) < 4:
+    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = Path(sys.argv[1])
+
+with open(dir_model / "vocab.txt", encoding="utf-8") as f:
+    vocab = f.readlines()
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+
+fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
+
+
+ARCH = gguf.MODEL_ARCH.BERT
+gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
+
+print("gguf: get model metadata")
+
+config = AutoConfig.from_pretrained(dir_model)
+
+block_count = config.num_hidden_layers
+gguf_writer.add_name("BERT")
+gguf_writer.add_context_length(config.max_position_embeddings)
+gguf_writer.add_embedding_length(config.hidden_size)
+gguf_writer.add_feed_forward_length(config.intermediate_size)
+gguf_writer.add_block_count(block_count)
+gguf_writer.add_head_count(config.num_attention_heads)
+gguf_writer.add_file_type(ftype)
+
+print("gguf: get tokenizer metadata")
+
+try:
+    with open(dir_model / "tokenizer.json", encoding="utf-8") as f:
+        tokenizer_json = json.load(f)
+except FileNotFoundError as e:
+    print(f'Error: Missing {e.filename!r}', file=sys.stderr)
+    sys.exit(1)
+
+print("gguf: get wordpiece tokenizer vocab")
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+print(tokenizer.encode('I believe the meaning of life is'))
+
+tokens: list[bytearray] = []
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+
+# The number of tokens in tokenizer.json can differ from the expected vocab size.
+# This causes downstream issues with mismatched tensor sizes when running the inference
+for i in range(config.vocab_size):
+    try:
+        text = reverse_vocab[i]
+    except KeyError:
+        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
+        pad_token = f"[PAD{i}]".encode("utf8")
+        text = bytearray(pad_token)
+
+    tokens.append(text)
+
+gguf_writer.add_tokenizer_model("bert")  # wordpiece
+gguf_writer.add_token_list(tokens)
+
+special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
+special_vocab.add_to_gguf(gguf_writer)
+
+print("gguf: get tensor metadata")
+
+model = AutoModel.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
+print(model)
+
+tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
+        continue
+    print("Processing variable:", name, "with shape:", data.shape)
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+        print("  Converting to float16")
+        data = data.astype(np.float16)
+        l_type = 1
+    else:
+        l_type = 0
+
+    # map tensor names
+    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
+    if new_name is None:
+        print("Can not map tensor '" + name + "'")
+        sys.exit()
+
+    gguf_writer.add_tensor(new_name, data)
+
+
+print("gguf: write header")
+gguf_writer.write_header_to_file()
+print("gguf: write metadata")
+gguf_writer.write_kv_data_to_file()
+print("gguf: write tensors")
+gguf_writer.write_tensors_to_file()
+
+gguf_writer.close()
+
+print(f"gguf: model successfully exported to '{fname_out}'")
+print()

gpt4all-backend/scripts/convert_falcon_hf_to_ggml.py

@@ -1,143 +0,0 @@
-# Based on: https://github.com/KerfuffleV2/ggml-falcon/blob/feat-improve-falcon-convert-hf/examples/falcon/convert-hf-to-ggml.py
-# Convert Hugging Face fine-tuned bloom-like models to ggml format
-#
-# Usage:
-#
-#   python3 convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]
-#
-# This script is similar to "convert-pt-to-ggml.py"
-#
-
-import io
-import os
-import sys
-import struct
-import json
-import code
-import torch
-import numpy as np
-import gc
-
-from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
-
-# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
-def bytes_to_unicode():
-    """
-    Returns list of utf-8 byte and a corresponding list of unicode strings.
-    The reversible bpe codes work on unicode strings.
-    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
-    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
-    This is a significant percentage of your normal, say, 32K bpe vocab.
-    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
-    And avoids mapping to whitespace/control characters the bpe code barfs on.
-    """
-    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
-    cs = bs[:]
-    n = 0
-    for b in range(2**8):
-        if b not in bs:
-            bs.append(b)
-            cs.append(2**8+n)
-            n += 1
-    cs = [chr(n) for n in cs]
-    return dict(zip(bs, cs))
-
-if len(sys.argv) < 3:
-    print("INFO: GGML V1 files produced are meant to be finalized through examples/falcon_quantize which will bring them to latest version and precision of choice");
-    print("Usage: python convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]")
-    print("  model_directory: name of the directory and model you convert (it should be a subdirectory)")
-    print("  output-directory: directory where the output file will be written")
-    print("  use-f32:    if present, use float32 instead of float16 (f32 is recommended)")
-    sys.exit(1)
-
-# num_parts = int(sys.argv[1])
-dir_model = sys.argv[1] # name and dir of model
-dir_out = sys.argv[2]   # output directory
-
-# make sure the output directory exists
-os.makedirs(dir_out, exist_ok=True)
-
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-ftype = 1
-if len(sys.argv) > 3:
-    ftype = 0
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-# print(tokenizer)
-config = AutoConfig.from_pretrained(dir_model, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(dir_model, trust_remote_code=True, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)
-hparams = config.to_dict()
-
-n_head = hparams["n_head"]
-n_head_kv = hparams["n_head_kv"] if "n_head_kv" in hparams else 1
-head_dim = hparams["hidden_size"] // n_head
-print("* Loading model from: ", dir_model)
-
-fname_out = dir_out + f"/ggml-model-{dir_model.split('/')[-1]}-{ftype_str[ftype]}.bin"
-fout = open(fname_out, "wb")
-fout.write(struct.pack("i", 0x67676a74)) # magic: ggmf in hex (version 1) - possibly change to ggfc ?
-fout.write(struct.pack("i", 1)) # version
-fout.write(struct.pack("i", hparams["vocab_size"]))
-fout.write(struct.pack("i", hparams["hidden_size"]))
-fout.write(struct.pack("i", n_head))
-fout.write(struct.pack("i", n_head_kv))
-fout.write(struct.pack("i", hparams["n_layer"]))
-fout.write(struct.pack("i", 40 if "n_head_kv" in hparams else 7)) # obsolete field that breaks ggml compatibility - todo again remove one day
-fout.write(struct.pack("i", ftype))
-
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-byte_encoder = bytes_to_unicode()
-byte_decoder = {v:k for k, v in byte_encoder.items()}
-
-for i in range(hparams["vocab_size"]):
-    text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
-    fout.write(struct.pack("i", len(text)))
-    fout.write(text)
-    fout.write(struct.pack("f", 0.0)) # falcon uses bpe on RefinedWeb - no probability scores used
-
-model = model.state_dict()
-for name in model.keys():
-    src = name
-    # The original query_key_value tensor contains n_head_kv "kv groups",
-    # each consisting of n_head/n_head_kv query weights followed by one key
-    # and one value weight (shared by all query heads in the kv group).
-    # This layout makes it a big pain to work with in GGML.
-    # So we rearrange them here,, so that we have n_head query weights
-    # followed by n_head_kv key weights followed by n_head_kv value weights,
-    # in contiguous fashion.
-
-    if "query_key_value" in src:
-        qkv = model[src].view(
-            n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)
-
-        q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head)
-        k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)
-        v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)
-
-        model[src] = torch.cat((q,k,v)).reshape_as(model[src])
-    data = model[src].squeeze()
-    n_dims = len(data.shape)
-    # default type is fp32
-    ftype_cur = 1 if ftype == 1 and n_dims > 1 else 0
-    data = data.to(dtype = torch.float16 if ftype_cur == 1 else torch.float32).numpy()
-    print(f'  |', name, data.shape, '->', data.dtype)
-    # header
-    str = name.encode('utf-8')
-    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str)
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-backend/scripts/convert_gptj_to_gguf.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+# Convert GPT-J-6B h5 transformer model to ggml format
+#
+# Load the model using GPTJForCausalLM.
+# Iterate over all variables and write them to a binary file.
+#
+# For each variable, write the following:
+#   - Number of dimensions (int)
+#   - Name length (int)
+#   - Dimensions (int[n_dims])
+#   - Name (char[name_length])
+#   - Data (float[n_dims])
+#
+# By default, the bigger matrices are converted to 16-bit floats.
+# This can be disabled by adding the "ftype" CLI argument.
+#
+# At the start of the ggml file we write the model parameters
+# and vocabulary.
+#
+
+from __future__ import annotations
+
+import sys
+import struct
+import json
+from pathlib import Path
+
+import gguf
+import numpy as np
+from transformers import AutoTokenizer, GPTJConfig, GPTJForCausalLM
+from transformers.models.gpt2 import tokenization_gpt2
+
+
+if not 2 <= len(sys.argv) < 4:
+    print("Usage: python {} dir-model [ftype]\n".format(Path(__file__).name))
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = Path(sys.argv[1])
+fname_out = dir_model / "ggml-model.gguf"
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+
+fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
+
+
+ARCH = gguf.MODEL_ARCH.GPTJ
+gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
+
+print("gguf: get model metadata")
+
+config = GPTJConfig(dir_model)
+
+block_count = config.n_layer
+gguf_writer.add_name("GPT-J")
+gguf_writer.add_context_length(config.n_positions)
+gguf_writer.add_embedding_length(config.n_embd)
+gguf_writer.add_block_count(block_count)
+gguf_writer.add_feed_forward_length(4 * config.n_embd)
+gguf_writer.add_head_count(config.n_head)
+gguf_writer.add_rope_dimension_count(config.rotary_dim)
+gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
+gguf_writer.add_file_type(ftype)
+
+print("gguf: get gpt2 tokenizer vocab")
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+byte_encoder = tokenization_gpt2.bytes_to_unicode()
+byte_decoder = {v: k for k, v in byte_encoder.items()}
+
+tokens: list[bytearray] = []
+
+for i in range(config.vocab_size):
+    if i in reverse_vocab:
+        try:
+            text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+        except KeyError:
+            text = bytearray()
+            for c in reverse_vocab[i]:
+                if ord(c) < 256:  # single byte character
+                    text.append(byte_decoder[c])
+                else:  # multibyte special token character
+                    text.extend(c.encode('utf-8'))
+    else:
+        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
+        pad_token = f"[PAD{i}]".encode("utf8")
+        text = bytearray(pad_token)
+
+    tokens.append(text)
+
+
+gguf_writer.add_tokenizer_model("gpt2")
+gguf_writer.add_token_list(tokens)
+
+special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
+special_vocab.add_to_gguf(gguf_writer)
+
+print("gguf: get tensor metadata")
+
+model = GPTJForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
+#print (model)
+
+tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
+
+list_vars = model.state_dict()
+#print (list_vars)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable:", name, "with shape:", data.shape)
+
+    # we don't need these
+    if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
+        print("  Skipping variable:", name)
+        continue
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+        print("  Converting to float16")
+        data = data.astype(np.float16)
+        ftype_cur = 1
+    elif ftype == 1 or data.dtype != np.float32:
+        print("  Converting to float32")
+        data = data.astype(np.float32)
+        ftype_cur = 0
+
+    # map tensor names
+    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
+    if new_name is None:
+        print("Can not map tensor '" + name + "'")
+        sys.exit()
+
+    gguf_writer.add_tensor(new_name, data)
+
+
+print("gguf: write header")
+gguf_writer.write_header_to_file()
+print("gguf: write metadata")
+gguf_writer.write_kv_data_to_file()
+print("gguf: write tensors")
+gguf_writer.write_tensors_to_file()
+
+gguf_writer.close()
+
+print(f"gguf: model successfully exported to '{fname_out}'")
+print()

gpt4all-backend/scripts/convert_mpt_hf_to_ggml.py

@@ -1,145 +0,0 @@
-# Convert Hugging Face fine-tuned bloom-like models to ggml format
-#
-# Usage:
-#
-#   python3 models/convert-h5-to-ggml.py 
-#
-# This script is similar to "convert-pt-to-ggml.py"
-#
-
-import io
-import os
-import sys
-import struct
-import json
-import code
-import torch
-import numpy as np
-
-from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BloomForCausalLM
-
-# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
-def bytes_to_unicode():
-    """
-    Returns list of utf-8 byte and a corresponding list of unicode strings.
-    The reversible bpe codes work on unicode strings.
-    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
-    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
-    This is a significant percentage of your normal, say, 32K bpe vocab.
-    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
-    And avoids mapping to whitespace/control characters the bpe code barfs on.
-    """
-    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
-    cs = bs[:]
-    n = 0
-    for b in range(2**8):
-        if b not in bs:
-            bs.append(b)
-            cs.append(2**8+n)
-            n += 1
-    cs = [chr(n) for n in cs]
-    return dict(zip(bs, cs))
-
-if len(sys.argv) < 3:
-    print("Usage: python convert-hf-to-ggml.py model_name dir-output [use-f32]")
-    print("  model_name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
-    print("  dir-output: directory where the output file will be written")
-    print("  use-f32:    if present, use float32 instead of float16")
-    sys.exit(1)
-
-model_name = sys.argv[1]
-dir_out = sys.argv[2]
-
-# make sure the output directory exists
-os.makedirs(dir_out, exist_ok=True)
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-ftype = 1
-if len(sys.argv) > 3:
-    ftype = 0
-
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
-hparams = config.to_dict()
-print("Loading model: ", model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32, low_cpu_mem_usage=True)
-print("Model loaded: ", model_name)
-
-
-fname_out = dir_out + f"/ggml-model-{model_name.split('/')[-1]}-{ftype_str[ftype]}.bin"
-fout = open(fname_out, "wb")
-vocab = tokenizer.vocab
-
-hparams["multiple_of"] = 1
-fout.write(struct.pack("I", 0x67676d6d)) # magic: ggml in hex
-fout.write(struct.pack("I", model.config.vocab_size))
-fout.write(struct.pack("I", model.config.max_seq_len))
-fout.write(struct.pack("I", model.config.n_layers))
-fout.write(struct.pack("I", model.config.n_heads))
-fout.write(struct.pack("I", model.config.d_model))
-fout.write(struct.pack("f", model.config.attn_config['alibi_bias_max']))
-clip_qkv = model.config.attn_config['clip_qkv']
-fout.write(struct.pack("f",  clip_qkv if clip_qkv is not None else 0))
-fout.write(struct.pack("I", ftype))
-
-# # Is this correct??
-# dot_token = tokenizer.encode(".")[0]
-# write tokens to ggml file 
-dot_token = tokenizer.encode('.')[0]
-fout.write(struct.pack("I", model.config.vocab_size))
-
-for i in range(model.config.vocab_size):
-    text = tokenizer.decode([dot_token, i]).encode('utf-8')
-    # remove the first byte (it's always '.')
-    text = text[1:]
-    enclen = len(text)
-    if i in tokenizer.all_special_ids:
-        print(f"special token: {text}")
-        enclen = enclen | 1<<31
-    fout.write(struct.pack("I", enclen))
-    fout.write(text)
-    
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable: " + name + " with shape: ", data.shape)
-
-    n_dims = len(data.shape);
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0;
-    if ftype != 0:
-        # Keep token embeddings in fp32
-        if name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
-            print("  Converting to float16")
-            data = data.astype(np.float16)
-            ftype_cur = 1
-        else:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-    else:
-        if data.dtype != np.float32:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-
-    # header
-    str = name.encode('utf-8')
-    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str);
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-backend/scripts/convert_mpt_hf_to_gguf.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+# Convert Hugging Face fine-tuned bloom-like models to ggml format
+#
+# Usage:
+#
+#   python3 models/convert-h5-to-ggml.py 
+#
+# This script is similar to "convert-pt-to-ggml.py"
+#
+
+from __future__ import annotations
+
+import json
+import struct
+import sys
+from pathlib import Path
+
+import gguf
+import numpy as np
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, MptConfig
+from transformers.models.gpt2 import tokenization_gpt2
+
+
+if not 3 <= len(sys.argv) < 5:
+    print("Usage: {} model-name dir-output [ftype]".format(Path(__file__).name))
+    print("  model-name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
+    print("  dir-output: directory where the output file will be written")
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+dir_model = Path(sys.argv[1])
+dir_out = Path(sys.argv[2])
+
+# make sure the output directory exists
+dir_out.mkdir(exist_ok=True)
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 3:
+    ftype = int(sys.argv[3])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+
+fname_out = dir_out / f"ggml-model-{dir_model.name}-{ftype_str[ftype]}.gguf"
+
+
+ARCH = gguf.MODEL_ARCH.MPT
+gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
+
+print("gguf: get model metadata")
+
+config = AutoConfig.from_pretrained(dir_model, trust_remote_code=True)
+
+block_count = config.n_layers
+gguf_writer.add_name("MPT")
+gguf_writer.add_context_length(config.max_seq_len)
+gguf_writer.add_embedding_length(config.d_model)
+gguf_writer.add_block_count(block_count)
+gguf_writer.add_feed_forward_length(4 * config.d_model)
+gguf_writer.add_head_count(config.n_heads)
+if kv_n_heads := config.attn_config.get('kv_n_heads'):
+    gguf_writer.add_head_count_kv(kv_n_heads)
+gguf_writer.add_max_alibi_bias(config.attn_config['alibi_bias_max'])
+gguf_writer.add_layer_norm_eps(MptConfig().layer_norm_epsilon)  # use default from upstream transformers
+gguf_writer.add_file_type(ftype)
+
+clip_qkv = config.attn_config['clip_qkv']
+if clip_qkv is not None:
+    gguf_writer.add_clamp_kqv(clip_qkv)
+
+print("gguf: get gpt2 tokenizer vocab")
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+
+special_ids = tokenizer.all_special_ids
+
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+added_tokens = tokenizer.get_added_vocab().values()
+byte_encoder = tokenization_gpt2.bytes_to_unicode()
+byte_decoder = {v: k for k, v in byte_encoder.items()}
+
+tokens: list[bytearray] = []
+toktypes: list[gguf.TokenType] = []
+
+# The number of tokens in tokenizer.json can differ from the expected vocab size.
+# This causes downstream issues with mismatched tensor sizes when running the inference
+for i in range(config.vocab_size):
+    if i not in reverse_vocab:
+        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
+        pad_token = f"[PAD{i}]".encode("utf8")
+        text = bytearray(pad_token)
+    elif i in added_tokens:
+        # these tokens are not encoded, for some reason
+        text = bytearray(reverse_vocab[i].encode('utf-8'))
+    else:
+        text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+
+    tokens.append(text)
+
+    # TODO(cebtenzzre): is there a better way to do this?
+    toktypes.append(gguf.TokenType.CONTROL if i in special_ids else gguf.TokenType.NORMAL)
+
+gguf_writer.add_tokenizer_model("gpt2")
+gguf_writer.add_token_list(tokens)
+gguf_writer.add_token_types(toktypes)
+
+special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
+special_vocab.add_to_gguf(gguf_writer)
+
+print("gguf: get tensor metadata")
+
+print("Loading model:", dir_model)
+model = AutoModelForCausalLM.from_pretrained(
+    dir_model, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32,
+    low_cpu_mem_usage=True, trust_remote_code=True,
+)
+print("Model loaded:", dir_model)
+
+tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable:", name, "with shape:", data.shape)
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0
+    # Keep token embeddings in fp32
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
+        print("  Converting to float16")
+        data = data.astype(np.float16)
+        ftype_cur = 1
+    elif ftype == 1 or data.dtype != np.float32:
+        print("  Converting to float32")
+        data = data.astype(np.float32)
+        ftype_cur = 0
+
+    # map tensor names
+    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
+    if new_name is None:
+        print("Can not map tensor '" + name + "'")
+        sys.exit()
+
+    gguf_writer.add_tensor(new_name, data)
+
+
+print("gguf: write header")
+gguf_writer.write_header_to_file()
+print("gguf: write metadata")
+gguf_writer.write_kv_data_to_file()
+print("gguf: write tensors")
+gguf_writer.write_tensors_to_file()
+
+gguf_writer.close()
+
+print(f"gguf: model successfully exported to '{fname_out}'")
+print()

gpt4all-backend/scripts/convert_replit_hf_to_ggml.py

@@ -1,113 +0,0 @@
-from pathlib import Path
-import sys
-import struct
-import json
-import numpy as np
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import sentencepiece.sentencepiece_model_pb2 as model
-
-if len(sys.argv) < 3:
-    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-
-# output in the same directory as the model
-dir_model = sys.argv[1]
-fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b.bin"
-
-
-with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
-    hparams = json.load(f)
-
-sp_proto = model.ModelProto()
-sp_proto.ParseFromString(open(Path(sys.argv[1]) / "spiece.model", "rb").read())
-
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-    fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".bin"
-
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(
-    dir_model, low_cpu_mem_usage=True, trust_remote_code=True
-)
-# print (model)
-
-# print(tokenizer.encode('I believe the meaning of life is'))
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-fout = open(fname_out, "wb")
-
-print(hparams)
-
-fout.write(struct.pack("i", 0x7265706c))  # magic: repl in hex
-fout.write(struct.pack("i", hparams["vocab_size"]))
-fout.write(struct.pack("i", hparams["max_seq_len"]))
-fout.write(struct.pack("i", hparams["d_model"]))
-fout.write(struct.pack("i", hparams["n_heads"]))
-fout.write(struct.pack("i", hparams["n_layers"]))
-fout.write(struct.pack("i", ftype))
-
-
-# TODO: temporary hack to not deal with implementing the tokenizer
-for piece in sp_proto.pieces:
-    encoded_piece = piece.piece.encode("utf-8")
-    fout.write(struct.pack("i", len(encoded_piece)))
-    fout.write(encoded_piece)
-    fout.write(struct.pack("f", piece.score))
-
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable: " + name + " with shape: ", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    if ftype != 0:
-        if name[-7:] == ".weight" and n_dims == 2:
-            print("  Converting to float16")
-            data = data.astype(np.float16)
-            ftype_cur = 1
-        else:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-    else:
-        if data.dtype != np.float32:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-
-    # header
-    str = name.encode("utf-8")
-    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str)
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-backend/scripts/convert_replit_v1_hf_to_gguf.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import struct
+import sys
+from pathlib import Path
+
+import gguf
+import numpy as np
+from sentencepiece import SentencePieceProcessor
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+
+if not 2 <= len(sys.argv) < 4:
+    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = Path(sys.argv[1])
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+
+fname_out = dir_model / ("ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".gguf")
+
+
+ARCH = gguf.MODEL_ARCH.MPT
+gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
+
+print("gguf: get model metadata")
+
+config = AutoConfig.from_pretrained(dir_model)
+
+block_count = config.n_layers
+gguf_writer.add_name("Replit")
+gguf_writer.add_context_length(config.max_seq_len)
+gguf_writer.add_embedding_length(config.d_model)
+gguf_writer.add_block_count(block_count)
+gguf_writer.add_feed_forward_length(4 * config.d_model)
+gguf_writer.add_head_count(config.n_heads)
+gguf_writer.add_max_alibi_bias(config.attn_config.alibi_bias_max)
+gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
+gguf_writer.add_file_type(ftype)
+
+clip_qkv = config.attn_config.clip_qkv
+if clip_qkv is not None:
+    gguf_writer.add_clamp_kqv(clip_qkv)
+
+print("gguf: get sentencepiece tokenizer vocab")
+
+tokenizer = SentencePieceProcessor(str(dir_model / "spiece.model"))
+#print(tokenizer.encode('I believe the meaning of life is'))
+
+tokens: list[bytearray] = []
+scores: list[float] = []
+toktypes: list[int] = []
+
+for i in range(tokenizer.vocab_size()):
+    tokens.append(tokenizer.id_to_piece(i).encode('utf-8'))
+    scores.append(tokenizer.get_score(i))
+
+    toktype = gguf.TokenType.NORMAL
+    if tokenizer.is_unknown(i):
+        toktype = gguf.TokenType.UNKNOWN
+    elif tokenizer.is_control(i):
+        toktype = gguf.TokenType.CONTROL
+    elif tokenizer.is_unused(i):
+        toktype = gguf.TokenType.UNUSED
+    elif tokenizer.is_byte(i):
+        toktype = gguf.TokenType.BYTE
+
+    toktypes.append(toktype)
+
+gguf_writer.add_tokenizer_model("llama")  # sentencepiece
+gguf_writer.add_token_list(tokens)
+gguf_writer.add_token_scores(scores)
+gguf_writer.add_token_types(toktypes)
+
+special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
+special_vocab.add_to_gguf(gguf_writer)
+
+print("gguf: get tensor metadata")
+
+model = AutoModelForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
+#print(model)
+
+tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+print(config)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable:", name, "with shape:", data.shape)
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+        print("  Converting to float16")
+        data = data.astype(np.float16)
+        ftype_cur = 1
+    elif ftype == 1 or data.dtype != np.float32:
+        print("  Converting to float32")
+        data = data.astype(np.float32)
+        ftype_cur = 0
+
+    # map tensor names
+    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
+    if new_name is None:
+        print("Can not map tensor '" + name + "'")
+        sys.exit()
+
+    gguf_writer.add_tensor(new_name, data)
+
+
+print("gguf: write header")
+gguf_writer.write_header_to_file()
+print("gguf: write metadata")
+gguf_writer.write_kv_data_to_file()
+print("gguf: write tensors")
+gguf_writer.write_tensors_to_file()
+
+gguf_writer.close()
+
+print(f"gguf: model successfully exported to '{fname_out}'")
+print()

gpt4all-backend/starcoder_impl.h

@@ -1,42 +0,0 @@
-#ifndef STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of starcoder.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef STARCODER_H
-#define STARCODER_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include <memory>
-#include "llmodel.h"
-
-struct StarcoderPrivate;
-class Starcoder : public LLModel {
-public:
-    Starcoder();
-    ~Starcoder();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    std::unique_ptr<StarcoderPrivate> d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // STARCODER_H

gpt4all-bindings/cli/README.md

@@ -40,5 +40,5 @@ directory, if necessary.
 If you have already saved a model beforehand, specify its path with the `-m`/`--model` argument,
 for example:
 ```shell
-python app.py repl --model /home/user/my-gpt4all-models/GPT4All-13B-snoozy.ggmlv3.q4_0.bin
+python app.py repl --model /home/user/my-gpt4all-models/gpt4all-13b-snoozy-q4_0.gguf
 ```

gpt4all-bindings/cli/app.py

@@ -1,16 +1,17 @@
+#!/usr/bin/env python3
 """GPT4All CLI
 
 The GPT4All CLI is a self-contained script based on the `gpt4all` and `typer` packages. It offers a
 REPL to communicate with a language model similar to the chat GUI application, but more basic.
 """
 
+import importlib.metadata
 import io
-import pkg_resources  # should be present as a dependency of gpt4all
 import sys
-import typer
-
 from collections import namedtuple
 from typing_extensions import Annotated
+
+import typer
 from gpt4all import GPT4All
 
 
@@ -53,7 +54,7 @@ def repl(
     model: Annotated[
         str,
         typer.Option("--model", "-m", help="Model to use for chatbot"),
-    ] = "ggml-gpt4all-j-v1.3-groovy",
+    ] = "mistral-7b-instruct-v0.1.Q4_0.gguf",
     n_threads: Annotated[
         int,
         typer.Option("--n-threads", "-t", help="Number of threads to use for chatbot"),
@@ -79,7 +80,7 @@ def repl(
 
     use_new_loop = False
     try:
-        version = pkg_resources.Environment()['gpt4all'][0].version
+        version = importlib.metadata.version('gpt4all')
         version_major = int(version.split('.')[0])
         if version_major >= 1:
             use_new_loop = True

gpt4all-bindings/csharp/README.md

@@ -23,6 +23,12 @@ gpt4all-bindings/
           └── linux-x64
 ```
 
+## Prerequisites
+
+On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+
+macOS users do not need Vulkan, as GPT4All will use Metal instead.
+
 ## Local Build Instructions
 > **Note** 
 > Tested On:
@@ -54,7 +60,7 @@ chmod +x ./build_linux.sh
 1. Setup
 ```
 choco install mingw
-$env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
+$env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
 choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
 ```
 2. Run the `./build_win-mingw.ps1` build script

gpt4all-bindings/csharp/build_linux.sh

@@ -1,3 +1,4 @@
+#!/bin/sh
 mkdir -p runtimes
 rm -rf runtimes/linux-x64
 mkdir -p runtimes/linux-x64/native
@@ -7,4 +8,3 @@ cmake --build runtimes/linux-x64/build --parallel --config Release
 cp runtimes/linux-x64/build/libllmodel.so  runtimes/linux-x64/native/libllmodel.so
 cp runtimes/linux-x64/build/libgptj*.so  runtimes/linux-x64/native/
 cp runtimes/linux-x64/build/libllama*.so  runtimes/linux-x64/native/
-cp runtimes/linux-x64/build/libmpt*.so  runtimes/linux-x64/native/

gpt4all-bindings/csharp/build_win-mingw.ps1

@@ -12,5 +12,5 @@ cmake -G "MinGW Makefiles" -S ..\..\gpt4all-backend -B $BUILD_DIR
 cmake --build $BUILD_DIR --parallel --config Release
 
 # copy native dlls
-cp "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll" $LIBS_DIR
-cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR
+cp "C:\ProgramData\mingw64\mingw64\bin\*dll" $LIBS_DIR
+cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR

gpt4all-bindings/java/README.md

@@ -22,7 +22,7 @@ implementation 'com.hexadevlabs:gpt4all-java-binding:1.1.5'
 
 To add the library dependency for another build system see [Maven Central Java bindings](https://central.sonatype.com/artifact/com.hexadevlabs/gpt4all-java-binding/).
 
-To download model binary weights file use a URL such as [`https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin`](https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin).
+To download model binary weights file use a URL such as [`https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf`](https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf).
 
 For information about other models available see the [model file list](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-chat#manual-download-of-models).
 
@@ -123,4 +123,4 @@ If this is the case you can easily download and install the latest x64 Microsoft
    - Falcon model support included.
 4. Version **1.1.5**:
    - Add a check for model file readability before loading model.
-   
+   

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java

@@ -8,9 +8,8 @@ import java.io.ByteArrayOutputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+import java.util.stream.Collectors;
 
 public  class LLModel implements AutoCloseable {
 
@@ -306,6 +305,197 @@ public  class LLModel implements AutoCloseable {
         };
     }
 
+    /**
+     * The array of messages for the conversation.
+     */
+    public static class Messages {
+
+        private final List<PromptMessage> messages = new ArrayList<>();
+
+        public Messages(PromptMessage...messages) {
+            this.messages.addAll(Arrays.asList(messages));
+        }
+
+        public Messages(List<PromptMessage> messages) {
+            this.messages.addAll(messages);
+        }
+
+        public Messages addPromptMessage(PromptMessage promptMessage) {
+            this.messages.add(promptMessage);
+            return this;
+        }
+
+        List<PromptMessage> toList() {
+            return Collections.unmodifiableList(this.messages);
+        }
+
+        List<Map<String, String>> toListMap() {
+            return messages.stream()
+                    .map(PromptMessage::toMap).collect(Collectors.toList());
+        }
+
+    }
+
+    /**
+     * A message in the conversation, identical to OpenAI's chat message.
+     */
+    public static class PromptMessage {
+
+        private static final String ROLE = "role";
+        private static final String CONTENT = "content";
+
+        private final Map<String, String> message = new HashMap<>();
+
+        public PromptMessage() {
+        }
+
+        public PromptMessage(Role role, String content) {
+            addRole(role);
+            addContent(content);
+        }
+
+        public PromptMessage addRole(Role role) {
+            return this.addParameter(ROLE, role.type());
+        }
+
+        public PromptMessage addContent(String content) {
+            return this.addParameter(CONTENT, content);
+        }
+
+        public PromptMessage addParameter(String key, String value) {
+            this.message.put(key, value);
+            return this;
+        }
+
+        public String content() {
+            return this.parameter(CONTENT);
+        }
+
+        public Role role() {
+            String role = this.parameter(ROLE);
+            return Role.from(role);
+        }
+
+        public String parameter(String key) {
+            return this.message.get(key);
+        }
+
+        Map<String, String> toMap() {
+            return Collections.unmodifiableMap(this.message);
+        }
+
+    }
+
+    public enum Role {
+
+        SYSTEM("system"), ASSISTANT("assistant"), USER("user");
+
+        private final String type;
+
+        String type() {
+            return this.type;
+        }
+
+        static Role from(String type) {
+
+            if (type == null) {
+                return null;
+            }
+
+            switch (type) {
+                case "system": return SYSTEM;
+                case "assistant": return ASSISTANT;
+                case "user": return USER;
+                default: throw new IllegalArgumentException(
+                        String.format("You passed %s type but only %s are supported",
+                                type, Arrays.toString(Role.values())
+                        )
+                );
+            }
+        }
+
+        Role(String type) {
+            this.type = type;
+        }
+
+        @Override
+        public String toString() {
+            return type();
+        }
+    }
+
+    /**
+     * The result of the completion, similar to OpenAI's format.
+     */
+    public static class CompletionReturn {
+        private String model;
+        private Usage usage;
+        private Choices choices;
+
+        public CompletionReturn(String model, Usage usage, Choices choices) {
+            this.model = model;
+            this.usage = usage;
+            this.choices = choices;
+        }
+
+        public Choices choices() {
+            return choices;
+        }
+
+        public String model() {
+            return model;
+        }
+
+        public Usage usage() {
+            return usage;
+        }
+    }
+
+    /**
+     * The generated completions.
+     */
+    public static class Choices {
+
+        private final List<CompletionChoice> choices = new ArrayList<>();
+
+        public Choices(List<CompletionChoice> choices) {
+            this.choices.addAll(choices);
+        }
+
+        public Choices(CompletionChoice...completionChoices){
+            this.choices.addAll(Arrays.asList(completionChoices));
+        }
+
+        public Choices addCompletionChoice(CompletionChoice completionChoice) {
+            this.choices.add(completionChoice);
+            return this;
+        }
+
+        public CompletionChoice first() {
+            return this.choices.get(0);
+        }
+
+        public int totalChoices() {
+            return this.choices.size();
+        }
+
+        public CompletionChoice get(int index) {
+            return this.choices.get(index);
+        }
+
+        public List<CompletionChoice> choices() {
+            return Collections.unmodifiableList(choices);
+        }
+    }
+
+    /**
+     * A completion choice, similar to OpenAI's format.
+     */
+    public static class CompletionChoice extends PromptMessage {
+        public CompletionChoice(Role role, String content) {
+            super(role, content);
+        }
+    }
 
     public static class ChatCompletionResponse {
         public String model;
@@ -323,6 +513,41 @@ public  class LLModel implements AutoCloseable {
         // Getters and setters
     }
 
+    public CompletionReturn chatCompletionResponse(Messages messages,
+                                                                GenerationConfig generationConfig) {
+        return chatCompletion(messages, generationConfig, false, false);
+    }
+
+    /**
+     * chatCompletion formats the existing chat conversation into a template to be
+     * easier to process for chat UIs. It is not absolutely necessary as generate method
+     * may be directly used to make generations with gpt models.
+     *
+     * @param messages object to create theMessages to send to GPT model
+     * @param generationConfig How to decode/process the generation.
+     * @param streamToStdOut Send tokens as they are calculated Standard output.
+     * @param outputFullPromptToStdOut Should full prompt built out of messages be sent to Standard output.
+     * @return CompletionReturn contains stats and generated Text.
+     */
+    public CompletionReturn chatCompletion(Messages messages,
+                                                  GenerationConfig generationConfig, boolean streamToStdOut,
+                                                  boolean outputFullPromptToStdOut) {
+
+        String fullPrompt = buildPrompt(messages.toListMap());
+
+        if(outputFullPromptToStdOut)
+            System.out.print(fullPrompt);
+
+        String generatedText = generate(fullPrompt, generationConfig, streamToStdOut);
+
+        final CompletionChoice promptMessage = new CompletionChoice(Role.ASSISTANT, generatedText);
+        final Choices choices = new Choices(promptMessage);
+
+        final Usage usage = getUsage(fullPrompt, generatedText);
+        return new CompletionReturn(this.modelName, usage, choices);
+
+    }
+
     public ChatCompletionResponse chatCompletion(List<Map<String, String>> messages,
                                                               GenerationConfig generationConfig) {
         return chatCompletion(messages, generationConfig, false, false);
@@ -352,19 +577,23 @@ public  class LLModel implements AutoCloseable {
         ChatCompletionResponse  response = new ChatCompletionResponse();
         response.model = this.modelName;
 
-        Usage usage = new Usage();
-        usage.promptTokens = fullPrompt.length();
-        usage.completionTokens = generatedText.length();
-        usage.totalTokens = fullPrompt.length() + generatedText.length();
-        response.usage =  usage;
+        response.usage = getUsage(fullPrompt, generatedText);
 
         Map<String, String> message = new HashMap<>();
         message.put("role", "assistant");
         message.put("content", generatedText);
 
         response.choices = List.of(message);
-
         return response;
+
+    }
+
+    private Usage getUsage(String fullPrompt, String generatedText) {
+        Usage usage = new Usage();
+        usage.promptTokens = fullPrompt.length();
+        usage.completionTokens = generatedText.length();
+        usage.totalTokens = fullPrompt.length() + generatedText.length();
+        return usage;
     }
 
     protected static String buildPrompt(List<Map<String, String>> messages) {

gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/BasicTests.java

@@ -28,6 +28,33 @@ import static org.mockito.Mockito.*;
 @ExtendWith(MockitoExtension.class)
 public class BasicTests {
 
+    @Test
+    public void simplePromptWithObject(){
+
+        LLModel model = Mockito.spy(new LLModel());
+
+        LLModel.GenerationConfig config =
+                LLModel.config()
+                        .withNPredict(20)
+                        .build();
+
+        // The generate method will return "4"
+        doReturn("4").when( model ).generate(anyString(), eq(config), eq(true));
+
+        LLModel.PromptMessage promptMessage1 = new LLModel.PromptMessage(LLModel.Role.SYSTEM, "You are a helpful assistant");
+        LLModel.PromptMessage promptMessage2 = new LLModel.PromptMessage(LLModel.Role.USER, "Add 2+2");
+
+        LLModel.Messages messages = new LLModel.Messages(promptMessage1, promptMessage2);
+
+        LLModel.CompletionReturn response = model.chatCompletion(
+                messages, config, true, true);
+
+        assertTrue( response.choices().first().content().contains("4") );
+
+        // Verifies the prompt and response are certain length.
+        assertEquals( 224 , response.usage().totalTokens );
+    }
+
     @Test
     public void simplePrompt(){
 

gpt4all-bindings/python/README.md

@@ -15,12 +15,20 @@ pip install gpt4all
 
 ## Local Build Instructions
 
+### Prerequisites
+
+On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+
+macOS users do not need Vulkan, as GPT4All will use Metal instead.
+
+### Building the python bindings
+
 **NOTE**: If you are doing this on a Windows machine, you must build the GPT4All backend using [MinGW64](https://www.mingw-w64.org/) compiler.
 
 1. Setup `llmodel`
 
 ```
-git clone --recurse-submodules git@github.com:nomic-ai/gpt4all.git
+git clone --recurse-submodules https://github.com/nomic-ai/gpt4all.git
 cd gpt4all/gpt4all-backend/
 mkdir build
 cd build
@@ -42,7 +50,7 @@ Test it out! In a Python script or console:
 
 ```python
 from gpt4all import GPT4All
-model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin")
+model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
 output = model.generate("The capital of France is ", max_tokens=3)
 print(output)
 ```
@@ -51,7 +59,7 @@ print(output)
 GPU Usage
 ```python
 from gpt4all import GPT4All
-model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin", device='gpu') # device='amd', device='intel'
+model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf", device='gpu') # device='amd', device='intel'
 output = model.generate("The capital of France is ", max_tokens=3)
 print(output)
 ```

gpt4all-bindings/python/docs/gpt4all_chat.md

@@ -7,7 +7,7 @@ It is optimized to run 7-13B parameter LLMs on the CPU's of any computer running
 ## Running LLMs on CPU
 The GPT4All Chat UI supports models from all newer versions of `GGML`, `llama.cpp` including the `LLaMA`, `MPT`, `replit`,  `GPT-J` and `falcon` architectures
 
-GPT4All maintains an official list of recommended models located in [models.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
+GPT4All maintains an official list of recommended models located in [models2.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
 
 #### Sideloading any GGML model
 If a model is compatible with the gpt4all-backend, you can sideload it into GPT4All Chat by:

gpt4all-bindings/python/docs/gpt4all_cli.md

@@ -166,7 +166,7 @@ If you want to use a different model, you can do so with the `-m`/`--model` para
 model file name is provided, it will again check in `.cache/gpt4all/` and might start downloading.
 If instead given a path to an existing model, the command could for example look like this:
 ```shell
-python app.py repl --model /home/user/my-gpt4all-models/GPT4All-13B-snoozy.ggmlv3.q4_0.bin
+python app.py repl --model /home/user/my-gpt4all-models/gpt4all-13b-snoozy-q4_0.gguf
 ```
 
 When you're done and want to end a session, simply type `/exit`.

gpt4all-bindings/python/docs/gpt4all_faq.md

@@ -61,12 +61,12 @@ or `allowDownload=true` (default), a model is automatically downloaded into `.ca
 unless it already exists.
 
 In case of connection issues or errors during the download, you might want to manually verify the model file's MD5
-checksum by comparing it with the one listed in [models.json].
+checksum by comparing it with the one listed in [models2.json].
 
 As an alternative to the basic downloader built into the bindings, you can choose to download from the 
 <https://gpt4all.io/> website instead. Scroll down to 'Model Explorer' and pick your preferred model.
 
-[models.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json
+[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
 
 #### I need the chat GUI and bindings to behave the same
 
@@ -93,7 +93,7 @@ The chat GUI and bindings are based on the same backend. You can make them behav
 - Next you'll have to compare the templates, adjusting them as necessary, based on how you're using the bindings.
     - Specifically, in Python:
         - With simple `generate()` calls, the input has to be surrounded with system and prompt templates.
-        - When using a chat session, it depends on whether the bindings are allowed to download [models.json]. If yes,
+        - When using a chat session, it depends on whether the bindings are allowed to download [models2.json]. If yes,
           and in the chat GUI the default templates are used, it'll be handled automatically. If no, use
           `chat_session()` template parameters to customize them.
 

gpt4all-bindings/python/docs/gpt4all_modal.md

@@ -8,7 +8,7 @@ import modal
 
 def download_model():
     import gpt4all
-    #you can use any model from https://gpt4all.io/models/models.json
+    #you can use any model from https://gpt4all.io/models/models2.json
     return gpt4all.GPT4All("ggml-gpt4all-j-v1.3-groovy.bin")
 
 image=modal.Image.debian_slim().pip_install("gpt4all").run_function(download_model)
@@ -31,4 +31,4 @@ def main():
     model = GPT4All()
     for i in range(10):
         model.generate.call()
-```
+```

gpt4all-bindings/python/docs/gpt4all_python.md

@@ -11,7 +11,7 @@ pip install gpt4all
 === "GPT4All Example"
     ``` py
     from gpt4all import GPT4All
-    model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin")
+    model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
     output = model.generate("The capital of France is ", max_tokens=3)
     print(output)
     ```
@@ -35,7 +35,7 @@ Use the GPT4All `chat_session` context manager to hold chat conversations with t
 
 === "GPT4All Example"
     ``` py
-    model = GPT4All(model_name='orca-mini-3b.ggmlv3.q4_0.bin')
+    model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf')
     with model.chat_session():
         response1 = model.generate(prompt='hello', temp=0)
         response2 = model.generate(prompt='write me a short poem', temp=0)
@@ -77,10 +77,10 @@ When using GPT4All models in the `chat_session` context:
 - Consecutive chat exchanges are taken into account and not discarded until the session ends; as long as the model has capacity.
 - Internal K/V caches are preserved from previous conversation history, speeding up inference.
 - The model is given a system and prompt template which make it chatty. Depending on `allow_download=True` (default),
-  it will obtain the latest version of [models.json] from the repository, which contains specifically tailored templates
+  it will obtain the latest version of [models2.json] from the repository, which contains specifically tailored templates
   for models. Conversely, if it is not allowed to download, it falls back to default templates instead.
 
-[models.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json
+[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
 
 
 ### Streaming Generations
@@ -89,7 +89,7 @@ To interact with GPT4All responses as the model generates, use the `streaming=Tr
 === "GPT4All Streaming Example"
     ``` py
     from gpt4all import GPT4All
-    model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin")
+    model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
     tokens = []
     for token in model.generate("The capital of France is", max_tokens=20, streaming=True):
         tokens.append(token)
@@ -135,7 +135,7 @@ is the same as if it weren't provided; that is, `~/.cache/gpt4all/` is the defau
     ``` py
     from pathlib import Path
     from gpt4all import GPT4All
-    model = GPT4All(model_name='orca-mini-3b.ggmlv3.q4_0.bin',
+    model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf',
                     model_path=(Path.home() / '.cache' / 'gpt4all'),
                     allow_download=False)
     response = model.generate('my favorite 3 fruits are:', temp=0)
@@ -152,7 +152,7 @@ If you want to point it at the chat GUI's default folder, it should be:
     from pathlib import Path
     from gpt4all import GPT4All
 
-    model_name = 'orca-mini-3b.ggmlv3.q4_0.bin'
+    model_name = 'orca-mini-3b-gguf2-q4_0.gguf'
     model_path = Path.home() / 'Library' / 'Application Support' / 'nomic.ai' / 'GPT4All'
     model = GPT4All(model_name, model_path)
     ```
@@ -161,7 +161,7 @@ If you want to point it at the chat GUI's default folder, it should be:
     from pathlib import Path
     from gpt4all import GPT4All
     import os
-    model_name = 'orca-mini-3b.ggmlv3.q4_0.bin'
+    model_name = 'orca-mini-3b-gguf2-q4_0.gguf'
     model_path = Path(os.environ['LOCALAPPDATA']) / 'nomic.ai' / 'GPT4All'
     model = GPT4All(model_name, model_path)
     ```
@@ -170,7 +170,7 @@ If you want to point it at the chat GUI's default folder, it should be:
     from pathlib import Path
     from gpt4all import GPT4All
 
-    model_name = 'orca-mini-3b.ggmlv3.q4_0.bin'
+    model_name = 'orca-mini-3b-gguf2-q4_0.gguf'
     model_path = Path.home() / '.local' / 'share' / 'nomic.ai' / 'GPT4All'
     model = GPT4All(model_name, model_path)
     ```
@@ -182,7 +182,7 @@ from pathlib import Path
 import gpt4all.gpt4all
 gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = Path.home() / 'my' / 'models-directory'
 from gpt4all import GPT4All
-model = GPT4All('orca-mini-3b.ggmlv3.q4_0.bin')
+model = GPT4All('orca-mini-3b-gguf2-q4_0.gguf')
 ...
 ```
 
@@ -193,7 +193,7 @@ Session templates can be customized when starting a `chat_session` context:
 === "GPT4All Custom Session Templates Example"
     ``` py
     from gpt4all import GPT4All
-    model = GPT4All('ggml-Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin')
+    model = GPT4All('wizardlm-13b-v1.2.Q4_0.gguf')
     system_template = 'A chat between a curious user and an artificial intelligence assistant.'
     # many models use triple hash '###' for keywords, Vicunas are simpler:
     prompt_template = 'USER: {0}\nASSISTANT: '
@@ -222,7 +222,7 @@ To do the same outside a session, the input has to be formatted manually. For ex
 
 === "GPT4All Templates Outside a Session Example"
     ``` py
-    model = GPT4All('ggml-Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin')
+    model = GPT4All('wizardlm-13b-v1.2.Q4_0.gguf')
     system_template = 'A chat between a curious user and an artificial intelligence assistant.'
     prompt_template = 'USER: {0}\nASSISTANT: '
     prompts = ['name 3 colors', 'now name 3 fruits', 'what were the 3 colors in your earlier response?']
@@ -285,7 +285,7 @@ customized in a subclass. As an example:
     ```
 === "GPT4All Custom Subclass Example"
     ``` py
-    model = RotatingTemplateGPT4All('ggml-Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin')
+    model = RotatingTemplateGPT4All('wizardlm-13b-v1.2.Q4_0.gguf')
     with model.chat_session():  # starting a session is optional in this example
         response1 = model.generate("hi, who are you?")
         print(response1)
@@ -345,7 +345,7 @@ logging infrastructure offers [many more customization options][py-logging-cookb
     import logging
     from gpt4all import GPT4All
     logging.basicConfig(level=logging.INFO)
-    model = GPT4All('nous-hermes-13b.ggmlv3.q4_0.bin')
+    model = GPT4All('nous-hermes-llama2-13b.Q4_0.gguf')
     with model.chat_session('You are a geography expert.\nBe terse.',
                             '### Instruction:\n{0}\n### Response:\n'):
         response = model.generate('who are you?', temp=0)
@@ -379,7 +379,7 @@ logging infrastructure offers [many more customization options][py-logging-cookb
 
 ### Without Online Connectivity
 To prevent GPT4All from accessing online resources, instantiate it with `allow_download=False`. This will disable both
-downloading missing models and [models.json], which contains information about them. As a result, predefined templates
+downloading missing models and [models2.json], which contains information about them. As a result, predefined templates
 are used instead of model-specific system and prompt templates:
 
 === "GPT4All Default Templates Example"
@@ -414,7 +414,7 @@ If you know exactly when a model should stop responding, you can add a custom ca
 === "GPT4All Custom Stop Callback"
     ``` py
     from gpt4all import GPT4All
-    model = GPT4All('orca-mini-3b.ggmlv3.q4_0.bin')
+    model = GPT4All('orca-mini-3b-gguf2-q4_0.gguf')
 
     def stop_on_token_callback(token_id, token_string):
         # one sentence is enough:

gpt4all-bindings/python/docs/gpt4all_typescript.md

@@ -58,6 +58,8 @@ const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional"
 *   (win) msvc version 143
     *   Can be obtained with visual studio 2022 build tools
 *   python 3
+*   On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+*   macOS users do not need Vulkan, as GPT4All will use Metal instead.
 
 ### Build (from source)
 
@@ -703,7 +705,7 @@ Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Glob
 
 ##### url
 
-Remote download url. Defaults to `https://gpt4all.io/models/<modelName>`
+Remote download url. Defaults to `https://gpt4all.io/models/gguf/<modelName>`
 
 Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
 

gpt4all-bindings/python/docs/index.md

@@ -9,7 +9,7 @@ GPT4All software is optimized to run inference of 3-13 billion parameter large l
 === "GPT4All Example"
     ``` py
     from gpt4all import GPT4All
-    model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin")
+    model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
     output = model.generate("The capital of France is ", max_tokens=3)
     print(output)
     ```
@@ -38,7 +38,7 @@ The GPT4All software ecosystem is compatible with the following Transformer arch
 - `MPT` (including `Replit`)
 - `GPT-J`
 
-You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json)
+You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json)
 
 
 GPT4All models are artifacts produced through a process known as neural network quantization.

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -1,14 +1,19 @@
 """
 Python only API for running all GPT4All models.
 """
+from __future__ import annotations
+
 import os
+import sys
 import time
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Any, Dict, Iterable, List, Optional, Union
 
 import requests
+from requests.exceptions import ChunkedEncodingError
 from tqdm import tqdm
+from urllib3.exceptions import IncompleteRead, ProtocolError
 
 from . import pyllmodel
 
@@ -29,17 +34,14 @@ class Embed4All:
     Python class that handles embeddings for GPT4All.
     """
 
-    def __init__(
-        self,
-        n_threads: Optional[int] = None,
-    ):
+    def __init__(self, model_name: Optional[str] = None, n_threads: Optional[int] = None, **kwargs):
         """
         Constructor
 
         Args:
             n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
         """
-        self.gpt4all = GPT4All(model_name='ggml-all-MiniLM-L6-v2-f16.bin', n_threads=n_threads)
+        self.gpt4all = GPT4All(model_name or 'all-MiniLM-L6-v2-f16.gguf', n_threads=n_threads, **kwargs)
 
     def embed(self, text: str) -> List[float]:
         """
@@ -62,17 +64,18 @@ class GPT4All:
     def __init__(
         self,
         model_name: str,
-        model_path: Optional[str] = None,
+        model_path: Optional[Union[str, os.PathLike[str]]] = None,
         model_type: Optional[str] = None,
         allow_download: bool = True,
         n_threads: Optional[int] = None,
         device: Optional[str] = "cpu",
+        verbose: bool = False,
     ):
         """
         Constructor
 
         Args:
-            model_name: Name of GPT4All or custom model. Including ".bin" file extension is optional but encouraged.
+            model_name: Name of GPT4All or custom model. Including ".gguf" file extension is optional but encouraged.
             model_path: Path to directory containing model file or, if file does not exist, where to download model.
                 Default is None, in which case models will be stored in `~/.cache/gpt4all/`.
             model_type: Model architecture. This argument currently does not have any functionality and is just used as
@@ -91,7 +94,7 @@ class GPT4All:
         self.model_type = model_type
         self.model = pyllmodel.LLModel()
         # Retrieve model and download if allowed
-        self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download)
+        self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download, verbose=verbose)
         if device is not None:
             if device != "cpu":
                 self.model.init_gpu(model_path=self.config["path"], device=device)
@@ -107,19 +110,22 @@ class GPT4All:
     @staticmethod
     def list_models() -> List[ConfigType]:
         """
-        Fetch model list from https://gpt4all.io/models/models.json.
+        Fetch model list from https://gpt4all.io/models/models2.json.
 
         Returns:
             Model list in JSON format.
         """
-        return requests.get("https://gpt4all.io/models/models.json").json()
+        resp = requests.get("https://gpt4all.io/models/models2.json")
+        if resp.status_code != 200:
+            raise ValueError(f'Request failed: HTTP {resp.status_code} {resp.reason}')
+        return resp.json()
 
     @staticmethod
     def retrieve_model(
         model_name: str,
-        model_path: Optional[str] = None,
+        model_path: Optional[Union[str, os.PathLike[str]]] = None,
         allow_download: bool = True,
-        verbose: bool = True,
+        verbose: bool = False,
     ) -> ConfigType:
         """
         Find model file, and if it doesn't exist, download the model.
@@ -135,7 +141,7 @@ class GPT4All:
             Model config.
         """
 
-        model_filename = append_bin_suffix_if_missing(model_name)
+        model_filename = append_extension_if_missing(model_name)
 
         # get the config for the model
         config: ConfigType = DEFAULT_MODEL_CONFIG
@@ -162,7 +168,7 @@ class GPT4All:
                 )
             model_path = DEFAULT_MODEL_DIRECTORY
         else:
-            model_path = model_path.replace("\\", "\\\\")
+            model_path = str(model_path).replace("\\", "\\\\")
 
         if not os.path.exists(model_path):
             raise ValueError(f"Invalid model directory: {model_path}")
@@ -172,7 +178,7 @@ class GPT4All:
             config.pop("url", None)
             config["path"] = model_dest
             if verbose:
-                print("Found model file at ", model_dest)
+                print("Found model file at", model_dest, file=sys.stderr)
 
         # If model file does not exist, download
         elif allow_download:
@@ -187,7 +193,7 @@ class GPT4All:
     @staticmethod
     def download_model(
         model_filename: str,
-        model_path: str,
+        model_path: Union[str, os.PathLike[str]],
         verbose: bool = True,
         url: Optional[str] = None,
     ) -> str:
@@ -195,7 +201,7 @@ class GPT4All:
         Download model from https://gpt4all.io.
 
         Args:
-            model_filename: Filename of model (with .bin extension).
+            model_filename: Filename of model (with .gguf extension).
             model_path: Path to download model to.
             verbose: If True (default), print debug messages.
             url: the models remote url (e.g. may be hosted on HF)
@@ -207,38 +213,67 @@ class GPT4All:
         def get_download_url(model_filename):
             if url:
                 return url
-            return f"https://gpt4all.io/models/{model_filename}"
+            return f"https://gpt4all.io/models/gguf/{model_filename}"
 
         # Download model
         download_path = os.path.join(model_path, model_filename).replace("\\", "\\\\")
         download_url = get_download_url(model_filename)
 
-        response = requests.get(download_url, stream=True)
+        def make_request(offset=None):
+            headers = {}
+            if offset:
+                print(f"\nDownload interrupted, resuming from byte position {offset}", file=sys.stderr)
+                headers['Range'] = f'bytes={offset}-'  # resume incomplete response
+            response = requests.get(download_url, stream=True, headers=headers)
+            if response.status_code not in (200, 206):
+                raise ValueError(f'Request failed: HTTP {response.status_code} {response.reason}')
+            if offset and (response.status_code != 206 or str(offset) not in response.headers.get('Content-Range', '')):
+                raise ValueError('Connection was interrupted and server does not support range requests')
+            return response
+
+        response = make_request()
+
         total_size_in_bytes = int(response.headers.get("content-length", 0))
         block_size = 2**20  # 1 MB
 
-        with tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar:
+        with open(download_path, "wb") as file, \
+                tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar:
             try:
-                with open(download_path, "wb") as file:
-                    for data in response.iter_content(block_size):
-                        progress_bar.update(len(data))
-                        file.write(data)
+                while True:
+                    last_progress = progress_bar.n
+                    try:
+                        for data in response.iter_content(block_size):
+                            file.write(data)
+                            progress_bar.update(len(data))
+                    except ChunkedEncodingError as cee:
+                        if cee.args and isinstance(pe := cee.args[0], ProtocolError):
+                            if len(pe.args) >= 2 and isinstance(ir := pe.args[1], IncompleteRead):
+                                assert progress_bar.n <= ir.partial  # urllib3 may be ahead of us but never behind
+                                # the socket was closed during a read - retry
+                                response = make_request(progress_bar.n)
+                                continue
+                        raise
+                    if total_size_in_bytes != 0 and progress_bar.n < total_size_in_bytes:
+                        if progress_bar.n == last_progress:
+                            raise RuntimeError('Download not making progress, aborting.')
+                        # server closed connection prematurely - retry
+                        response = make_request(progress_bar.n)
+                        continue
+                    break
             except Exception:
-                if os.path.exists(download_path):
-                    if verbose:
-                        print("Cleaning up the interrupted download...")
+                if verbose:
+                    print("Cleaning up the interrupted download...", file=sys.stderr)
+                try:
                     os.remove(download_path)
+                except OSError:
+                    pass
                 raise
 
-        # Validate download was successful
-        if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
-            raise RuntimeError("An error occurred during download. Downloaded file may not work.")
-
-        # Sleep for a little bit so OS can remove file lock
-        time.sleep(2)
+        if os.name == 'nt':
+            time.sleep(2)  # Sleep for a little bit so Windows can remove file lock
 
         if verbose:
-            print("Model downloaded at: ", download_path)
+            print("Model downloaded at:", download_path, file=sys.stderr)
         return download_path
 
     def generate(
@@ -314,7 +349,6 @@ class GPT4All:
             callback: pyllmodel.ResponseCallbackType,
             output_collector: List[MessageType],
         ) -> pyllmodel.ResponseCallbackType:
-
             def _callback(token_id: int, response: str) -> bool:
                 nonlocal callback, output_collector
 
@@ -422,7 +456,7 @@ def empty_chat_session(system_prompt: str = "") -> List[MessageType]:
     return [{"role": "system", "content": system_prompt}]
 
 
-def append_bin_suffix_if_missing(model_name):
-    if not model_name.endswith(".bin"):
-        model_name += ".bin"
+def append_extension_if_missing(model_name):
+    if not model_name.endswith((".bin", ".gguf")):
+        model_name += ".gguf"
     return model_name

gpt4all-bindings/python/gpt4all/pyllmodel.py

@@ -1,48 +1,42 @@
+import atexit
 import ctypes
+import importlib.resources
 import logging
 import os
 import platform
-from queue import Queue
 import re
 import subprocess
 import sys
 import threading
+from contextlib import ExitStack
+from queue import Queue
 from typing import Callable, Iterable, List
 
-import pkg_resources
-
 logger: logging.Logger = logging.getLogger(__name__)
 
 
+file_manager = ExitStack()
+atexit.register(file_manager.close)  # clean up files on exit
+
 # TODO: provide a config file to make this more robust
-LLMODEL_PATH = os.path.join("llmodel_DO_NOT_MODIFY", "build").replace("\\", "\\\\")
-MODEL_LIB_PATH = str(pkg_resources.resource_filename("gpt4all", LLMODEL_PATH)).replace("\\", "\\\\")
+MODEL_LIB_PATH = file_manager.enter_context(importlib.resources.as_file(
+    importlib.resources.files("gpt4all") / "llmodel_DO_NOT_MODIFY" / "build",
+))
 
 
 def load_llmodel_library():
-    system = platform.system()
+    ext = {"Darwin": "dylib", "Linux": "so", "Windows": "dll"}[platform.system()]
 
-    def get_c_shared_lib_extension():
-        if system == "Darwin":
-            return "dylib"
-        elif system == "Linux":
-            return "so"
-        elif system == "Windows":
-            return "dll"
-        else:
-            raise Exception("Operating System not supported")
+    try:
+        # Linux, Windows, MinGW
+        lib = ctypes.CDLL(str(MODEL_LIB_PATH / f"libllmodel.{ext}"))
+    except FileNotFoundError:
+        if ext != 'dll':
+            raise
+        # MSVC
+        lib = ctypes.CDLL(str(MODEL_LIB_PATH / "llmodel.dll"))
 
-    c_lib_ext = get_c_shared_lib_extension()
-
-    llmodel_file = "libllmodel" + "." + c_lib_ext
-
-    llmodel_dir = str(pkg_resources.resource_filename("gpt4all", os.path.join(LLMODEL_PATH, llmodel_file))).replace(
-        "\\", "\\\\"
-    )
-
-    llmodel_lib = ctypes.CDLL(llmodel_dir)
-
-    return llmodel_lib
+    return lib
 
 
 llmodel = load_llmodel_library()
@@ -131,7 +125,7 @@ llmodel.llmodel_set_implementation_search_path.restype = None
 llmodel.llmodel_threadCount.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_threadCount.restype = ctypes.c_int32
 
-llmodel.llmodel_set_implementation_search_path(MODEL_LIB_PATH.encode("utf-8"))
+llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).replace("\\", r"\\").encode("utf-8"))
 
 llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
 llmodel.llmodel_available_gpu_devices.restype = ctypes.POINTER(LLModelGPUDevice)
@@ -259,12 +253,13 @@ class LLModel:
         True if model loaded successfully, False otherwise
         """
         model_path_enc = model_path.encode("utf-8")
-        self.model = llmodel.llmodel_model_create(model_path_enc)
+        err = LLModelError()
+        self.model = llmodel.llmodel_model_create2(model_path_enc, b"auto", ctypes.byref(err))
 
-        if self.model is not None:
-            llmodel.llmodel_loadModel(self.model, model_path_enc)
-        else:
-            raise ValueError("Unable to instantiate model")
+        if self.model is None:
+            raise ValueError(f"Unable to instantiate model: code={err.code}, {err.message.decode()}")
+
+        llmodel.llmodel_loadModel(self.model, model_path_enc)
 
         filename = os.path.basename(model_path)
         self.model_name = os.path.splitext(filename)[0]

gpt4all-bindings/python/gpt4all/tests/test_embed_timings.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import sys
 import time
 from io import StringIO

gpt4all-bindings/python/gpt4all/tests/test_gpt4all.py

@@ -8,7 +8,7 @@ import pytest
 
 
 def test_inference():
-    model = GPT4All(model_name='orca-mini-3b.ggmlv3.q4_0.bin')
+    model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf')
     output_1 = model.generate('hello', top_k=1)
 
     with model.chat_session():
@@ -47,49 +47,44 @@ def do_long_input(model):
 
 
 def test_inference_long_orca_3b():
-    model = GPT4All(model_name="orca-mini-3b.ggmlv3.q4_0.bin")
+    model = GPT4All(model_name="orca-mini-3b-gguf2-q4_0.gguf")
     do_long_input(model)
 
 
 def test_inference_long_falcon():
-    model = GPT4All(model_name='ggml-model-gpt4all-falcon-q4_0.bin')
+    model = GPT4All(model_name='gpt4all-falcon-q4_0.gguf')
     do_long_input(model)
 
 
 def test_inference_long_llama_7b():
-    model = GPT4All(model_name="orca-mini-7b.ggmlv3.q4_0.bin")
+    model = GPT4All(model_name="mistral-7b-openorca.Q4_0.gguf")
     do_long_input(model)
 
 
 def test_inference_long_llama_13b():
-    model = GPT4All(model_name='ggml-nous-hermes-13b.ggmlv3.q4_0.bin')
+    model = GPT4All(model_name='nous-hermes-llama2-13b.Q4_0.gguf')
     do_long_input(model)
 
 
 def test_inference_long_mpt():
-    model = GPT4All(model_name='ggml-mpt-7b-chat.bin')
+    model = GPT4All(model_name='mpt-7b-chat-q4_0.gguf')
     do_long_input(model)
 
 
 def test_inference_long_replit():
-    model = GPT4All(model_name='ggml-replit-code-v1-3b.bin')
-    do_long_input(model)
-
-
-def test_inference_long_groovy():
-    model = GPT4All(model_name='ggml-gpt4all-j-v1.3-groovy.bin')
+    model = GPT4All(model_name='replit-code-v1_5-3b-q4_0.gguf')
     do_long_input(model)
 
 
 def test_inference_hparams():
-    model = GPT4All(model_name='orca-mini-3b.ggmlv3.q4_0.bin')
+    model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf')
 
     output = model.generate("The capital of france is ", max_tokens=3)
     assert 'Paris' in output
 
 
 def test_inference_falcon():
-    model = GPT4All(model_name='ggml-model-gpt4all-falcon-q4_0.bin')
+    model = GPT4All(model_name='gpt4all-falcon-q4_0.gguf')
     prompt = 'hello'
     output = model.generate(prompt)
     assert isinstance(output, str)
@@ -97,7 +92,7 @@ def test_inference_falcon():
 
 
 def test_inference_mpt():
-    model = GPT4All(model_name='ggml-mpt-7b-chat.bin')
+    model = GPT4All(model_name='mpt-7b-chat-q4_0.gguf')
     prompt = 'hello'
     output = model.generate(prompt)
     assert isinstance(output, str)

gpt4all-bindings/python/setup.py

@@ -61,7 +61,7 @@ copy_prebuilt_C_lib(SRC_CLIB_DIRECtORY,
 
 setup(
     name=package_name,
-    version="1.0.12",
+    version="2.0.1",
     description="Python bindings for GPT4All",
     author="Nomic and the Open Source Community",
     author_email="support@nomic.ai",

gpt4all-bindings/typescript/README.md

@@ -58,6 +58,8 @@ const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional"
 *   (win) msvc version 143
     *   Can be obtained with visual studio 2022 build tools
 *   python 3
+*   On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+*   macOS users do not need Vulkan, as GPT4All will use Metal instead.
 
 ### Build (from source)
 
@@ -703,7 +705,7 @@ Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Glob
 
 ##### url
 
-Remote download url. Defaults to `https://gpt4all.io/models/<modelName>`
+Remote download url. Defaults to `https://gpt4all.io/models/gguf/<modelName>`
 
 Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
 

gpt4all-bindings/typescript/scripts/build_mingw.ps1

@@ -12,5 +12,5 @@ cmake -G "MinGW Makefiles" -S ..\..\gpt4all-backend -B $BUILD_DIR -DLLAMA_AVX2=O
 cmake --build $BUILD_DIR --parallel --config Release
 
 # copy native dlls
-# cp "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll" $LIBS_DIR
+# cp "C:\ProgramData\mingw64\mingw64\bin\*dll" $LIBS_DIR
 cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR

gpt4all-bindings/typescript/scripts/build_unix.sh

@@ -25,9 +25,6 @@ mkdir -p "$NATIVE_DIR" "$BUILD_DIR"
 cmake -S ../../gpt4all-backend -B "$BUILD_DIR" &&
 cmake --build "$BUILD_DIR" -j --config Release && {
   cp "$BUILD_DIR"/libbert*.$LIB_EXT   "$NATIVE_DIR"/
-  cp "$BUILD_DIR"/libfalcon*.$LIB_EXT "$NATIVE_DIR"/
-  cp "$BUILD_DIR"/libreplit*.$LIB_EXT "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libgptj*.$LIB_EXT   "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libllama*.$LIB_EXT  "$NATIVE_DIR"/
-  cp "$BUILD_DIR"/libmpt*.$LIB_EXT    "$NATIVE_DIR"/
 }

gpt4all-bindings/typescript/spec/chat.mjs

@@ -1,7 +1,7 @@
 import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY, loadModel } from '../src/gpt4all.js'
 
 const model = await loadModel(
-    'orca-mini-3b.ggmlv3.q4_0.bin',
+    'orca-mini-3b-gguf2-q4_0.gguf',
     { verbose: true }
 );
 const ll = model.llm;

gpt4all-bindings/typescript/src/config.js

@@ -21,7 +21,7 @@ const DEFAULT_MODEL_CONFIG = {
     promptTemplate: "### Human: \n%1\n### Assistant:\n",
 }
 
-const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models.json";
+const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models2.json";
 
 const DEFAULT_PROMPT_CONTEXT = {
     temp: 0.7,

gpt4all-bindings/typescript/src/gpt4all.d.ts

@@ -444,8 +444,8 @@ interface DownloadModelOptions {
     verbose?: boolean;
 
     /**
-     * Remote download url. Defaults to `https://gpt4all.io/models/<modelName>`
-     * @default https://gpt4all.io/models/<modelName>
+     * Remote download url. Defaults to `https://gpt4all.io/models/gguf/<modelName>`
+     * @default https://gpt4all.io/models/gguf/<modelName>
      */
     url?: string;
     /**

gpt4all-bindings/typescript/src/util.js

@@ -113,7 +113,7 @@ function downloadModel(modelName, options = {}) {
     );
     const finalModelPath = path.join(downloadOptions.modelPath, modelFileName);
     const modelUrl =
-        downloadOptions.url ?? `https://gpt4all.io/models/${modelFileName}`;
+        downloadOptions.url ?? `https://gpt4all.io/models/gguf/${modelFileName}`;
 
     mkdirp.sync(downloadOptions.modelPath)
 
@@ -236,7 +236,7 @@ async function retrieveModel(modelName, options = {}) {
         file: retrieveOptions.modelConfigFile,
         url:
             retrieveOptions.allowDownload &&
-            "https://gpt4all.io/models/models.json",
+            "https://gpt4all.io/models/models2.json",
     });
 
     const loadedModelConfig = availableModels.find(

gpt4all-chat/CMakeLists.txt

@@ -17,8 +17,8 @@ if(APPLE)
 endif()
 
 set(APP_VERSION_MAJOR 2)
-set(APP_VERSION_MINOR 4)
-set(APP_VERSION_PATCH 17)
+set(APP_VERSION_MINOR 5)
+set(APP_VERSION_PATCH 2)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file
@@ -180,8 +180,8 @@ install(TARGETS llmodel DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 
 # We should probably iterate through the list of the cmake for backend, but these need to be installed
 # to the this component's dir for the finicky qt installer to work
-#install(TARGETS gptj-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-#install(TARGETS gptj-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS gptj-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS gptj-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llama-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llama-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llamamodel-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
@@ -189,30 +189,19 @@ install(TARGETS llamamodel-mainline-default DESTINATION lib COMPONENT ${COMPONEN
 if(APPLE)
 install(TARGETS llamamodel-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 endif()
-install(TARGETS falcon-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS falcon-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-#install(TARGETS mpt-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-#install(TARGETS mpt-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS replit-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS replit-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-if(APPLE)
-install(TARGETS replit-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-endif()
 install(TARGETS bert-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS bert-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS starcoder-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS starcoder-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 
 set(CPACK_GENERATOR "IFW")
 set(CPACK_VERBATIM_VARIABLES YES)
 set(CPACK_IFW_VERBOSE ON)
 
 if(${CMAKE_SYSTEM_NAME} MATCHES Linux)
-    set(LINUXDEPLOYQT "$ENV{HOME}/dev/linuxdeployqt/build/tools/linuxdeployqt/linuxdeployqt")
+    find_program(LINUXDEPLOYQT linuxdeployqt HINTS "$ENV{HOME}/dev/linuxdeployqt/build/tools/linuxdeployqt" "$ENV{HOME}/project/linuxdeployqt/bin")
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/deploy-qt-linux.cmake.in"
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-linux.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-linux.cmake)
-    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.5")
+    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.6")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-linux")
     set(CPACK_IFW_TARGET_DIRECTORY "@HomeDir@/${COMPONENT_NAME_MAIN}")
 elseif(${CMAKE_SYSTEM_NAME} MATCHES Windows)
@@ -220,7 +209,7 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES Windows)
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/deploy-qt-windows.cmake.in"
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-windows.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-windows.cmake)
-    set(CPACK_IFW_ROOT "C:/Qt/Tools/QtInstallerFramework/4.5")
+    set(CPACK_IFW_ROOT "C:/Qt/Tools/QtInstallerFramework/4.6")
     set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.ico")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-win64")
     set(CPACK_IFW_TARGET_DIRECTORY "@HomeDir@\\${COMPONENT_NAME_MAIN}")
@@ -229,7 +218,7 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/deploy-qt-mac.cmake.in"
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-mac.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-mac.cmake)
-    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.5")
+    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.6")
     set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-darwin")
     set(CPACK_IFW_TARGET_DIRECTORY "@ApplicationsDir@/${COMPONENT_NAME_MAIN}")
@@ -259,7 +248,11 @@ set(CPACK_IFW_PACKAGE_WIZARD_SHOW_PAGE_LIST OFF)
 include(InstallRequiredSystemLibraries)
 include(CPack)
 include(CPackIFW)
-cpack_add_component(${COMPONENT_NAME_MAIN} DOWNLOADED)
+if(GPT4ALL_OFFLINE_INSTALLER)
+  cpack_add_component(${COMPONENT_NAME_MAIN})
+else()
+  cpack_add_component(${COMPONENT_NAME_MAIN} DOWNLOADED)
+endif()
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} ESSENTIAL FORCED_INSTALLATION)
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} VERSION ${APP_VERSION})
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} LICENSES "MIT LICENSE" ${CPACK_RESOURCE_FILE_LICENSE})
@@ -269,7 +262,7 @@ cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} REPLACES "gpt4all-chat") #W
 if (GPT4ALL_LOCALHOST)
     cpack_ifw_add_repository("GPT4AllRepository" URL "http://localhost/repository")
 elseif(GPT4ALL_OFFLINE_INSTALLER)
-    cpack_ifw_add_repository("GPT4AllRepository" URL "file://${CMAKE_BINARY_DIR}/packages")
+  add_compile_definitions(GPT4ALL_OFFLINE_INSTALLER)
 else()
   if(${CMAKE_SYSTEM_NAME} MATCHES Linux)
       cpack_ifw_add_repository("GPT4AllRepository" URL "https://gpt4all.io/installer_repos/linux/repository")

gpt4all-chat/README.md

@@ -32,13 +32,8 @@ One click installers for macOS, Linux, and Windows at https://gpt4all.io
 * Multi-chat - a list of current and past chats and the ability to save/delete/export and switch between
 * Text to speech - have the AI response with voice
 * Speech to text - give the prompt with your voice
-* Python bindings
-* Typescript bindings
 * Plugin support for langchain other developer tools
-* Save your prompt/responses to disk
-* Upload prompt/response manually/automatically to nomic.ai to aid future training runs
-* Syntax highlighting support for programming languages, etc.
-* REST API with a built-in webserver in the chat gui itself with a headless operation mode as well
+* chat gui headless operation mode
 * Advanced settings for changing temperature, topk, etc. (DONE)
 * * Improve the accessibility of the installer for screen reader users
 * YOUR IDEA HERE

gpt4all-chat/build_and_run.md

@@ -1,57 +1,104 @@
-# Install Qt 6.x and setup/build gpt4all-chat from source
+# Building gpt4all-chat from source
 
 Depending upon your operating system, there are many ways that Qt is distributed. 
 Here is the recommended method for getting the Qt dependency installed to setup and build 
 gpt4all-chat from source.
 
-## Create a [Qt account](https://login.qt.io/register) 
+## Prerequisites
 
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/d1e44cab-4245-4144-a91c-7b02267df2b2)
+On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
 
-## Go to the Qt open source [download page](https://www.qt.io/download-qt-installer-oss)
+macOS users do not need Vulkan, as GPT4All will use Metal instead.
 
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/d68f5f45-cca3-4fe9-acf4-cabdcb95f669)
+## Note for Linux users
 
-## Start the installer and sign in
+Linux users may install Qt via their distro's official packages instead of using the Qt installer. You need at least Qt 6.5, with support for QPdf and the Qt HTTP Server. It should be straightforward to build with just cmake and make, but you may continue to follow these instructions to build with Qt Creator.
 
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/899b1422-51ae-4bb5-acc9-b9027a8e9b19)
+On Arch Linux, this looks like:
+```
+sudo pacman -S --needed base-devel qt6-base qt6-httpserver qtcreator cmake ninja
+```
 
-## After some screens about license, select custom
+On Ubuntu 23.04, this looks like:
+```
+sudo apt install build-essential libqt6gui6 qt6-base-dev libqt6httpserver6 qt6-httpserver-dev qtcreator cmake ninja-build
+```
 
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/2290031a-fdb0-4f47-a7f1-d77ad5451068)
+## Download Qt
 
-## Select the following
+- Go to https://login.qt.io/register to create a free Qt account.
+- Download the Qt Online Installer for your OS from here: https://www.qt.io/download-qt-installer-oss
+- Sign into the installer.
+- Agree to the terms of the (L)GPL 3 license.
+- Select whether you would like to send anonymous usage statistics to Qt.
+- On the Installation Folder page, leave the default installation path, and select "Custom Installation".
+
+## Customize the installation
 
 ![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/c6e999e5-cc8a-4dfc-8065-b59139e8c7ae)
 
-NOTE: This is for macOS. For Linux it is similar, but you need MSVC for Windows, not the mingw install
+Under "Qt", find the latest Qt 6.x release.
 
-## Open up QtCreator
+Under this release (e.g. Qt 6.5.0), select the target platform:
+- On macOS, it is just called "macOS".
+- On Windows, it is called "MSVC 2019 64-bit" (for 64-bit x86 CPUs). MinGW has not been tested.
 
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/a34978f4-a220-459c-af66-e901d7ccd7bb)
+Under this release, select the following additional components:
+- Qt Quick 3D
+- Qt 5 Compatibility Module
+- Qt Shader Tools
+- Additional Libraries:
+  - Qt HTTP Server
+  - Qt PDF
+- Qt Debug information Files
+- Qt Quick Timeline
 
-## Clone the git repo for gpt4all-chat
+Under Developer and Designer Tools, select the following components:
+- Qt Creator
+- Qt Creator CDB Debugger Support (for Windows only)
+- Debugging Tools for Windows (for Windows only)
+- CMake
+- Ninja
 
+Agree to the license and complete the installation.
+
+## Download the source code
+
+You must use git to download the source code for gpt4all:
 ```
 git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
 ```
 
-## Open the gpt4all-chat project in QtCreator
+Note the use of --recurse-submodules, which makes sure the necessary dependencies are downloaded inside the repo. This is why you cannot simply download a zip archive.
+
+Windows users: To install git for Windows, see https://git-scm.com/downloads. Once it is installed, you should be able to shift-right click in any folder, "Open PowerShell window here" (or similar, depending on the version of Windows), and run the above command.
+
+## Open gpt4all-chat in Qt Creator
+
+Open Qt Creator. Navigate to File > Open File or Project, find the "gpt4all-chat" folder inside the freshly cloned repository, and select CMakeLists.txt.
 
 ![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/3d3e2743-2a1d-43d6-9e55-62f7f4306de7)
 
-NOTE: File->Open File or Project and navigate to the gpt4all-chat repo and choose the CMakeLists.txt
-
 ## Configure project
 
+You can now expand the "Details" section next to the build kit. It is best to uncheck all but one build configuration, e.g. "Release", which will produce optimized binaries that are not useful for debugging.
+
+Click "Configure Project", and wait for it to complete.
+
 ![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/44d5aafb-a95d-434b-ba2a-a3138c0e49a0)
 
 ## Build project
 
+Now that the project has been configured, click the hammer button on the left sidebar to build the project.
+
 ![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/43cd7b42-32f0-4efa-9612-d51f85637103)
 
 ## Run project
 
+Click the play button on the left sidebar to run the Chat UI.
+
 ![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/611ea795-bdcd-4feb-a466-eb1c2e936e7e)
 
+## Updating the downloaded source code
 
+You do not need to make a fresh clone of the source code every time. To update it, you may open a terminal/command prompt in the repository, run `git pull`, and then `git submodule update --init --recursive`.

gpt4all-chat/chat.cpp

@@ -57,6 +57,7 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::generatedNameChanged, this, &Chat::generatedNameChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportSpeed, this, &Chat::handleTokenSpeedChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportDevice, this, &Chat::handleDeviceChanged, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::reportFallbackReason, this, &Chat::handleFallbackReasonChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::databaseResultsChanged, this, &Chat::handleDatabaseResultsChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelInfoChanged, this, &Chat::handleModelInfoChanged, Qt::QueuedConnection);
 
@@ -141,17 +142,9 @@ QString Chat::response() const
     return m_response;
 }
 
-QString Chat::responseState() const
+Chat::ResponseState Chat::responseState() const
 {
-    switch (m_responseState) {
-    case ResponseStopped: return QStringLiteral("response stopped");
-    case LocalDocsRetrieval: return QStringLiteral("retrieving ") + m_collections.join(", ");
-    case LocalDocsProcessing: return QStringLiteral("processing ") + m_collections.join(", ");
-    case PromptProcessing: return QStringLiteral("processing");
-    case ResponseGeneration: return QStringLiteral("generating response");
-    };
-    Q_UNREACHABLE();
-    return QString();
+    return m_responseState;
 }
 
 void Chat::handleResponseChanged(const QString &response)
@@ -352,6 +345,12 @@ void Chat::handleDeviceChanged(const QString &device)
     emit deviceChanged();
 }
 
+void Chat::handleFallbackReasonChanged(const QString &fallbackReason)
+{
+    m_fallbackReason = fallbackReason;
+    emit fallbackReasonChanged();
+}
+
 void Chat::handleDatabaseResultsChanged(const QList<ResultInfo> &results)
 {
     m_databaseResults = results;
@@ -378,7 +377,11 @@ bool Chat::serialize(QDataStream &stream, int version) const
         stream << m_modelInfo.filename();
     if (version > 2)
         stream << m_collections;
-    if (!m_llmodel->serialize(stream, version))
+
+    const bool serializeKV = MySettings::globalInstance()->saveChatsContext();
+    if (version > 5)
+        stream << serializeKV;
+    if (!m_llmodel->serialize(stream, version, serializeKV))
         return false;
     if (!m_chatModel->serialize(stream, version))
         return false;
@@ -392,34 +395,46 @@ bool Chat::deserialize(QDataStream &stream, int version)
     emit idChanged(m_id);
     stream >> m_name;
     stream >> m_userName;
+    m_generatedName = QLatin1String("nonempty");
     emit nameChanged();
 
     QString modelId;
     stream >> modelId;
     if (version > 4) {
-        if (!ModelList::globalInstance()->contains(modelId))
-            return false;
-        m_modelInfo = ModelList::globalInstance()->modelInfo(modelId);
+        if (ModelList::globalInstance()->contains(modelId))
+            m_modelInfo = ModelList::globalInstance()->modelInfo(modelId);
     } else {
-        if (!ModelList::globalInstance()->containsByFilename(modelId))
-            return false;
-        m_modelInfo = ModelList::globalInstance()->modelInfoByFilename(modelId);
+        if (ModelList::globalInstance()->containsByFilename(modelId))
+            m_modelInfo = ModelList::globalInstance()->modelInfoByFilename(modelId);
     }
-    emit modelInfoChanged();
+    if (!m_modelInfo.id().isEmpty())
+        emit modelInfoChanged();
+
+    bool discardKV = m_modelInfo.id().isEmpty();
 
     // Prior to version 2 gptj models had a bug that fixed the kv_cache to F32 instead of F16 so
     // unfortunately, we cannot deserialize these
     if (version < 2 && m_modelInfo.filename().contains("gpt4all-j"))
-        return false;
+        discardKV = true;
+
     if (version > 2) {
         stream >> m_collections;
         emit collectionListChanged(m_collections);
     }
+
+    bool deserializeKV = true;
+    if (version > 5)
+        stream >> deserializeKV;
+
     m_llmodel->setModelInfo(m_modelInfo);
-    if (!m_llmodel->deserialize(stream, version))
+    if (!m_llmodel->deserialize(stream, version, deserializeKV, discardKV))
         return false;
     if (!m_chatModel->deserialize(stream, version))
         return false;
+
+    if (!deserializeKV || discardKV)
+        m_llmodel->setStateFromText(m_chatModel->text());
+
     emit chatModelChanged();
     return stream.status() == QDataStream::Ok;
 }

gpt4all-chat/chat.h

@@ -21,11 +21,12 @@ class Chat : public QObject
     Q_PROPERTY(bool responseInProgress READ responseInProgress NOTIFY responseInProgressChanged)
     Q_PROPERTY(bool isRecalc READ isRecalc NOTIFY recalcChanged)
     Q_PROPERTY(bool isServer READ isServer NOTIFY isServerChanged)
-    Q_PROPERTY(QString responseState READ responseState NOTIFY responseStateChanged)
+    Q_PROPERTY(ResponseState responseState READ responseState NOTIFY responseStateChanged)
     Q_PROPERTY(QList<QString> collectionList READ collectionList NOTIFY collectionListChanged)
     Q_PROPERTY(QString modelLoadingError READ modelLoadingError NOTIFY modelLoadingErrorChanged)
     Q_PROPERTY(QString tokenSpeed READ tokenSpeed NOTIFY tokenSpeedChanged);
     Q_PROPERTY(QString device READ device NOTIFY deviceChanged);
+    Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY fallbackReasonChanged);
     QML_ELEMENT
     QML_UNCREATABLE("Only creatable from c++!")
 
@@ -53,6 +54,8 @@ public:
     }
     ChatModel *chatModel() { return m_chatModel; }
 
+    bool isNewChat() const { return m_name == tr("New Chat") && !m_chatModel->count(); }
+
     Q_INVOKABLE void reset();
     Q_INVOKABLE void processSystemPrompt();
     Q_INVOKABLE bool isModelLoaded() const;
@@ -65,7 +68,7 @@ public:
 
     QString response() const;
     bool responseInProgress() const { return m_responseInProgress; }
-    QString responseState() const;
+    ResponseState responseState() const;
     ModelInfo modelInfo() const;
     void setModelInfo(const ModelInfo &modelInfo);
     bool isRecalc() const;
@@ -90,6 +93,7 @@ public:
 
     QString tokenSpeed() const { return m_tokenSpeed; }
     QString device() const { return m_device; }
+    QString fallbackReason() const { return m_fallbackReason; }
 
 public Q_SLOTS:
     void serverNewPromptResponsePair(const QString &prompt);
@@ -118,6 +122,7 @@ Q_SIGNALS:
     void collectionListChanged(const QList<QString> &collectionList);
     void tokenSpeedChanged();
     void deviceChanged();
+    void fallbackReasonChanged();
 
 private Q_SLOTS:
     void handleResponseChanged(const QString &response);
@@ -129,6 +134,7 @@ private Q_SLOTS:
     void handleModelLoadingError(const QString &error);
     void handleTokenSpeedChanged(const QString &tokenSpeed);
     void handleDeviceChanged(const QString &device);
+    void handleFallbackReasonChanged(const QString &device);
     void handleDatabaseResultsChanged(const QList<ResultInfo> &results);
     void handleModelInfoChanged(const ModelInfo &modelInfo);
     void handleModelInstalled();
@@ -142,6 +148,7 @@ private:
     QString m_modelLoadingError;
     QString m_tokenSpeed;
     QString m_device;
+    QString m_fallbackReason;
     QString m_response;
     QList<QString> m_collections;
     ChatModel *m_chatModel;

gpt4all-chat/chatlistmodel.cpp

@@ -5,7 +5,7 @@
 #include <QDataStream>
 
 #define CHAT_FORMAT_MAGIC 0xF5D553CC
-#define CHAT_FORMAT_VERSION 5
+#define CHAT_FORMAT_VERSION 6
 
 class MyChatListModel: public ChatListModel { };
 Q_GLOBAL_STATIC(MyChatListModel, chatListModelInstance)
@@ -17,11 +17,10 @@ ChatListModel *ChatListModel::globalInstance()
 ChatListModel::ChatListModel()
     : QAbstractListModel(nullptr)
     , m_newChat(nullptr)
-    , m_dummyChat(nullptr)
     , m_serverChat(nullptr)
     , m_currentChat(nullptr)
 {
-    addDummyChat();
+    addChat();
 
     ChatsRestoreThread *thread = new ChatsRestoreThread;
     connect(thread, &ChatsRestoreThread::chatRestored, this, &ChatListModel::restoreChat);
@@ -59,10 +58,7 @@ void ChatListModel::saveChats()
     for (Chat *chat : m_chats) {
         if (chat == m_serverChat)
             continue;
-        const bool isChatGPT = chat->modelInfo().isChatGPT;
-        if (!isChatGPT && !MySettings::globalInstance()->saveChats())
-            continue;
-        if (isChatGPT && !MySettings::globalInstance()->saveChatGPTChats())
+        if (chat->isNewChat())
             continue;
         toSave.append(chat);
     }
@@ -84,13 +80,16 @@ void ChatSaver::saveChats(const QVector<Chat *> &chats)
     const QString savePath = MySettings::globalInstance()->modelPath();
     for (Chat *chat : chats) {
         QString fileName = "gpt4all-" + chat->id() + ".chat";
-        QFile file(savePath + "/" + fileName);
-        bool success = file.open(QIODevice::WriteOnly);
+        QString filePath = savePath + "/" + fileName;
+        QFile originalFile(filePath);
+        QFile tempFile(filePath + ".tmp"); // Temporary file
+
+        bool success = tempFile.open(QIODevice::WriteOnly);
         if (!success) {
-            qWarning() << "ERROR: Couldn't save chat to file:" << file.fileName();
+            qWarning() << "ERROR: Couldn't save chat to temporary file:" << tempFile.fileName();
             continue;
         }
-        QDataStream out(&file);
+        QDataStream out(&tempFile);
 
         out << (quint32)CHAT_FORMAT_MAGIC;
         out << (qint32)CHAT_FORMAT_VERSION;
@@ -98,11 +97,16 @@ void ChatSaver::saveChats(const QVector<Chat *> &chats)
 
         qDebug() << "serializing chat" << fileName;
         if (!chat->serialize(out, CHAT_FORMAT_VERSION)) {
-            qWarning() << "ERROR: Couldn't serialize chat to file:" << file.fileName();
-            file.remove();
+            qWarning() << "ERROR: Couldn't serialize chat to file:" << tempFile.fileName();
+            tempFile.remove();
+            continue;
         }
-        file.close();
+
+        if (originalFile.exists())
+            originalFile.remove();
+        tempFile.rename(filePath);
     }
+
     qint64 elapsedTime = timer.elapsed();
     qDebug() << "serializing chats took:" << elapsedTime << "ms";
     emit saveChatsFinished();
@@ -189,48 +193,47 @@ void ChatsRestoreThread::run()
     });
 
     for (FileInfo &f : files) {
-            QFile file(f.file);
-            bool success = file.open(QIODevice::ReadOnly);
-            if (!success) {
-                qWarning() << "ERROR: Couldn't restore chat from file:" << file.fileName();
+        QFile file(f.file);
+        bool success = file.open(QIODevice::ReadOnly);
+        if (!success) {
+            qWarning() << "ERROR: Couldn't restore chat from file:" << file.fileName();
+            continue;
+        }
+        QDataStream in(&file);
+
+        qint32 version = 0;
+        if (!f.oldFile) {
+            // Read and check the header
+            quint32 magic;
+            in >> magic;
+            if (magic != CHAT_FORMAT_MAGIC) {
+                qWarning() << "ERROR: Chat file has bad magic:" << file.fileName();
                 continue;
             }
-            QDataStream in(&file);
 
-            qint32 version = 0;
-            if (!f.oldFile) {
-                // Read and check the header
-                quint32 magic;
-                in >> magic;
-                if (magic != CHAT_FORMAT_MAGIC) {
-                    qWarning() << "ERROR: Chat file has bad magic:" << file.fileName();
-                    continue;
-                }
-
-                // Read the version
-                in >> version;
-                if (version < 1) {
-                    qWarning() << "ERROR: Chat file has non supported version:" << file.fileName();
-                    continue;
-                }
-
-                if (version <= 1)
-                    in.setVersion(QDataStream::Qt_6_2);
+            // Read the version
+            in >> version;
+            if (version < 1) {
+                qWarning() << "ERROR: Chat file has non supported version:" << file.fileName();
+                continue;
             }
 
-            qDebug() << "deserializing chat" << f.file;
+            if (version <= 1)
+                in.setVersion(QDataStream::Qt_6_2);
+        }
 
-            Chat *chat = new Chat;
-            chat->moveToThread(qApp->thread());
-            if (!chat->deserialize(in, version)) {
-                qWarning() << "ERROR: Couldn't deserialize chat from file:" << file.fileName();
-                file.remove();
-            } else {
-                emit chatRestored(chat);
-            }
-            if (f.oldFile)
-               file.remove(); // No longer storing in this directory
-            file.close();
+        qDebug() << "deserializing chat" << f.file;
+
+        Chat *chat = new Chat;
+        chat->moveToThread(qApp->thread());
+        if (!chat->deserialize(in, version)) {
+            qWarning() << "ERROR: Couldn't deserialize chat from file:" << file.fileName();
+        } else {
+            emit chatRestored(chat);
+        }
+        if (f.oldFile)
+           file.remove(); // No longer storing in this directory
+        file.close();
     }
 
     qint64 elapsedTime = timer.elapsed();
@@ -242,35 +245,13 @@ void ChatListModel::restoreChat(Chat *chat)
     chat->setParent(this);
     connect(chat, &Chat::nameChanged, this, &ChatListModel::nameChanged);
 
-    if (m_dummyChat) {
-        beginResetModel();
-        m_chats = QList<Chat*>({chat});
-        setCurrentChat(chat);
-        delete m_dummyChat;
-        m_dummyChat = nullptr;
-        endResetModel();
-    } else {
-        beginInsertRows(QModelIndex(), m_chats.size(), m_chats.size());
-        m_chats.append(chat);
-        endInsertRows();
-    }
+    beginInsertRows(QModelIndex(), m_chats.size(), m_chats.size());
+    m_chats.append(chat);
+    endInsertRows();
 }
 
 void ChatListModel::chatsRestoredFinished()
 {
-    if (m_dummyChat) {
-        beginResetModel();
-        Chat *dummy = m_dummyChat;
-        m_dummyChat = nullptr;
-        m_chats.clear();
-        addChat();
-        delete dummy;
-        endResetModel();
-    }
-
-    if (m_chats.isEmpty())
-        addChat();
-
     addServerChat();
 }
 

gpt4all-chat/chatlistmodel.h

@@ -84,7 +84,7 @@ public:
     Q_INVOKABLE void addChat()
     {
         // Don't add a new chat if we already have one
-        if (m_newChat || m_dummyChat)
+        if (m_newChat)
             return;
 
         // Create a new chat pointer and connect it to determine when it is populated
@@ -101,18 +101,6 @@ public:
         setCurrentChat(m_newChat);
     }
 
-    Q_INVOKABLE void addDummyChat()
-    {
-        // Create a new dummy chat pointer and don't connect it
-        m_dummyChat = new Chat(this);
-        beginInsertRows(QModelIndex(), 0, 0);
-        m_chats.prepend(m_dummyChat);
-        endInsertRows();
-        emit countChanged();
-        m_currentChat = m_dummyChat;
-        emit currentChatChanged();
-    }
-
     Q_INVOKABLE void addServerChat()
     {
         // Create a new dummy chat pointer and don't connect it
@@ -252,7 +240,6 @@ private Q_SLOTS:
 
 private:
     Chat* m_newChat;
-    Chat* m_dummyChat;
     Chat* m_serverChat;
     Chat* m_currentChat;
     QList<Chat*> m_chats;

gpt4all-chat/chatllm.cpp

@@ -9,13 +9,9 @@
 //#define DEBUG
 //#define DEBUG_MODEL_LOADING
 
-#define MPT_INTERNAL_STATE_VERSION 0
 #define GPTJ_INTERNAL_STATE_VERSION 0
-#define REPLIT_INTERNAL_STATE_VERSION 0
 #define LLAMA_INTERNAL_STATE_VERSION 0
-#define FALCON_INTERNAL_STATE_VERSION 0
 #define BERT_INTERNAL_STATE_VERSION 0
-#define STARCODER_INTERNAL_STATE_VERSION 0
 
 class LLModelStore {
 public:
@@ -72,6 +68,7 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_forceMetal(MySettings::globalInstance()->forceMetal())
     , m_reloadingToChangeVariant(false)
     , m_processedSystemPrompt(false)
+    , m_restoreStateFromText(false)
 {
     moveToThread(&m_llmThread);
     connect(this, &ChatLLM::sendStartup, Network::globalInstance(), &Network::sendStartup);
@@ -230,7 +227,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         if (!m_isServer)
             LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
         m_llModelInfo = LLModelInfo();
-        emit modelLoadingError(QString("Previous attempt to load model resulted in crash for `%1` most likely due to out of memory. You should either remove this model or decrease your system RAM by closing other applications.").arg(modelInfo.filename()));
+        emit modelLoadingError(QString("Previous attempt to load model resulted in crash for `%1` most likely due to insufficient memory. You should either remove this model or decrease your system RAM by closing other applications.").arg(modelInfo.filename()));
     }
 
     if (fileInfo.exists()) {
@@ -263,40 +260,56 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             if (m_llModelInfo.model) {
                 // Update the settings that a model is being loaded and update the device list
                 MySettings::globalInstance()->setAttemptModelLoad(filePath);
-                std::vector<LLModel::GPUDevice> devices = m_llModelInfo.model->availableGPUDevices(0);
-                QVector<QString> deviceList{ "Auto" };
-                for (LLModel::GPUDevice &d : devices)
-                    deviceList << QString::fromStdString(d.name);
-                deviceList << "CPU";
-                MySettings::globalInstance()->setDeviceList(deviceList);
 
                 // Pick the best match for the device
                 QString actualDevice = m_llModelInfo.model->implementation().buildVariant() == "metal" ? "Metal" : "CPU";
                 const QString requestedDevice = MySettings::globalInstance()->device();
-                if (requestedDevice != "CPU") {
+                if (requestedDevice == "CPU") {
+                    emit reportFallbackReason(""); // fallback not applicable
+                } else {
                     const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString());
                     std::vector<LLModel::GPUDevice> availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
+                    LLModel::GPUDevice *device = nullptr;
+
                     if (!availableDevices.empty() && requestedDevice == "Auto" && availableDevices.front().type == 2 /*a discrete gpu*/) {
-                        m_llModelInfo.model->initializeGPUDevice(availableDevices.front());
-                        actualDevice = QString::fromStdString(availableDevices.front().name);
+                        device = &availableDevices.front();
                     } else {
                         for (LLModel::GPUDevice &d : availableDevices) {
                             if (QString::fromStdString(d.name) == requestedDevice) {
-                                m_llModelInfo.model->initializeGPUDevice(d);
-                                actualDevice = QString::fromStdString(d.name);
+                                device = &d;
                                 break;
                             }
                         }
                     }
+
+                    emit reportFallbackReason(""); // no fallback yet
+                    std::string unavail_reason;
+                    if (!device) {
+                        // GPU not available
+                    } else if (!m_llModelInfo.model->initializeGPUDevice(*device, &unavail_reason)) {
+                        emit reportFallbackReason(QString::fromStdString("<br>" + unavail_reason));
+                    } else {
+                        actualDevice = QString::fromStdString(device->name);
+                    }
                 }
 
                 // Report which device we're actually using
                 emit reportDevice(actualDevice);
 
                 bool success = m_llModelInfo.model->loadModel(filePath.toStdString());
-                if (!success && actualDevice != "CPU") {
+                if (actualDevice == "CPU") {
+                    // we asked llama.cpp to use the CPU
+                } else if (!success) {
+                    // llama_init_from_file returned nullptr
                     emit reportDevice("CPU");
+                    emit reportFallbackReason("<br>GPU loading failed (out of VRAM?)");
                     success = m_llModelInfo.model->loadModel(filePath.toStdString());
+                } else if (!m_llModelInfo.model->usingGPUDevice()) {
+                    // ggml_vk_init was not called in llama.cpp
+                    // We might have had to fallback to CPU after load if the model is not possible to accelerate
+                    // for instance if the quantization method is not supported on Vulkan yet
+                    emit reportDevice("CPU");
+                    emit reportFallbackReason("<br>model or quant has no GPU support");
                 }
 
                 MySettings::globalInstance()->setAttemptModelLoad(QString());
@@ -308,19 +321,10 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     m_llModelInfo = LLModelInfo();
                     emit modelLoadingError(QString("Could not load model due to invalid model file for %1").arg(modelInfo.filename()));
                 } else {
-                    // We might have had to fallback to CPU after load if the model is not possible to accelerate
-                    // for instance if the quantization method is not supported on Vulkan yet
-                    if (actualDevice != "CPU" && !m_llModelInfo.model->usingGPUDevice())
-                        emit reportDevice("CPU");
-
                     switch (m_llModelInfo.model->implementation().modelType()[0]) {
                     case 'L': m_llModelType = LLModelType::LLAMA_; break;
                     case 'G': m_llModelType = LLModelType::GPTJ_; break;
-                    case 'M': m_llModelType = LLModelType::MPT_; break;
-                    case 'R': m_llModelType = LLModelType::REPLIT_; break;
-                    case 'F': m_llModelType = LLModelType::FALCON_; break;
                     case 'B': m_llModelType = LLModelType::BERT_; break;
-                    case 'S': m_llModelType = LLModelType::STARCODER_; break;
                     default:
                         {
                             delete m_llModelInfo.model;
@@ -362,10 +366,10 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         emit modelLoadingError(QString("Could not find file for model %1").arg(modelInfo.filename()));
     }
 
-    if (m_llModelInfo.model)
+    if (m_llModelInfo.model) {
         setModelInfo(modelInfo);
-
-    processSystemPrompt();
+        processSystemPrompt();
+    }
     return m_llModelInfo.model;
 }
 
@@ -374,6 +378,32 @@ bool ChatLLM::isModelLoaded() const
     return m_llModelInfo.model && m_llModelInfo.model->isModelLoaded();
 }
 
+std::string remove_leading_whitespace(const std::string& input) {
+    auto first_non_whitespace = std::find_if(input.begin(), input.end(), [](unsigned char c) {
+        return !std::isspace(c);
+    });
+
+    if (first_non_whitespace == input.end())
+        return std::string();
+
+    return std::string(first_non_whitespace, input.end());
+}
+
+std::string trim_whitespace(const std::string& input) {
+    auto first_non_whitespace = std::find_if(input.begin(), input.end(), [](unsigned char c) {
+        return !std::isspace(c);
+    });
+
+    if (first_non_whitespace == input.end())
+        return std::string();
+
+    auto last_non_whitespace = std::find_if(input.rbegin(), input.rend(), [](unsigned char c) {
+        return !std::isspace(c);
+    }).base();
+
+    return std::string(first_non_whitespace, last_non_whitespace);
+}
+
 void ChatLLM::regenerateResponse()
 {
     // ChatGPT uses a different semantic meaning for n_past than local models. For ChatGPT, the meaning
@@ -383,7 +413,7 @@ void ChatLLM::regenerateResponse()
     else
         m_ctx.n_past -= m_promptResponseTokens;
     m_ctx.n_past = std::max(0, m_ctx.n_past);
-    m_ctx.tokens.erase(m_ctx.tokens.end() -= m_promptResponseTokens, m_ctx.tokens.end());
+    m_ctx.tokens.erase(m_ctx.tokens.end() - m_promptResponseTokens, m_ctx.tokens.end());
     m_promptResponseTokens = 0;
     m_promptTokens = 0;
     m_response = std::string();
@@ -405,26 +435,6 @@ void ChatLLM::resetContext()
     m_ctx = LLModel::PromptContext();
 }
 
-std::string remove_leading_whitespace(const std::string& input) {
-    auto first_non_whitespace = std::find_if(input.begin(), input.end(), [](unsigned char c) {
-        return !std::isspace(c);
-    });
-
-    return std::string(first_non_whitespace, input.end());
-}
-
-std::string trim_whitespace(const std::string& input) {
-    auto first_non_whitespace = std::find_if(input.begin(), input.end(), [](unsigned char c) {
-        return !std::isspace(c);
-    });
-
-    auto last_non_whitespace = std::find_if(input.rbegin(), input.rend(), [](unsigned char c) {
-        return !std::isspace(c);
-    }).base();
-
-    return std::string(first_non_whitespace, last_non_whitespace);
-}
-
 QString ChatLLM::response() const
 {
     return QString::fromStdString(remove_leading_whitespace(m_response));
@@ -469,7 +479,7 @@ bool ChatLLM::handleResponse(int32_t token, const std::string &response)
     // check for error
     if (token < 0) {
         m_response.append(response);
-        emit responseChanged(QString::fromStdString(m_response));
+        emit responseChanged(QString::fromStdString(remove_leading_whitespace(m_response)));
         return false;
     }
 
@@ -479,7 +489,7 @@ bool ChatLLM::handleResponse(int32_t token, const std::string &response)
     m_timer->inc();
     Q_ASSERT(!response.empty());
     m_response.append(response);
-    emit responseChanged(QString::fromStdString(m_response));
+    emit responseChanged(QString::fromStdString(remove_leading_whitespace(m_response)));
     return !m_stopGenerating;
 }
 
@@ -496,6 +506,11 @@ bool ChatLLM::handleRecalculate(bool isRecalc)
 }
 bool ChatLLM::prompt(const QList<QString> &collectionList, const QString &prompt)
 {
+    if (m_restoreStateFromText) {
+        Q_ASSERT(m_state.isEmpty());
+        processRestoreStateFromText();
+    }
+
     if (!m_processedSystemPrompt)
         processSystemPrompt();
     const QString promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
@@ -718,26 +733,58 @@ bool ChatLLM::handleSystemRecalculate(bool isRecalc)
     return false;
 }
 
-bool ChatLLM::serialize(QDataStream &stream, int version)
+bool ChatLLM::handleRestoreStateFromTextPrompt(int32_t token)
+{
+#if defined(DEBUG)
+    qDebug() << "restore state from text prompt" << m_llmThread.objectName() << token << m_stopGenerating;
+#endif
+    Q_UNUSED(token);
+    return !m_stopGenerating;
+}
+
+bool ChatLLM::handleRestoreStateFromTextResponse(int32_t token, const std::string &response)
+{
+#if defined(DEBUG)
+    qDebug() << "restore state from text response" << m_llmThread.objectName() << token << response << m_stopGenerating;
+#endif
+    Q_UNUSED(token);
+    Q_UNUSED(response);
+    return false;
+}
+
+bool ChatLLM::handleRestoreStateFromTextRecalculate(bool isRecalc)
+{
+#if defined(DEBUG)
+    qDebug() << "restore state from text recalc" << m_llmThread.objectName() << isRecalc;
+#endif
+    Q_UNUSED(isRecalc);
+    return false;
+}
+
+bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
 {
     if (version > 1) {
         stream << m_llModelType;
         switch (m_llModelType) {
-        case REPLIT_: stream << REPLIT_INTERNAL_STATE_VERSION; break;
-        case MPT_: stream << MPT_INTERNAL_STATE_VERSION; break;
         case GPTJ_: stream << GPTJ_INTERNAL_STATE_VERSION; break;
         case LLAMA_: stream << LLAMA_INTERNAL_STATE_VERSION; break;
-        case FALCON_: stream << FALCON_INTERNAL_STATE_VERSION; break;
         case BERT_: stream << BERT_INTERNAL_STATE_VERSION; break;
-        case STARCODER_: stream << STARCODER_INTERNAL_STATE_VERSION; break;
         default: Q_UNREACHABLE();
         }
     }
     stream << response();
     stream << generatedName();
     stream << m_promptResponseTokens;
+
+    if (!serializeKV) {
+#if defined(DEBUG)
+        qDebug() << "serialize" << m_llmThread.objectName() << m_state.size();
+#endif
+        return stream.status() == QDataStream::Ok;
+    }
+
     if (version <= 3) {
-        int responseLogits;
+        int responseLogits = 0;
         stream << responseLogits;
     }
     stream << m_ctx.n_past;
@@ -754,7 +801,7 @@ bool ChatLLM::serialize(QDataStream &stream, int version)
     return stream.status() == QDataStream::Ok;
 }
 
-bool ChatLLM::deserialize(QDataStream &stream, int version)
+bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV, bool discardKV)
 {
     if (version > 1) {
         int internalStateVersion;
@@ -768,26 +815,60 @@ bool ChatLLM::deserialize(QDataStream &stream, int version)
     stream >> nameResponse;
     m_nameResponse = nameResponse.toStdString();
     stream >> m_promptResponseTokens;
+
+    // If we do not deserialize the KV or it is discarded, then we need to restore the state from the
+    // text only. This will be a costly operation, but the chat has to be restored from the text archive
+    // alone.
+    m_restoreStateFromText = !deserializeKV || discardKV;
+
+    if (!deserializeKV) {
+#if defined(DEBUG)
+        qDebug() << "deserialize" << m_llmThread.objectName();
+#endif
+        return stream.status() == QDataStream::Ok;
+    }
+
     if (version <= 3) {
         int responseLogits;
         stream >> responseLogits;
     }
-    stream >> m_ctx.n_past;
+
+    int32_t n_past;
+    stream >> n_past;
+    if (!discardKV) m_ctx.n_past = n_past;
+
     quint64 logitsSize;
     stream >> logitsSize;
-    m_ctx.logits.resize(logitsSize);
-    stream.readRawData(reinterpret_cast<char*>(m_ctx.logits.data()), logitsSize * sizeof(float));
+    if (!discardKV) {
+        m_ctx.logits.resize(logitsSize);
+        stream.readRawData(reinterpret_cast<char*>(m_ctx.logits.data()), logitsSize * sizeof(float));
+    } else {
+        stream.skipRawData(logitsSize * sizeof(float));
+    }
+
     quint64 tokensSize;
     stream >> tokensSize;
-    m_ctx.tokens.resize(tokensSize);
-    stream.readRawData(reinterpret_cast<char*>(m_ctx.tokens.data()), tokensSize * sizeof(int));
+    if (!discardKV) {
+        m_ctx.tokens.resize(tokensSize);
+        stream.readRawData(reinterpret_cast<char*>(m_ctx.tokens.data()), tokensSize * sizeof(int));
+    } else {
+        stream.skipRawData(tokensSize * sizeof(int));
+    }
+
     if (version > 0) {
         QByteArray compressed;
         stream >> compressed;
-        m_state = qUncompress(compressed);
+        if (!discardKV)
+            m_state = qUncompress(compressed);
     } else {
-        stream >> m_state;
+        if (!discardKV)
+            stream >> m_state;
+        else {
+            QByteArray state;
+            stream >> m_state;
+        }
     }
+
 #if defined(DEBUG)
     qDebug() << "deserialize" << m_llmThread.objectName();
 #endif
@@ -818,7 +899,7 @@ void ChatLLM::saveState()
 
 void ChatLLM::restoreState()
 {
-    if (!isModelLoaded() || m_state.isEmpty())
+    if (!isModelLoaded())
         return;
 
     if (m_llModelType == LLModelType::CHATGPT_) {
@@ -837,6 +918,10 @@ void ChatLLM::restoreState()
     qDebug() << "restoreState" << m_llmThread.objectName() << "size:" << m_state.size();
 #endif
     m_processedSystemPrompt = true;
+
+    if (m_state.isEmpty())
+        return;
+
     m_llModelInfo.model->restoreState(static_cast<const uint8_t*>(reinterpret_cast<void*>(m_state.data())));
     m_state.clear();
     m_state.resize(0);
@@ -854,7 +939,10 @@ void ChatLLM::processSystemPrompt()
         return;
     }
 
+    // Start with a whole new context
     m_stopGenerating = false;
+    m_ctx = LLModel::PromptContext();
+
     auto promptFunc = std::bind(&ChatLLM::handleSystemPrompt, this, std::placeholders::_1);
     auto responseFunc = std::bind(&ChatLLM::handleSystemResponse, this, std::placeholders::_1,
         std::placeholders::_2);
@@ -885,5 +973,54 @@ void ChatLLM::processSystemPrompt()
     printf("\n");
     fflush(stdout);
 #endif
-    m_processedSystemPrompt = true;
-}
+
+    m_processedSystemPrompt = !m_stopGenerating;
+}
+
+void ChatLLM::processRestoreStateFromText()
+{
+    Q_ASSERT(isModelLoaded());
+    if (!isModelLoaded() || !m_restoreStateFromText || m_isServer)
+        return;
+
+    m_isRecalc = true;
+    emit recalcChanged();
+
+    m_stopGenerating = false;
+    m_ctx = LLModel::PromptContext();
+
+    auto promptFunc = std::bind(&ChatLLM::handleRestoreStateFromTextPrompt, this, std::placeholders::_1);
+    auto responseFunc = std::bind(&ChatLLM::handleRestoreStateFromTextResponse, this, std::placeholders::_1,
+        std::placeholders::_2);
+    auto recalcFunc = std::bind(&ChatLLM::handleRestoreStateFromTextRecalculate, this, std::placeholders::_1);
+
+    const QString promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
+    const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
+    const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
+    const float top_p = MySettings::globalInstance()->modelTopP(m_modelInfo);
+    const float temp = MySettings::globalInstance()->modelTemperature(m_modelInfo);
+    const int32_t n_batch = MySettings::globalInstance()->modelPromptBatchSize(m_modelInfo);
+    const float repeat_penalty = MySettings::globalInstance()->modelRepeatPenalty(m_modelInfo);
+    const int32_t repeat_penalty_tokens = MySettings::globalInstance()->modelRepeatPenaltyTokens(m_modelInfo);
+    int n_threads = MySettings::globalInstance()->threadCount();
+    m_ctx.n_predict = n_predict;
+    m_ctx.top_k = top_k;
+    m_ctx.top_p = top_p;
+    m_ctx.temp = temp;
+    m_ctx.n_batch = n_batch;
+    m_ctx.repeat_penalty = repeat_penalty;
+    m_ctx.repeat_last_n = repeat_penalty_tokens;
+    m_llModelInfo.model->setThreadCount(n_threads);
+    for (auto pair : m_stateFromText) {
+        const QString str = pair.first == "Prompt: " ? promptTemplate.arg(pair.second) : pair.second;
+        m_llModelInfo.model->prompt(str.toStdString(), promptFunc, responseFunc, recalcFunc, m_ctx);
+    }
+
+    if (!m_stopGenerating) {
+        m_restoreStateFromText = false;
+        m_stateFromText.clear();
+    }
+
+    m_isRecalc = false;
+    emit recalcChanged();
+}

gpt4all-chat/chatllm.h

@@ -10,14 +10,10 @@
 #include "../gpt4all-backend/llmodel.h"
 
 enum LLModelType {
-    MPT_,
     GPTJ_,
     LLAMA_,
     CHATGPT_,
-    REPLIT_,
-    FALCON_,
     BERT_,
-    STARCODER_
 };
 
 struct LLModelInfo {
@@ -95,8 +91,9 @@ public:
 
     QString generatedName() const { return QString::fromStdString(m_nameResponse); }
 
-    bool serialize(QDataStream &stream, int version);
-    bool deserialize(QDataStream &stream, int version);
+    bool serialize(QDataStream &stream, int version, bool serializeKV);
+    bool deserialize(QDataStream &stream, int version, bool deserializeKV, bool discardKV);
+    void setStateFromText(const QVector<QPair<QString, QString>> &stateFromText) { m_stateFromText = stateFromText; }
 
 public Q_SLOTS:
     bool prompt(const QList<QString> &collectionList, const QString &prompt);
@@ -113,6 +110,7 @@ public Q_SLOTS:
     void handleForceMetalChanged(bool forceMetal);
     void handleDeviceChanged();
     void processSystemPrompt();
+    void processRestoreStateFromText();
 
 Q_SIGNALS:
     void recalcChanged();
@@ -130,6 +128,7 @@ Q_SIGNALS:
     void requestRetrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
     void reportSpeed(const QString &speed);
     void reportDevice(const QString &device);
+    void reportFallbackReason(const QString &fallbackReason);
     void databaseResultsChanged(const QList<ResultInfo>&);
     void modelInfoChanged(const ModelInfo &modelInfo);
 
@@ -146,6 +145,9 @@ protected:
     bool handleSystemPrompt(int32_t token);
     bool handleSystemResponse(int32_t token, const std::string &response);
     bool handleSystemRecalculate(bool isRecalc);
+    bool handleRestoreStateFromTextPrompt(int32_t token);
+    bool handleRestoreStateFromTextResponse(int32_t token, const std::string &response);
+    bool handleRestoreStateFromTextRecalculate(bool isRecalc);
     void saveState();
     void restoreState();
 
@@ -170,6 +172,8 @@ private:
     bool m_forceMetal;
     bool m_reloadingToChangeVariant;
     bool m_processedSystemPrompt;
+    bool m_restoreStateFromText;
+    QVector<QPair<QString, QString>> m_stateFromText;
 };
 
 #endif // CHATLLM_H

gpt4all-chat/chatmodel.h

@@ -285,6 +285,14 @@ public:
         return stream.status() == QDataStream::Ok;
     }
 
+    QVector<QPair<QString, QString>> text() const
+    {
+        QVector<QPair<QString, QString>> result;
+        for (const auto &c : m_chatItems)
+            result << qMakePair(c.name, c.value);
+        return result;
+    }
+
 Q_SIGNALS:
     void countChanged();
 

gpt4all-chat/cmake/deploy-qt-mac.cmake.in

@@ -3,29 +3,15 @@ set(COMPONENT_NAME_MAIN "@COMPONENT_NAME_MAIN@")
 set(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@")
 execute_process(COMMAND ${MACDEPLOYQT} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -verbose=2)
 file(GLOB MYGPTJLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libgptj*)
-file(GLOB MYMPTLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libmpt*)
 file(GLOB MYLLAMALIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllama*)
-file(GLOB MYREPLITLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libreplit*)
-file(GLOB MYFALCONLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libfalcon*)
 file(GLOB MYBERTLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libbert*)
-file(GLOB MYSTARCODERLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libstarcoder*)
 file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllmodel.*)
 file(COPY ${MYGPTJLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYMPTLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLAMALIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYREPLITLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYFALCONLLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYBERTLLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYSTARCODERLLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYLLAMALIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLMODELLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns"

gpt4all-chat/cmake/sign_dmg.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import os
 import subprocess
 import tempfile

gpt4all-chat/download.cpp

@@ -129,7 +129,7 @@ void Download::downloadModel(const QString &modelFile)
 
     ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::DownloadingRole, true);
     ModelInfo info = ModelList::globalInstance()->modelInfoByFilename(modelFile);
-    QString url = !info.url.isEmpty() ? info.url : "http://gpt4all.io/models/" + modelFile;
+    QString url = !info.url.isEmpty() ? info.url : "http://gpt4all.io/models/gguf/" + modelFile;
     Network::globalInstance()->sendDownloadStarted(modelFile);
     QNetworkRequest request(url);
     request.setAttribute(QNetworkRequest::User, modelFile);

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.appdata.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<component type="desktop">
+    <id>io.gpt4all.gpt4all</id>
+    <metadata_license>CC0-1.0</metadata_license>
+    <project_license>MIT License</project_license>
+    <name>GPT4ALL</name>
+    <summary>Open-source assistant-style large language models that run locally on your CPU and GPU</summary>
+    <description>
+        <p>Cross platform Qt based GUI for GPT4All</p>
+        <ul>
+            <li>Fast CPU and GPU based inference using ggml for open source LLM's</li>
+            <li>The UI is made to look and feel like you've come to expect from a chatty gpt</li>
+            <li>Check for updates so you can always stay fresh with latest models</li>
+            <li>Easy to install with precompiled binaries available for all three major desktop platforms</li>
+            <li>Multi-model - Ability to load more than one model and switch between them</li>
+            <li>Supports both llama.cpp and gptj.cpp style models</li>
+            <li>Model downloader in GUI featuring many popular open source models</li>
+            <li>Settings dialog to change temp, top_p, top_k, threads, etc</li>
+            <li>Copy your conversation to clipboard</li>
+        </ul>
+    </description>
+    <screenshots>
+        <screenshot type="default">
+            <caption>Main Window</caption>
+            <image>https://user-images.githubusercontent.com/50458173/231464085-da9edff6-a593-410e-8f38-7513f75c8aab.png</image>
+        </screenshot>
+    </screenshots>
+    <url type="homepage">https://gpt4all.io</url>
+    <url type="bugtracker">https://github.com/nomic-ai/gpt4all/issues</url>
+    <url type="vcs-browser">https://github.com/nomic-ai/gpt4all</url>
+    <releases>
+      <release version="2.4.19" date="2023-09-16">
+        <description>
+          <p>
+            <ul>
+              <li>A bugfix for crashes on systems that have a corrupted Vulkan driver or a corrupted version of the Vulkan shared library</li>
+            </ul>
+          </p>
+        </description>
+        </release>              
+    </releases>
+    <launchable type="desktop-id">io.gpt4all.gpt4all.desktop</launchable>
+    <content_rating type="oars-1.0">
+        <content_attribute id="language-profanity">mild</content_attribute>
+        <content_attribute id="language-humor">moderate</content_attribute>
+        <content_attribute id="language-discrimination">mild</content_attribute>
+    </content_rating>
+</component>

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.desktop

@@ -0,0 +1,9 @@
+[Desktop Entry]
+Name=GPT4ALL
+GenericName=Open-source assistant-style large language models that run locally on your CPU
+Comment=Run any GPT4All model natively on your home desktop with the auto-updating desktop chat client. See GPT4All Website for a full list of open-source models you can run with this powerful desktop application.
+Exec=chat
+Icon=io.gpt4all.gpt4all
+Type=Application
+Categories=Utility;Office;
+Keywords=GPT,Chat;AI

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.yml

@@ -0,0 +1,166 @@
+app-id: io.gpt4all.gpt4all
+default-branch: stable
+runtime: org.kde.Platform
+runtime-version: '6.5'
+sdk: org.kde.Sdk
+sdk-extensions:
+  - org.freedesktop.Sdk.Extension.node14
+finish-args:
+  - --share=ipc
+  - --socket=wayland
+  - --socket=x11
+  - --share=network
+  - --device=dri
+  - --env=LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/lib/x86_64-linux-gnu/
+  - --filesystem=xdg-documents:ro
+command: chat
+cleanup:
+  - /include
+
+modules:
+  - name: qthttpserver
+    buildsystem: cmake
+    sources:
+      - type: archive
+        url: https://invent.kde.org/qt/qt/qthttpserver/-/archive/6.5.2/qthttpserver-6.5.2.zip
+        sha256: 9fb7b14774b4ed62fe9097e13fa593af06ba75537783fc62f34652bada26abee
+
+  - name: python-html5lib
+    buildsystem: simple
+    build-commands:
+    - 'pip3 install --verbose --exists-action=i --no-index --find-links="file://${PWD}" --prefix=${FLATPAK_DEST} "html5lib" --no-build-isolation'
+    sources:
+      - type: file
+        url: https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl
+        sha256: a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78
+        x-checker-data:
+            type: pypi
+            name: webencodings
+            packagetype: bdist_wheel
+      - type: file
+        url: https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl
+        sha256: 0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d
+        x-checker-data:
+            type: pypi
+            name: html5lib
+            packagetype: bdist_wheel
+    cleanup:
+    - '*'
+
+  - name: qtwebengine
+    buildsystem: cmake
+    builddir: true
+    config-opts:
+      - -DQT_FEATURE_qtwebengine_build=OFF
+      - -DQT_FEATURE_qtpdf_build=ON
+    build-options:
+      append-path: /usr/lib/sdk/node14/bin
+      env:
+        - npm_config_nodedir=/usr/lib/sdk/node14
+    sources:
+      - type: git
+        url: https://invent.kde.org/qt/qt/qtwebengine.git
+        tag: v6.5.2
+        commit: ac887518e8243828333e923b5a1e61a007babde5
+  
+  - name: vulkan-headers
+    buildsystem: cmake
+    builddir: true
+    sources:
+      - type: git
+        url: https://github.com/KhronosGroup/Vulkan-Headers.git
+        tag: v1.3.224
+        commit: 2b55157592bf4c639b76cc16d64acaef565cc4b5
+
+  - name: fmt
+    buildsystem: cmake
+    builddir: true
+    sources:
+      - type: git
+        url: https://github.com/fmtlib/fmt.git
+        tag: 10.1.1
+        commit: f5e54359df4c26b6230fc61d38aa294581393084
+
+  - name: vulkan-tools
+    buildsystem: cmake
+    builddir: true
+    sources:
+      - type: git
+        url: https://github.com/KhronosGroup/Vulkan-Tools.git
+        tag: v1.3.224
+        commit: 497f232680b046db34ba9e9da065e6303a125851
+
+    
+
+    modules:
+      - name: shaderc
+        buildsystem: cmake-ninja
+        builddir: true
+        config-opts:
+          - -DSHADERC_SKIP_COPYRIGHT_CHECK=ON
+          - -DSHADERC_SKIP_EXAMPLES=ON
+          - -DSHADERC_SKIP_TESTS=ON
+          - -DSPIRV_SKIP_EXECUTABLES=ON
+          - -DENABLE_GLSLANG_BINARIES=OFF
+        cleanup:
+          - /bin
+          - /include
+          - /lib/cmake
+          - /lib/pkgconfig
+        sources:
+          - type: git
+            url: https://github.com/google/shaderc.git
+            tag: v2023.4
+            commit: 45b735dfddefe26a99b77e5a74e30d860713ac64
+#           x-checker-data:
+#             type: git
+#             tag-pattern: ^v(\d{4}\.\d{1,2})$
+          - type: git
+            url: https://github.com/KhronosGroup/SPIRV-Tools.git
+            tag: v2023.2
+            commit: 44d72a9b36702f093dd20815561a56778b2d181e
+            dest: third_party/spirv-tools
+            x-checker-data:
+              type: git
+              tag-pattern: ^v(\d{4}\.\d{1})$
+          - type: git
+            url: https://github.com/KhronosGroup/SPIRV-Headers.git
+            tag: sdk-1.3.250.1
+            commit: 268a061764ee69f09a477a695bf6a11ffe311b8d
+            dest: third_party/spirv-headers
+            x-checker-data:
+              type: git
+              tag-pattern: ^sdk-([\d.]+)$
+          - type: git
+            url: https://github.com/KhronosGroup/glslang.git
+            tag: 12.2.0
+            commit: d1517d64cfca91f573af1bf7341dc3a5113349c0
+            dest: third_party/glslang
+          
+          
+
+
+  - name: gpt4all
+    buildsystem: simple
+    build-commands:
+      - git submodule update --init --recursive
+      - mkdir build
+      - cmake  -S ./gpt4all-chat -B build -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF -DKOMPUTE_OPT_USE_BUILT_IN_FMT=OFF -DCMAKE_INSTALL_PREFIX=/app
+      - cmake --build build --config Release -- -j
+      - cmake --install build --prefix "/app"
+      - install -Dm644 logo.svg /app/share/icons/hicolor/scalable/apps/io.gpt4all.gpt4all.svg
+      - install -Dm644 io.gpt4all.gpt4all.desktop /app/share/applications/io.gpt4all.gpt4all.desktop
+      - install -Dm644 io.gpt4all.gpt4all.appdata.xml  /app/share/appdata/io.gpt4all.gpt4all.appdata.xml
+    sources:
+      - type: git
+        url: https://github.com/nomic-ai/gpt4all
+        tag: v2.4.19
+        commit: 84905aa28171545542fc653dbeca501ae5af383e
+      - type: file
+        url: https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/icons/logo.svg
+        sha256: 4c4e8476d0e2020585b69c6e2fc9e7d0cb12cbb36aa7b83c3a2e48ed4a9a424c
+      - type: file
+        path: io.gpt4all.gpt4all.desktop
+      - type: file
+        path: io.gpt4all.gpt4all.appdata.xml
+

gpt4all-chat/llm.cpp

@@ -9,6 +9,7 @@
 #include <QProcess>
 #include <QResource>
 #include <QSettings>
+#include <QDesktopServices>
 #include <fstream>
 
 class MyLLM: public LLM { };
@@ -60,6 +61,10 @@ bool LLM::hasSettingsAccess() const
 
 bool LLM::checkForUpdates() const
 {
+    #ifdef GPT4ALL_OFFLINE_INSTALLER
+    #pragma message "offline installer build will not check for updates!"
+    return QDesktopServices::openUrl(QUrl("https://gpt4all.io/"));
+    #else
     Network::globalInstance()->sendCheckForUpdates();
 
 #if defined(Q_OS_LINUX)
@@ -78,6 +83,7 @@ bool LLM::checkForUpdates() const
     }
 
     return QProcess::startDetached(fileName);
+    #endif
 }
 
 bool LLM::directoryExists(const QString &path) const

gpt4all-chat/main.qml

@@ -189,7 +189,7 @@ Window {
             + "causes include a bad file format, an incomplete or corrupted download, the wrong file "
             + "type, not enough system RAM or an incompatible model type. Here are some suggestions for resolving the problem:"
             + "<br><ul>"
-            + "<li>Ensure the model file has a compatible ggml format and type"
+            + "<li>Ensure the model file has a compatible format and type"
             + "<li>Check the model file is complete in the download folder"
             + "<li>You can find the download folder in the settings dialog"
             + "<li>If you've sideloaded the model ensure the file is not corrupt by checking md5sum"
@@ -282,8 +282,8 @@ Window {
                     highlighted: comboBox.highlightedIndex === index
                 }
                 Accessible.role: Accessible.ComboBox
-                Accessible.name: qsTr("ComboBox for displaying/picking the current model")
-                Accessible.description: qsTr("Use this for picking the current model to use; the first item is the current model")
+                Accessible.name: qsTr("List of available models")
+                Accessible.description: qsTr("The top item is the current model")
                 onActivated: function (index) {
                     currentChat.stopGenerating()
                     currentChat.reset();
@@ -307,7 +307,7 @@ Window {
                     running: parent.visible
                     Accessible.role: Accessible.Animation
                     Accessible.name: qsTr("Busy indicator")
-                    Accessible.description: qsTr("Displayed when the model is loading")
+                    Accessible.description: qsTr("loading model...")
                 }
 
                 Label {
@@ -339,8 +339,8 @@ Window {
         padding: 15
 
         Accessible.role: Accessible.ButtonMenu
-        Accessible.name: qsTr("Hamburger button")
-        Accessible.description: qsTr("Hamburger button that reveals a drawer on the left of the application")
+        Accessible.name: qsTr("Main menu")
+        Accessible.description: qsTr("Navigation drawer with options")
 
         background: Item {
             anchors.centerIn: parent
@@ -389,7 +389,7 @@ Window {
         Item {
             Accessible.role: Accessible.Dialog
             Accessible.name: qsTr("Network dialog")
-            Accessible.description: qsTr("Dialog for opt-in to sharing feedback/conversations")
+            Accessible.description: qsTr("opt-in to share feedback/conversations")
         }
     }
 
@@ -405,7 +405,7 @@ Window {
         padding: 15
         toggled: MySettings.networkIsActive
         source: "qrc:/gpt4all/icons/network.svg"
-        Accessible.name: qsTr("Network button")
+        Accessible.name: qsTr("Network")
         Accessible.description: qsTr("Reveals a dialogue where you can opt-in for sharing data over network")
 
         onClicked: {
@@ -441,8 +441,8 @@ Window {
         padding: 15
         toggled: currentChat.collectionList.length
         source: "qrc:/gpt4all/icons/db.svg"
-        Accessible.name: qsTr("Add collections of documents to the chat")
-        Accessible.description: qsTr("Provides a button to add collections of documents to the chat")
+        Accessible.name: qsTr("Add documents")
+        Accessible.description: qsTr("add collections of documents to the chat")
 
         onClicked: {
             collectionsDialog.open()
@@ -460,8 +460,8 @@ Window {
         z: 200
         padding: 15
         source: "qrc:/gpt4all/icons/settings.svg"
-        Accessible.name: qsTr("Settings button")
-        Accessible.description: qsTr("Reveals a dialogue where you can change various settings")
+        Accessible.name: qsTr("Settings")
+        Accessible.description: qsTr("Reveals a dialogue with settings")
 
         onClicked: {
             settingsDialog.open()
@@ -528,7 +528,7 @@ Window {
         z: 200
         padding: 15
         source: "qrc:/gpt4all/icons/copy.svg"
-        Accessible.name: qsTr("Copy button")
+        Accessible.name: qsTr("Copy")
         Accessible.description: qsTr("Copy the conversation to the clipboard")
 
         TextEdit{
@@ -595,7 +595,7 @@ Window {
         source: "qrc:/gpt4all/icons/regenerate.svg"
 
         Accessible.name: text
-        Accessible.description: qsTr("Reset the context which erases current conversation")
+        Accessible.description: qsTr("Reset the context and erase current conversation")
 
         onClicked: {
             Network.sendResetContext(chatModel.count)
@@ -623,7 +623,7 @@ Window {
             font.pixelSize: theme.fontSizeLarge
             Accessible.role: Accessible.Dialog
             Accessible.name: text
-            Accessible.description: qsTr("Dialog indicating an error")
+            Accessible.description: qsTr("Error dialog")
         }
         background: Rectangle {
             anchors.fill: parent
@@ -641,7 +641,7 @@ Window {
         height: window.height - (window.height * .1)
         Item {
             Accessible.role: Accessible.Dialog
-            Accessible.name: qsTr("Download new models dialog")
+            Accessible.name: qsTr("Download new models")
             Accessible.description: qsTr("Dialog for downloading new models")
         }
     }
@@ -740,8 +740,8 @@ Window {
                     ScrollBar.vertical: ScrollBar { policy: ScrollBar.AlwaysOn }
 
                     Accessible.role: Accessible.List
-                    Accessible.name: qsTr("List of prompt/response pairs")
-                    Accessible.description: qsTr("This is the list of prompt/response pairs comprising the actual conversation with the model")
+                    Accessible.name: qsTr("Conversation with the model")
+                    Accessible.description: qsTr("prompt / response pairs from the conversation")
 
                     delegate: TextArea {
                         id: myTextArea
@@ -811,12 +811,21 @@ Window {
                                     running: (currentResponse ? true : false) && value === "" && currentChat.responseInProgress
                                     Accessible.role: Accessible.Animation
                                     Accessible.name: qsTr("Busy indicator")
-                                    Accessible.description: qsTr("Displayed when the model is thinking")
+                                    Accessible.description: qsTr("The model is thinking")
                                 }
                                 Label {
                                     anchors.verticalCenter: parent.verticalCenter
-                                    text: currentChat.responseState + "..."
                                     color: theme.textAccent
+                                    text: {
+                                        switch (currentChat.responseState) {
+                                        case Chat.ResponseStopped: return "response stopped ...";
+                                        case Chat.LocalDocsRetrieval: return "retrieving " + currentChat.collectionList.join(", ") + " ...";
+                                        case Chat.LocalDocsProcessing: return "processing " + currentChat.collectionList.join(", ") + " ...";
+                                        case Chat.PromptProcessing: return "processing ..."
+                                        case Chat.ResponseGeneration: return "generating response ...";
+                                        default: return ""; // handle unexpected values
+                                        }
+                                    }
                                 }
                             }
                         }
@@ -1013,7 +1022,7 @@ Window {
             anchors.rightMargin: 30
             color: theme.mutedTextColor
             visible: currentChat.tokenSpeed !== ""
-            text: qsTr("Speed: ") + currentChat.tokenSpeed + "<br>" + qsTr("Device: ") + currentChat.device
+            text: qsTr("Speed: ") + currentChat.tokenSpeed + "<br>" + qsTr("Device: ") + currentChat.device + currentChat.fallbackReason
             font.pixelSize: theme.fontSizeLarge
         }
 
@@ -1053,7 +1062,7 @@ Window {
                 }
                 Accessible.role: Accessible.EditableText
                 Accessible.name: placeholderText
-                Accessible.description: qsTr("Textfield for sending messages/prompts to the model")
+                Accessible.description: qsTr("Send messages/prompts to the model")
                 Keys.onReturnPressed: (event)=> {
                     if (event.modifiers & Qt.ControlModifier || event.modifiers & Qt.ShiftModifier)
                         event.accepted = false;
@@ -1090,7 +1099,7 @@ Window {
             height: 30
             visible: !currentChat.isServer
             source: "qrc:/gpt4all/icons/send_message.svg"
-            Accessible.name: qsTr("Send the message button")
+            Accessible.name: qsTr("Send message")
             Accessible.description: qsTr("Sends the message/prompt contained in textfield to the model")
 
             onClicked: {

gpt4all-chat/metadata/models2.json

@@ -0,0 +1,210 @@
+[
+  {
+    "order": "a",
+    "md5sum": "48de9538c774188eb25a7e9ee024bbd3",
+    "name": "Mistral OpenOrca",
+    "filename": "mistral-7b-openorca.Q4_0.gguf",
+    "filesize": "4108927744",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "systemPrompt": " ",
+    "description": "<strong>Best overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
+    "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.Q4_0.gguf"
+  },
+  {
+    "order": "b",
+    "md5sum": "97463be739b50525df56d33b26b00852",
+    "name": "Mistral Instruct",
+    "filename": "mistral-7b-instruct-v0.1.Q4_0.gguf",
+    "filesize": "4108916384",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "systemPrompt": " ",
+    "description": "<strong>Best overall fast instruction following model</strong><br><ul><li>Fast responses</li><li>Trained by Mistral AI<li>Uncensored</li><li>Licensed for commercial use</li></ul>",
+    "url": "https://gpt4all.io/models/gguf/mistral-7b-instruct-v0.1.Q4_0.gguf",
+    "promptTemplate": "[INST] %1 [/INST]"
+  },
+  {
+    "order": "c",
+    "md5sum": "31cb6d527bd3bfb5e73c2e9dfbc75033",
+    "name": "GPT4All Falcon",
+    "filename": "gpt4all-falcon-q4_0.gguf",
+    "filesize": "4210419040",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Falcon",
+    "systemPrompt": " ",
+    "description": "<strong>Very fast model with good quality</strong><br><ul><li>Fastest responses</li><li>Instruction based</li><li>Trained by TII<li>Finetuned by Nomic AI<li>Licensed for commercial use</ul>",
+    "url": "https://gpt4all.io/models/gguf/gpt4all-falcon-q4_0.gguf",
+    "promptTemplate": "### Instruction:\n%1\n### Response:\n"
+  },
+  {
+    "order": "e",
+    "md5sum": "5aff90007499bce5c64b1c0760c0b186",
+    "name": "Wizard v1.2",
+    "filename": "wizardlm-13b-v1.2.Q4_0.gguf",
+    "filesize": "7365834624",
+    "requires": "2.5.0",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_0",
+    "type": "LLaMA2",
+    "systemPrompt": " ",
+    "description": "<strong>Best overall larger model</strong><br><ul><li>Instruction based<li>Gives very long responses<li>Finetuned with only 1k of high-quality data<li>Trained by Microsoft and Peking University<li>Cannot be used commercially</ul",
+    "url": "https://gpt4all.io/models/gguf/wizardlm-13b-v1.2.Q4_0.gguf"
+  },
+  {
+    "order": "f",
+    "md5sum": "3d12810391d04d1153b692626c0c6e16",
+    "name": "Hermes",
+    "filename": "nous-hermes-llama2-13b.Q4_0.gguf",
+    "filesize": "7366062080",
+    "requires": "2.5.0",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_0",
+    "type": "LLaMA2",
+    "systemPrompt": " ",
+    "description": "<strong>Extremely good model</strong><br><ul><li>Instruction based<li>Gives long responses<li>Curated with 300,000 uncensored instructions<li>Trained by Nous Research<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/nous-hermes-llama2-13b.Q4_0.gguf",
+    "promptTemplate": "### Instruction:\n%1\n### Response:\n"
+  },
+  {
+    "order": "g",
+    "md5sum": "40388eb2f8d16bb5d08c96fdfaac6b2c",
+    "name": "Snoozy",
+    "filename": "gpt4all-13b-snoozy-q4_0.gguf",
+    "filesize": "7365834624",
+    "requires": "2.5.0",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_0",
+    "type": "LLaMA",
+    "systemPrompt": " ",
+    "description": "<strong>Very good overall model</strong><br><ul><li>Instruction based<li>Based on the same dataset as Groovy<li>Slower than Groovy, with higher quality responses<li>Trained by Nomic AI<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf"
+  },
+  {
+    "order": "h",
+    "md5sum": "cf5e8f73747f9d7c6fe72a629808c1de",
+    "name": "MPT Chat",
+    "filename": "mpt-7b-chat-merges-q4_0.gguf",
+    "filesize": "3796133728",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "MPT",
+    "description": "<strong>Good model with novel architecture</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/mpt-7b-chat-merges-q4_0.gguf",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|><|im_start|>assistant\n",
+    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
+  },
+  {
+    "order": "i",
+    "md5sum": "0e769317b90ac30d6e09486d61fefa26",
+    "name": "Mini Orca (Small)",
+    "filename": "orca-mini-3b-gguf2-q4_0.gguf",
+    "filesize": "1979946720",
+    "requires": "2.5.0",
+    "ramrequired": "4",
+    "parameters": "3 billion",
+    "quant": "q4_0",
+    "type": "OpenLLaMa",
+    "description": "<strong>Small version of new model with novel dataset</strong><br><ul><li>Instruction based<li>Explain tuned datasets<li>Orca Research Paper dataset construction approaches<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/orca-mini-3b-gguf2-q4_0.gguf",
+    "promptTemplate": "### User:\n%1\n### Response:\n",
+    "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
+  },
+  {
+    "order": "j",
+    "md5sum": "e30579a1b109882f10e2a5e75ea388fb",
+    "disableGUI": "true",
+    "name": "Replit",
+    "filename": "replit-code-v1_5-3b-q4_0.gguf",
+    "filesize": "1870449696",
+    "requires": "2.5.0",
+    "ramrequired": "4",
+    "parameters": "3 billion",
+    "quant": "q4_0",
+    "type": "Replit",
+    "systemPrompt": " ",
+    "promptTemplate": "%1",
+    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>Licensed for commercial use</ul>",
+    "url": "https://gpt4all.io/models/gguf/replit-code-v1_5-3b-q4_0.gguf"
+  },
+  {
+    "order": "k",
+    "md5sum": "556fc3e13df42286997fb58e6f4c639f",
+    "disableGUI": "true",
+    "name": "Starcoder",
+    "filename": "starcoder-q4_0.gguf",
+    "filesize": "8987166880",
+    "requires": "2.5.0",
+    "ramrequired": "4",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Starcoder",
+    "systemPrompt": " ",
+    "promptTemplate": "%1",
+    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based</ul>",
+    "url": "https://gpt4all.io/models/gguf/starcoder-q4_0.gguf"
+  },
+  {
+    "order": "l",
+    "md5sum": "e973dd26f0ffa6e46783feaea8f08c83",
+    "disableGUI": "true",
+    "name": "Rift coder",
+    "filename": "rift-coder-v0-7b-q4_0.gguf",
+    "filesize": "3825903776",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "LLaMA",
+    "systemPrompt": " ",
+    "promptTemplate": "%1",
+    "description": "Code completion model",
+    "url": "https://gpt4all.io/models/gguf/rift-coder-v0-7b-q4_0.gguf"
+  },
+  {
+    "order": "m",
+    "md5sum": "e479e6f38b59afc51a470d1953a6bfc7",
+    "disableGUI": "true",
+    "name": "SBert",
+    "filename": "all-MiniLM-L6-v2-f16.gguf",
+    "filesize": "45887744",
+    "requires": "2.5.0",
+    "ramrequired": "1",
+    "parameters": "1 million",
+    "quant": "f16",
+    "type": "Bert",
+    "systemPrompt": " ",
+    "description": "<strong>Sbert</strong><br><ul><li>For embeddings",
+    "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf"
+  },
+  {
+    "order": "n",
+    "md5sum": "919de4dd6f25351bcb0223790db1932d",
+    "name": "EM German Mistral",
+    "filename": "em_german_mistral_v01.Q4_0.gguf",
+    "filesize": "4108916352",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "description": "<strong>Mistral-based model for German-language applications</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by ellamind<li>Finetuned on German instruction and chat data</a><li>Licensed for commercial use</ul>",
+    "url": "https://huggingface.co/TheBloke/em_german_mistral_v01-GGUF/resolve/main/em_german_mistral_v01.Q4_0.gguf",
+    "promptTemplate": "USER: %1 ASSISTANT: ",
+    "systemPrompt": "Du bist ein hilfreicher Assistent. "
+  }
+]

gpt4all-chat/metadata/release.json

@@ -75,7 +75,7 @@
 * resumable downloads for models
 * chat list in the drawer drop down
 * add/remove/rename chats
-* perist chats to disk and restore them with full context (WARNING: the average size of each chat on disk is ~1.5GB)
+* persist chats to disk and restore them with full context (WARNING: the average size of each chat on disk is ~1.5GB)
 * NOTE: to turn on the persistent chats feature you need to do so via the settings dialog as it is off by default
 * automatically rename chats using the AI after the first prompt/response pair
 * new usage statistics including more detailed hardware info to help debug problems on older hardware
@@ -506,6 +506,65 @@
 "
 * Adam Treat (Nomic AI)
 * Aaron Miller (Nomic AI)
+"
+  },
+  {
+    "version": "2.4.18",
+    "notes":
+"
+* Bugfix for devices to show up in the settings combobox on application start and not just on model load
+* Send information on requested device and actual device on model load to help assess which model/gpu/os combos are working
+",
+    "contributors":
+"
+* Adam Treat (Nomic AI)
+"
+  },
+  {
+    "version": "2.4.19",
+    "notes":
+"
+* Fix a crash on systems with corrupted vulkan drivers or corrupted vulkan dlls
+",
+    "contributors":
+"
+* Adam Treat (Nomic AI)
+"
+  },
+  {
+    "version": "2.5.0",
+    "notes":
+"
+* Major new release supports GGUF models only!
+* New models like Mistral Instruct, Replit 1.5, Rift Coder and more
+* All previous version of ggml-based models are no longer supported
+* Extensive changes to vulkan support
+* Better GPU error messages
+* Prompt processing on the GPU
+* Save chats now saves to text (less harddrive space)
+* Many more changes
+",
+    "contributors":
+"
+* Aaron Miller (Nomic AI)
+* Jared Van Bortel (Nomic AI)
+* Adam Treat (Nomic AI)
+* Community (beta testers, bug reporters, bindings authors)
+"
+  },
+  {
+    "version": "2.5.1",
+    "notes":
+"
+* Accessibility fixes
+* Bugfix for crasher on Windows
+",
+    "contributors":
+"
+* Aaron Miller (Nomic AI)
+* Jared Van Bortel (Nomic AI)
+* Victor Tsaran <vtsaran@yahoo.com>
+* Community (beta testers, bug reporters, bindings authors)
 "
   }
 ]

gpt4all-chat/modellist.cpp

@@ -796,7 +796,7 @@ void ModelList::updateModelsFromDirectory()
                 QString filename = it.fileName();
 
                 // All files that end with .bin and have 'ggml' somewhere in the name
-                if ((filename.endsWith(".bin") && filename.contains("ggml") && !filename.startsWith("incomplete"))
+                if (((filename.endsWith(".bin") || filename.endsWith(".gguf")) && (/*filename.contains("ggml") ||*/ filename.contains("gguf")) && !filename.startsWith("incomplete"))
                     || (filename.endsWith(".txt") && filename.startsWith("chatgpt-"))) {
 
                     QString filePath = it.filePath();
@@ -834,12 +834,14 @@ void ModelList::updateModelsFromDirectory()
         processDirectory(localPath);
 }
 
+#define MODELS_VERSION 2
+
 void ModelList::updateModelsFromJson()
 {
 #if defined(USE_LOCAL_MODELSJSON)
-    QUrl jsonUrl("file://" + QDir::homePath() + "/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models.json");
+    QUrl jsonUrl("file://" + QDir::homePath() + QString("/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models%1.json").arg(MODELS_VERSION));
 #else
-    QUrl jsonUrl("http://gpt4all.io/models/models.json");
+    QUrl jsonUrl(QString("http://gpt4all.io/models/models%1.json").arg(MODELS_VERSION));
 #endif
     QNetworkRequest request(jsonUrl);
     QSslConfiguration conf = request.sslConfiguration();
@@ -881,9 +883,9 @@ void ModelList::updateModelsFromJsonAsync()
     emit asyncModelRequestOngoingChanged();
 
 #if defined(USE_LOCAL_MODELSJSON)
-    QUrl jsonUrl("file://" + QDir::homePath() + "/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models.json");
+    QUrl jsonUrl("file://" + QDir::homePath() + QString("/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models%1.json").arg(MODELS_VERSION));
 #else
-    QUrl jsonUrl("http://gpt4all.io/models/models.json");
+    QUrl jsonUrl(QString("http://gpt4all.io/models/models%1.json").arg(MODELS_VERSION));
 #endif
     QNetworkRequest request(jsonUrl);
     QSslConfiguration conf = request.sslConfiguration();

gpt4all-chat/modellist.h

@@ -115,7 +115,7 @@ private:
     double  m_repeatPenalty       = 1.18;
     int     m_repeatPenaltyTokens = 64;
     QString m_promptTemplate      = "### Human:\n%1\n### Assistant:\n";
-    QString m_systemPrompt        = "### System:\nYou are an AI assistant who gives quality response to whatever humans ask of you.\n";
+    QString m_systemPrompt        = "### System:\nYou are an AI assistant who gives a quality response to whatever humans ask of you.\n";
     friend class MySettings;
 };
 Q_DECLARE_METATYPE(ModelInfo)

gpt4all-chat/mysettings.cpp

@@ -1,5 +1,6 @@
 #include "mysettings.h"
 #include "modellist.h"
+#include "../gpt4all-backend/llmodel.h"
 
 #include <QDir>
 #include <QFile>
@@ -9,8 +10,7 @@
 #include <QUrl>
 
 static int      default_threadCount         = std::min(4, (int32_t) std::thread::hardware_concurrency());
-static bool     default_saveChats           = false;
-static bool     default_saveChatGPTChats    = true;
+static bool     default_saveChatsContext    = false;
 static bool     default_serverChat          = false;
 static QString  default_userDefaultModel    = "Application default";
 static bool     default_forceMetal          = false;
@@ -63,6 +63,13 @@ MySettings::MySettings()
     : QObject{nullptr}
 {
     QSettings::setDefaultFormat(QSettings::IniFormat);
+
+    std::vector<LLModel::GPUDevice> devices = LLModel::availableGPUDevices();
+    QVector<QString> deviceList{ "Auto" };
+    for (LLModel::GPUDevice &d : devices)
+        deviceList << QString::fromStdString(d.name);
+    deviceList << "CPU";
+    setDeviceList(deviceList);
 }
 
 Q_INVOKABLE QVector<QString> MySettings::deviceList() const
@@ -95,8 +102,7 @@ void MySettings::restoreApplicationDefaults()
     setFontSize(default_fontSize);
     setDevice(default_device);
     setThreadCount(default_threadCount);
-    setSaveChats(default_saveChats);
-    setSaveChatGPTChats(default_saveChatGPTChats);
+    setSaveChatsContext(default_saveChatsContext);
     setServerChat(default_serverChat);
     setModelPath(defaultLocalModelsPath());
     setUserDefaultModel(default_userDefaultModel);
@@ -389,40 +395,22 @@ void MySettings::setThreadCount(int c)
     emit threadCountChanged();
 }
 
-bool MySettings::saveChats() const
+bool MySettings::saveChatsContext() const
 {
     QSettings setting;
     setting.sync();
-    return setting.value("saveChats", default_saveChats).toBool();
+    return setting.value("saveChatsContext", default_saveChatsContext).toBool();
 }
 
-void MySettings::setSaveChats(bool b)
+void MySettings::setSaveChatsContext(bool b)
 {
-    if (saveChats() == b)
+    if (saveChatsContext() == b)
         return;
 
     QSettings setting;
-    setting.setValue("saveChats", b);
+    setting.setValue("saveChatsContext", b);
     setting.sync();
-    emit saveChatsChanged();
-}
-
-bool MySettings::saveChatGPTChats() const
-{
-    QSettings setting;
-    setting.sync();
-    return setting.value("saveChatGPTChats", default_saveChatGPTChats).toBool();
-}
-
-void MySettings::setSaveChatGPTChats(bool b)
-{
-    if (saveChatGPTChats() == b)
-        return;
-
-    QSettings setting;
-    setting.setValue("saveChatGPTChats", b);
-    setting.sync();
-    emit saveChatGPTChatsChanged();
+    emit saveChatsContextChanged();
 }
 
 bool MySettings::serverChat() const

gpt4all-chat/mysettings.h

@@ -10,8 +10,7 @@ class MySettings : public QObject
 {
     Q_OBJECT
     Q_PROPERTY(int threadCount READ threadCount WRITE setThreadCount NOTIFY threadCountChanged)
-    Q_PROPERTY(bool saveChats READ saveChats WRITE setSaveChats NOTIFY saveChatsChanged)
-    Q_PROPERTY(bool saveChatGPTChats READ saveChatGPTChats WRITE setSaveChatGPTChats NOTIFY saveChatGPTChatsChanged)
+    Q_PROPERTY(bool saveChatsContext READ saveChatsContext WRITE setSaveChatsContext NOTIFY saveChatsContextChanged)
     Q_PROPERTY(bool serverChat READ serverChat WRITE setServerChat NOTIFY serverChatChanged)
     Q_PROPERTY(QString modelPath READ modelPath WRITE setModelPath NOTIFY modelPathChanged)
     Q_PROPERTY(QString userDefaultModel READ userDefaultModel WRITE setUserDefaultModel NOTIFY userDefaultModelChanged)
@@ -64,10 +63,8 @@ public:
     // Application settings
     int threadCount() const;
     void setThreadCount(int c);
-    bool saveChats() const;
-    void setSaveChats(bool b);
-    bool saveChatGPTChats() const;
-    void setSaveChatGPTChats(bool b);
+    bool saveChatsContext() const;
+    void setSaveChatsContext(bool b);
     bool serverChat() const;
     void setServerChat(bool b);
     QString modelPath() const;
@@ -122,8 +119,7 @@ Q_SIGNALS:
     void promptTemplateChanged(const ModelInfo &model);
     void systemPromptChanged(const ModelInfo &model);
     void threadCountChanged();
-    void saveChatsChanged();
-    void saveChatGPTChatsChanged();
+    void saveChatsContextChanged();
     void serverChatChanged();
     void modelPathChanged();
     void userDefaultModelChanged();

gpt4all-chat/network.cpp

@@ -317,16 +317,6 @@ void Network::sendNetworkToggled(bool isActive)
     sendMixpanelEvent("network_toggled", QVector<KeyValue>{kv});
 }
 
-void Network::sendSaveChatsToggled(bool isActive)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("isActive");
-    kv.value = QJsonValue(isActive);
-    sendMixpanelEvent("savechats_toggled", QVector<KeyValue>{kv});
-}
-
 void Network::sendNewChat(int count)
 {
     if (!MySettings::globalInstance()->networkUsageStatsActive())
@@ -393,6 +383,8 @@ void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &valu
     properties.insert("name", QCoreApplication::applicationName() + " v"
         + QCoreApplication::applicationVersion());
     properties.insert("model", ChatListModel::globalInstance()->currentChat()->modelInfo().filename());
+    properties.insert("requestedDevice", MySettings::globalInstance()->device());
+    properties.insert("actualDevice", ChatListModel::globalInstance()->currentChat()->device());
 
     // Some additional startup information
     if (ev == "startup") {

gpt4all-chat/network.h

@@ -38,7 +38,6 @@ public Q_SLOTS:
     void sendDownloadFinished(const QString &model, bool success);
     Q_INVOKABLE void sendSettingsDialog();
     Q_INVOKABLE void sendNetworkToggled(bool active);
-    Q_INVOKABLE void sendSaveChatsToggled(bool active);
     Q_INVOKABLE void sendNewChat(int count);
     Q_INVOKABLE void sendRemoveChat();
     Q_INVOKABLE void sendRenameChat();

gpt4all-chat/qml/ApplicationSettings.qml

@@ -35,8 +35,8 @@ MySettingsTab {
             Layout.fillWidth: false
             model: ["Dark", "Light"]
             Accessible.role: Accessible.ComboBox
-            Accessible.name: qsTr("ComboBox for displaying/picking the color theme")
-            Accessible.description: qsTr("Use this for picking the color theme for the chat client to use")
+            Accessible.name: qsTr("Color theme")
+            Accessible.description: qsTr("Color theme for the chat client to use")
             function updateModel() {
                 themeBox.currentIndex = themeBox.indexOfValue(MySettings.chatTheme);
             }
@@ -70,8 +70,8 @@ MySettingsTab {
             Layout.fillWidth: false
             model: ["Small", "Medium", "Large"]
             Accessible.role: Accessible.ComboBox
-            Accessible.name: qsTr("ComboBox for displaying/picking the font size")
-            Accessible.description: qsTr("Use this for picking the font size of the chat client")
+            Accessible.name: qsTr("Font size")
+            Accessible.description: qsTr("Font size of the chat client")
             function updateModel() {
                 fontBox.currentIndex = fontBox.indexOfValue(MySettings.fontSize);
             }
@@ -105,8 +105,8 @@ MySettingsTab {
             Layout.fillWidth: false
             model: MySettings.deviceList
             Accessible.role: Accessible.ComboBox
-            Accessible.name: qsTr("ComboBox for displaying/picking the device")
-            Accessible.description: qsTr("Use this for picking the device of the chat client")
+            Accessible.name: qsTr("Device")
+            Accessible.description: qsTr("Device of the chat client")
             function updateModel() {
                 deviceBox.currentIndex = deviceBox.indexOfValue(MySettings.device);
             }
@@ -143,8 +143,8 @@ MySettingsTab {
             Layout.fillWidth: true
             model: ModelList.userDefaultModelList
             Accessible.role: Accessible.ComboBox
-            Accessible.name: qsTr("ComboBox for displaying/picking the default model")
-            Accessible.description: qsTr("Use this for picking the default model to use; the first item is the current default model")
+            Accessible.name: qsTr("Default model")
+            Accessible.description: qsTr("Default model to use; the first item is the current default model")
             function updateModel() {
                 comboBox.currentIndex = comboBox.indexOfValue(MySettings.userDefaultModel);
             }
@@ -194,7 +194,7 @@ MySettingsTab {
             Layout.row: 5
             Layout.column: 2
             text: qsTr("Browse")
-            Accessible.description: qsTr("Opens a folder picker dialog to choose where to save model files")
+            Accessible.description: qsTr("Choose where to save model files")
             onClicked: {
                 openFolderDialog("file://" + MySettings.modelPath, function(selectedFolder) {
                     MySettings.modelPath = selectedFolder
@@ -234,53 +234,35 @@ MySettingsTab {
             Accessible.description: ToolTip.text
         }
         Label {
-            id: saveChatsLabel
-            text: qsTr("Save chats to disk:")
+            id: saveChatsContextLabel
+            text: qsTr("Save chats context to disk:")
             color: theme.textColor
             font.pixelSize: theme.fontSizeLarge
             Layout.row: 7
             Layout.column: 0
         }
         MyCheckBox {
-            id: saveChatsBox
+            id: saveChatsContextBox
             Layout.row: 7
             Layout.column: 1
-            checked: MySettings.saveChats
+            checked: MySettings.saveChatsContext
             onClicked: {
-                Network.sendSaveChatsToggled(saveChatsBox.checked);
-                MySettings.saveChats = !MySettings.saveChats
+                MySettings.saveChatsContext = !MySettings.saveChatsContext
             }
             ToolTip.text: qsTr("WARNING: Saving chats to disk can be ~2GB per chat")
             ToolTip.visible: hovered
         }
-        Label {
-            id: saveChatGPTChatsLabel
-            text: qsTr("Save ChatGPT chats to disk:")
-            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
-            Layout.row: 8
-            Layout.column: 0
-        }
-        MyCheckBox {
-            id: saveChatGPTChatsBox
-            Layout.row: 8
-            Layout.column: 1
-            checked: MySettings.saveChatGPTChats
-            onClicked: {
-                MySettings.saveChatGPTChats = !MySettings.saveChatGPTChats
-            }
-        }
         Label {
             id: serverChatLabel
             text: qsTr("Enable API server:")
             color: theme.textColor
             font.pixelSize: theme.fontSizeLarge
-            Layout.row: 9
+            Layout.row: 8
             Layout.column: 0
         }
         MyCheckBox {
             id: serverChatBox
-            Layout.row: 9
+            Layout.row: 8
             Layout.column: 1
             checked: MySettings.serverChat
             onClicked: {
@@ -290,7 +272,7 @@ MySettingsTab {
             ToolTip.visible: hovered
         }
         Rectangle {
-            Layout.row: 10
+            Layout.row: 9
             Layout.column: 0
             Layout.columnSpan: 3
             Layout.fillWidth: true

gpt4all-chat/qml/ChatDrawer.qml

@@ -31,8 +31,8 @@ Drawer {
         anchors.margins: 10
 
         Accessible.role: Accessible.Pane
-        Accessible.name: qsTr("Drawer on the left of the application")
-        Accessible.description: qsTr("Drawer that is revealed by pressing the hamburger button")
+        Accessible.name: qsTr("Drawer")
+        Accessible.description: qsTr("Main navigation drawer")
 
         MyButton {
             id: newChat
@@ -42,7 +42,7 @@ Drawer {
             topPadding: 20
             bottomPadding: 20
             text: qsTr("\uFF0B New chat")
-            Accessible.description: qsTr("Use this to create a new chat")
+            Accessible.description: qsTr("Create a new chat")
             background: Rectangle {
                 border.color: newChat.down ? theme.backgroundLightest : theme.buttonBorder
                 border.width: 2
@@ -135,7 +135,7 @@ Drawer {
                         }
                         Accessible.role: Accessible.Button
                         Accessible.name: qsTr("Select the current chat")
-                        Accessible.description: qsTr("Provides a button to select the current chat or edit the chat when in edit mode")
+                        Accessible.description: qsTr("Select the current chat or edit the chat when in edit mode")
                     }
                     Row {
                         id: buttons
@@ -155,8 +155,7 @@ Drawer {
                                 chatName.readOnly = false
                                 chatName.selectByMouse = true
                             }
-                            Accessible.name: qsTr("Edit the chat name")
-                            Accessible.description: qsTr("Provides a button to edit the chat name")
+                            Accessible.name: qsTr("Edit chat name")
                         }
                         MyToolButton {
                             id: trashButton
@@ -168,8 +167,7 @@ Drawer {
                                 trashQuestionDisplayed = true
                                 timer.start()
                             }
-                            Accessible.name: qsTr("Delete of the chat")
-                            Accessible.description: qsTr("Provides a button to delete the chat")
+                            Accessible.name: qsTr("Delete chat")
                         }
                     }
                     Rectangle {
@@ -207,8 +205,7 @@ Drawer {
                                     Network.sendRemoveChat()
                                 }
                                 Accessible.role: Accessible.Button
-                                Accessible.name: qsTr("Confirm delete of the chat")
-                                Accessible.description: qsTr("Provides a button to confirm delete of the chat")
+                                Accessible.name: qsTr("Confirm chat deletion")
                             }
                             Button {
                                 id: cancel
@@ -230,8 +227,7 @@ Drawer {
                                     trashQuestionDisplayed = false
                                 }
                                 Accessible.role: Accessible.Button
-                                Accessible.name: qsTr("Cancel the delete of the chat")
-                                Accessible.description: qsTr("Provides a button to cancel delete of the chat")
+                                Accessible.name: qsTr("Cancel chat deletion")
                             }
                         }
                     }
@@ -256,7 +252,7 @@ Drawer {
             anchors.bottomMargin: 10
             text: qsTr("Updates")
             font.pixelSize: theme.fontSizeLarge
-            Accessible.description: qsTr("Use this to launch an external application that will check for updates to the installer")
+            Accessible.description: qsTr("Launch an external application that will check for updates to the installer")
             onClicked: {
                 if (!LLM.checkForUpdates())
                     checkForUpdatesError.open()
@@ -270,7 +266,7 @@ Drawer {
             anchors.bottom: aboutButton.top
             anchors.bottomMargin: 10
             text: qsTr("Downloads")
-            Accessible.description: qsTr("Use this to launch a dialog to download new models")
+            Accessible.description: qsTr("Launch a dialog to download new models")
             onClicked: {
                 downloadClicked()
             }
@@ -282,7 +278,7 @@ Drawer {
             anchors.right: parent.right
             anchors.bottom: parent.bottom
             text: qsTr("About")
-            Accessible.description: qsTr("Use this to launch a dialog to show the about page")
+            Accessible.description: qsTr("Launch a dialog to show the about page")
             onClicked: {
                 aboutClicked()
             }

gpt4all-chat/qml/LocalDocsSettings.qml

@@ -83,7 +83,7 @@ MySettingsTab {
                     text: qsTr("Add")
                     Accessible.role: Accessible.Button
                     Accessible.name: text
-                    Accessible.description: qsTr("Add button")
+                    Accessible.description: qsTr("Add collection")
                     onClicked: {
                         var isError = false;
                         if (root.collection === "") {
@@ -242,13 +242,13 @@ MySettingsTab {
             Layout.column: 0
             color: theme.textColor
             font.pixelSize: theme.fontSizeLarge
-            text: qsTr("Document snippets per prompt:")
+            text: qsTr("Max document snippets per prompt:")
         }
 
         MyTextField {
             Layout.row: 2
             Layout.column: 1
-            ToolTip.text: qsTr("Best N matches of retrieved document snippets to add to the context for prompt.\nNOTE: larger numbers increase likelihood of factual responses, but also result in slower generation.")
+            ToolTip.text: qsTr("Max best N matches of retrieved document snippets to add to the context for prompt.\nNOTE: larger numbers increase likelihood of factual responses, but also result in slower generation.")
             ToolTip.visible: hovered
             text: MySettings.localDocsRetrievalSize
             validator: IntValidator {

gpt4all-chat/qml/ModelDownloaderDialog.qml

@@ -47,7 +47,7 @@ MyDialog {
             Layout.fillHeight: true
             horizontalAlignment: Qt.AlignHCenter
             verticalAlignment: Qt.AlignVCenter
-            text: qsTr("Network error: could not retrieve http://gpt4all.io/models/models.json")
+            text: qsTr("Network error: could not retrieve http://gpt4all.io/models/models2.json")
             font.pixelSize: theme.fontSizeLarge
             color: theme.mutedTextColor
         }
@@ -125,7 +125,7 @@ MyDialog {
                                     Layout.fillWidth: true
                                     Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
                                     visible: !isChatGPT && !installed && !calcHash && downloadError === ""
-                                    Accessible.description: qsTr("Cancel/Resume/Download button to stop/restart/start the download")
+                                    Accessible.description: qsTr("Stop/restart/start the download")
                                     background: Rectangle {
                                         border.color: downloadButton.down ? theme.backgroundLightest : theme.buttonBorder
                                         border.width: 2
@@ -151,7 +151,7 @@ MyDialog {
                                     Layout.fillWidth: true
                                     Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
                                     visible: installed || downloadError !== ""
-                                    Accessible.description: qsTr("Remove button to remove model from filesystem")
+                                    Accessible.description: qsTr("Remove model from filesystem")
                                     background: Rectangle {
                                         border.color: removeButton.down ? theme.backgroundLightest : theme.buttonBorder
                                         border.width: 2
@@ -186,8 +186,8 @@ MyDialog {
                                             Download.installModel(filename, openaiKey.text);
                                     }
                                     Accessible.role: Accessible.Button
-                                    Accessible.name: qsTr("Install button")
-                                    Accessible.description: qsTr("Install button to install chatgpt model")
+                                    Accessible.name: qsTr("Install")
+                                    Accessible.description: qsTr("Install chatGPT model")
                                 }
 
                                 ColumnLayout {
@@ -385,7 +385,7 @@ MyDialog {
                             linkColor: theme.textColor
                             Accessible.role: Accessible.Paragraph
                             Accessible.name: qsTr("Description")
-                            Accessible.description: qsTr("The description of the file")
+                            Accessible.description: qsTr("File description")
                             onLinkActivated: Qt.openUrlExternally(link)
                         }
                     }
@@ -456,7 +456,7 @@ MyDialog {
             }
             MyButton {
                 text: qsTr("Browse")
-                Accessible.description: qsTr("Opens a folder picker dialog to choose where to save model files")
+                Accessible.description: qsTr("Choose where to save model files")
                 onClicked: modelPathDialog.open()
             }
         }

gpt4all-chat/qml/MySettingsTab.qml

@@ -69,7 +69,7 @@ Item {
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.Button
                     Accessible.name: text
-                    Accessible.description: qsTr("Restores the settings dialog to a default state")
+                    Accessible.description: qsTr("Restores settings dialog to a default state")
                     onClicked: {
                         root.restoreDefaultsClicked();
                     }

gpt4all-chat/qml/NetworkDialog.qml

@@ -89,7 +89,7 @@ NOTE: By turning on this feature, you will be sending your data to the GPT4All O
             }
             Accessible.role: Accessible.EditableText
             Accessible.name: qsTr("Attribution (optional)")
-            Accessible.description: qsTr("Textfield for providing attribution")
+            Accessible.description: qsTr("Provide attribution")
             onEditingFinished: {
                 MySettings.networkAttribution = attribution.text;
             }
@@ -103,12 +103,12 @@ NOTE: By turning on this feature, you will be sending your data to the GPT4All O
         spacing: 10
         MyButton {
             text: qsTr("Enable")
-            Accessible.description: qsTr("Enable opt-in button")
+            Accessible.description: qsTr("Enable opt-in")
             DialogButtonBox.buttonRole: DialogButtonBox.AcceptRole
         }
         MyButton {
             text: qsTr("Cancel")
-            Accessible.description: qsTr("Cancel opt-in button")
+            Accessible.description: qsTr("Cancel opt-in")
             DialogButtonBox.buttonRole: DialogButtonBox.RejectRole
         }
         background: Rectangle {

gpt4all-chat/qml/SettingsDialog.qml

@@ -21,8 +21,8 @@ MyDialog {
 
     Item {
         Accessible.role: Accessible.Dialog
-        Accessible.name: qsTr("Settings dialog")
-        Accessible.description: qsTr("Dialog containing various application settings")
+        Accessible.name: qsTr("Settings")
+        Accessible.description: qsTr("Contains various application settings")
     }
 
     ListModel {

gpt4all-chat/qml/StartupDialog.qml

@@ -133,7 +133,6 @@ model release that uses your data!")
                 font.pixelSize: theme.fontSizeLarge
                 Accessible.role: Accessible.Paragraph
                 Accessible.name: qsTr("Opt-in for anonymous usage statistics")
-                Accessible.description: qsTr("Label for opt-in")
             }
 
             ButtonGroup {
@@ -162,7 +161,7 @@ model release that uses your data!")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton
                     Accessible.name: qsTr("Opt-in for anonymous usage statistics")
-                    Accessible.description: qsTr("Radio button to allow opt-in for anonymous usage statistics")
+                    Accessible.description: qsTr("Allow opt-in for anonymous usage statistics")
 
                     background: Rectangle {
                         color: "transparent"
@@ -203,7 +202,7 @@ model release that uses your data!")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton
                     Accessible.name: qsTr("Opt-out for anonymous usage statistics")
-                    Accessible.description: qsTr("Radio button to allow opt-out for anonymous usage statistics")
+                    Accessible.description: qsTr("Allow opt-out for anonymous usage statistics")
 
                     background: Rectangle {
                         color: "transparent"
@@ -249,7 +248,7 @@ model release that uses your data!")
                 font.pixelSize: theme.fontSizeLarge
                 Accessible.role: Accessible.Paragraph
                 Accessible.name: qsTr("Opt-in for network")
-                Accessible.description: qsTr("Checkbox to allow opt-in for network")
+                Accessible.description: qsTr("Allow opt-in for network")
             }
 
             ButtonGroup {
@@ -276,7 +275,7 @@ model release that uses your data!")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton
                     Accessible.name: qsTr("Opt-in for network")
-                    Accessible.description: qsTr("Radio button to allow opt-in anonymous sharing of chats to the GPT4All Datalake")
+                    Accessible.description: qsTr("Allow opt-in anonymous sharing of chats to the GPT4All Datalake")
 
                     background: Rectangle {
                         color: "transparent"
@@ -317,7 +316,7 @@ model release that uses your data!")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton
                     Accessible.name: qsTr("Opt-out for network")
-                    Accessible.description: qsTr("Radio button to allow opt-out anonymous sharing of chats to the GPT4All Datalake")
+                    Accessible.description: qsTr("Allow opt-out anonymous sharing of chats to the GPT4All Datalake")
 
                     background: Rectangle {
                         color: "transparent"

gpt4all-training/build_map.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import numpy as np
 from nomic import atlas
 import glob
@@ -51,4 +52,4 @@ atlas.map_embeddings(embeddings,
                      colorable_fields=["source", "loss", "trained_on"],
                      build_topic_model=True,
                      topic_label_field="inputs",
-                     reset_project_if_exists=True,)
+                     reset_project_if_exists=True,)