WIP: refactor for subgroups on mat * vec kernel.

.circleci/continue_config.yml

@@ -27,176 +27,7 @@ jobs:
       - image: circleci/python:3.7
     steps:
       - run: echo "CircleCI pipeline triggered"
-  build-offline-chat-installer-macos:
-    macos:
-      xcode: 14.0.0
-    steps:
-      - checkout
-      - run:
-          name: Update Submodules
-          command: |
-            git submodule sync
-            git submodule update --init --recursive
-      - restore_cache:  # this is the new step to restore cache
-          keys:
-            - macos-qt-cache_v2
-      - run:
-          name: Installing Qt
-          command: |
-            if [ ! -d ~/Qt ]; then
-              curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
-              hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
-              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
-              hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
-            fi
-      - save_cache:  # this is the new step to save cache
-          key: macos-qt-cache_v2
-          paths:
-            - ~/Qt
-      - run:
-          name: Build
-          command: |
-            mkdir build
-            cd build
-            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
-            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
-              -DCMAKE_GENERATOR:STRING=Ninja \
-              -DBUILD_UNIVERSAL=ON \
-              -DMACDEPLOYQT=~/Qt/6.5.1/macos/bin/macdeployqt \
-              -DGPT4ALL_OFFLINE_INSTALLER=ON \
-              -DCMAKE_BUILD_TYPE=Release \
-              -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
-              -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-              -S ../gpt4all-chat \
-              -B .
-            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
-            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install
-            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package
-            mkdir upload
-            cp gpt4all-installer-* upload
-      - store_artifacts:
-          path: build/upload
-  build-offline-chat-installer-linux:
-    machine:
-      image: ubuntu-2204:2023.04.2
-    steps:
-      - checkout
-      - run:
-          name: Update Submodules
-          command: |
-            git submodule sync
-            git submodule update --init --recursive
-      - restore_cache:  # this is the new step to restore cache
-          keys:
-            - linux-qt-cache
-      - run:
-          name: Setup Linux and Dependencies
-          command: |
-            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf
-      - run:
-          name: Installing Qt
-          command: |
-            if [ ! -d ~/Qt ]; then
-              wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
-              chmod +x qt-unified-linux-x64-4.6.0-online.run
-              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
-            fi
-      - save_cache:  # this is the new step to save cache
-          key: linux-qt-cache
-          paths:
-            - ~/Qt
-      - run:
-          name: Build linuxdeployqt
-          command: |
-            git clone https://github.com/nomic-ai/linuxdeployqt
-            cd linuxdeployqt && qmake && sudo make install
-      - run:
-          name: Build
-          command: |
-            set -eo pipefail
-            export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
-            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
-            mkdir build
-            cd build
-            mkdir upload
-            ~/Qt/Tools/CMake/bin/cmake -DGPT4ALL_OFFLINE_INSTALLER=ON -DCMAKE_BUILD_TYPE=Release -S ../gpt4all-chat -B .
-            ~/Qt/Tools/CMake/bin/cmake --build . --target all
-            ~/Qt/Tools/CMake/bin/cmake --build . --target install
-            ~/Qt/Tools/CMake/bin/cmake --build . --target package
-            cp gpt4all-installer-* upload
-      - store_artifacts:
-          path: build/upload
-  build-offline-chat-installer-windows:
-    machine:
-      image: 'windows-server-2019-vs2019:2022.08.1'
-      resource_class: windows.large
-      shell: powershell.exe -ExecutionPolicy Bypass
-    steps:
-      - checkout
-      - run:
-          name: Update Submodules
-          command: |
-            git submodule sync
-            git submodule update --init --recursive
-      - restore_cache:  # this is the new step to restore cache
-          keys:
-            - windows-qt-cache
-      - run:
-          name: Installing Qt
-          command: |
-            if (-not (Test-Path C:\Qt)) {
-              Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
-              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
-            }
-      - save_cache:  # this is the new step to save cache
-          key: windows-qt-cache
-          paths:
-            - C:\Qt
-      - run:
-          name: Install VulkanSDK
-          command: |
-            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
-            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
-      - run:
-          name: Build
-          command: |
-            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
-            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
-            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
-            $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
-            $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.6\bin"
-            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
-            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
-            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
-            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64"
-            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt"
-            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um"
-            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared"
-            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt"
-            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt"
-            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
-            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
-            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
-            mkdir build
-            cd build
-            & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
-              "-DCMAKE_GENERATOR:STRING=Ninja" `
-              "-DCMAKE_BUILD_TYPE=Release" `
-              "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
-              "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
-              "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
-              "-DGPT4ALL_OFFLINE_INSTALLER=ON" `
-              "-S ..\gpt4all-chat" `
-              "-B ."
-            & "C:\Qt\Tools\Ninja\ninja.exe"
-            & "C:\Qt\Tools\Ninja\ninja.exe" install
-            & "C:\Qt\Tools\Ninja\ninja.exe" package
-            mkdir upload
-            copy gpt4all-installer-win64.exe upload
-      - store_artifacts:
-          path: build/upload
+
   build-gpt4all-chat-linux:
     machine:
       image: ubuntu-2204:2023.04.2
@@ -332,7 +163,6 @@ jobs:
             cd build
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
               -DCMAKE_GENERATOR:STRING=Ninja \
-              -DBUILD_UNIVERSAL=ON \
               -DCMAKE_BUILD_TYPE=Release \
               -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
               -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
@@ -414,8 +244,6 @@ jobs:
           command: |
             cd gpt4all-bindings/python/
             python setup.py bdist_wheel --plat-name=manylinux1_x86_64
-      - store_artifacts:
-          path: gpt4all-bindings/python/dist
       - persist_to_workspace:
           root: gpt4all-bindings/python/dist
           paths:
@@ -446,9 +274,7 @@ jobs:
           name: Build wheel
           command: |
             cd gpt4all-bindings/python
-            python setup.py bdist_wheel --plat-name=macosx_10_15_universal2
-      - store_artifacts:
-          path: gpt4all-bindings/python/dist
+            python setup.py bdist_wheel --plat-name=macosx_10_9_universal2
       - persist_to_workspace:
           root: gpt4all-bindings/python/dist
           paths:
@@ -462,6 +288,9 @@ jobs:
       - run:
           name: Install MinGW64
           command: choco install -y mingw --force --no-progress
+      - run:
+          name: Add MinGW64 to PATH
+          command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
       - run:
           name: Install VulkanSDK
           command: |
@@ -482,7 +311,6 @@ jobs:
             cd gpt4all-backend
             mkdir build
             cd build
-            $env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
             $env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
             cmake -G "MinGW Makefiles" .. -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
             cmake --build . --parallel
@@ -495,11 +323,9 @@ jobs:
             cd gpt4all
             mkdir llmodel_DO_NOT_MODIFY
             mkdir llmodel_DO_NOT_MODIFY/build/
-            cp 'C:\ProgramData\mingw64\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
+            cp 'C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
             cd ..
             python setup.py bdist_wheel --plat-name=win_amd64
-      - store_artifacts:
-          path: gpt4all-bindings/python/dist
       - persist_to_workspace:
           root: gpt4all-bindings/python/dist
           paths:
@@ -616,7 +442,7 @@ jobs:
       - run:
           name: Build Libraries
           command: |
-            $MinGWBin = "C:\ProgramData\mingw64\mingw64\bin"
+            $MinGWBin = "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
             $Env:Path += ";$MinGwBin"
             $Env:Path += ";C:\Program Files\CMake\bin"
             $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
@@ -1002,20 +828,6 @@ workflows:
     when: << pipeline.parameters.run-default-workflow >>
     jobs:
       - default-job
-  build-chat-offline-installers:
-    when: << pipeline.parameters.run-chat-workflow >>
-    jobs:
-      - hold:
-          type: approval
-      - build-offline-chat-installer-macos:
-          requires:
-            - hold
-      - build-offline-chat-installer-windows:
-          requires:
-            - hold
-      - build-offline-chat-installer-linux:
-          requires:
-            - hold
   build-and-test-gpt4all-chat:
     when: << pipeline.parameters.run-chat-workflow >>
     jobs:

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
-ignore-words-list = blong, belong, afterall, som, assistent
+ignore-words-list = blong, belong, afterall, som
 skip = .git,*.pdf,*.svg,*.lock

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -27,6 +27,21 @@ body:
         - label: "The official example notebooks/scripts"
         - label: "My own modified scripts"
 
+  - type: checkboxes
+    id: related-components
+    attributes:
+      label: Related Components
+      description: "Select the components related to the issue (if applicable):"
+      options:
+        - label: "backend"
+        - label: "bindings"
+        - label: "python-bindings"
+        - label: "chat-ui"
+        - label: "models"
+        - label: "circleci"
+        - label: "docker"
+        - label: "api"
+
   - type: textarea
     id: reproduction
     validations:

.gitmodules

@@ -1,4 +1,9 @@
+[submodule "llama.cpp-230519"]
+	path = gpt4all-backend/llama.cpp-230519
+	url = https://github.com/ggerganov/llama.cpp.git
+[submodule "llama.cpp-230511"]
+	path = gpt4all-backend/llama.cpp-230511
+	url = https://github.com/nomic-ai/llama.cpp
 [submodule "llama.cpp-mainline"]
 	path = gpt4all-backend/llama.cpp-mainline
 	url = https://github.com/nomic-ai/llama.cpp.git
-	branch = gguf

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -26,7 +26,7 @@ router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
 async def list_engines():
     '''
     List all available GPT4All models from
-    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json
+    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json
     '''
     raise NotImplementedError()
     return ListEnginesResponse(data=[])

gpt4all-backend/CMakeLists.txt

@@ -20,7 +20,7 @@ endif()
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 
 set(LLMODEL_VERSION_MAJOR 0)
-set(LLMODEL_VERSION_MINOR 5)
+set(LLMODEL_VERSION_MINOR 4)
 set(LLMODEL_VERSION_PATCH 0)
 set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
 project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
@@ -97,19 +97,35 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    if (NOT LLAMA_METAL)
-        add_library(gptj-${BUILD_VARIANT} SHARED
-            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        prepare_target(gptj llama-mainline)
+    add_library(replit-mainline-${BUILD_VARIANT} SHARED
+    replit.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+    target_compile_definitions(replit-mainline-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+    prepare_target(replit-mainline llama-mainline)
 
-        add_library(mpt-${BUILD_VARIANT} SHARED
-            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        prepare_target(mpt llama-mainline)
+    if (NOT LLAMA_METAL)
+# FIXME: These need to be forward ported to latest ggml
+#        add_library(gptj-${BUILD_VARIANT} SHARED
+#            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+#        prepare_target(gptj ggml-230511)
+
+        add_library(falcon-${BUILD_VARIANT} SHARED
+            falcon.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(falcon-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+        prepare_target(falcon llama-mainline)
+# FIXME: These need to be forward ported to latest ggml
+#        add_library(mpt-${BUILD_VARIANT} SHARED
+#            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+#        prepare_target(mpt ggml-230511)
 
         add_library(bert-${BUILD_VARIANT} SHARED
             bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
         target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
         prepare_target(bert llama-mainline)
+
+        add_library(starcoder-${BUILD_VARIANT} SHARED
+            starcoder.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(starcoder-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+        prepare_target(starcoder llama-mainline)
     endif()
 endforeach()
 

gpt4all-backend/bert.cpp

@@ -4,10 +4,10 @@
 #include "ggml.h"
 
 #include <cassert>
-#include <cinttypes>
 #include <cmath>
 #include <cstdio>
 #include <cstring>
+#include <fstream>
 #include <map>
 #include <string>
 #include <vector>
@@ -34,6 +34,7 @@ struct bert_hparams
     int32_t n_intermediate = 1536;
     int32_t n_head = 12;
     int32_t n_layer = 6;
+    int32_t f16 = 1;
 };
 
 struct bert_layer
@@ -87,6 +88,7 @@ struct bert_model
     std::vector<bert_layer> layers;
 
     struct ggml_context *ctx;
+    std::map<std::string, struct ggml_tensor *> tensors;
 };
 
 // Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
@@ -343,7 +345,7 @@ void bert_eval(
 
     // embd norm
     {
-        inpL = ggml_norm(ctx0, inpL, 1e-5f);
+        inpL = ggml_norm(ctx0, inpL);
 
         inpL = ggml_add(ctx0,
                         ggml_mul(ctx0,
@@ -404,7 +406,7 @@ void bert_eval(
 
         // attention norm
         {
-            cur = ggml_norm(ctx0, cur, 1e-5f);
+            cur = ggml_norm(ctx0, cur);
 
             cur = ggml_add(ctx0,
                            ggml_mul(ctx0,
@@ -430,7 +432,7 @@ void bert_eval(
 
         // output norm
         {
-            cur = ggml_norm(ctx0, cur, 1e-5f);
+            cur = ggml_norm(ctx0, cur);
 
             cur = ggml_add(ctx0,
                            ggml_mul(ctx0,
@@ -480,6 +482,7 @@ void bert_eval(
 //
 
 void bert_free(bert_ctx * ctx) {
+    ggml_free(ctx->model.ctx);
     delete ctx;
 }
 
@@ -489,130 +492,63 @@ struct bert_ctx * bert_load_from_file(const char *fname)
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname);
 #endif
 
-    bert_ctx * new_bert = new bert_ctx;
-    bert_model & model = new_bert->model;
-    bert_vocab & vocab = new_bert->vocab;
-
-    struct gguf_init_params params = {
-        /*.no_alloc = */ false,
-        /*.ctx      = */ &model.ctx,
-    };
-    gguf_context *ggufctx = gguf_init_from_file(fname, params);
-    if (!ggufctx) {
-        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
+    auto fin = std::ifstream(fname, std::ios::binary);
+    if (!fin)
+    {
+        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname);
         return nullptr;
     }
 
-    printf("%s: gguf version     = %d\n", __func__, gguf_get_version(ggufctx));
-    printf("%s: gguf alignment   = %zu\n", __func__, gguf_get_alignment(ggufctx));
-    printf("%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx));
-
-    // print some standard metadata
+    // verify magic
     {
-        int keyidx;
-
-        keyidx = gguf_find_key(ggufctx, "general.name");
-        if (keyidx != -1) { printf("%s: model name           = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.description");
-        if (keyidx != -1) { printf("%s: model description    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.author");
-        if (keyidx != -1) { printf("%s: model author         = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.license");
-        if (keyidx != -1) { printf("%s: model license        = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx != -1) { printf("%s: model architecture   = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.file_type");
-        if (keyidx != -1) { printf("%s: model file type      = %" PRIu32 "\n", __func__, gguf_get_val_u32(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "gptneox.tensor_data_layout");
-        if (keyidx != -1) { printf("%s: model data layout    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.source.huggingface.repository");
-        if (keyidx != -1) { printf("%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-    }
-
-    // check required metadata
-    {
-        // check model architecture kv
-        int keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: gguf model architecture not found!\n", __func__);
-            return nullptr;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
-            fprintf(stderr, "%s: model architecture not supported!\n", __func__);
+        uint32_t magic;
+        fin.read((char *)&magic, sizeof(magic));
+        if (magic != 0x62657274)
+        {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname);
             return nullptr;
         }
     }
 
+    bert_ctx * new_bert = new bert_ctx;
+    bert_model & model = new_bert->model;
+    bert_vocab & vocab = new_bert->vocab;
+
     // load hparams
     {
         auto &hparams = model.hparams;
 
-        bool ok = false;
-        int keyidx;
-
-        do {
-            keyidx = gguf_find_key(ggufctx, "bert.context_length");
-            if (keyidx == -1) { break; }
-            hparams.n_max_tokens = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.embedding_length");
-            if (keyidx == -1) { break; }
-            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.feed_forward_length");
-            if (keyidx == -1) { break; }
-            hparams.n_intermediate = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.attention.head_count");
-            if (keyidx == -1) { break; }
-            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.block_count");
-            if (keyidx == -1) { break; }
-            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
-
-            ok = true;
-        } while (false);
-
-        if (!ok) {
-            fprintf(stderr, "%s: required hparam missing!\n", __func__);
-            return nullptr;
-        }
+        fin.read((char *)&hparams.n_vocab, sizeof(hparams.n_vocab));
+        fin.read((char *)&hparams.n_max_tokens, sizeof(hparams.n_max_tokens));
+        fin.read((char *)&hparams.n_embd, sizeof(hparams.n_embd));
+        fin.read((char *)&hparams.n_intermediate, sizeof(hparams.n_intermediate));
+        fin.read((char *)&hparams.n_head, sizeof(hparams.n_head));
+        fin.read((char *)&hparams.n_layer, sizeof(hparams.n_layer));
+        fin.read((char *)&hparams.f16, sizeof(hparams.f16));
 
 #if defined(DEBUG_BERT)
+        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
         printf("%s: n_max_tokens   = %d\n", __func__, hparams.n_max_tokens);
-        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_intermediate = %d\n", __func__, hparams.n_intermediate);
-        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
+        printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
+        printf("%s: n_intermediate  = %d\n", __func__, hparams.n_intermediate);
+        printf("%s: n_head  = %d\n", __func__, hparams.n_head);
+        printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
+        printf("%s: f16     = %d\n", __func__, hparams.f16);
 #endif
     }
 
     // load vocab
     {
-        auto & hparams = model.hparams;
+        int32_t n_vocab = model.hparams.n_vocab;
 
-        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
-            return nullptr;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
-            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
-            return nullptr;
-        }
+        std::string word;
+        for (int i = 0; i < n_vocab; i++)
+        {
+            uint32_t len;
+            fin.read((char *)&len, sizeof(len));
 
-        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
-        if (tokens_keyidx == -1) {
-            fprintf(stderr, "%s: bert tokenizer vocab not found!\n", __func__);
-            return nullptr;
-        }
-
-        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
-        printf("%s: bert tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
-
-        for (int i = 0; i < hparams.n_vocab; i++) {
-            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
+            word.resize(len);
+            fin.read((char *)word.data(), len);
 
             if (word[0] == '#' && word[1] == '#')
             {
@@ -628,52 +564,290 @@ struct bert_ctx * bert_load_from_file(const char *fname)
         }
     }
 
+    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
+    // in order to save memory and also to speed up the computation
+    ggml_type wtype = GGML_TYPE_COUNT;
+    switch (model.hparams.f16)
+    {
+    case 0:
+        wtype = GGML_TYPE_F32;
+        break;
+    case 1:
+        wtype = GGML_TYPE_F16;
+        break;
+    case 2:
+        wtype = GGML_TYPE_Q4_0;
+        break;
+    case 3:
+        wtype = GGML_TYPE_Q4_1;
+        break;
+    default:
+    {
+        fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
+                __func__, fname, model.hparams.f16);
+        bert_free(new_bert);
+        return nullptr;
+    }
+    }
+
     auto &ctx = model.ctx;
 
+    size_t model_mem_req = 0;
+
+    {
+        const auto &hparams = model.hparams;
+
+        const int n_embd = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_max_tokens = hparams.n_max_tokens;
+        const int n_intermediate = hparams.n_intermediate;
+        const int n_vocab = hparams.n_vocab;
+
+        // Calculate size requirements
+
+        model_mem_req += n_embd * n_vocab * ggml_type_sizef(wtype); // word_embeddings
+        model_mem_req += n_embd * 2 * ggml_type_sizef(wtype); // token_type_embeddings
+        model_mem_req += n_embd * n_max_tokens * ggml_type_sizef(wtype); // position_embeddings
+
+        model_mem_req += 2 * n_embd * ggml_type_sizef(GGML_TYPE_F32); // ln_e_*
+
+        model_mem_req += 4 * n_layer * (n_embd * ggml_type_sizef(GGML_TYPE_F32)); // ln_*
+
+        model_mem_req += 4 * n_layer * (n_embd * n_embd * ggml_type_sizef(wtype)); // kqvo weights
+        model_mem_req += 4 * n_layer * (n_embd * ggml_type_sizef(GGML_TYPE_F32)); // kqvo bias
+
+        model_mem_req += 2 * n_layer * (n_embd * n_intermediate * ggml_type_sizef(wtype)); // ff_*_w
+        model_mem_req += n_layer * (n_intermediate * ggml_type_sizef(GGML_TYPE_F32)); // ff_i_b
+        model_mem_req += n_layer * (n_embd * ggml_type_sizef(GGML_TYPE_F32)); // ff_o_b
+
+        model_mem_req += (5 + 16 * n_layer) * ggml_tensor_overhead(); // object overhead
+
 #if defined(DEBUG_BERT)
-    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ggml_get_mem_size(ctx) / (1024.0 * 1024.0));
+        printf("%s: ggml ctx size = %6.2f MB\n", __func__, model_mem_req / (1024.0 * 1024.0));
 #endif
+    }
+
+    // create the ggml context
+    {
+        struct ggml_init_params params = {
+            .mem_size = model_mem_req,
+            .mem_buffer = NULL,
+            .no_alloc = false,
+        };
+
+        model.ctx = ggml_init(params);
+        if (!model.ctx)
+        {
+            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
+            bert_free(new_bert);
+            return nullptr;
+        }
+    }
 
     // prepare memory for the weights
     {
-        const int n_layer = model.hparams.n_layer;
+        const auto &hparams = model.hparams;
+
+        const int n_embd = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_intermediate = hparams.n_intermediate;
+        const int n_max_tokens = hparams.n_max_tokens;
+        const int n_vocab = hparams.n_vocab;
+
         model.layers.resize(n_layer);
 
-        model.word_embeddings       = ggml_get_tensor(ctx, "token_embd.weight");
-        model.token_type_embeddings = ggml_get_tensor(ctx, "token_types.weight");
-        model.position_embeddings   = ggml_get_tensor(ctx, "position_embd.weight");
-        model.ln_e_w                = ggml_get_tensor(ctx, "output_norm.weight");
-        model.ln_e_b                = ggml_get_tensor(ctx, "output_norm.bias");
+        model.word_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
+        model.token_type_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, 2);
+        model.position_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_max_tokens);
 
-        auto name = [](int i, std::string n) {
-            static std::string key;
-            key = "blk." + std::to_string(i) + "." + n;
-            return key.c_str();
-        };
+        model.ln_e_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.ln_e_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+        // map by name
+        model.tensors["embeddings.word_embeddings.weight"] = model.word_embeddings;
+        model.tensors["embeddings.token_type_embeddings.weight"] = model.token_type_embeddings;
+        model.tensors["embeddings.position_embeddings.weight"] = model.position_embeddings;
+
+        model.tensors["embeddings.LayerNorm.weight"] = model.ln_e_w;
+        model.tensors["embeddings.LayerNorm.bias"] = model.ln_e_b;
 
         for (int i = 0; i < n_layer; ++i)
         {
             auto &layer = model.layers[i];
 
-            layer.ln_att_w = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
-            layer.ln_att_b = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
-            layer.ln_out_w = ggml_get_tensor(ctx, name(i, "ffn_norm.weight"));
-            layer.ln_out_b = ggml_get_tensor(ctx, name(i, "ffn_norm.bias"));
-            layer.q_w      = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
-            layer.q_b      = ggml_get_tensor(ctx, name(i, "attn_q.bias"));
-            layer.k_w      = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
-            layer.k_b      = ggml_get_tensor(ctx, name(i, "attn_k.bias"));
-            layer.v_w      = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
-            layer.v_b      = ggml_get_tensor(ctx, name(i, "attn_v.bias"));
-            layer.o_w      = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
-            layer.o_b      = ggml_get_tensor(ctx, name(i, "attn_output.bias"));
-            layer.ff_i_w   = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
-            layer.ff_i_b   = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
-            layer.ff_o_w   = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
-            layer.ff_o_b   = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
+            layer.ln_att_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.ln_att_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.ln_out_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.ln_out_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+            layer.q_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
+            layer.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.k_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
+            layer.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.v_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
+            layer.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.o_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
+            layer.o_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+            layer.ff_i_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_intermediate);
+            layer.ff_i_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_intermediate);
+
+            layer.ff_o_w = ggml_new_tensor_2d(ctx, wtype, n_intermediate, n_embd);
+            layer.ff_o_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+            // map by name
+
+            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.query.weight"] = layer.q_w;
+            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.query.bias"] = layer.q_b;
+            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.key.weight"] = layer.k_w;
+            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.key.bias"] = layer.k_b;
+            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.value.weight"] = layer.v_w;
+            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.value.bias"] = layer.v_b;
+            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.LayerNorm.weight"] = layer.ln_att_w;
+            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.LayerNorm.bias"] = layer.ln_att_b;
+            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.dense.weight"] = layer.o_w;
+            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.dense.bias"] = layer.o_b;
+
+            model.tensors["encoder.layer." + std::to_string(i) + ".intermediate.dense.weight"] = layer.ff_i_w;
+            model.tensors["encoder.layer." + std::to_string(i) + ".intermediate.dense.bias"] = layer.ff_i_b;
+
+            model.tensors["encoder.layer." + std::to_string(i) + ".output.LayerNorm.weight"] = layer.ln_out_w;
+            model.tensors["encoder.layer." + std::to_string(i) + ".output.LayerNorm.bias"] = layer.ln_out_b;
+            model.tensors["encoder.layer." + std::to_string(i) + ".output.dense.weight"] = layer.ff_o_w;
+            model.tensors["encoder.layer." + std::to_string(i) + ".output.dense.bias"] = layer.ff_o_b;
         }
     }
 
+    // load weights
+    {
+        int n_tensors = 0;
+#if defined(DEBUG_BERT)
+        size_t total_size = 0;
+#endif
+
+#if defined(DEBUG_BERT)
+        printf("%s: ", __func__);
+#endif
+
+        while (true)
+        {
+            int32_t n_dims;
+            int32_t length;
+            int32_t ftype;
+
+            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
+            fin.read(reinterpret_cast<char *>(&ftype), sizeof(ftype));
+
+            if (fin.eof())
+            {
+                break;
+            }
+
+            int64_t nelements = 1;
+            int64_t ne[2] = {1, 1};
+            for (int i = 0; i < n_dims; ++i)
+            {
+                int32_t ne_cur;
+                fin.read(reinterpret_cast<char *>(&ne_cur), sizeof(ne_cur));
+                ne[i] = ne_cur;
+                nelements *= ne[i];
+            }
+
+            std::string name(length, 0);
+            fin.read(&name[0], length);
+
+            if (model.tensors.find(name.data()) == model.tensors.end())
+            {
+                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
+                bert_free(new_bert);
+                return nullptr;
+            }
+
+            auto tensor = model.tensors[name.data()];
+            if (ggml_nelements(tensor) != nelements)
+            {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
+                bert_free(new_bert);
+                return nullptr;
+            }
+
+            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1])
+            {
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%ld, %ld], expected [%ld, %ld]\n",
+                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
+                bert_free(new_bert);
+                return nullptr;
+            }
+
+#if defined(DEBUG_BERT)
+            static const char *ftype_str[] = {
+                "f32",
+                "f16",
+                "q4_0",
+                "q4_1",
+            };
+            printf("%24s - [%5ld, %5ld], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ftype_str[ftype], ggml_nbytes(tensor) / 1024.0 / 1024.0, ggml_nbytes(tensor));
+#endif
+
+            size_t bpe = 0;
+
+            switch (ftype)
+            {
+            case 0:
+                bpe = ggml_type_size(GGML_TYPE_F32);
+                break;
+            case 1:
+                bpe = ggml_type_size(GGML_TYPE_F16);
+                break;
+            case 2:
+                bpe = ggml_type_size(GGML_TYPE_Q4_0);
+                assert(ne[0] % 64 == 0);
+                break;
+            case 3:
+                bpe = ggml_type_size(GGML_TYPE_Q4_1);
+                assert(ne[0] % 64 == 0);
+                break;
+            default:
+            {
+                fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype);
+                bert_free(new_bert);
+                return nullptr;
+            }
+            };
+
+            if ((nelements * bpe) / ggml_blck_size(tensor->type) != ggml_nbytes(tensor))
+            {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %lu\n",
+                        __func__, name.data(), ggml_nbytes(tensor), nelements * bpe);
+                bert_free(new_bert);
+                return nullptr;
+            }
+
+            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
+
+#if defined(DEBUG_BERT)
+            // printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
+            total_size += ggml_nbytes(tensor);
+#endif
+
+            if (++n_tensors % 8 == 0)
+            {
+#if defined(DEBUG_BERT)
+                printf(".");
+                fflush(stdout);
+#endif
+            }
+        }
+
+#if defined(DEBUG_BERT)
+        printf(" done\n");
+        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size / 1024.0 / 1024.0, n_tensors);
+#endif
+    }
+
+    fin.close();
+
     // Calculate space requirements for setting up context buffers later
     {
         bert_vocab_id tokens[] = {0, 1, 2, 3};
@@ -845,16 +1019,6 @@ const std::vector<LLModel::Token> &Bert::endTokens() const
     return out;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -874,21 +1038,13 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
-
-    bool isValid = gguf_get_version(ctx_gguf) <= 2;
-    isValid = isValid && get_arch_name(ctx_gguf) == "bert";
-
-    gguf_free(ctx_gguf);
-    return isValid;
+DLL_EXPORT bool magic_match(std::istream& f) {
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    if (magic != 0x62657274) {
+         return false;
+    }
+    return true;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/falcon.cpp

@@ -0,0 +1,985 @@
+#include "ggml.h"
+#define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#include "falcon_impl.h"
+#include "llama.h"
+#include "llama-util.h"
+#include "utils.h"
+#include "llmodel_shared.h"
+
+#include <cassert>
+#include <cinttypes>
+#include <iostream>
+#include <sstream>
+
+namespace {
+const char *modelType_ = "Falcon";
+}
+
+// commented out 40B support as it presently would require forking ggml/llama.cpp
+// can re-add once mainline ggml supports it
+
+#define FALCON_MAGIC 0x67676a74
+
+// default hparams (Falcon 7B)
+struct falcon_hparams {
+    int32_t n_vocab = 65024;
+    int32_t n_embd  = 4544;
+    int32_t n_head  = 71;
+    int32_t n_head_kv = 1;
+    int32_t n_layer = 32;
+    int32_t falcon_version = 7; // 7 for Falcon-7B, 40 for Falcon-40B
+    int32_t ftype   = 1;
+    int32_t n_ctx   = 2048;
+};
+
+struct falcon_layer {
+    // normalization
+    struct ggml_tensor* input_layernorm;
+    struct ggml_tensor* input_layernorm_b;
+    //struct ggml_tensor* attention_norm;    // Falcon-40B only
+    //struct ggml_tensor* attention_norm_b;  // Falcon-40B only
+
+    // attention
+    struct ggml_tensor* query_key_value;
+    struct ggml_tensor* wo;
+
+    // ff
+    struct ggml_tensor* ffn_up;
+    struct ggml_tensor* ffn_down;
+};
+
+struct falcon_model {
+    falcon_hparams hparams;
+
+    struct ggml_tensor* tok_embeddings;
+    struct ggml_tensor* output_norm;
+    struct ggml_tensor* output_norm_b;
+    struct ggml_tensor* lm_head;
+
+    std::vector<falcon_layer> layers;
+
+    // key + value memory
+    llm_kv_cache kv_self;
+
+    struct ggml_context* ctx;
+    std::map<std::string, struct ggml_tensor*> tensors;
+
+    llm_buffer eval_buf;
+    llm_buffer work_buf;
+    llm_buffer scr0_buf;
+    llm_buffer scr1_buf;
+};
+
+static bool kv_cache_init(
+        const struct falcon_hparams & hparams,
+              struct llm_kv_cache & cache,
+                         ggml_type   wtype,
+                               int   n_ctx) {
+    const int n_embd  = hparams.n_embd;
+    const int dim_head  = n_embd / hparams.n_head;
+    const int dim_kv = dim_head * hparams.n_head_kv;
+    const int n_layer = hparams.n_layer;
+
+    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
+    const int64_t n_elements = dim_kv * n_mem;
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+    struct ggml_init_params params;
+    params.mem_size   = cache.buf.size;
+    params.mem_buffer = cache.buf.addr;
+    params.no_alloc   = false;
+
+    cache.ctx = ggml_init(params);
+    if (!cache.ctx) {
+        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
+        return false;
+    }
+
+    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+    return true;
+}
+
+// load the model's weights from a file
+bool falcon_model_load(const std::string & fname, falcon_model & model, gpt_vocab & vocab, size_t *mem_req) {
+    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+    if (mem_req) {
+        *mem_req = 0;
+    }
+
+    auto fin = std::ifstream(fname, std::ios::binary);
+    if (!fin) {
+        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
+        return false;
+    }
+
+    // verify magic
+    {
+        uint32_t magic;
+        fin.read((char *) &magic, sizeof(magic));
+        if (magic != FALCON_MAGIC) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
+            return false;
+        }
+    }
+
+    uint32_t format_version;
+    fin.read((char *) &format_version, sizeof(format_version));
+
+    // load hparams
+    {
+        auto & hparams = model.hparams;
+
+        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
+        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
+        fin.read((char *) &hparams.n_head_kv, sizeof(hparams.n_head_kv));
+        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
+        fin.read((char *) &hparams.falcon_version, sizeof(hparams.falcon_version));
+        fin.read((char *) &hparams.ftype,   sizeof(hparams.ftype));
+
+        if (hparams.falcon_version != 7) { // && hparams.falcon_version != 40) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad Falcon version: %d)\n", __func__, fname.c_str(), hparams.falcon_version);
+            return false;
+        }
+
+        const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
+
+        printf("%s: n_vocab   = %d\n", __func__, hparams.n_vocab);
+        printf("%s: n_embd    = %d\n", __func__, hparams.n_embd);
+        printf("%s: n_head    = %d\n", __func__, hparams.n_head);
+        printf("%s: n_head_kv = %d\n", __func__, hparams.n_head_kv);
+        printf("%s: n_layer   = %d\n", __func__, hparams.n_layer);
+        printf("%s: ftype     = %d\n", __func__, hparams.ftype);
+        printf("%s: qntvr     = %d\n", __func__, qntvr);
+
+        hparams.ftype %= GGML_QNT_VERSION_FACTOR;
+    }
+
+    // load vocab
+    {
+        const int32_t n_vocab = model.hparams.n_vocab;
+
+        std::string word;
+        std::vector<char> buf(128);
+
+        for (int i = 0; i < n_vocab; i++) {
+            uint32_t len;
+            fin.read((char *) &len, sizeof(len));
+
+            buf.resize(len);
+            fin.read((char *) buf.data(), len);
+            word.assign(buf.data(), len);
+
+            uint32_t dummy;
+            fin.read((char *) &dummy, sizeof(dummy));
+
+            vocab.token_to_id[word] = i;
+            vocab.id_to_token[i] = word;
+        }
+    }
+
+    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
+    // in order to save memory and also to speed up the computation
+    ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
+    if (wtype == GGML_TYPE_COUNT) {
+        fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
+                __func__, fname.c_str(), model.hparams.ftype);
+        return false;
+    }
+
+    auto & ctx = model.ctx;
+
+    size_t ctx_size = 0;
+
+    {
+        const auto& hparams = model.hparams;
+
+        const int n_embd = hparams.n_embd;
+        const int n_head = hparams.n_head;
+        const int n_head_kv = hparams.n_head_kv;
+        const int n_layer = hparams.n_layer;
+        const int n_ctx = hparams.n_ctx;
+        const int n_ff = 4 * model.hparams.n_embd;
+        const int n_vocab = hparams.n_vocab;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // tok_embeddings
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm_b
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // lm_head
+
+        // if (hparams.version == 40) { // Falcon-40B
+        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm
+        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm_b
+        // }
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm_b
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * (n_head_kv * 2 + n_head) * head_dim);  // query_key_value
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_embd);  // wo
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_ff);  // ffn_up
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_ff * n_embd);  // ffn_down
+ 
+        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
+    }
+
+    if (mem_req) {
+        const int n_embd  = model.hparams.n_embd;
+        const int dim_head  = n_embd / model.hparams.n_head;
+        const int dim_kv = dim_head * model.hparams.n_head_kv;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = dim_kv * n_mem;
+        size_t kv_cache_size = 2u*n_elements*ggml_type_size(wtype) + 2_MiB;
+        *mem_req = ctx_size + kv_cache_size;
+        return false;
+    }
+
+    // create the ggml context
+    {
+        struct ggml_init_params params = {
+            .mem_size   = ctx_size,
+            .mem_buffer = NULL,
+            .no_alloc   = false,
+        };
+
+        model.ctx = ggml_init(params);
+        if (!model.ctx) {
+            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
+            return false;
+        }
+    }
+
+    // prepare memory for the weights
+    {
+        const auto& hparams = model.hparams;
+
+        const int n_embd = hparams.n_embd;
+        const int n_head = hparams.n_head;
+        const int n_head_kv = hparams.n_head_kv;
+        const int n_layer = hparams.n_layer;
+        const int n_ff = 4 * model.hparams.n_embd;
+        const int n_vocab = hparams.n_vocab;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        model.layers.resize(n_layer);
+
+        model.tok_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
+
+        model.output_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.output_norm_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
+
+        // map by name
+        model.tensors["transformer.word_embeddings.weight"] =
+            model.tok_embeddings;
+
+        model.tensors["transformer.ln_f.weight"] = model.output_norm;
+        model.tensors["transformer.ln_f.bias"] = model.output_norm_b;
+        model.tensors["lm_head.weight"] = model.lm_head;
+
+        for (int i = 0; i < n_layer; ++i) {
+            auto& layer = model.layers[i];
+
+            layer.input_layernorm =
+                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.input_layernorm_b =
+                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+            // if (hparams.version == 40) { // for Falcon-40B only
+            //     layer.attention_norm =
+            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            //     layer.attention_norm_b =
+            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            // }
+
+            // query_key_value shape for config.multi_query == True:
+            layer.query_key_value = ggml_new_tensor_2d(
+                ctx, wtype, n_embd, (n_head_kv * 2 + n_head) * head_dim);
+            layer.wo = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
+
+            layer.ffn_up = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);
+            layer.ffn_down = ggml_new_tensor_2d(ctx, wtype, n_ff, n_embd);
+
+            // map by name
+            // if (hparams.version == 40) {
+            //     // Falcon-40B:
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_mlp.weight"] = layer.input_layernorm;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_mlp.bias"] = layer.input_layernorm_b;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_attn.weight"] = layer.attention_norm;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_attn.bias"] = layer.attention_norm_b;
+            // } else {
+            // Falcon-7B:
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".input_layernorm.weight"] = layer.input_layernorm;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".input_layernorm.bias"] = layer.input_layernorm_b;
+            //}
+
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".self_attention.query_key_value.weight"] =
+                layer.query_key_value;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".self_attention.dense.weight"] = layer.wo;
+
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".mlp.dense_h_to_4h.weight"] = layer.ffn_up;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".mlp.dense_4h_to_h.weight"] = layer.ffn_down;
+        }
+    }
+
+    // key + value memory
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_layer = hparams.n_layer;
+        const int n_ctx   = hparams.n_ctx;
+        const int n_head_kv = hparams.n_head_kv;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        const int64_t n_mem      = n_layer*n_ctx;
+        const int64_t n_elements = head_dim*n_mem;
+
+        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F32, model.hparams.n_ctx)) {
+            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
+            ggml_free(ctx);
+            return false;
+        }
+        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
+
+        printf("%s: memory_size = %8.2f MB, n_mem = %" PRId64 "\n", __func__, memory_size/1024.0/1024.0, n_mem);
+    }
+
+    // load weights
+    {
+        int n_tensors = 0;
+        size_t total_size = 0;
+
+        printf("%s: ", __func__);
+
+        while (true) {
+            int32_t n_dims;
+            int32_t length;
+            int32_t ttype;
+
+            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
+            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
+
+            if (fin.eof()) {
+                break;
+            }
+
+            int32_t nelements = 1;
+            int32_t ne[2] = { 1, 1 };
+            for (int i = 0; i < n_dims; ++i) {
+                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+                nelements *= ne[i];
+            }
+
+            std::string name(length, 0);
+            fin.read(&name[0], length);
+            fin.seekg(-static_cast<ptrdiff_t>(fin.tellg()) & 31, std::ios_base::cur);
+
+            if (model.tensors.find(name.data()) == model.tensors.end()) {
+                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
+                return false;
+            }
+
+            auto tensor = model.tensors[name.data()];
+            if (ggml_nelements(tensor) != nelements) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
+                return false;
+            }
+
+            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%5d, %5d], expected [%5d, %5d]\n",
+                        __func__, name.data(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
+                return false;
+            }
+
+            // for debugging
+            if (0) {
+                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
+            }
+
+            const size_t bpe = ggml_type_size(ggml_type(ttype));
+
+            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
+                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
+                return false;
+            }
+
+            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
+
+            total_size += ggml_nbytes(tensor);
+            if (++n_tensors % 8 == 0) {
+                printf(".");
+                fflush(stdout);
+            }
+        }
+
+        printf(" done\n");
+
+        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
+    }
+
+    fin.close();
+
+    model.eval_buf.resize(1280u * 1024 * 1024);
+    model.scr0_buf.resize(256u * 1024 * 1024);
+    model.scr1_buf.resize(256u * 1024 * 1024);
+    return true;
+}
+
+
+// evaluate the transformer
+//
+//   - model:     the model
+//   - n_threads: number of threads to use
+//   - n_past:    the context size so far
+//   - embd_inp:  the embeddings of the tokens in the context
+//   - embd_w:    the predicted logits for the next token
+//
+bool falcon_eval(
+        falcon_model & model,
+        const int n_threads,
+        const int n_past,
+        const std::vector<gpt_vocab::id> & embd_inp,
+              std::vector<float>         & embd_w,
+              size_t                     & mem_per_token) {
+    const int N = embd_inp.size();
+
+    const auto & hparams = model.hparams;
+
+    const int n_embd  = hparams.n_embd;
+    const int n_layer = hparams.n_layer;
+    const int n_ctx   = hparams.n_ctx;
+    const int n_head  = hparams.n_head;
+    const int n_head_kv = hparams.n_head_kv;
+    const int n_vocab = hparams.n_vocab;
+    const int version = hparams.falcon_version;
+    const size_t head_dim = n_embd / n_head;
+
+   struct ggml_init_params eval_ctx_params = {
+        .mem_size = model.eval_buf.size,
+        .mem_buffer = model.eval_buf.addr,
+        .no_alloc = false,
+    };
+
+    struct ggml_context * ctx0 = ggml_init(eval_ctx_params);
+    struct ggml_cgraph gf = {};
+
+    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
+
+    // wte
+    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.tok_embeddings, embd);
+    struct ggml_tensor* repeat_dummy = ggml_new_tensor_3d(ctx0, inpL->type, head_dim, N + n_past, n_head);
+
+    ggml_type wtype = GGML_TYPE_F32;
+    const int sizeof_wtype = ggml_type_sizef(wtype);
+
+    for (int il = 0; il < n_layer; ++il) {
+        struct ggml_tensor * cur;
+        struct ggml_tensor * layernorm_output;
+
+        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
+        // self-attention
+        {
+            layernorm_output = ggml_norm(ctx0, inpL);
+
+            layernorm_output = ggml_add(ctx0,
+                    ggml_mul(ctx0,
+                        ggml_repeat(ctx0, model.layers[il].input_layernorm, layernorm_output),
+                        layernorm_output),
+                    ggml_repeat(ctx0, model.layers[il].input_layernorm_b, layernorm_output));
+
+            // if (version == 40) { // Falcon-40B only
+            //     cur = ggml_norm(ctx0, inpL);
+
+            //     cur = ggml_add(ctx0,
+            //             ggml_mul(ctx0,
+            //                 ggml_repeat(ctx0, model.layers[il].attention_norm, cur),
+            //                 cur),
+            //             ggml_repeat(ctx0, model.layers[il].attention_norm_b, cur));
+            // }
+            // else {
+            cur = layernorm_output;
+            // }
+
+            // compute QKV
+
+            cur = ggml_mul_mat(ctx0, model.layers[il].query_key_value, cur);
+
+            // Note that the strides for Kcur, Vcur are set up so that the
+            // resulting views are misaligned with the tensor's storage
+            // (by applying the K/V offset we shift the tensor's original
+            // view to stick out behind the viewed QKV tensor's allocated
+            // memory, so to say). This is ok because no actual accesses
+            // happen to that out-of-range memory, but it can require some
+            // trickery when trying to accurately dump these views for
+            // debugging.
+
+            struct ggml_tensor * Qcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                0);
+
+            struct ggml_tensor * Kcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head_kv, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                head_dim * n_head * sizeof_wtype);
+
+            struct ggml_tensor * Vcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head_kv, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                head_dim * (n_head + n_head_kv) * sizeof_wtype);
+
+            // using mode = 2 for neox mode
+            Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, head_dim, 2, n_ctx);
+            Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, head_dim, 2, n_ctx);
+
+            // store key and value to memory
+            {
+                struct ggml_tensor* k = ggml_view_1d(
+                    ctx0, model.kv_self.k, N * n_head_kv * head_dim,
+                    (ggml_element_size(model.kv_self.k) * n_head_kv * head_dim) *
+                        (il * n_ctx + n_past));
+                struct ggml_tensor* v = ggml_view_1d(
+                    ctx0, model.kv_self.v, N * n_head_kv * head_dim,
+                    (ggml_element_size(model.kv_self.v) * n_head_kv * head_dim) *
+                        (il * n_ctx + n_past));
+
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
+            }
+
+            struct ggml_tensor * K = ggml_permute(
+                ctx0,
+                ggml_view_3d(
+                    ctx0,
+                    model.kv_self.k,
+                    head_dim, n_head_kv, n_past + N,
+                    head_dim * sizeof_wtype,
+                    head_dim * n_head_kv * sizeof_wtype,
+                    il * n_ctx * ggml_element_size(model.kv_self.k) * n_head_kv * head_dim),
+                0, 2, 1, 3);
+
+            // K * Q
+
+            // changed from repeat2 back to repeat, will not support 40B!
+            K = ggml_cont(ctx0, ggml_repeat(ctx0, K, repeat_dummy));
+
+            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
+
+            // KQ_scaled = KQ / sqrt(n_embd/n_head)
+            struct ggml_tensor * KQ_scaled =
+                ggml_scale_inplace(ctx0,
+                        KQ,
+                        ggml_new_f32(ctx0, 1.0f/sqrt(float(head_dim)))
+                        );
+
+            // KQ_masked = mask_past(KQ_scaled)
+            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
+
+            // KQ = soft_max(KQ_masked)
+            struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
+
+            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
+            struct ggml_tensor* V = ggml_permute(
+                ctx0,
+                ggml_view_3d(
+                    ctx0,
+                    model.kv_self.v,
+                    head_dim, n_head_kv, n_past + N,
+                    head_dim * sizeof_wtype,
+                    head_dim * n_head_kv * sizeof_wtype,
+                    il * n_ctx * ggml_element_size(model.kv_self.v) * n_head_kv * head_dim),
+                0, 2, 1, 3);
+
+            // changed from repeat2 back to repeat, will not support 40B!
+            V = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_repeat(ctx0, V, repeat_dummy)));
+
+            // KQV = transpose(V) * KQ_soft_max
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
+
+            // KQV_merged = KQV.permute(0, 2, 1, 3)
+            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
+
+            // cur = KQV_merged.contiguous().view(n_embd, N)
+            cur = ggml_cpy(ctx0,
+                    KQV_merged,
+                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+
+            // projection
+            {
+                cur = ggml_mul_mat(ctx0,
+                        model.layers[il].wo,
+                        cur);
+            }
+        }
+
+        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
+
+        struct ggml_tensor* inpFF = layernorm_output;
+        struct ggml_tensor* attn_out = ggml_cpy(
+            ctx0, cur, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+
+        {
+            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_up, inpFF);
+            cur = ggml_gelu(ctx0, cur);
+            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_down, cur);
+        }
+
+        cur = ggml_add(ctx0, cur, attn_out);
+        cur = ggml_add(ctx0, cur, inpL);
+        // input for next layer
+        inpL = cur;
+    }
+
+    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
+    // norm
+    {
+        inpL = ggml_norm(ctx0, inpL);
+
+        // inpL = ln_f_g*inpL + ln_f_b
+        inpL = ggml_add(ctx0,
+                ggml_mul(ctx0,
+                    ggml_repeat(ctx0, model.output_norm, inpL),
+                    inpL),
+                ggml_repeat(ctx0, model.output_norm_b, inpL));
+    }
+
+    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
+
+    // lm_head
+    {
+        inpL = ggml_mul_mat(ctx0, model.lm_head, inpL);
+
+        //inpL = ggml_add(ctx0,
+        //        ggml_repeat(ctx0, model.lmh_b, inpL),
+        //        inpL);
+    }
+
+    // logits -> probs
+    //inpL = ggml_soft_max_inplace(ctx0, inpL);
+
+    // run the computation
+    ggml_build_forward_expand(&gf, inpL);
+    ggml_graph_compute_g4a(model.work_buf, &gf, n_threads);
+  
+
+    //if (n_past%100 == 0) {
+    //    ggml_graph_print   (&gf);
+    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
+    //}
+
+    //embd_w.resize(n_vocab*N);
+    //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
+
+    // return result for just the last token
+    embd_w.resize(n_vocab);
+    memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
+
+    if (mem_per_token == 0) {
+        mem_per_token = ggml_used_mem(ctx0)/N;
+    }
+    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
+
+    ggml_free(ctx0);
+
+    return true;
+}
+
+
+#define MAX_RNG_STATE 64*1024
+size_t falcon_get_state_size(const falcon_model &model) {
+    const size_t s_rng_size        = sizeof(size_t);
+    const size_t s_rng             = MAX_RNG_STATE;
+    const size_t s_kv_size         = sizeof(size_t);
+    const size_t s_kv_ntok         = sizeof(int);
+    const size_t s_kv              = model.kv_self.buf.size;
+    const size_t s_total = (
+        + s_rng_size
+        + s_rng
+        + s_kv_size
+        + s_kv_ntok
+        + s_kv
+    );
+    return s_total;
+}
+
+size_t falcon_copy_state_data(const falcon_model &model, const std::mt19937 &rng, uint8_t *dest)
+{
+    uint8_t * out = dest;
+    // copy rng
+    {
+        std::stringstream rng_ss;
+        rng_ss << rng;
+
+        const size_t rng_size = rng_ss.str().size();
+        char rng_buf[MAX_RNG_STATE];
+
+        memset(&rng_buf[0], 0, MAX_RNG_STATE);
+        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
+
+        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
+        memcpy(out, &rng_buf[0], MAX_RNG_STATE); out += MAX_RNG_STATE;
+    }
+
+    // copy kv cache
+    {
+        const size_t kv_size = model.kv_self.buf.size;
+        const int    kv_ntok = model.kv_self.n;
+
+        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
+        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
+
+        if (kv_size) {
+            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
+        }
+    }
+
+    const size_t written  = out - dest;
+    assert(written == falcon_get_state_size(model));
+    fflush(stdout);
+    return written;
+}
+
+size_t falcon_set_state_data(falcon_model *model, std::mt19937 *rng, const uint8_t *src)
+{
+    const uint8_t * in = src;
+
+    // set rng
+    {
+        size_t rng_size;
+        char   rng_buf[MAX_RNG_STATE];
+
+        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
+        memcpy(&rng_buf[0], in, MAX_RNG_STATE); in += MAX_RNG_STATE;
+
+        std::stringstream rng_ss;
+        rng_ss.str(std::string(&rng_buf[0], rng_size));
+        rng_ss >> *rng;
+
+        assert(rng_ss.fail() == false);
+    }
+
+    // set kv cache
+    {
+        size_t kv_size;
+        int kv_ntok;
+
+        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
+        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
+
+        if (kv_size) {
+            assert(model->kv_self.buf.size == kv_size);
+
+            void * k_data = model->kv_self.k->data; // remember data pointers
+            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
+
+            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
+
+            model->kv_self.k->data = k_data; // restore correct data pointers
+            model->kv_self.v->data = v_data;
+
+        }
+
+        model->kv_self.n = kv_ntok;
+    }
+
+    const size_t nread    = in - src;
+    assert(nread == falcon_get_state_size(*model));
+    fflush(stdout);
+    return nread;
+}
+
+struct FalconPrivate {
+    const std::string modelPath;
+    bool modelLoaded;
+    gpt_vocab vocab;
+    falcon_model *model = nullptr;
+    int64_t n_threads = 0;
+    size_t mem_per_token = 0;
+    std::mt19937 rng;
+};
+
+Falcon::Falcon() : d_ptr(new FalconPrivate) {
+    d_ptr->model = new falcon_model;
+    d_ptr->model->ctx = nullptr;
+    d_ptr->modelLoaded = false;
+}
+
+Falcon::~Falcon() {
+    if(d_ptr->model->ctx) {
+        ggml_free(d_ptr->model->ctx);
+        d_ptr->model->ctx = nullptr;
+    }
+    delete d_ptr->model;
+}
+
+bool Falcon::loadModel(const std::string &modelPath)
+{
+    std::mt19937 rng(time(NULL));
+    d_ptr->rng = rng;
+
+    // load the model
+    if (!falcon_model_load(modelPath, *d_ptr->model, d_ptr->vocab, nullptr)) {
+        std::cerr << "FALCON ERROR: failed to load model from " <<  modelPath;
+        return false;
+    }
+
+    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    d_ptr->modelLoaded = true;
+    fflush(stdout);
+    return true;
+}
+
+bool Falcon::isModelLoaded() const
+{
+    return d_ptr -> modelLoaded;
+}
+
+size_t Falcon::requiredMem(const std::string &modelPath)
+{
+    falcon_model dummy_model;
+    gpt_vocab dummy_vocab;
+    size_t mem_req;
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    falcon_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
+    return mem_req;
+}
+
+size_t Falcon::stateSize() const
+{
+    return falcon_get_state_size(*d_ptr->model);
+}
+
+size_t Falcon::saveState(uint8_t *dest) const
+{
+    return falcon_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
+}
+
+size_t Falcon::restoreState(const uint8_t *src)
+{
+    return falcon_set_state_data(d_ptr->model, &d_ptr->rng, src);
+}
+
+void Falcon::setThreadCount(int32_t n_threads)
+{
+    d_ptr->n_threads = n_threads;
+}
+
+int32_t Falcon::threadCount() const
+{
+    return d_ptr->n_threads;
+}
+
+std::vector<LLModel::Token> Falcon::tokenize(PromptContext &, const std::string &str) const
+{
+    return ::gpt_tokenize(d_ptr->vocab, str);
+}
+
+LLModel::Token Falcon::sampleToken(PromptContext &promptCtx) const
+{
+    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
+    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
+        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
+        n_prev_toks,
+        promptCtx.logits,
+        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
+        promptCtx.repeat_penalty,
+        d_ptr->rng);
+}
+
+std::string Falcon::tokenToString(Token id) const
+{
+    return d_ptr->vocab.id_to_token[id];
+}
+
+bool Falcon::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
+{
+    // determine the required inference memory per token:
+    static bool initialized = false;
+    if (!initialized) {
+        falcon_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
+            d_ptr->mem_per_token);
+        initialized = true;
+    }
+
+    return falcon_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
+}
+
+int32_t Falcon::contextLength() const
+{
+    return d_ptr->model->hparams.n_ctx;
+}
+
+const std::vector<LLModel::Token> &Falcon::endTokens() const
+{
+    static const std::vector<LLModel::Token> out = { 11 };
+    return out;
+}
+
+#if defined(_WIN32)
+#define DLL_EXPORT __declspec(dllexport)
+#else
+#define DLL_EXPORT __attribute__ ((visibility ("default")))
+#endif
+
+extern "C" {
+DLL_EXPORT bool is_g4a_backend_model_implementation() {
+    return true;
+}
+
+DLL_EXPORT const char *get_model_type() {
+    return modelType_;
+}
+
+DLL_EXPORT const char *get_build_variant() {
+    return GGML_BUILD_VARIANT;
+}
+
+DLL_EXPORT bool magic_match(std::istream& f) {
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    uint32_t version = 0;
+    f.read(reinterpret_cast<char*>(&version), sizeof(version));
+    if (magic != FALCON_MAGIC) {
+         return false; 
+    }
+    falcon_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    // we're matching the file format of existing pre-converted models
+    // compatible with ctransformers llama.cpp based format, which also
+    // unfortunately shares its magic number what llama uses, so we now
+    // differentiate by n_vocab
+    // give some wiggle room over the max to allow for finetunes that expand the
+    // vocabulary
+    if (!(hparams.n_vocab >= 65024 && hparams.n_vocab <= 65100)) {
+        return false;
+    }
+    if (hparams.falcon_version != 7) {
+        return false;
+    }
+    return true;
+}
+
+DLL_EXPORT LLModel *construct() {
+    return new Falcon;
+}
+}

gpt4all-backend/falcon_impl.h

@@ -0,0 +1,42 @@
+#ifndef FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of falcon.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef FALCON_H
+#define FALCON_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <memory>
+#include "llmodel.h"
+
+struct FalconPrivate;
+class Falcon : public LLModel {
+public:
+    Falcon();
+    ~Falcon();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    std::unique_ptr<FalconPrivate> d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // Falcon_H

gpt4all-backend/gptj.cpp

@@ -9,6 +9,7 @@
 #include <cmath>
 #include <cstdio>
 #include <cstring>
+#include <fstream>
 #include <map>
 #include <string>
 #include <vector>
@@ -41,7 +42,7 @@ struct gptj_hparams {
     int32_t n_head  = 16;
     int32_t n_layer = 28;
     int32_t n_rot   = 64;
-    float norm_eps  = 1e-5;
+    int32_t f16     = 1;
 };
 
 struct gptj_layer {
@@ -127,149 +128,216 @@ static bool kv_cache_init(
     return true;
 }
 
-// load the model's weights from a file path
-bool gptj_model_load(const std::string &fname, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
+// load the model's weights from a stream
+bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
     if(mem_req != nullptr) {
         *mem_req = 0;
     }
 
-    // create the ggml context
-    struct gguf_init_params params = {
-        /*.no_alloc = */ false,
-        /*.ctx      = */ &model.ctx,
-    };
-
-    gguf_context *ggufctx = gguf_init_from_file(fname.c_str(), params);
-    if (!ggufctx) {
-        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
-        return false;
+    // verify magic
+    {
+        uint32_t magic;
+        fin.read((char *) &magic, sizeof(magic));
+        if (magic != 0x67676d6c) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
+            return false;
+        }
     }
 
     // load hparams
     {
         auto & hparams = model.hparams;
 
-        bool ok = false;
-        int keyidx;
-
-        do {
-            keyidx = gguf_find_key(ggufctx, "gptj.context_length");
-            if (keyidx == -1) { break; }
-            hparams.n_ctx = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.embedding_length");
-            if (keyidx == -1) { break; }
-            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.attention.head_count");
-            if (keyidx == -1) { break; }
-            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.block_count");
-            if (keyidx == -1) { break; }
-            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.rope.dimension_count");
-            if (keyidx == -1) { break; }
-            hparams.n_rot = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.attention.layer_norm_epsilon");
-            if (keyidx == -1) { break; }
-            hparams.norm_eps = gguf_get_val_f32(ggufctx, keyidx);
-
-            ok = true;
-        } while (false);
-
-        if (!ok) {
-            fprintf(stderr, "%s: required hparam missing!\n", __func__);
-            return false;
-        }
+        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
+        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
+        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
+        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
+        fin.read((char *) &hparams.n_rot,   sizeof(hparams.n_rot));
+        fin.read((char *) &hparams.f16,     sizeof(hparams.f16));
 
+        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
         printf("%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
         printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
         printf("%s: n_head  = %d\n", __func__, hparams.n_head);
         printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
         printf("%s: n_rot   = %d\n", __func__, hparams.n_rot);
+        printf("%s: f16     = %d\n", __func__, hparams.f16);
     }
 
     // load vocab
     {
-        auto & hparams = model.hparams;
+        int32_t n_vocab = 0;
+        fin.read((char *) &n_vocab, sizeof(n_vocab));
 
-        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
-            return false;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "gpt2") != 0) {
-            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
+        if (n_vocab != model.hparams.n_vocab) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
+                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
             return false;
         }
 
-        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
-        if (tokens_keyidx == -1) {
-            fprintf(stderr, "%s: gpt2 tokenizer vocab not found!\n", __func__);
-            return false;
-        }
+        std::string word;
+        for (int i = 0; i < n_vocab; i++) {
+            uint32_t len;
+            fin.read((char *) &len, sizeof(len));
 
-        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
-        printf("%s: gpt2 tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
+            word.resize(len);
+            fin.read((char *) word.data(), len);
 
-        for (int i = 0; i < hparams.n_vocab; i++) {
-            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
             vocab.token_to_id[word] = i;
             vocab.id_to_token[i] = word;
         }
     }
 
+    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
+    // in order to save memory and also to speed up the computation
+    ggml_type wtype = GGML_TYPE_COUNT;
+    switch (model.hparams.f16) {
+        case 0: wtype = GGML_TYPE_F32;  break;
+        case 1: wtype = GGML_TYPE_F16;  break;
+        case 2: wtype = GGML_TYPE_Q4_0; break;
+        case 3: wtype = GGML_TYPE_Q4_1; break;
+        case 5: wtype = GGML_TYPE_Q4_2; break;
+        default:
+                {
+                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
+                            __func__, fname.c_str(), model.hparams.f16);
+                    return false;
+                }
+    }
+
     auto & ctx = model.ctx;
 
-    size_t ctx_size = ggml_get_mem_size(ctx);
-    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size / (1024.0 * 1024.0));
+    size_t ctx_size = 0;
+
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_ctx   = hparams.n_ctx;
+        const int n_vocab = hparams.n_vocab;
+
+        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_g
+        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_b
+
+        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype); // wte
+
+        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype);         // lmh_g
+        ctx_size +=        n_vocab*ggml_type_sizef(GGML_TYPE_F32); // lmh_b
+
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_g
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_b
+
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_q_proj_w
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_k_proj_w
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_v_proj_w
+
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_proj_w
+
+        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_fc_w
+        ctx_size += n_layer*(       4*n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_fc_b
+
+        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_proj_w
+        ctx_size += n_layer*(         n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_proj_b
+
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_k
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_v
+
+        ctx_size += (5 + 10*n_layer)*256; // object overhead
+
+        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
+    }
 
     if (mem_req != nullptr) {
-        *mem_req = ctx_size;
-        gguf_free(ggufctx);
+        *mem_req += ctx_size;
+        const int n_embd  = model.hparams.n_embd;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = n_embd*n_mem;
+
+        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
         return false;
     }
 
+
+    // create the ggml context
+    {
+        struct ggml_init_params params = {
+            .mem_size   = ctx_size,
+            .mem_buffer = NULL,
+            .no_alloc = false
+        };
+
+        model.ctx = ggml_init(params);
+        if (!model.ctx) {
+            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
+            return false;
+        }
+    }
+
     // prepare memory for the weights
     {
         const auto & hparams = model.hparams;
-        model.layers.resize(hparams.n_layer);
 
-        model.wte    = ggml_get_tensor(ctx, "token_embd.weight");
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_vocab = hparams.n_vocab;
 
-        model.ln_f_g = ggml_get_tensor(ctx, "output_norm.weight");
-        model.ln_f_b = ggml_get_tensor(ctx, "output_norm.bias");
+        model.layers.resize(n_layer);
 
-        model.lmh_g  = ggml_get_tensor(ctx, "output.weight");
-        model.lmh_b  = ggml_get_tensor(ctx, "output.bias");
+        model.wte    = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
 
-        auto name = [](int i, std::string n) {
-            static std::string key;
-            key = "blk." + std::to_string(i) + "." + n;
-            return key.c_str();
-        };
+        model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
 
-        for (int i = 0; i < hparams.n_layer; ++i) {
+        model.lmh_g  = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
+        model.lmh_b  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab);
+
+        // map by name
+        model.tensors["transformer.wte.weight"] = model.wte;
+
+        model.tensors["transformer.ln_f.weight"] = model.ln_f_g;
+        model.tensors["transformer.ln_f.bias"]   = model.ln_f_b;
+
+        model.tensors["lm_head.weight"] = model.lmh_g;
+        model.tensors["lm_head.bias"]   = model.lmh_b;
+
+        for (int i = 0; i < n_layer; ++i) {
             auto & layer = model.layers[i];
 
-            layer.ln_1_g          = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
-            layer.ln_1_b          = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
+            layer.ln_1_g          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
+            layer.ln_1_b          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
 
-            layer.c_attn_q_proj_w = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
-            layer.c_attn_k_proj_w = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
-            layer.c_attn_v_proj_w = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
+            layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
 
-            layer.c_attn_proj_w   = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
+            layer.c_attn_proj_w   = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
 
-            layer.c_mlp_fc_w      = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
-            layer.c_mlp_fc_b      = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
+            layer.c_mlp_fc_w      = ggml_new_tensor_2d(ctx, wtype,           n_embd, 4*n_embd);
+            layer.c_mlp_fc_b      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
 
-            layer.c_mlp_proj_w    = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
-            layer.c_mlp_proj_b    = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
+            layer.c_mlp_proj_w    = ggml_new_tensor_2d(ctx, wtype,         4*n_embd,   n_embd);
+            layer.c_mlp_proj_b    = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
+
+            // map by name
+            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.weight"]          = layer.ln_1_g;
+            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.bias"]            = layer.ln_1_b;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.q_proj.weight"]   = layer.c_attn_q_proj_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.k_proj.weight"]   = layer.c_attn_k_proj_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.v_proj.weight"]   = layer.c_attn_v_proj_w;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.out_proj.weight"] = layer.c_attn_proj_w;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.weight"]     = layer.c_mlp_fc_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.bias"]       = layer.c_mlp_fc_b;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.weight"]    = layer.c_mlp_proj_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.bias"]      = layer.c_mlp_proj_b;
         }
     }
 
@@ -286,12 +354,113 @@ bool gptj_model_load(const std::string &fname, gptj_model & model, gpt_vocab & v
         printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
     }
 
+    // load weights
+    {
+        int n_tensors = 0;
+        size_t total_size = 0;
+
+        printf("%s: ", __func__);
+
+        while (true) {
+            int32_t n_dims;
+            int32_t length;
+            int32_t ftype;
+
+            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
+            fin.read(reinterpret_cast<char *>(&ftype),  sizeof(ftype));
+
+            if (fin.eof()) {
+                break;
+            }
+
+            int32_t nelements = 1;
+            int32_t ne[2] = { 1, 1 };
+            for (int i = 0; i < n_dims; ++i) {
+                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+                nelements *= ne[i];
+            }
+
+            std::string name(length, 0);
+            fin.read(&name[0], length);
+
+            if (model.tensors.find(name.data()) == model.tensors.end()) {
+                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
+                return false;
+            }
+
+            auto tensor = model.tensors[name.data()];
+            if (ggml_nelements(tensor) != nelements) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
+                return false;
+            }
+
+            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
+                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
+                return false;
+            }
+
+            if (0) {
+                static const char * ftype_str[] = { "f32", "f16", "q4_0", "q4_1", };
+                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ftype_str[ftype], ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
+            }
+
+            size_t bpe = 0;
+
+            switch (ftype) {
+                case 0: bpe = ggml_type_size(GGML_TYPE_F32);  break;
+                case 1: bpe = ggml_type_size(GGML_TYPE_F16);  break;
+                case 2: bpe = ggml_type_size(GGML_TYPE_Q4_0); assert(ne[0] % 64 == 0); break;
+                case 3: bpe = ggml_type_size(GGML_TYPE_Q4_1); assert(ne[0] % 64 == 0); break;
+                default:
+                        {
+                            fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype);
+                            return false;
+                        }
+            };
+
+            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
+                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
+                return false;
+            }
+
+            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
+
+            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
+            total_size += ggml_nbytes(tensor);
+            if (++n_tensors % 8 == 0) {
+                printf(".");
+                fflush(stdout);
+            }
+        }
+
+        printf(" done\n");
+
+        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
+    }
+
     model.scr0_buf.resize(256u * 1024 * 1024);
     model.scr1_buf.resize(256u * 1024 * 1024);
 
     return true;
 }
 
+// load the model's weights from a file path
+bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) {
+
+    auto fin = std::ifstream(fname, std::ios::binary);
+    if (!fin) {
+        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
+        return false;
+    }
+
+    bool loaded = gptj_model_load(fname, fin, model, vocab);
+    fin.close();
+    return loaded;
+}
+
 // evaluate the transformer
 //
 //   - model:     the model
@@ -344,6 +513,7 @@ bool gptj_eval(
 
     struct ggml_context * ctx0 = ggml_init(params);
     struct ggml_cgraph gf = {};
+    gf.n_threads = n_threads;
 
     struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
     memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -356,7 +526,7 @@ bool gptj_eval(
         ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
         // norm
         {
-            cur = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
+            cur = ggml_norm(ctx0, inpL);
 
             // cur = ln_1_g*cur + ln_1_b
             cur = ggml_add(ctx0,
@@ -370,31 +540,37 @@ bool gptj_eval(
 
         // self-attention
         {
-            struct ggml_tensor * Qcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
-            struct ggml_tensor * Kcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
+            struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur);
+            struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur);
+            struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur);
 
             // store key and value to memory
             {
-                struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur));
-
                 struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
-                        (   n_ctx)*ggml_element_size(model.kv_self.v),
-                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
+                struct ggml_tensor * v = ggml_view_1d(ctx0, model.kv_self.v, N*n_embd, (ggml_element_size(model.kv_self.v)*n_embd)*(il*n_ctx + n_past));
 
                 ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
                 ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
             }
 
             // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+            struct ggml_tensor * Q =
+                ggml_permute(ctx0,
+                        ggml_rope(ctx0,
+                            ggml_cpy(ctx0,
+                                Qcur,
+                                ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),
+                            n_past, n_rot, 0),
+                        0, 2, 1, 3);
 
             // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
             struct ggml_tensor * K =
                 ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0,
-                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                            n_embd/n_head, n_head, n_past + N),
+                        ggml_rope(ctx0,
+                            ggml_reshape_3d(ctx0,
+                                ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
+                                n_embd/n_head, n_head, n_past + N),
+                            n_past, n_rot, 1),
                         0, 2, 1, 3);
 
             // K * Q
@@ -414,15 +590,17 @@ bool gptj_eval(
             struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
 
             // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V =
-                ggml_view_3d(ctx0, model.kv_self.v,
-                        n_past + N, n_embd/n_head, n_head,
-                        n_ctx*ggml_element_size(model.kv_self.v),
-                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
-                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
+            struct ggml_tensor * V_trans =
+                ggml_cpy(ctx0,
+                        ggml_permute(ctx0,
+                            ggml_reshape_3d(ctx0,
+                                ggml_view_1d(ctx0, model.kv_self.v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd),
+                                n_embd/n_head, n_head, n_past + N),
+                            1, 2, 0, 3),
+                        ggml_new_tensor_3d(ctx0, model.kv_self.v->type, n_past + N, n_embd/n_head, n_head));
 
             // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);
 
             // KQV_merged = KQV.permute(0, 2, 1, 3)
             struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
@@ -478,7 +656,7 @@ bool gptj_eval(
 
     // norm
     {
-        inpL = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
+        inpL = ggml_norm(ctx0, inpL);
 
         // inpL = ln_f_g*inpL + ln_f_b
         inpL = ggml_add(ctx0,
@@ -502,18 +680,9 @@ bool gptj_eval(
     // logits -> probs
     //inpL = ggml_soft_max(ctx0, inpL);
 
-    ggml_build_forward_expand(&gf, inpL);
-
     // run the computation
-    {
-        std::unique_ptr<uint8_t []> data;
-        auto plan = ggml_graph_plan(&gf, n_threads);
-        if (plan.work_size > 0) {
-            data.reset(new uint8_t[plan.work_size]);
-            plan.work_data = data.get();
-        }
-        ggml_graph_compute(&gf, &plan);
-    }
+    ggml_build_forward_expand(&gf, inpL);
+    ggml_graph_compute       (ctx0, &gf);
 
     //if (n_past%100 == 0) {
     //    ggml_graph_print   (&gf);
@@ -667,7 +836,8 @@ size_t GPTJ::requiredMem(const std::string &modelPath) {
     gptj_model dummy_model;
     gpt_vocab dummy_vocab;
     size_t mem_req;
-    gptj_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    gptj_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
     return mem_req;
 }
 
@@ -675,8 +845,10 @@ bool GPTJ::loadModel(const std::string &modelPath) {
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
 
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+
     // load the model
-    if (!gptj_model_load(modelPath, *d_ptr->model, d_ptr->vocab)) {
+    if (!gptj_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab)) {
         std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
         return false;
     }
@@ -767,16 +939,6 @@ const std::vector<LLModel::Token> &GPTJ::endTokens() const
     return fres;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -796,21 +958,15 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
-
-    bool isValid = gguf_get_version(ctx_gguf) <= 2;
-    isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
-
-    gguf_free(ctx_gguf);
-    return isValid;
+DLL_EXPORT bool magic_match(std::istream& f) {
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    gptj_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    if (!(hparams.n_vocab >= 50300 && hparams.n_vocab <= 50400)) {
+        return false; // not a gptj.
+    }
+    return magic == 0x67676d6c;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llama.cpp.cmake

@@ -173,10 +173,7 @@ if (LLAMA_KOMPUTE)
         set(spv_file ${CMAKE_CURRENT_BINARY_DIR}/${OP_FILE}.spv)
         add_custom_command(
             OUTPUT ${spv_file}
-            DEPENDS ${LLAMA_DIR}/${source}
-              ${LLAMA_DIR}/kompute/common.comp
-              ${LLAMA_DIR}/kompute/op_getrows.comp
-              ${LLAMA_DIR}/kompute/op_mul_mv_q_n.comp
+            DEPENDS ${LLAMA_DIR}/${source} ${LLAMA_DIR}/kompute/common.comp
             COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
             COMMENT "Compiling ${source} to ${source}.spv"
         )
@@ -196,11 +193,11 @@ if (LLAMA_KOMPUTE)
               COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
               DEPENDS ${spv_file} xxd
-              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd"
+              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}/xxd"
             )
         else()
             add_custom_command(
@@ -222,7 +219,6 @@ if (LLAMA_KOMPUTE)
 
     if (EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
         message(STATUS "Kompute found")
-        set(KOMPUTE_OPT_LOG_LEVEL Critical CACHE STRING "Kompute log level")
         add_subdirectory(${LLAMA_DIR}/kompute)
 
         # Compile our shaders
@@ -239,16 +235,12 @@ if (LLAMA_KOMPUTE)
           kompute/op_norm.comp
           kompute/op_rmsnorm.comp
           kompute/op_diagmask.comp
-          kompute/op_mul_mat_mat_f32.comp
           kompute/op_mul_mat_f16.comp
-          kompute/op_mul_mat_q8_0.comp
           kompute/op_mul_mat_q4_0.comp
           kompute/op_mul_mat_q4_1.comp
-          kompute/op_mul_mat_q6_k.comp
           kompute/op_getrows_f16.comp
           kompute/op_getrows_q4_0.comp
           kompute/op_getrows_q4_1.comp
-          kompute/op_getrows_q6_k.comp
           kompute/op_rope.comp
           kompute/op_cpy_f16_f16.comp
           kompute/op_cpy_f16_f32.comp
@@ -270,16 +262,12 @@ if (LLAMA_KOMPUTE)
           shaderop_norm.h
           shaderop_rmsnorm.h
           shaderop_diagmask.h
-          shaderop_mul_mat_mat_f32.h
           shaderop_mul_mat_f16.h
-          shaderop_mul_mat_q8_0.h
           shaderop_mul_mat_q4_0.h
           shaderop_mul_mat_q4_1.h
-          shaderop_mul_mat_q6_k.h
           shaderop_getrows_f16.h
           shaderop_getrows_q4_0.h
           shaderop_getrows_q4_1.h
-          shaderop_getrows_q6_k.h
           shaderop_rope.h
           shaderop_cpy_f16_f16.h
           shaderop_cpy_f16_f32.h
@@ -358,13 +346,6 @@ endif()
 # TODO: probably these flags need to be tweaked on some architectures
 #       feel free to update the Makefile for your architecture and send a pull request or issue
 message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
-if (MSVC)
-  string(TOLOWER "${CMAKE_GENERATOR_PLATFORM}" CMAKE_GENERATOR_PLATFORM_LWR)
-  message(STATUS "CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}")
-else ()
-  set(CMAKE_GENERATOR_PLATFORM_LWR "")
-endif ()
-
 if (NOT MSVC)
     if (LLAMA_STATIC)
         add_link_options(-static)
@@ -380,139 +361,6 @@ if (NOT MSVC)
     endif()
 endif()
 
-if ((${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm") OR (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") OR ("${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "arm64"))
-    message(STATUS "ARM detected")
-    if (MSVC)
-        add_compile_definitions(__ARM_NEON)
-        add_compile_definitions(__ARM_FEATURE_FMA)
-        add_compile_definitions(__ARM_FEATURE_DOTPROD)
-        # add_compile_definitions(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) # MSVC doesn't support vdupq_n_f16, vld1q_f16, vst1q_f16
-        add_compile_definitions(__aarch64__) # MSVC defines _M_ARM64 instead
-    else()
-        include(CheckCXXCompilerFlag)
-        check_cxx_compiler_flag(-mfp16-format=ieee COMPILER_SUPPORTS_FP16_FORMAT_I3E)
-        if (NOT "${COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
-            add_compile_options(-mfp16-format=ieee)
-        endif()
-        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv6")
-            # Raspberry Pi 1, Zero
-            add_compile_options(-mfpu=neon-fp-armv8 -mno-unaligned-access)
-        endif()
-        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7")
-            # Raspberry Pi 2
-            add_compile_options(-mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations)
-        endif()
-        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv8")
-            # Raspberry Pi 3, 4, Zero 2 (32-bit)
-            add_compile_options(-mno-unaligned-access)
-        endif()
-    endif()
-elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$" OR "${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "^(x86_64|i686|amd64|x64)$" )
-    message(STATUS "x86 detected")
-    if (MSVC)
-        if (LLAMA_AVX512)
-            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX512>)
-            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
-            # MSVC has no compile-time flags enabling specific
-            # AVX512 extensions, neither it defines the
-            # macros corresponding to the extensions.
-            # Do it manually.
-            if (LLAMA_AVX512_VBMI)
-                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
-                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
-            endif()
-            if (LLAMA_AVX512_VNNI)
-                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
-                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
-            endif()
-        elseif (LLAMA_AVX2)
-            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX2>)
-            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
-        elseif (LLAMA_AVX)
-            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX>)
-            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
-        endif()
-    else()
-        if (LLAMA_F16C)
-            add_compile_options(-mf16c)
-        endif()
-        if (LLAMA_FMA)
-            add_compile_options(-mfma)
-        endif()
-        if (LLAMA_AVX)
-            add_compile_options(-mavx)
-        endif()
-        if (LLAMA_AVX2)
-            add_compile_options(-mavx2)
-        endif()
-        if (LLAMA_AVX512)
-            add_compile_options(-mavx512f)
-            add_compile_options(-mavx512bw)
-        endif()
-        if (LLAMA_AVX512_VBMI)
-            add_compile_options(-mavx512vbmi)
-        endif()
-        if (LLAMA_AVX512_VNNI)
-            add_compile_options(-mavx512vnni)
-        endif()
-    endif()
-elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
-    message(STATUS "PowerPC detected")
-    add_compile_options(-mcpu=native -mtune=native)
-    #TODO: Add  targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be)
-else()
-    message(STATUS "Unknown architecture")
-endif()
-
-#
-# POSIX conformance
-#
-
-# clock_gettime came in POSIX.1b (1993)
-# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional
-# posix_memalign came in POSIX.1-2001 / SUSv3
-# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985)
-add_compile_definitions(_XOPEN_SOURCE=600)
-
-# Somehow in OpenBSD whenever POSIX conformance is specified
-# some string functions rely on locale_t availability,
-# which was introduced in POSIX.1-2008, forcing us to go higher
-if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
-    remove_definitions(-D_XOPEN_SOURCE=600)
-    add_compile_definitions(_XOPEN_SOURCE=700)
-endif()
-
-# Data types, macros and functions related to controlling CPU affinity and
-# some memory allocation are available on Linux through GNU extensions in libc
-if (CMAKE_SYSTEM_NAME MATCHES "Linux")
-    add_compile_definitions(_GNU_SOURCE)
-endif()
-
-# RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1,
-# and on macOS its availability depends on enabling Darwin extensions
-# similarly on DragonFly, enabling BSD extensions is necessary
-if (
-    CMAKE_SYSTEM_NAME MATCHES "Darwin" OR
-    CMAKE_SYSTEM_NAME MATCHES "iOS" OR
-    CMAKE_SYSTEM_NAME MATCHES "tvOS" OR
-    CMAKE_SYSTEM_NAME MATCHES "DragonFly"
-)
-    add_compile_definitions(_DARWIN_C_SOURCE)
-endif()
-
-# alloca is a non-standard interface that is not visible on BSDs when
-# POSIX conformance is specified, but not all of them provide a clean way
-# to enable it in such cases
-if (CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
-    add_compile_definitions(__BSD_VISIBLE)
-endif()
-if (CMAKE_SYSTEM_NAME MATCHES "NetBSD")
-    add_compile_definitions(_NETBSD_SOURCE)
-endif()
-if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
-    add_compile_definitions(_BSD_SOURCE)
-endif()
-
 function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
     message(STATUS "Configuring ggml implementation target llama${SUFFIX} in ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}")
 
@@ -620,14 +468,15 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
 
     if (WITH_LLAMA)
         # Backwards compatibility with old llama.cpp versions
-#        set(LLAMA_UTIL_SOURCE_FILE llama-util.h)
+        set(LLAMA_UTIL_SOURCE_FILE llama-util.h)
         if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
             set(LLAMA_UTIL_SOURCE_FILE llama_util.h)
         endif()
 
         add_library(llama${SUFFIX} STATIC
                     ${DIRECTORY}/llama.cpp
-                    ${DIRECTORY}/llama.h)
+                    ${DIRECTORY}/llama.h
+                    ${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
 
         if (LLAMA_METAL AND GGML_METAL_SOURCES)
             target_compile_definitions(llama${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)

gpt4all-backend/llamamodel.cpp

@@ -36,25 +36,18 @@ namespace {
 const char *modelType_ = "LLaMA";
 }
 
-static bool llama_verbose() {
-    const char* var = getenv("GPT4ALL_VERBOSE_LLAMACPP");
-    return var && *var;
-}
-
-static void llama_log_callback(enum ggml_log_level level, const char *text, void *userdata) {
-    (void)userdata;
-    if (llama_verbose() || level <= GGML_LOG_LEVEL_ERROR) {
-        fputs(text, stderr);
-    }
-}
-
 struct gpt_params {
     int32_t seed          = -1;   // RNG seed
     int32_t n_keep        = 0;    // number of tokens to keep from initial prompt
+#if LLAMA_DATE <= 230511
+    int32_t n_parts       = -1;   // amount of model parts (-1 = determine from model dimensions)
+#endif
 
+#if LLAMA_DATE >= 230519
     // sampling parameters
     float   tfs_z         = 1.0f; // 1.0 = disabled
     float   typical_p     = 1.0f; // 1.0 = disabled
+#endif
 
     std::string prompt = "";
 
@@ -64,6 +57,7 @@ struct gpt_params {
     bool use_mlock         = false; // use mlock to keep model in memory
 };
 
+#if LLAMA_DATE >= 230519
 static int llama_sample_top_p_top_k(
         llama_context *ctx,
         const llama_token *last_n_tokens_data,
@@ -91,6 +85,7 @@ static int llama_sample_top_p_top_k(
     llama_sample_temperature(ctx, &candidates_p, temp);
     return llama_sample_token(ctx, &candidates_p);
 }
+#endif
 
 struct LLamaPrivate {
     const std::string modelPath;
@@ -98,7 +93,6 @@ struct LLamaPrivate {
     llama_context *ctx = nullptr;
     llama_context_params params;
     int64_t n_threads = 0;
-    std::vector<LLModel::Token> end_tokens;
 };
 
 LLamaModel::LLamaModel()
@@ -155,10 +149,11 @@ bool LLamaModel::loadModel(const std::string &modelPath)
 #else
     d_ptr->params.use_mlock  = params.use_mlock;
 #endif
+#if LLAMA_DATE <= 230511
+    d_ptr->params.n_parts  = params.n_parts;
+#endif
 #ifdef GGML_USE_METAL
-    if (llama_verbose()) {
-        std::cerr << "llama.cpp: using Metal" << std::endl;
-    }
+    std::cerr << "llama.cpp: using Metal" << std::endl;
     // metal always runs the whole model if n_gpu_layers is not 0, at least
     // currently
     d_ptr->params.n_gpu_layers = 1;
@@ -181,8 +176,6 @@ bool LLamaModel::loadModel(const std::string &modelPath)
         return false;
     }
 
-    d_ptr->end_tokens = {llama_token_eos(d_ptr->ctx)};
-
 #ifdef GGML_USE_KOMPUTE
     if (ggml_vk_has_device()) {
         std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
@@ -233,9 +226,9 @@ size_t LLamaModel::restoreState(const uint8_t *src)
 
 std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str) const
 {
-    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos(d_ptr->ctx));
+    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
     std::vector<LLModel::Token> fres(str.size()+4);
-    auto fres_len = llama_tokenize(d_ptr->ctx, str.c_str(), str.length(), fres.data(), fres.size(), useBOS);
+    auto fres_len = llama_tokenize(d_ptr->ctx, str.c_str(), fres.data(), fres.size(), useBOS);
     fres.resize(fres_len);
     return fres;
 }
@@ -256,7 +249,16 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
 
 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
 {
-    return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+    // When we recalculate context we could have erased the original BOS token... we need to replace it
+    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
+    if (useBOS) {
+        std::vector<int32_t> myTokens;
+        myTokens.push_back(llama_token_bos());
+        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
+        ctx.n_past += 1;
+        return llama_eval(d_ptr->ctx, myTokens.data(), myTokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+    } else
+        return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
 }
 
 int32_t LLamaModel::contextLength() const
@@ -266,7 +268,8 @@ int32_t LLamaModel::contextLength() const
 
 const std::vector<LLModel::Token> &LLamaModel::endTokens() const
 {
-    return d_ptr->end_tokens;
+    static const std::vector<LLModel::Token> fres = {llama_token_eos()};
+    return fres;
 }
 
 #if defined(GGML_USE_KOMPUTE)
@@ -305,9 +308,8 @@ bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string& d
 #endif
 }
 
-bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device, std::string *unavail_reason)
+bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device)
 {
-    bool result = false;
 #if defined(GGML_USE_KOMPUTE)
     ggml_vk_device vkDevice;
     vkDevice.index = device.index;
@@ -315,16 +317,10 @@ bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device, std::stri
     vkDevice.heapSize = device.heapSize;
     vkDevice.name = device.name;
     vkDevice.vendor = device.vendor;
-    result = ggml_vk_init_device(vkDevice);
-    if (!result && unavail_reason) {
-        *unavail_reason = "failed to init GPU";
-    }
+    return ggml_vk_init_device(vkDevice);
 #else
-    if (unavail_reason) {
-        *unavail_reason = "built without Kompute";
-    }
+    return false;
 #endif
-    return result;
 }
 
 bool LLamaModel::initializeGPUDevice(int device)
@@ -355,16 +351,6 @@ bool LLamaModel::usingGPUDevice()
     return false;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != (GGUF_TYPE_STRING)) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -384,27 +370,42 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
-
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
+DLL_EXPORT bool magic_match(std::istream& f) {
+    // Check magic
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    if (magic != 0x67676a74) return false;
+    // Check version
+    uint32_t version = 0;
+    f.read(reinterpret_cast<char*>(&version), sizeof(version));
+    if (!(version LLAMA_VERSIONS)) {
         return false;
-
-    bool isValid = gguf_get_version(ctx_gguf) <= 2;
-    auto arch = get_arch_name(ctx_gguf);
-    isValid = isValid && (arch == "llama" || arch == "starcoder" || arch == "falcon");
-
-    gguf_free(ctx_gguf);
-    return isValid;
+    }
+    llama_file_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    if (!(hparams.n_vocab >= 32000 && hparams.n_vocab <= 32100)) {
+        return false; // not a llama.
+    }
+#ifdef GGML_USE_METAL
+    // Check quant supported on metal
+    // skip fields
+    switch(hparams.ftype) {
+        // currently supported on Metal https://github.com/ggerganov/llama.cpp/blob/ae9663f1887513e152839e91f61c513075a19422/ggml-metal.m#L51-L55
+        case LLAMA_FTYPE_MOSTLY_F16:
+        case LLAMA_FTYPE_MOSTLY_Q2_K:
+        case LLAMA_FTYPE_MOSTLY_Q4_0:
+        case LLAMA_FTYPE_MOSTLY_Q6_K:
+        case LLAMA_FTYPE_MOSTLY_Q4_K_S:
+        case LLAMA_FTYPE_MOSTLY_Q4_K_M:
+            return true;
+        default: // unsupported quant-type for Metal
+            return false;
+    }
+#endif
+    return true;
 }
 
 DLL_EXPORT LLModel *construct() {
-    llama_log_set(llama_log_callback, nullptr);
     return new LLamaModel;
 }
 }

gpt4all-backend/llamamodel_impl.h

@@ -27,7 +27,7 @@ public:
     int32_t threadCount() const override;
     std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) override;
     bool initializeGPUDevice(size_t memoryRequired, const std::string& device) override;
-    bool initializeGPUDevice(const GPUDevice &device, std::string *unavail_reason) override;
+    bool initializeGPUDevice(const GPUDevice &device) override;
     bool initializeGPUDevice(int device) override;
     bool hasGPUDevice() override;
     bool usingGPUDevice() override;

gpt4all-backend/llmodel.cpp

@@ -52,7 +52,7 @@ LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
     m_buildVariant = get_build_variant();
-    m_magicMatch = m_dlhandle->get<bool(const char*)>("magic_match");
+    m_magicMatch = m_dlhandle->get<bool(std::ifstream&)>("magic_match");
     assert(m_magicMatch);
     m_construct = m_dlhandle->get<LLModel *()>("construct");
     assert(m_construct);
@@ -111,28 +111,31 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     return *libs;
 }
 
-const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
+const LLModel::Implementation* LLModel::Implementation::implementation(std::ifstream& f, const std::string& buildVariant) {
     for (const auto& i : implementationList()) {
+        f.seekg(0);
+        if (!i.m_magicMatch(f)) continue;
         if (buildVariant != i.m_buildVariant) continue;
-        if (!i.m_magicMatch(fname)) continue;
         return &i;
     }
     return nullptr;
 }
 
 LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant) {
-    if (!has_at_least_minimal_hardware()) {
-        std::cerr << "LLModel ERROR: CPU does not support AVX\n";
-        return nullptr;
-    }
 
+    if (!has_at_least_minimal_hardware())
+        return nullptr;
+
+    // Read magic
+    std::ifstream f(modelPath, std::ios::binary);
+    if (!f) return nullptr;
     // Get correct implementation
     const Implementation* impl = nullptr;
 
     #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
         if (buildVariant == "auto") {
             size_t total_mem = getSystemTotalRAMInBytes();
-            impl = implementation(modelPath.c_str(), "metal");
+            impl = implementation(f, "metal");
             if(impl) {
                 LLModel* metalimpl = impl->m_construct();
                 metalimpl->m_implementation = impl;
@@ -158,9 +161,10 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
                 buildVariant = "default";
             }
         }
-        impl = implementation(modelPath.c_str(), buildVariant);
+        impl = implementation(f, buildVariant);
         if (!impl) return nullptr;
     }
+    f.close();
 
     // Construct and return llmodel implementation
     auto fres = impl->m_construct();

gpt4all-backend/llmodel.h

@@ -27,13 +27,13 @@ public:
 
         static bool isImplementation(const Dlhandle&);
         static const std::vector<Implementation>& implementationList();
-        static const Implementation *implementation(const char *fname, const std::string& buildVariant);
+        static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
         static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
         static void setImplementationsSearchPath(const std::string& path);
         static const std::string& implementationsSearchPath();
 
     private:
-        bool (*m_magicMatch)(const char *fname);
+        bool (*m_magicMatch)(std::ifstream& f);
         LLModel *(*m_construct)();
 
     private:
@@ -97,12 +97,7 @@ public:
 
     virtual std::vector<GPUDevice> availableGPUDevices(size_t /*memoryRequired*/) { return std::vector<GPUDevice>(); }
     virtual bool initializeGPUDevice(size_t /*memoryRequired*/, const std::string& /*device*/) { return false; }
-    virtual bool initializeGPUDevice(const GPUDevice &/*device*/, std::string *unavail_reason = nullptr) {
-        if (unavail_reason) {
-            *unavail_reason = "model has no GPU support";
-        }
-        return false;
-    }
+    virtual bool initializeGPUDevice(const GPUDevice &/*device*/) { return false; }
     virtual bool initializeGPUDevice(int /*device*/) { return false; }
     virtual bool hasGPUDevice() { return false; }
     virtual bool usingGPUDevice() { return false; }

gpt4all-backend/llmodel_shared.cpp

@@ -92,10 +92,10 @@ void LLModel::prompt(const std::string &prompt,
             if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
                 promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(batch.at(t));
-            promptCtx.n_past += 1;
             if (!promptCallback(batch.at(t)))
                 return;
         }
+        promptCtx.n_past += batch.size();
         i = batch_end;
     }
 
@@ -126,6 +126,8 @@ void LLModel::prompt(const std::string &prompt,
             return;
         }
 
+        promptCtx.n_past += 1;
+
         // display text
         for (const auto token : endTokens()) {
             if (id == token) return;
@@ -160,7 +162,6 @@ void LLModel::prompt(const std::string &prompt,
             if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
                 promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(t);
-            promptCtx.n_past += 1;
             //TODO: Conversion to std::string can be avoided here...
             if (!responseCallback(t, std::string(tokenToString(t))))
                 return;

gpt4all-backend/llmodel_shared.h

@@ -80,6 +80,7 @@ struct llm_kv_cache {
     }
 };
 
+#if LLAMA_DATE >= 230519
 inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads) {
     struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
     if (plan.work_size > 0) {
@@ -88,3 +89,4 @@ inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_t
     }
     ggml_graph_compute(graph, &plan);
 }
+#endif

gpt4all-backend/mpt.cpp

@@ -9,6 +9,7 @@
 #include <cmath>
 #include <cstdio>
 #include <cstring>
+#include <fstream>
 #include <map>
 #include <random>
 #include <string>
@@ -28,7 +29,6 @@
 #include <sstream>
 #include <thread>
 #include <unordered_set>
-#include <unordered_map>
 #include <regex>
 #include <ggml.h>
 
@@ -46,8 +46,8 @@ struct mpt_hparams {
     int32_t n_layer      = 32;
     float alibi_bias_max = 8;
     float clip_qkv       = 0;
-    float norm_eps       = 1e-5;
     int32_t expand       = 4;
+    int32_t f16          = 1;
 };
 
 struct mpt_layer {
@@ -78,6 +78,7 @@ struct mpt_model {
 
     struct llm_kv_cache kv_self;
     struct ggml_context * ctx;
+    std::map<std::string, struct ggml_tensor *> tensors;
 
 
     llm_buffer eval_buf;
@@ -91,121 +92,6 @@ struct mpt_model {
     }
 };
 
-enum mpt_token_type {
-    MPT_TOKEN_TYPE_NORMAL  = 1,
-    MPT_TOKEN_TYPE_CONTROL = 3,
-};
-
-using replit_piece_t = std::pair<std::size_t, float>;
-using replit_piece_map_t = std::unordered_map<std::string, replit_piece_t>;
-
-static const std::string replit_ws_symbol = "\342\226\201";
-
-struct mpt_vocab {
-    bool is_replit = false;
-    gpt_vocab raw;
-    replit_piece_map_t piece_map;
-    std::vector<std::string> vocab;
-
-    const char * end_of_text() const {
-        return is_replit ? "<|endoftext|>" : "<|im_end|>";
-    }
-};
-
-std::pair<std::vector<LLModel::Token>, float> encode_word(const std::string & word, const replit_piece_map_t & model) {
-    std::vector<int> best_segmentations_starts(word.length() + 1, -1);
-    best_segmentations_starts[0] = 0;
-
-    std::vector<float> best_segmentations_scores(word.length() + 1, -std::numeric_limits<float>::infinity());
-    best_segmentations_scores[0] = 1.0;
-
-    for (size_t start_idx = 0; start_idx < word.length(); ++start_idx) {
-        float best_score_at_start = best_segmentations_scores[start_idx];
-        for (size_t end_idx = start_idx + 1; end_idx <= word.length(); ++end_idx) {
-            std::string token = word.substr(start_idx, end_idx - start_idx);
-            if (model.count(token) && best_score_at_start != -std::numeric_limits<float>::infinity()) {
-                float token_score = model.at(token).second;
-                float score = token_score + best_score_at_start;
-                if (best_segmentations_scores[end_idx] == -std::numeric_limits<float>::infinity() ||
-                    best_segmentations_scores[end_idx] > score) {
-                    best_segmentations_starts[end_idx] = start_idx;
-                    best_segmentations_scores[end_idx] = score;
-                }
-            }
-        }
-    }
-
-    if (best_segmentations_scores.back() == -std::numeric_limits<float>::infinity()) {
-        return std::make_pair(std::vector<LLModel::Token>{0}, 0.0f);
-    }
-
-    float score = best_segmentations_scores.back();
-    int start = best_segmentations_starts.back();
-    int end = word.length();
-    std::vector<LLModel::Token> tokens;
-    while (start != 0) {
-        const auto token_id = model.at(word.substr(start, end - start)).first;
-        tokens.insert(tokens.begin(), token_id);
-        int next_start = best_segmentations_starts[start];
-        end = start;
-        start = next_start;
-    }
-    const auto token_id = model.at(word.substr(start, end - start)).first;
-    tokens.insert(tokens.begin(), token_id);
-    return std::make_pair(tokens, score);
-}
-
-bool replit_tokenizer_load(mpt_vocab & tokenizer, gguf_context * ggufctx, int tokens_keyidx, int max_vocab_size) {
-    int scores_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.scores");
-    if (scores_keyidx == -1) {
-        fprintf(stderr, "%s: llama token scores not found!\n", __func__);
-        return false;
-    }
-    const auto *scores = reinterpret_cast<const float *>(gguf_get_arr_data(ggufctx, scores_keyidx));
-
-    for (LLModel::Token i = 0; i < max_vocab_size; i++) {
-        std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
-        tokenizer.piece_map[word] = std::make_pair(i, -scores[i]);
-        tokenizer.raw.id_to_token[i] = word;
-        tokenizer.raw.token_to_id[word] = i;
-    }
-
-    return true;
-}
-
-std::string replace_all(const std::string & str,    // where to work
-                        const std::string & find,   // substitute 'find'
-                        const std::string & replace //      by 'replace'
-) {
-    std::string result;
-    size_t find_len = find.size();
-    size_t pos, from = 0;
-    while (std::string::npos != (pos = str.find(find, from))) {
-        result.append(str, from, pos - from);
-        result.append(replace);
-        from = pos + find_len;
-    }
-    result.append(str, from, std::string::npos);
-    return result;
-}
-
-std::vector<LLModel::Token> replit_tokenizer_tokenize(mpt_vocab & tokenizer, const std::string & text) {
-    std::vector<LLModel::Token> tokens;
-    auto normalized_text = replace_all(text, " ", replit_ws_symbol);
-    auto tokenized = encode_word(normalized_text, tokenizer.piece_map);
-
-    return tokenized.first;
-}
-
-std::string replit_tokenizer_detokenize(mpt_vocab & tokenizer, const std::vector<LLModel::Token> & tokens) {
-    std::string text;
-    for (auto token : tokens) {
-        text += tokenizer.raw.id_to_token[token];
-    }
-    return replace_all(text, replit_ws_symbol, " ");
-}
-
-
 static bool kv_cache_init(
         const struct mpt_hparams & hparams,
              struct llm_kv_cache & cache,
@@ -237,62 +123,20 @@ static bool kv_cache_init(
     return true;
 }
 
-// load the model's weights from a file path. if mem_req ptr is passed the model is
+// load the model's weights from a stream. if mem_req ptr is passed the model is
 // only partially parsed to estimate required memory
-bool mpt_model_load(const std::string &fname, mpt_model & model, mpt_vocab & vocab, size_t * mem_req) {
+bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & model, gpt_vocab & vocab, size_t * mem_req) {
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
     if (mem_req != nullptr) {
         *mem_req = 0;
     }
 
-    // create the ggml context
-    struct gguf_init_params params = {
-        /*.no_alloc = */ false,
-        /*.ctx      = */ &model.ctx,
-    };
-
-    gguf_context *ggufctx = gguf_init_from_file(fname.c_str(), params);
-    if (!ggufctx) {
-        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
-        return false;
-    }
-
-    printf("%s: gguf version     = %d\n", __func__, gguf_get_version(ggufctx));
-    printf("%s: gguf alignment   = %zu\n", __func__, gguf_get_alignment(ggufctx));
-    printf("%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx));
-
-    // print some standard metadata
+    // verify magic
     {
-        int keyidx;
-
-        keyidx = gguf_find_key(ggufctx, "general.name");
-        if (keyidx != -1) { printf("%s: model name           = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.description");
-        if (keyidx != -1) { printf("%s: model description    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.author");
-        if (keyidx != -1) { printf("%s: model author         = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.license");
-        if (keyidx != -1) { printf("%s: model license        = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx != -1) { printf("%s: model architecture   = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.file_type");
-        if (keyidx != -1) { printf("%s: model file type      = %" PRIu32 "\n", __func__, gguf_get_val_u32(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "gptneox.tensor_data_layout");
-        if (keyidx != -1) { printf("%s: model data layout    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.source.huggingface.repository");
-        if (keyidx != -1) { printf("%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-    }
-
-    // check required metadata
-    {
-        // check model architecture kv
-        int keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: gguf model architecture not found!\n", __func__);
-            return false;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "mpt") != 0) {
-            fprintf(stderr, "%s: model architecture not supported!\n", __func__);
+        uint32_t magic;
+        fin.read((char *) &magic, sizeof(magic));
+        if (magic != 0x67676d6d) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
             return false;
         }
     }
@@ -301,153 +145,182 @@ bool mpt_model_load(const std::string &fname, mpt_model & model, mpt_vocab & voc
     {
         auto & hparams = model.hparams;
 
-        bool ok = false;
-        int keyidx;
-
-        do {
-            keyidx = gguf_find_key(ggufctx, "mpt.context_length");
-            if (keyidx == -1) { break; }
-            hparams.n_ctx = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "mpt.embedding_length");
-            if (keyidx == -1) { break; }
-            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "mpt.attention.head_count");
-            if (keyidx == -1) { break; }
-            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "mpt.block_count");
-            if (keyidx == -1) { break; }
-            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "mpt.attention.max_alibi_bias");
-            if (keyidx == -1) { break; }
-            hparams.alibi_bias_max = gguf_get_val_f32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "mpt.attention.clamp_kqv");
-            if (keyidx != -1) { // optional
-                hparams.clip_qkv = gguf_get_val_f32(ggufctx, keyidx);
-            }
-
-            keyidx = gguf_find_key(ggufctx, "mpt.attention.layer_norm_epsilon");
-            if (keyidx == -1) { break; }
-            hparams.norm_eps = gguf_get_val_f32(ggufctx, keyidx);
-
-            ok = true;
-        } while (false);
-
-        if (!ok) {
-            fprintf(stderr, "%s: required hparam missing!\n", __func__);
-            return false;
-        }
+        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
+        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
+        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
+        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
+        fin.read((char *) &hparams.alibi_bias_max,  sizeof(hparams.alibi_bias_max));
+        fin.read((char *) &hparams.clip_qkv,  sizeof(hparams.clip_qkv));
+        fin.read((char *) &hparams.f16,   sizeof(hparams.f16));
 
+        printf("%s: n_vocab        = %d\n", __func__, hparams.n_vocab);
         printf("%s: n_ctx          = %d\n", __func__, hparams.n_ctx);
         printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
         printf("%s: n_head         = %d\n", __func__, hparams.n_head);
         printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
         printf("%s: alibi_bias_max = %f\n", __func__, hparams.alibi_bias_max);
         printf("%s: clip_qkv       = %f\n", __func__, hparams.clip_qkv);
+        printf("%s: ftype          = %d\n", __func__, hparams.f16);
     }
 
     // load vocab
     {
-        auto & hparams = model.hparams;
+        int32_t n_vocab = model.hparams.n_vocab;
+        fin.read((char *) &n_vocab, sizeof(n_vocab));
 
-        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
-        if (tokens_keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer vocab not found!\n", __func__);
+        if (n_vocab != model.hparams.n_vocab) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
+                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
             return false;
         }
 
-        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
-            return false;
+        std::string word;
+        for (int i = 0; i < n_vocab; i++) {
+            uint32_t len;
+            fin.read((char *) &len, sizeof(len));
+            bool special = false;
+            if (len & (1<<31)) {
+                len = len &~ (1<<31);
+                special = true;
+            }
+
+            if (len > 0) {
+                word.resize(len);
+                fin.read((char *) word.data(), len);
+                vocab.token_to_id[word] = i;
+                vocab.id_to_token[i] = word;
+            }
+
+            if(special) {
+                vocab.add_special_token(word);
+            }
         }
-        std::string tokenizer_model(gguf_get_val_str(ggufctx, keyidx));
+    }
 
-        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
-        printf("%s: %s tokenizer vocab = %d\n", __func__, tokenizer_model.c_str(), int(hparams.n_vocab));
-
-        if (tokenizer_model == "llama") { // Replit
-            vocab.is_replit = true;
-            if (!replit_tokenizer_load(vocab, ggufctx, tokens_keyidx, hparams.n_vocab)) {
-                return false;
-            }
-        } else if (tokenizer_model == "gpt2") {
-            int toktypes_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.token_type");
-            if (toktypes_keyidx == -1) {
-                fprintf(stderr, "%s: gpt2 token types not found!\n", __func__);
-                return false;
-            }
-            const auto *toktypes = reinterpret_cast<const uint32_t *>(gguf_get_arr_data(ggufctx, toktypes_keyidx));
-
-            for (int i = 0; i < hparams.n_vocab; i++) {
-                std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
-
-                bool special = false;
-                if (toktypes[i] == MPT_TOKEN_TYPE_CONTROL) {
-                    special = true;
-                } else if (toktypes[i] != MPT_TOKEN_TYPE_NORMAL) {
-                    fprintf(stderr, "%s: unknown token type: %d\n", __func__, int(toktypes[i]));
+    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
+    // in order to save memory and also to speed up the computation
+    ggml_type wtype = GGML_TYPE_COUNT;
+    switch (model.hparams.f16) {
+        case 0: wtype = GGML_TYPE_F32;  break;
+        case 1: wtype = GGML_TYPE_F16;  break;
+        case 2: wtype = GGML_TYPE_Q4_0; break;
+        case 3: wtype = GGML_TYPE_Q4_1; break;
+        case 5: wtype = GGML_TYPE_Q4_2; break;
+        default:
+                {
+                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
+                            __func__, fname.c_str(), model.hparams.f16);
                     return false;
                 }
-
-                vocab.raw.token_to_id[word] = i;
-                vocab.raw.id_to_token[i] = word;
-
-                if (special) {
-                    vocab.raw.add_special_token(word);
-                }
-            }
-        } else {
-            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
-            return false;
-        }
     }
 
     auto & ctx = model.ctx;
 
-    size_t ctx_size = ggml_get_mem_size(ctx);
-    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size / (1024.0 * 1024.0));
+    size_t ctx_size = 0;
+
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_ctx   = hparams.n_ctx;
+        const int n_vocab = hparams.n_vocab;
+        const int expand  = hparams.expand;
+
+
+        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_w
+
+        ctx_size += n_embd*n_vocab*ggml_type_sizef(GGML_TYPE_F32); // wte
+
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_1_w
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_2_w
+
+        ctx_size += n_layer*(3*n_embd*n_embd*ggml_type_sizef(wtype)); // attn_Wqkv_w
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // attn_out_proj_w
+
+        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype));  // ffn_up_proj_w
+        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype)); // ffn_down_proj_w
+
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_k
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_v
+
+        // TODO probably less now?
+        ctx_size += (5 + 10*n_layer)*256; // object overhead
+
+        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
+    }
 
     if (mem_req != nullptr) {
-        *mem_req = ctx_size;
-        gguf_free(ggufctx);
+        *mem_req += ctx_size;
+        const int n_embd  = model.hparams.n_embd;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = n_embd*n_mem;
+
+        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
         return false;
     }
 
+    // create the ggml context
+    {
+        struct ggml_init_params params = {
+            .mem_size   = ctx_size,
+            .mem_buffer = NULL,
+            .no_alloc   = false,
+        };
+
+        model.ctx = ggml_init(params);
+        if (!model.ctx) {
+            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
+            return false;
+        }
+    }
+
     // prepare memory for the weights
     {
         const auto & hparams = model.hparams;
-        model.layers.resize(hparams.n_layer);
 
-        model.wte      = ggml_get_tensor(ctx, "token_embd.weight");
-        model.norm_f_w = ggml_get_tensor(ctx, "output_norm.weight");
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_vocab = hparams.n_vocab;
+        const int expand  = hparams.expand;
 
-        auto name = [](int i, std::string n) {
-            static std::string key;
-            key = "blk." + std::to_string(i) + "." + n;
-            return key.c_str();
-        };
+        model.layers.resize(n_layer);
 
-        for (int i = 0; i < hparams.n_layer; ++i) {
-            auto &layer = model.layers[i];
+        model.wte    = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab);
+        model.norm_f_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
 
-            layer.norm_1_w        = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
-            layer.norm_2_w        = ggml_get_tensor(ctx, name(i, "ffn_norm.weight"));
+        // map by name
+        model.tensors["transformer.wte.weight"] = model.wte;
+        model.tensors["transformer.norm_f.weight"] = model.norm_f_w;
 
-            layer.attn_Wqkv_w     = ggml_get_tensor(ctx, name(i, "attn_qkv.weight"));
-            layer.attn_out_proj_w = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
-            layer.ffn_up_proj_w   = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
-            layer.ffn_down_proj_w = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
+        for (int i = 0; i < n_layer; ++i) {
+            auto & layer = model.layers[i];
+
+            layer.norm_1_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.norm_2_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+            layer.attn_Wqkv_w     = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd * 3);
+            layer.attn_out_proj_w = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd);
+            layer.ffn_up_proj_w   = ggml_new_tensor_2d(ctx, wtype,        n_embd, expand*n_embd);
+            layer.ffn_down_proj_w = ggml_new_tensor_2d(ctx, wtype, expand*n_embd, n_embd);
+
+            // map by name
+            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_1.weight"]        = layer.norm_1_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_2.weight"]        = layer.norm_2_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.Wqkv.weight"]     = layer.attn_Wqkv_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.out_proj.weight"] = layer.attn_out_proj_w;
+
+            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.up_proj.weight"]   = layer.ffn_up_proj_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.down_proj.weight"] = layer.ffn_down_proj_w;
         }
     }
 
     // key + value memory
     {
-        const auto &hparams = model.hparams;
+        const auto & hparams = model.hparams;
         if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
             fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
             ggml_free(ctx);
@@ -458,12 +331,101 @@ bool mpt_model_load(const std::string &fname, mpt_model & model, mpt_vocab & voc
         printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
     }
 
+    // load weights
+    {
+        int n_tensors = 0;
+        size_t total_size = 0;
+
+        printf("%s: ", __func__);
+
+        while (true) {
+            int32_t n_dims;
+            int32_t length;
+            int32_t ttype;
+
+            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
+            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
+
+            if (fin.eof()) {
+                break;
+            }
+
+            int32_t nelements = 1;
+            int32_t ne[2] = { 1, 1 };
+            for (int i = 0; i < n_dims; ++i) {
+                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+                nelements *= ne[i];
+            }
+
+            std::string name(length, 0);
+            fin.read(&name[0], length);
+
+            if (model.tensors.find(name.data()) == model.tensors.end()) {
+                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
+                return false;
+            }
+
+            auto tensor = model.tensors[name.data()];
+            if (ggml_nelements(tensor) != nelements) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
+                return false;
+            }
+
+            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
+                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
+                return false;
+            }
+
+            // for debugging
+            if (0) {
+                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
+            }
+
+            const size_t bpe = ggml_type_size(ggml_type(ttype));
+
+            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
+                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
+                return false;
+            }
+
+            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
+
+            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ttype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
+            total_size += ggml_nbytes(tensor);
+            if (++n_tensors % 8 == 0) {
+                printf(".");
+                fflush(stdout);
+            }
+        }
+
+        printf(" done\n");
+
+        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
+    }
+
     model.scr0_buf.resize(256u * 1024 * 1024);
     model.scr1_buf.resize(256u * 1024 * 1024);
 
     return true;
 }
 
+// load the model's weights from a file path
+bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vocab) {
+
+    auto fin = std::ifstream(fname, std::ios::binary);
+    if (!fin) {
+        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
+        return false;
+    }
+
+    bool loaded = mpt_model_load(fname, fin, model, vocab, nullptr);
+    fin.close();
+    return loaded;
+}
+
 bool mpt_eval(
         mpt_model & model,
         const int n_threads,
@@ -505,6 +467,7 @@ bool mpt_eval(
 
     struct ggml_context * ctx0 = ggml_init(params);
     struct ggml_cgraph gf = {};
+    gf.n_threads = n_threads;
 
     struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
     memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -521,7 +484,7 @@ bool mpt_eval(
         {
 
             // norm1
-            cur = ggml_norm(ctx0, cur, model.hparams.norm_eps);
+            cur = ggml_norm(ctx0, cur);
             cur = ggml_mul(ctx0,
                     ggml_repeat(ctx0, model.layers[il].norm_1_w, cur),
                     cur);
@@ -572,9 +535,7 @@ bool mpt_eval(
 
 
             // Alibi
-            struct ggml_tensor * KQ_scaled_biased = ggml_alibi(
-                ctx0, ggml_cont(ctx0, KQ_scaled), n_past, n_head, model.hparams.alibi_bias_max
-            );
+            struct ggml_tensor * KQ_scaled_biased = ggml_alibi(ctx0, ggml_cont(ctx0, KQ_scaled), n_past, n_head);
 
             // KQ_masked = mask_past(KQ_scaled)
             struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled_biased, n_past);
@@ -614,7 +575,7 @@ bool mpt_eval(
         {
             cur = resSA;
             // norm2
-            cur = ggml_norm(ctx0, cur, model.hparams.norm_eps);
+            cur = ggml_norm(ctx0, cur);
             cur = ggml_mul(ctx0,
                     ggml_repeat(ctx0, model.layers[il].norm_2_w, cur),
                     cur);
@@ -637,7 +598,7 @@ bool mpt_eval(
     struct ggml_tensor * out = inpL;
     // -> logits
     {
-        out = ggml_norm(ctx0, out, model.hparams.norm_eps);
+        out = ggml_norm(ctx0, out);
         out = ggml_mul(ctx0,
                     ggml_repeat(ctx0, model.norm_f_w, out),
                     out);
@@ -645,19 +606,10 @@ bool mpt_eval(
         out = ggml_mul_mat(ctx0, model.wte, out);
     }
 
-    ggml_build_forward_expand(&gf, out);
-
 
     // run the computation
-    {
-        std::unique_ptr<uint8_t []> data;
-        auto plan = ggml_graph_plan(&gf, n_threads);
-        if (plan.work_size > 0) {
-            data.reset(new uint8_t[plan.work_size]);
-            plan.work_data = data.get();
-        }
-        ggml_graph_compute(&gf, &plan);
-    }
+    ggml_build_forward_expand(&gf, out);
+    ggml_graph_compute       (ctx0, &gf);
 
 
     // return result for just the last token
@@ -787,12 +739,12 @@ size_t mpt_set_state_data(mpt_model *model, std::mt19937 *rng, const uint8_t *sr
 struct MPTPrivate {
     const std::string modelPath;
     bool modelLoaded;
-    mpt_vocab vocab;
+    gpt_vocab vocab;
     mpt_model *model = nullptr;
     int64_t n_threads = 0;
     size_t mem_per_token = 0;
     std::mt19937 rng;
-    bool has_end_of_text = false;
+    bool has_im_end = false;
 };
 
 MPT::MPT()
@@ -804,9 +756,10 @@ MPT::MPT()
 
 size_t MPT::requiredMem(const std::string &modelPath) {
     mpt_model dummy_model;
-    mpt_vocab dummy_vocab;
+    gpt_vocab dummy_vocab;
     size_t mem_req;
-    mpt_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    mpt_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
     return mem_req;
 }
 
@@ -814,16 +767,17 @@ bool MPT::loadModel(const std::string &modelPath) {
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
 
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+
     // load the model
-    if (!mpt_model_load(modelPath, *d_ptr->model, d_ptr->vocab, nullptr)) {
+    if (!mpt_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab, nullptr)) {
         std::cerr << "MPT ERROR: failed to load model from " <<  modelPath;
         return false;
     }
 
     d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
     d_ptr->modelLoaded = true;
-    const auto & vocab = d_ptr->vocab;
-    d_ptr->has_end_of_text = vocab.raw.token_to_id.find(vocab.end_of_text()) != vocab.raw.token_to_id.end();
+    d_ptr->has_im_end = d_ptr->vocab.token_to_id.find("<|im_end|>") != d_ptr->vocab.token_to_id.end();
     fflush(stdout);
     return true;
 }
@@ -864,18 +818,12 @@ size_t MPT::restoreState(const uint8_t *src)
 
 std::vector<LLModel::Token> MPT::tokenize(PromptContext &, const std::string &str) const
 {
-    if (d_ptr->vocab.is_replit) {
-        return replit_tokenizer_tokenize(d_ptr->vocab, str);
-    }
-    return ::gpt_tokenize(d_ptr->vocab.raw, str);
+    return ::gpt_tokenize(d_ptr->vocab, str);
 }
 
 std::string MPT::tokenToString(Token id) const
 {
-    if (d_ptr->vocab.is_replit) {
-        return replit_tokenizer_detokenize(d_ptr->vocab, {id});
-    }
-    return d_ptr->vocab.raw.id_to_token[id];
+    return d_ptr->vocab.id_to_token[id];
 }
 
 LLModel::Token MPT::sampleToken(PromptContext &promptCtx) const
@@ -910,23 +858,10 @@ int32_t MPT::contextLength() const
 
 const std::vector<LLModel::Token> &MPT::endTokens() const
 {
-    static std::vector<LLModel::Token> fres;
-    if (fres.empty()) {
-        fres = {0, d_ptr->vocab.raw.token_to_id[d_ptr->vocab.end_of_text()]};
-    }
+    static const std::vector<LLModel::Token> fres = {0, d_ptr->vocab.token_to_id["<|im_end|>"]};
     return fres;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -946,21 +881,10 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
-
-    bool isValid = gguf_get_version(ctx_gguf) <= 2;
-    isValid = isValid && get_arch_name(ctx_gguf) == "mpt";
-
-    gguf_free(ctx_gguf);
-    return isValid;
+DLL_EXPORT bool magic_match(std::istream& f) {
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    return magic == 0x67676d6d;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/replit_impl.h

@@ -0,0 +1,44 @@
+#ifndef REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of replit.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef REPLIT_H
+#define REPLIT_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include "llmodel.h"
+
+#define GGML_QNT_VERSION_FACTOR 1000 // do not change this
+
+struct ReplitPrivate;
+class Replit : public LLModel {
+public:
+    Replit();
+    ~Replit();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string & modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+    bool usingGPUDevice() override;
+
+private:
+    ReplitPrivate *d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::string tokenToString(Token) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // REPLIT_H

gpt4all-backend/scripts/convert_bert_hf_to_ggml.py

@@ -0,0 +1,102 @@
+import sys
+import struct
+import json
+import torch
+import numpy as np
+
+from transformers import AutoModel, AutoTokenizer
+
+if len(sys.argv) < 3:
+    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = sys.argv[1]
+fname_out = sys.argv[1] + "/ggml-model.bin"
+
+with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f:
+    encoder = json.load(f)
+
+with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
+    hparams = json.load(f)
+
+with open(dir_model + "/vocab.txt", "r", encoding="utf-8") as f:
+    vocab = f.readlines()
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+    fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
+
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+model = AutoModel.from_pretrained(dir_model, low_cpu_mem_usage=True)
+print (model)
+
+print(tokenizer.encode('I believe the meaning of life is'))
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+fout = open(fname_out, "wb")
+
+print(hparams)
+
+fout.write(struct.pack("i", 0x62657274)) # magic: ggml in hex
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["max_position_embeddings"]))
+fout.write(struct.pack("i", hparams["hidden_size"]))
+fout.write(struct.pack("i", hparams["intermediate_size"]))
+fout.write(struct.pack("i", hparams["num_attention_heads"]))
+fout.write(struct.pack("i", hparams["num_hidden_layers"]))
+fout.write(struct.pack("i", ftype))
+
+for i in range(hparams["vocab_size"]):
+    text = vocab[i][:-1] # strips newline at the end
+    #print(f"{i}:{text}")
+    data = bytes(text, 'utf-8')
+    fout.write(struct.pack("i", len(data)))
+    fout.write(data)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
+        continue
+    print("Processing variable: " + name + " with shape: ", data.shape)
+
+    n_dims = len(data.shape);
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+            print("  Converting to float16")
+            data = data.astype(np.float16)
+            l_type = 1
+    else:
+        l_type = 0
+
+    # header
+    str = name.encode('utf-8')
+    fout.write(struct.pack("iii", n_dims, len(str), l_type))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str);
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/scripts/convert_bert_hf_to_gguf.py

@@ -1,140 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import json
-import struct
-import sys
-from pathlib import Path
-
-import gguf
-import numpy as np
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-
-
-if not 2 <= len(sys.argv) < 4:
-    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = Path(sys.argv[1])
-
-with open(dir_model / "vocab.txt", encoding="utf-8") as f:
-    vocab = f.readlines()
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
-
-
-ARCH = gguf.MODEL_ARCH.BERT
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(dir_model)
-
-block_count = config.num_hidden_layers
-gguf_writer.add_name("BERT")
-gguf_writer.add_context_length(config.max_position_embeddings)
-gguf_writer.add_embedding_length(config.hidden_size)
-gguf_writer.add_feed_forward_length(config.intermediate_size)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_head_count(config.num_attention_heads)
-gguf_writer.add_file_type(ftype)
-
-print("gguf: get tokenizer metadata")
-
-try:
-    with open(dir_model / "tokenizer.json", encoding="utf-8") as f:
-        tokenizer_json = json.load(f)
-except FileNotFoundError as e:
-    print(f'Error: Missing {e.filename!r}', file=sys.stderr)
-    sys.exit(1)
-
-print("gguf: get wordpiece tokenizer vocab")
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-print(tokenizer.encode('I believe the meaning of life is'))
-
-tokens: list[bytearray] = []
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-
-# The number of tokens in tokenizer.json can differ from the expected vocab size.
-# This causes downstream issues with mismatched tensor sizes when running the inference
-for i in range(config.vocab_size):
-    try:
-        text = reverse_vocab[i]
-    except KeyError:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-
-    tokens.append(text)
-
-gguf_writer.add_tokenizer_model("bert")  # wordpiece
-gguf_writer.add_token_list(tokens)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-model = AutoModel.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
-print(model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
-        continue
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        l_type = 1
-    else:
-        l_type = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-backend/scripts/convert_falcon_hf_to_ggml.py

@@ -0,0 +1,143 @@
+# Based on: https://github.com/KerfuffleV2/ggml-falcon/blob/feat-improve-falcon-convert-hf/examples/falcon/convert-hf-to-ggml.py
+# Convert Hugging Face fine-tuned bloom-like models to ggml format
+#
+# Usage:
+#
+#   python3 convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]
+#
+# This script is similar to "convert-pt-to-ggml.py"
+#
+
+import io
+import os
+import sys
+import struct
+import json
+import code
+import torch
+import numpy as np
+import gc
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
+
+# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a significant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+if len(sys.argv) < 3:
+    print("INFO: GGML V1 files produced are meant to be finalized through examples/falcon_quantize which will bring them to latest version and precision of choice");
+    print("Usage: python convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]")
+    print("  model_directory: name of the directory and model you convert (it should be a subdirectory)")
+    print("  output-directory: directory where the output file will be written")
+    print("  use-f32:    if present, use float32 instead of float16 (f32 is recommended)")
+    sys.exit(1)
+
+# num_parts = int(sys.argv[1])
+dir_model = sys.argv[1] # name and dir of model
+dir_out = sys.argv[2]   # output directory
+
+# make sure the output directory exists
+os.makedirs(dir_out, exist_ok=True)
+
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+ftype = 1
+if len(sys.argv) > 3:
+    ftype = 0
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+# print(tokenizer)
+config = AutoConfig.from_pretrained(dir_model, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(dir_model, trust_remote_code=True, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)
+hparams = config.to_dict()
+
+n_head = hparams["n_head"]
+n_head_kv = hparams["n_head_kv"] if "n_head_kv" in hparams else 1
+head_dim = hparams["hidden_size"] // n_head
+print("* Loading model from: ", dir_model)
+
+fname_out = dir_out + f"/ggml-model-{dir_model.split('/')[-1]}-{ftype_str[ftype]}.bin"
+fout = open(fname_out, "wb")
+fout.write(struct.pack("i", 0x67676a74)) # magic: ggmf in hex (version 1) - possibly change to ggfc ?
+fout.write(struct.pack("i", 1)) # version
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["hidden_size"]))
+fout.write(struct.pack("i", n_head))
+fout.write(struct.pack("i", n_head_kv))
+fout.write(struct.pack("i", hparams["n_layer"]))
+fout.write(struct.pack("i", 40 if "n_head_kv" in hparams else 7)) # obsolete field that breaks ggml compatibility - todo again remove one day
+fout.write(struct.pack("i", ftype))
+
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+byte_encoder = bytes_to_unicode()
+byte_decoder = {v:k for k, v in byte_encoder.items()}
+
+for i in range(hparams["vocab_size"]):
+    text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+    fout.write(struct.pack("i", len(text)))
+    fout.write(text)
+    fout.write(struct.pack("f", 0.0)) # falcon uses bpe on RefinedWeb - no probability scores used
+
+model = model.state_dict()
+for name in model.keys():
+    src = name
+    # The original query_key_value tensor contains n_head_kv "kv groups",
+    # each consisting of n_head/n_head_kv query weights followed by one key
+    # and one value weight (shared by all query heads in the kv group).
+    # This layout makes it a big pain to work with in GGML.
+    # So we rearrange them here,, so that we have n_head query weights
+    # followed by n_head_kv key weights followed by n_head_kv value weights,
+    # in contiguous fashion.
+
+    if "query_key_value" in src:
+        qkv = model[src].view(
+            n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)
+
+        q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head)
+        k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)
+        v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)
+
+        model[src] = torch.cat((q,k,v)).reshape_as(model[src])
+    data = model[src].squeeze()
+    n_dims = len(data.shape)
+    # default type is fp32
+    ftype_cur = 1 if ftype == 1 and n_dims > 1 else 0
+    data = data.to(dtype = torch.float16 if ftype_cur == 1 else torch.float32).numpy()
+    print(f'  |', name, data.shape, '->', data.dtype)
+    # header
+    str = name.encode('utf-8')
+    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str)
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/scripts/convert_gptj_to_gguf.py

@@ -1,165 +0,0 @@
-#!/usr/bin/env python3
-# Convert GPT-J-6B h5 transformer model to ggml format
-#
-# Load the model using GPTJForCausalLM.
-# Iterate over all variables and write them to a binary file.
-#
-# For each variable, write the following:
-#   - Number of dimensions (int)
-#   - Name length (int)
-#   - Dimensions (int[n_dims])
-#   - Name (char[name_length])
-#   - Data (float[n_dims])
-#
-# By default, the bigger matrices are converted to 16-bit floats.
-# This can be disabled by adding the "ftype" CLI argument.
-#
-# At the start of the ggml file we write the model parameters
-# and vocabulary.
-#
-
-from __future__ import annotations
-
-import sys
-import struct
-import json
-from pathlib import Path
-
-import gguf
-import numpy as np
-from transformers import AutoTokenizer, GPTJConfig, GPTJForCausalLM
-from transformers.models.gpt2 import tokenization_gpt2
-
-
-if not 2 <= len(sys.argv) < 4:
-    print("Usage: python {} dir-model [ftype]\n".format(Path(__file__).name))
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = Path(sys.argv[1])
-fname_out = dir_model / "ggml-model.gguf"
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
-
-
-ARCH = gguf.MODEL_ARCH.GPTJ
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = GPTJConfig(dir_model)
-
-block_count = config.n_layer
-gguf_writer.add_name("GPT-J")
-gguf_writer.add_context_length(config.n_positions)
-gguf_writer.add_embedding_length(config.n_embd)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_feed_forward_length(4 * config.n_embd)
-gguf_writer.add_head_count(config.n_head)
-gguf_writer.add_rope_dimension_count(config.rotary_dim)
-gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
-gguf_writer.add_file_type(ftype)
-
-print("gguf: get gpt2 tokenizer vocab")
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-byte_encoder = tokenization_gpt2.bytes_to_unicode()
-byte_decoder = {v: k for k, v in byte_encoder.items()}
-
-tokens: list[bytearray] = []
-
-for i in range(config.vocab_size):
-    if i in reverse_vocab:
-        try:
-            text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
-        except KeyError:
-            text = bytearray()
-            for c in reverse_vocab[i]:
-                if ord(c) < 256:  # single byte character
-                    text.append(byte_decoder[c])
-                else:  # multibyte special token character
-                    text.extend(c.encode('utf-8'))
-    else:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-
-    tokens.append(text)
-
-
-gguf_writer.add_tokenizer_model("gpt2")
-gguf_writer.add_token_list(tokens)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-model = GPTJForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
-#print (model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-#print (list_vars)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    # we don't need these
-    if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
-        print("  Skipping variable:", name)
-        continue
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        ftype_cur = 1
-    elif ftype == 1 or data.dtype != np.float32:
-        print("  Converting to float32")
-        data = data.astype(np.float32)
-        ftype_cur = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-backend/scripts/convert_mpt_hf_to_ggml.py

@@ -0,0 +1,145 @@
+# Convert Hugging Face fine-tuned bloom-like models to ggml format
+#
+# Usage:
+#
+#   python3 models/convert-h5-to-ggml.py 
+#
+# This script is similar to "convert-pt-to-ggml.py"
+#
+
+import io
+import os
+import sys
+import struct
+import json
+import code
+import torch
+import numpy as np
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BloomForCausalLM
+
+# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a significant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+if len(sys.argv) < 3:
+    print("Usage: python convert-hf-to-ggml.py model_name dir-output [use-f32]")
+    print("  model_name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
+    print("  dir-output: directory where the output file will be written")
+    print("  use-f32:    if present, use float32 instead of float16")
+    sys.exit(1)
+
+model_name = sys.argv[1]
+dir_out = sys.argv[2]
+
+# make sure the output directory exists
+os.makedirs(dir_out, exist_ok=True)
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+ftype = 1
+if len(sys.argv) > 3:
+    ftype = 0
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
+hparams = config.to_dict()
+print("Loading model: ", model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32, low_cpu_mem_usage=True)
+print("Model loaded: ", model_name)
+
+
+fname_out = dir_out + f"/ggml-model-{model_name.split('/')[-1]}-{ftype_str[ftype]}.bin"
+fout = open(fname_out, "wb")
+vocab = tokenizer.vocab
+
+hparams["multiple_of"] = 1
+fout.write(struct.pack("I", 0x67676d6d)) # magic: ggml in hex
+fout.write(struct.pack("I", model.config.vocab_size))
+fout.write(struct.pack("I", model.config.max_seq_len))
+fout.write(struct.pack("I", model.config.n_layers))
+fout.write(struct.pack("I", model.config.n_heads))
+fout.write(struct.pack("I", model.config.d_model))
+fout.write(struct.pack("f", model.config.attn_config['alibi_bias_max']))
+clip_qkv = model.config.attn_config['clip_qkv']
+fout.write(struct.pack("f",  clip_qkv if clip_qkv is not None else 0))
+fout.write(struct.pack("I", ftype))
+
+# # Is this correct??
+# dot_token = tokenizer.encode(".")[0]
+# write tokens to ggml file 
+dot_token = tokenizer.encode('.')[0]
+fout.write(struct.pack("I", model.config.vocab_size))
+
+for i in range(model.config.vocab_size):
+    text = tokenizer.decode([dot_token, i]).encode('utf-8')
+    # remove the first byte (it's always '.')
+    text = text[1:]
+    enclen = len(text)
+    if i in tokenizer.all_special_ids:
+        print(f"special token: {text}")
+        enclen = enclen | 1<<31
+    fout.write(struct.pack("I", enclen))
+    fout.write(text)
+    
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable: " + name + " with shape: ", data.shape)
+
+    n_dims = len(data.shape);
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0;
+    if ftype != 0:
+        # Keep token embeddings in fp32
+        if name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
+            print("  Converting to float16")
+            data = data.astype(np.float16)
+            ftype_cur = 1
+        else:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+    else:
+        if data.dtype != np.float32:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+
+    # header
+    str = name.encode('utf-8')
+    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str);
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/scripts/convert_mpt_hf_to_gguf.py

@@ -1,162 +0,0 @@
-#!/usr/bin/env python3
-# Convert Hugging Face fine-tuned bloom-like models to ggml format
-#
-# Usage:
-#
-#   python3 models/convert-h5-to-ggml.py 
-#
-# This script is similar to "convert-pt-to-ggml.py"
-#
-
-from __future__ import annotations
-
-import json
-import struct
-import sys
-from pathlib import Path
-
-import gguf
-import numpy as np
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
-from transformers.models.gpt2 import tokenization_gpt2
-
-
-if not 3 <= len(sys.argv) < 5:
-    print("Usage: {} model-name dir-output [ftype]".format(Path(__file__).name))
-    print("  model-name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
-    print("  dir-output: directory where the output file will be written")
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-model_name = sys.argv[1]
-dir_out = Path(sys.argv[2])
-
-# make sure the output directory exists
-dir_out.mkdir(exist_ok=True)
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 3:
-    ftype = int(sys.argv[3])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_out / f"ggml-model-{Path(model_name).name}-{ftype_str[ftype]}.gguf"
-
-
-ARCH = gguf.MODEL_ARCH.MPT
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(model_name)
-
-block_count = config.n_layers
-gguf_writer.add_name("MPT")
-gguf_writer.add_context_length(config.max_seq_len)
-gguf_writer.add_embedding_length(config.d_model)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_feed_forward_length(4 * config.d_model)
-gguf_writer.add_head_count(config.n_heads)
-gguf_writer.add_max_alibi_bias(config.attn_config.alibi_bias_max)
-gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
-gguf_writer.add_file_type(ftype)
-
-clip_qkv = config.attn_config.clip_qkv
-if clip_qkv is not None:
-    gguf_writer.add_clamp_kqv(clip_qkv)
-
-print("gguf: get gpt2 tokenizer vocab")
-
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-
-special_ids = tokenizer.all_special_ids
-
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-added_tokens = tokenizer.get_added_vocab().values()
-byte_encoder = tokenization_gpt2.bytes_to_unicode()
-byte_decoder = {v: k for k, v in byte_encoder.items()}
-
-tokens: list[bytearray] = []
-toktypes: list[gguf.TokenType] = []
-
-# The number of tokens in tokenizer.json can differ from the expected vocab size.
-# This causes downstream issues with mismatched tensor sizes when running the inference
-for i in range(config.vocab_size):
-    if i not in reverse_vocab:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-    elif i in added_tokens:
-        # these tokens are not encoded, for some reason
-        text = bytearray(reverse_vocab[i].encode('utf-8'))
-    else:
-        text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
-
-    tokens.append(text)
-
-    # TODO(cebtenzzre): is there a better way to do this?
-    toktypes.append(gguf.TokenType.CONTROL if i in special_ids else gguf.TokenType.NORMAL)
-
-gguf_writer.add_tokenizer_model("gpt2")
-gguf_writer.add_token_list(tokens)
-gguf_writer.add_token_types(toktypes)
-
-print("gguf: get tensor metadata")
-
-print("Loading model:", model_name)
-model = AutoModelForCausalLM.from_pretrained(
-    model_name, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32, low_cpu_mem_usage=True,
-)
-print("Model loaded:", model_name)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    # Keep token embeddings in fp32
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        ftype_cur = 1
-    elif ftype == 1 or data.dtype != np.float32:
-        print("  Converting to float32")
-        data = data.astype(np.float32)
-        ftype_cur = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-backend/scripts/convert_replit_hf_to_ggml.py

@@ -0,0 +1,113 @@
+from pathlib import Path
+import sys
+import struct
+import json
+import numpy as np
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import sentencepiece.sentencepiece_model_pb2 as model
+
+if len(sys.argv) < 3:
+    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+
+# output in the same directory as the model
+dir_model = sys.argv[1]
+fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b.bin"
+
+
+with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
+    hparams = json.load(f)
+
+sp_proto = model.ModelProto()
+sp_proto.ParseFromString(open(Path(sys.argv[1]) / "spiece.model", "rb").read())
+
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+    fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".bin"
+
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    dir_model, low_cpu_mem_usage=True, trust_remote_code=True
+)
+# print (model)
+
+# print(tokenizer.encode('I believe the meaning of life is'))
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+fout = open(fname_out, "wb")
+
+print(hparams)
+
+fout.write(struct.pack("i", 0x7265706c))  # magic: repl in hex
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["max_seq_len"]))
+fout.write(struct.pack("i", hparams["d_model"]))
+fout.write(struct.pack("i", hparams["n_heads"]))
+fout.write(struct.pack("i", hparams["n_layers"]))
+fout.write(struct.pack("i", ftype))
+
+
+# TODO: temporary hack to not deal with implementing the tokenizer
+for piece in sp_proto.pieces:
+    encoded_piece = piece.piece.encode("utf-8")
+    fout.write(struct.pack("i", len(encoded_piece)))
+    fout.write(encoded_piece)
+    fout.write(struct.pack("f", piece.score))
+
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable: " + name + " with shape: ", data.shape)
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0
+    if ftype != 0:
+        if name[-7:] == ".weight" and n_dims == 2:
+            print("  Converting to float16")
+            data = data.astype(np.float16)
+            ftype_cur = 1
+        else:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+    else:
+        if data.dtype != np.float32:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+
+    # header
+    str = name.encode("utf-8")
+    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str)
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/scripts/convert_replit_v1_hf_to_gguf.py

@@ -1,145 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import json
-import struct
-import sys
-from pathlib import Path
-
-import gguf
-import numpy as np
-from sentencepiece import SentencePieceProcessor
-from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
-
-
-if not 2 <= len(sys.argv) < 4:
-    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = Path(sys.argv[1])
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_model / ("ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".gguf")
-
-
-ARCH = gguf.MODEL_ARCH.MPT
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(dir_model)
-
-block_count = config.n_layers
-gguf_writer.add_name("Replit")
-gguf_writer.add_context_length(config.max_seq_len)
-gguf_writer.add_embedding_length(config.d_model)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_feed_forward_length(4 * config.d_model)
-gguf_writer.add_head_count(config.n_heads)
-gguf_writer.add_max_alibi_bias(config.attn_config.alibi_bias_max)
-gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
-gguf_writer.add_file_type(ftype)
-
-clip_qkv = config.attn_config.clip_qkv
-if clip_qkv is not None:
-    gguf_writer.add_clamp_kqv(clip_qkv)
-
-print("gguf: get sentencepiece tokenizer vocab")
-
-tokenizer = SentencePieceProcessor(str(dir_model / "spiece.model"))
-#print(tokenizer.encode('I believe the meaning of life is'))
-
-tokens: list[bytearray] = []
-scores: list[float] = []
-toktypes: list[int] = []
-
-for i in range(tokenizer.vocab_size()):
-    tokens.append(tokenizer.id_to_piece(i).encode('utf-8'))
-    scores.append(tokenizer.get_score(i))
-
-    toktype = gguf.TokenType.NORMAL
-    if tokenizer.is_unknown(i):
-        toktype = gguf.TokenType.UNKNOWN
-    elif tokenizer.is_control(i):
-        toktype = gguf.TokenType.CONTROL
-    elif tokenizer.is_unused(i):
-        toktype = gguf.TokenType.UNUSED
-    elif tokenizer.is_byte(i):
-        toktype = gguf.TokenType.BYTE
-
-    toktypes.append(toktype)
-
-gguf_writer.add_tokenizer_model("llama")  # sentencepiece
-gguf_writer.add_token_list(tokens)
-gguf_writer.add_token_scores(scores)
-gguf_writer.add_token_types(toktypes)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-model = AutoModelForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
-#print(model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-print(config)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        ftype_cur = 1
-    elif ftype == 1 or data.dtype != np.float32:
-        print("  Converting to float32")
-        data = data.astype(np.float32)
-        ftype_cur = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-backend/starcoder_impl.h

@@ -0,0 +1,42 @@
+#ifndef STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of starcoder.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef STARCODER_H
+#define STARCODER_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <memory>
+#include "llmodel.h"
+
+struct StarcoderPrivate;
+class Starcoder : public LLModel {
+public:
+    Starcoder();
+    ~Starcoder();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    std::unique_ptr<StarcoderPrivate> d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // STARCODER_H

gpt4all-bindings/cli/app.py

@@ -4,13 +4,13 @@ The GPT4All CLI is a self-contained script based on the `gpt4all` and `typer` pa
 REPL to communicate with a language model similar to the chat GUI application, but more basic.
 """
 
-import importlib.metadata
 import io
+import pkg_resources  # should be present as a dependency of gpt4all
 import sys
+import typer
+
 from collections import namedtuple
 from typing_extensions import Annotated
-
-import typer
 from gpt4all import GPT4All
 
 
@@ -79,7 +79,7 @@ def repl(
 
     use_new_loop = False
     try:
-        version = importlib.metadata.version('gpt4all')
+        version = pkg_resources.Environment()['gpt4all'][0].version
         version_major = int(version.split('.')[0])
         if version_major >= 1:
             use_new_loop = True

gpt4all-bindings/csharp/README.md

@@ -23,12 +23,6 @@ gpt4all-bindings/
           └── linux-x64
 ```
 
-## Prerequisites
-
-On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
-
-macOS users do not need Vulkan, as GPT4All will use Metal instead.
-
 ## Local Build Instructions
 > **Note** 
 > Tested On:
@@ -60,7 +54,7 @@ chmod +x ./build_linux.sh
 1. Setup
 ```
 choco install mingw
-$env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
+$env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
 choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
 ```
 2. Run the `./build_win-mingw.ps1` build script

gpt4all-bindings/csharp/build_win-mingw.ps1

@@ -12,5 +12,5 @@ cmake -G "MinGW Makefiles" -S ..\..\gpt4all-backend -B $BUILD_DIR
 cmake --build $BUILD_DIR --parallel --config Release
 
 # copy native dlls
-cp "C:\ProgramData\mingw64\mingw64\bin\*dll" $LIBS_DIR
+cp "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll" $LIBS_DIR
 cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java

@@ -8,8 +8,9 @@ import java.io.ByteArrayOutputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.*;
-import java.util.stream.Collectors;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 public  class LLModel implements AutoCloseable {
 
@@ -305,197 +306,6 @@ public  class LLModel implements AutoCloseable {
         };
     }
 
-    /**
-     * The array of messages for the conversation.
-     */
-    public static class Messages {
-
-        private final List<PromptMessage> messages = new ArrayList<>();
-
-        public Messages(PromptMessage...messages) {
-            this.messages.addAll(Arrays.asList(messages));
-        }
-
-        public Messages(List<PromptMessage> messages) {
-            this.messages.addAll(messages);
-        }
-
-        public Messages addPromptMessage(PromptMessage promptMessage) {
-            this.messages.add(promptMessage);
-            return this;
-        }
-
-        List<PromptMessage> toList() {
-            return Collections.unmodifiableList(this.messages);
-        }
-
-        List<Map<String, String>> toListMap() {
-            return messages.stream()
-                    .map(PromptMessage::toMap).collect(Collectors.toList());
-        }
-
-    }
-
-    /**
-     * A message in the conversation, identical to OpenAI's chat message.
-     */
-    public static class PromptMessage {
-
-        private static final String ROLE = "role";
-        private static final String CONTENT = "content";
-
-        private final Map<String, String> message = new HashMap<>();
-
-        public PromptMessage() {
-        }
-
-        public PromptMessage(Role role, String content) {
-            addRole(role);
-            addContent(content);
-        }
-
-        public PromptMessage addRole(Role role) {
-            return this.addParameter(ROLE, role.type());
-        }
-
-        public PromptMessage addContent(String content) {
-            return this.addParameter(CONTENT, content);
-        }
-
-        public PromptMessage addParameter(String key, String value) {
-            this.message.put(key, value);
-            return this;
-        }
-
-        public String content() {
-            return this.parameter(CONTENT);
-        }
-
-        public Role role() {
-            String role = this.parameter(ROLE);
-            return Role.from(role);
-        }
-
-        public String parameter(String key) {
-            return this.message.get(key);
-        }
-
-        Map<String, String> toMap() {
-            return Collections.unmodifiableMap(this.message);
-        }
-
-    }
-
-    public enum Role {
-
-        SYSTEM("system"), ASSISTANT("assistant"), USER("user");
-
-        private final String type;
-
-        String type() {
-            return this.type;
-        }
-
-        static Role from(String type) {
-
-            if (type == null) {
-                return null;
-            }
-
-            switch (type) {
-                case "system": return SYSTEM;
-                case "assistant": return ASSISTANT;
-                case "user": return USER;
-                default: throw new IllegalArgumentException(
-                        String.format("You passed %s type but only %s are supported",
-                                type, Arrays.toString(Role.values())
-                        )
-                );
-            }
-        }
-
-        Role(String type) {
-            this.type = type;
-        }
-
-        @Override
-        public String toString() {
-            return type();
-        }
-    }
-
-    /**
-     * The result of the completion, similar to OpenAI's format.
-     */
-    public static class CompletionReturn {
-        private String model;
-        private Usage usage;
-        private Choices choices;
-
-        public CompletionReturn(String model, Usage usage, Choices choices) {
-            this.model = model;
-            this.usage = usage;
-            this.choices = choices;
-        }
-
-        public Choices choices() {
-            return choices;
-        }
-
-        public String model() {
-            return model;
-        }
-
-        public Usage usage() {
-            return usage;
-        }
-    }
-
-    /**
-     * The generated completions.
-     */
-    public static class Choices {
-
-        private final List<CompletionChoice> choices = new ArrayList<>();
-
-        public Choices(List<CompletionChoice> choices) {
-            this.choices.addAll(choices);
-        }
-
-        public Choices(CompletionChoice...completionChoices){
-            this.choices.addAll(Arrays.asList(completionChoices));
-        }
-
-        public Choices addCompletionChoice(CompletionChoice completionChoice) {
-            this.choices.add(completionChoice);
-            return this;
-        }
-
-        public CompletionChoice first() {
-            return this.choices.get(0);
-        }
-
-        public int totalChoices() {
-            return this.choices.size();
-        }
-
-        public CompletionChoice get(int index) {
-            return this.choices.get(index);
-        }
-
-        public List<CompletionChoice> choices() {
-            return Collections.unmodifiableList(choices);
-        }
-    }
-
-    /**
-     * A completion choice, similar to OpenAI's format.
-     */
-    public static class CompletionChoice extends PromptMessage {
-        public CompletionChoice(Role role, String content) {
-            super(role, content);
-        }
-    }
 
     public static class ChatCompletionResponse {
         public String model;
@@ -513,41 +323,6 @@ public  class LLModel implements AutoCloseable {
         // Getters and setters
     }
 
-    public CompletionReturn chatCompletionResponse(Messages messages,
-                                                                GenerationConfig generationConfig) {
-        return chatCompletion(messages, generationConfig, false, false);
-    }
-
-    /**
-     * chatCompletion formats the existing chat conversation into a template to be
-     * easier to process for chat UIs. It is not absolutely necessary as generate method
-     * may be directly used to make generations with gpt models.
-     *
-     * @param messages object to create theMessages to send to GPT model
-     * @param generationConfig How to decode/process the generation.
-     * @param streamToStdOut Send tokens as they are calculated Standard output.
-     * @param outputFullPromptToStdOut Should full prompt built out of messages be sent to Standard output.
-     * @return CompletionReturn contains stats and generated Text.
-     */
-    public CompletionReturn chatCompletion(Messages messages,
-                                                  GenerationConfig generationConfig, boolean streamToStdOut,
-                                                  boolean outputFullPromptToStdOut) {
-
-        String fullPrompt = buildPrompt(messages.toListMap());
-
-        if(outputFullPromptToStdOut)
-            System.out.print(fullPrompt);
-
-        String generatedText = generate(fullPrompt, generationConfig, streamToStdOut);
-
-        final CompletionChoice promptMessage = new CompletionChoice(Role.ASSISTANT, generatedText);
-        final Choices choices = new Choices(promptMessage);
-
-        final Usage usage = getUsage(fullPrompt, generatedText);
-        return new CompletionReturn(this.modelName, usage, choices);
-
-    }
-
     public ChatCompletionResponse chatCompletion(List<Map<String, String>> messages,
                                                               GenerationConfig generationConfig) {
         return chatCompletion(messages, generationConfig, false, false);
@@ -577,23 +352,19 @@ public  class LLModel implements AutoCloseable {
         ChatCompletionResponse  response = new ChatCompletionResponse();
         response.model = this.modelName;
 
-        response.usage = getUsage(fullPrompt, generatedText);
+        Usage usage = new Usage();
+        usage.promptTokens = fullPrompt.length();
+        usage.completionTokens = generatedText.length();
+        usage.totalTokens = fullPrompt.length() + generatedText.length();
+        response.usage =  usage;
 
         Map<String, String> message = new HashMap<>();
         message.put("role", "assistant");
         message.put("content", generatedText);
 
         response.choices = List.of(message);
+
         return response;
-
-    }
-
-    private Usage getUsage(String fullPrompt, String generatedText) {
-        Usage usage = new Usage();
-        usage.promptTokens = fullPrompt.length();
-        usage.completionTokens = generatedText.length();
-        usage.totalTokens = fullPrompt.length() + generatedText.length();
-        return usage;
     }
 
     protected static String buildPrompt(List<Map<String, String>> messages) {

gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/BasicTests.java

@@ -28,33 +28,6 @@ import static org.mockito.Mockito.*;
 @ExtendWith(MockitoExtension.class)
 public class BasicTests {
 
-    @Test
-    public void simplePromptWithObject(){
-
-        LLModel model = Mockito.spy(new LLModel());
-
-        LLModel.GenerationConfig config =
-                LLModel.config()
-                        .withNPredict(20)
-                        .build();
-
-        // The generate method will return "4"
-        doReturn("4").when( model ).generate(anyString(), eq(config), eq(true));
-
-        LLModel.PromptMessage promptMessage1 = new LLModel.PromptMessage(LLModel.Role.SYSTEM, "You are a helpful assistant");
-        LLModel.PromptMessage promptMessage2 = new LLModel.PromptMessage(LLModel.Role.USER, "Add 2+2");
-
-        LLModel.Messages messages = new LLModel.Messages(promptMessage1, promptMessage2);
-
-        LLModel.CompletionReturn response = model.chatCompletion(
-                messages, config, true, true);
-
-        assertTrue( response.choices().first().content().contains("4") );
-
-        // Verifies the prompt and response are certain length.
-        assertEquals( 224 , response.usage().totalTokens );
-    }
-
     @Test
     public void simplePrompt(){
 

gpt4all-bindings/python/README.md

@@ -15,14 +15,6 @@ pip install gpt4all
 
 ## Local Build Instructions
 
-### Prerequisites
-
-On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
-
-macOS users do not need Vulkan, as GPT4All will use Metal instead.
-
-### Building the python bindings
-
 **NOTE**: If you are doing this on a Windows machine, you must build the GPT4All backend using [MinGW64](https://www.mingw-w64.org/) compiler.
 
 1. Setup `llmodel`

gpt4all-bindings/python/docs/gpt4all_chat.md

@@ -7,7 +7,7 @@ It is optimized to run 7-13B parameter LLMs on the CPU's of any computer running
 ## Running LLMs on CPU
 The GPT4All Chat UI supports models from all newer versions of `GGML`, `llama.cpp` including the `LLaMA`, `MPT`, `replit`,  `GPT-J` and `falcon` architectures
 
-GPT4All maintains an official list of recommended models located in [models2.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
+GPT4All maintains an official list of recommended models located in [models.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
 
 #### Sideloading any GGML model
 If a model is compatible with the gpt4all-backend, you can sideload it into GPT4All Chat by:

gpt4all-bindings/python/docs/gpt4all_faq.md

@@ -61,12 +61,12 @@ or `allowDownload=true` (default), a model is automatically downloaded into `.ca
 unless it already exists.
 
 In case of connection issues or errors during the download, you might want to manually verify the model file's MD5
-checksum by comparing it with the one listed in [models2.json].
+checksum by comparing it with the one listed in [models.json].
 
 As an alternative to the basic downloader built into the bindings, you can choose to download from the 
 <https://gpt4all.io/> website instead. Scroll down to 'Model Explorer' and pick your preferred model.
 
-[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
+[models.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json
 
 #### I need the chat GUI and bindings to behave the same
 
@@ -93,7 +93,7 @@ The chat GUI and bindings are based on the same backend. You can make them behav
 - Next you'll have to compare the templates, adjusting them as necessary, based on how you're using the bindings.
     - Specifically, in Python:
         - With simple `generate()` calls, the input has to be surrounded with system and prompt templates.
-        - When using a chat session, it depends on whether the bindings are allowed to download [models2.json]. If yes,
+        - When using a chat session, it depends on whether the bindings are allowed to download [models.json]. If yes,
           and in the chat GUI the default templates are used, it'll be handled automatically. If no, use
           `chat_session()` template parameters to customize them.
 

gpt4all-bindings/python/docs/gpt4all_modal.md

@@ -8,7 +8,7 @@ import modal
 
 def download_model():
     import gpt4all
-    #you can use any model from https://gpt4all.io/models/models2.json
+    #you can use any model from https://gpt4all.io/models/models.json
     return gpt4all.GPT4All("ggml-gpt4all-j-v1.3-groovy.bin")
 
 image=modal.Image.debian_slim().pip_install("gpt4all").run_function(download_model)

gpt4all-bindings/python/docs/gpt4all_python.md

@@ -77,10 +77,10 @@ When using GPT4All models in the `chat_session` context:
 - Consecutive chat exchanges are taken into account and not discarded until the session ends; as long as the model has capacity.
 - Internal K/V caches are preserved from previous conversation history, speeding up inference.
 - The model is given a system and prompt template which make it chatty. Depending on `allow_download=True` (default),
-  it will obtain the latest version of [models2.json] from the repository, which contains specifically tailored templates
+  it will obtain the latest version of [models.json] from the repository, which contains specifically tailored templates
   for models. Conversely, if it is not allowed to download, it falls back to default templates instead.
 
-[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
+[models.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json
 
 
 ### Streaming Generations
@@ -379,7 +379,7 @@ logging infrastructure offers [many more customization options][py-logging-cookb
 
 ### Without Online Connectivity
 To prevent GPT4All from accessing online resources, instantiate it with `allow_download=False`. This will disable both
-downloading missing models and [models2.json], which contains information about them. As a result, predefined templates
+downloading missing models and [models.json], which contains information about them. As a result, predefined templates
 are used instead of model-specific system and prompt templates:
 
 === "GPT4All Default Templates Example"

gpt4all-bindings/python/docs/gpt4all_typescript.md

@@ -58,8 +58,6 @@ const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional"
 *   (win) msvc version 143
     *   Can be obtained with visual studio 2022 build tools
 *   python 3
-*   On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
-*   macOS users do not need Vulkan, as GPT4All will use Metal instead.
 
 ### Build (from source)
 

gpt4all-bindings/python/docs/index.md

@@ -38,7 +38,7 @@ The GPT4All software ecosystem is compatible with the following Transformer arch
 - `MPT` (including `Replit`)
 - `GPT-J`
 
-You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json)
+You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json)
 
 
 GPT4All models are artifacts produced through a process known as neural network quantization.

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -1,19 +1,14 @@
 """
 Python only API for running all GPT4All models.
 """
-from __future__ import annotations
-
 import os
-import sys
 import time
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Any, Dict, Iterable, List, Optional, Union
 
 import requests
-from requests.exceptions import ChunkedEncodingError
 from tqdm import tqdm
-from urllib3.exceptions import IncompleteRead, ProtocolError
 
 from . import pyllmodel
 
@@ -34,14 +29,17 @@ class Embed4All:
     Python class that handles embeddings for GPT4All.
     """
 
-    def __init__(self, model_name: Optional[str] = None, n_threads: Optional[int] = None, **kwargs):
+    def __init__(
+        self,
+        n_threads: Optional[int] = None,
+    ):
         """
         Constructor
 
         Args:
             n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
         """
-        self.gpt4all = GPT4All(model_name or 'all-MiniLM-L6-v2-f16.gguf', n_threads=n_threads, **kwargs)
+        self.gpt4all = GPT4All(model_name='ggml-all-MiniLM-L6-v2-f16.bin', n_threads=n_threads)
 
     def embed(self, text: str) -> List[float]:
         """
@@ -64,12 +62,11 @@ class GPT4All:
     def __init__(
         self,
         model_name: str,
-        model_path: Optional[Union[str, os.PathLike[str]]] = None,
+        model_path: Optional[str] = None,
         model_type: Optional[str] = None,
         allow_download: bool = True,
         n_threads: Optional[int] = None,
         device: Optional[str] = "cpu",
-        verbose: bool = False,
     ):
         """
         Constructor
@@ -94,7 +91,7 @@ class GPT4All:
         self.model_type = model_type
         self.model = pyllmodel.LLModel()
         # Retrieve model and download if allowed
-        self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download, verbose=verbose)
+        self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download)
         if device is not None:
             if device != "cpu":
                 self.model.init_gpu(model_path=self.config["path"], device=device)
@@ -110,22 +107,19 @@ class GPT4All:
     @staticmethod
     def list_models() -> List[ConfigType]:
         """
-        Fetch model list from https://gpt4all.io/models/models2.json.
+        Fetch model list from https://gpt4all.io/models/models.json.
 
         Returns:
             Model list in JSON format.
         """
-        resp = requests.get("https://gpt4all.io/models/models2.json")
-        if resp.status_code != 200:
-            raise ValueError(f'Request failed: HTTP {resp.status_code} {resp.reason}')
-        return resp.json()
+        return requests.get("https://gpt4all.io/models/models.json").json()
 
     @staticmethod
     def retrieve_model(
         model_name: str,
-        model_path: Optional[Union[str, os.PathLike[str]]] = None,
+        model_path: Optional[str] = None,
         allow_download: bool = True,
-        verbose: bool = False,
+        verbose: bool = True,
     ) -> ConfigType:
         """
         Find model file, and if it doesn't exist, download the model.
@@ -168,7 +162,7 @@ class GPT4All:
                 )
             model_path = DEFAULT_MODEL_DIRECTORY
         else:
-            model_path = str(model_path).replace("\\", "\\\\")
+            model_path = model_path.replace("\\", "\\\\")
 
         if not os.path.exists(model_path):
             raise ValueError(f"Invalid model directory: {model_path}")
@@ -178,7 +172,7 @@ class GPT4All:
             config.pop("url", None)
             config["path"] = model_dest
             if verbose:
-                print("Found model file at", model_dest, file=sys.stderr)
+                print("Found model file at ", model_dest)
 
         # If model file does not exist, download
         elif allow_download:
@@ -193,7 +187,7 @@ class GPT4All:
     @staticmethod
     def download_model(
         model_filename: str,
-        model_path: Union[str, os.PathLike[str]],
+        model_path: str,
         verbose: bool = True,
         url: Optional[str] = None,
     ) -> str:
@@ -219,61 +213,32 @@ class GPT4All:
         download_path = os.path.join(model_path, model_filename).replace("\\", "\\\\")
         download_url = get_download_url(model_filename)
 
-        def make_request(offset=None):
-            headers = {}
-            if offset:
-                print(f"\nDownload interrupted, resuming from byte position {offset}", file=sys.stderr)
-                headers['Range'] = f'bytes={offset}-'  # resume incomplete response
-            response = requests.get(download_url, stream=True, headers=headers)
-            if response.status_code not in (200, 206):
-                raise ValueError(f'Request failed: HTTP {response.status_code} {response.reason}')
-            if offset and (response.status_code != 206 or str(offset) not in response.headers.get('Content-Range', '')):
-                raise ValueError('Connection was interrupted and server does not support range requests')
-            return response
-
-        response = make_request()
-
+        response = requests.get(download_url, stream=True)
         total_size_in_bytes = int(response.headers.get("content-length", 0))
         block_size = 2**20  # 1 MB
 
-        with open(download_path, "wb") as file, \
-                tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar:
+        with tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar:
             try:
-                while True:
-                    last_progress = progress_bar.n
-                    try:
-                        for data in response.iter_content(block_size):
-                            file.write(data)
-                            progress_bar.update(len(data))
-                    except ChunkedEncodingError as cee:
-                        if cee.args and isinstance(pe := cee.args[0], ProtocolError):
-                            if len(pe.args) >= 2 and isinstance(ir := pe.args[1], IncompleteRead):
-                                assert progress_bar.n <= ir.partial  # urllib3 may be ahead of us but never behind
-                                # the socket was closed during a read - retry
-                                response = make_request(progress_bar.n)
-                                continue
-                        raise
-                    if total_size_in_bytes != 0 and progress_bar.n < total_size_in_bytes:
-                        if progress_bar.n == last_progress:
-                            raise RuntimeError('Download not making progress, aborting.')
-                        # server closed connection prematurely - retry
-                        response = make_request(progress_bar.n)
-                        continue
-                    break
+                with open(download_path, "wb") as file:
+                    for data in response.iter_content(block_size):
+                        progress_bar.update(len(data))
+                        file.write(data)
             except Exception:
-                if verbose:
-                    print("Cleaning up the interrupted download...", file=sys.stderr)
-                try:
+                if os.path.exists(download_path):
+                    if verbose:
+                        print("Cleaning up the interrupted download...")
                     os.remove(download_path)
-                except OSError:
-                    pass
                 raise
 
-        if os.name == 'nt':
-            time.sleep(2)  # Sleep for a little bit so Windows can remove file lock
+        # Validate download was successful
+        if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
+            raise RuntimeError("An error occurred during download. Downloaded file may not work.")
+
+        # Sleep for a little bit so OS can remove file lock
+        time.sleep(2)
 
         if verbose:
-            print("Model downloaded at:", download_path, file=sys.stderr)
+            print("Model downloaded at: ", download_path)
         return download_path
 
     def generate(
@@ -349,6 +314,7 @@ class GPT4All:
             callback: pyllmodel.ResponseCallbackType,
             output_collector: List[MessageType],
         ) -> pyllmodel.ResponseCallbackType:
+
             def _callback(token_id: int, response: str) -> bool:
                 nonlocal callback, output_collector
 
@@ -457,6 +423,6 @@ def empty_chat_session(system_prompt: str = "") -> List[MessageType]:
 
 
 def append_bin_suffix_if_missing(model_name):
-    if not model_name.endswith((".bin", ".gguf")):
+    if not model_name.endswith(".bin"):
         model_name += ".bin"
     return model_name

gpt4all-bindings/python/gpt4all/pyllmodel.py

@@ -1,42 +1,48 @@
-import atexit
 import ctypes
-import importlib.resources
 import logging
 import os
 import platform
+from queue import Queue
 import re
 import subprocess
 import sys
 import threading
-from contextlib import ExitStack
-from queue import Queue
 from typing import Callable, Iterable, List
 
+import pkg_resources
+
 logger: logging.Logger = logging.getLogger(__name__)
 
 
-file_manager = ExitStack()
-atexit.register(file_manager.close)  # clean up files on exit
-
 # TODO: provide a config file to make this more robust
-MODEL_LIB_PATH = file_manager.enter_context(importlib.resources.as_file(
-    importlib.resources.files("gpt4all") / "llmodel_DO_NOT_MODIFY" / "build",
-))
+LLMODEL_PATH = os.path.join("llmodel_DO_NOT_MODIFY", "build").replace("\\", "\\\\")
+MODEL_LIB_PATH = str(pkg_resources.resource_filename("gpt4all", LLMODEL_PATH)).replace("\\", "\\\\")
 
 
 def load_llmodel_library():
-    ext = {"Darwin": "dylib", "Linux": "so", "Windows": "dll"}[platform.system()]
+    system = platform.system()
 
-    try:
-        # Linux, Windows, MinGW
-        lib = ctypes.CDLL(str(MODEL_LIB_PATH / f"libllmodel.{ext}"))
-    except FileNotFoundError:
-        if ext != 'dll':
-            raise
-        # MSVC
-        lib = ctypes.CDLL(str(MODEL_LIB_PATH / "llmodel.dll"))
+    def get_c_shared_lib_extension():
+        if system == "Darwin":
+            return "dylib"
+        elif system == "Linux":
+            return "so"
+        elif system == "Windows":
+            return "dll"
+        else:
+            raise Exception("Operating System not supported")
 
-    return lib
+    c_lib_ext = get_c_shared_lib_extension()
+
+    llmodel_file = "libllmodel" + "." + c_lib_ext
+
+    llmodel_dir = str(pkg_resources.resource_filename("gpt4all", os.path.join(LLMODEL_PATH, llmodel_file))).replace(
+        "\\", "\\\\"
+    )
+
+    llmodel_lib = ctypes.CDLL(llmodel_dir)
+
+    return llmodel_lib
 
 
 llmodel = load_llmodel_library()
@@ -125,7 +131,7 @@ llmodel.llmodel_set_implementation_search_path.restype = None
 llmodel.llmodel_threadCount.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_threadCount.restype = ctypes.c_int32
 
-llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).replace("\\", r"\\").encode("utf-8"))
+llmodel.llmodel_set_implementation_search_path(MODEL_LIB_PATH.encode("utf-8"))
 
 llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
 llmodel.llmodel_available_gpu_devices.restype = ctypes.POINTER(LLModelGPUDevice)
@@ -253,13 +259,12 @@ class LLModel:
         True if model loaded successfully, False otherwise
         """
         model_path_enc = model_path.encode("utf-8")
-        err = LLModelError()
-        self.model = llmodel.llmodel_model_create2(model_path_enc, b"auto", ctypes.byref(err))
+        self.model = llmodel.llmodel_model_create(model_path_enc)
 
-        if self.model is None:
-            raise ValueError(f"Unable to instantiate model: code={err.code}, {err.message.decode()}")
-
-        llmodel.llmodel_loadModel(self.model, model_path_enc)
+        if self.model is not None:
+            llmodel.llmodel_loadModel(self.model, model_path_enc)
+        else:
+            raise ValueError("Unable to instantiate model")
 
         filename = os.path.basename(model_path)
         self.model_name = os.path.splitext(filename)[0]

gpt4all-bindings/python/setup.py

@@ -61,7 +61,7 @@ copy_prebuilt_C_lib(SRC_CLIB_DIRECtORY,
 
 setup(
     name=package_name,
-    version="2.0.0rc2",
+    version="1.0.12",
     description="Python bindings for GPT4All",
     author="Nomic and the Open Source Community",
     author_email="support@nomic.ai",

gpt4all-bindings/typescript/README.md

@@ -58,8 +58,6 @@ const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional"
 *   (win) msvc version 143
     *   Can be obtained with visual studio 2022 build tools
 *   python 3
-*   On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
-*   macOS users do not need Vulkan, as GPT4All will use Metal instead.
 
 ### Build (from source)
 

gpt4all-bindings/typescript/scripts/build_mingw.ps1

@@ -12,5 +12,5 @@ cmake -G "MinGW Makefiles" -S ..\..\gpt4all-backend -B $BUILD_DIR -DLLAMA_AVX2=O
 cmake --build $BUILD_DIR --parallel --config Release
 
 # copy native dlls
-# cp "C:\ProgramData\mingw64\mingw64\bin\*dll" $LIBS_DIR
+# cp "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll" $LIBS_DIR
 cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR

gpt4all-bindings/typescript/src/config.js

@@ -21,7 +21,7 @@ const DEFAULT_MODEL_CONFIG = {
     promptTemplate: "### Human: \n%1\n### Assistant:\n",
 }
 
-const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models2.json";
+const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models.json";
 
 const DEFAULT_PROMPT_CONTEXT = {
     temp: 0.7,

gpt4all-bindings/typescript/src/util.js

@@ -236,7 +236,7 @@ async function retrieveModel(modelName, options = {}) {
         file: retrieveOptions.modelConfigFile,
         url:
             retrieveOptions.allowDownload &&
-            "https://gpt4all.io/models/models2.json",
+            "https://gpt4all.io/models/models.json",
     });
 
     const loadedModelConfig = availableModels.find(

gpt4all-chat/CMakeLists.txt

@@ -17,8 +17,8 @@ if(APPLE)
 endif()
 
 set(APP_VERSION_MAJOR 2)
-set(APP_VERSION_MINOR 5)
-set(APP_VERSION_PATCH 0)
+set(APP_VERSION_MINOR 4)
+set(APP_VERSION_PATCH 20)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file
@@ -180,8 +180,8 @@ install(TARGETS llmodel DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 
 # We should probably iterate through the list of the cmake for backend, but these need to be installed
 # to the this component's dir for the finicky qt installer to work
-install(TARGETS gptj-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS gptj-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+#install(TARGETS gptj-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+#install(TARGETS gptj-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llama-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llama-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llamamodel-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
@@ -189,21 +189,30 @@ install(TARGETS llamamodel-mainline-default DESTINATION lib COMPONENT ${COMPONEN
 if(APPLE)
 install(TARGETS llamamodel-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 endif()
-install(TARGETS mpt-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS mpt-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS falcon-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS falcon-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+#install(TARGETS mpt-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+#install(TARGETS mpt-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS replit-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS replit-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+if(APPLE)
+install(TARGETS replit-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+endif()
 install(TARGETS bert-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS bert-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS starcoder-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS starcoder-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 
 set(CPACK_GENERATOR "IFW")
 set(CPACK_VERBATIM_VARIABLES YES)
 set(CPACK_IFW_VERBOSE ON)
 
 if(${CMAKE_SYSTEM_NAME} MATCHES Linux)
-    find_program(LINUXDEPLOYQT linuxdeployqt HINTS "$ENV{HOME}/dev/linuxdeployqt/build/tools/linuxdeployqt" "$ENV{HOME}/project/linuxdeployqt/bin")
+    set(LINUXDEPLOYQT "$ENV{HOME}/dev/linuxdeployqt/build/tools/linuxdeployqt/linuxdeployqt")
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/deploy-qt-linux.cmake.in"
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-linux.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-linux.cmake)
-    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.6")
+    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.5")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-linux")
     set(CPACK_IFW_TARGET_DIRECTORY "@HomeDir@/${COMPONENT_NAME_MAIN}")
 elseif(${CMAKE_SYSTEM_NAME} MATCHES Windows)
@@ -211,7 +220,7 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES Windows)
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/deploy-qt-windows.cmake.in"
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-windows.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-windows.cmake)
-    set(CPACK_IFW_ROOT "C:/Qt/Tools/QtInstallerFramework/4.6")
+    set(CPACK_IFW_ROOT "C:/Qt/Tools/QtInstallerFramework/4.5")
     set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.ico")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-win64")
     set(CPACK_IFW_TARGET_DIRECTORY "@HomeDir@\\${COMPONENT_NAME_MAIN}")
@@ -220,7 +229,7 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/deploy-qt-mac.cmake.in"
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-mac.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-mac.cmake)
-    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.6")
+    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.5")
     set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-darwin")
     set(CPACK_IFW_TARGET_DIRECTORY "@ApplicationsDir@/${COMPONENT_NAME_MAIN}")
@@ -264,7 +273,7 @@ cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} REPLACES "gpt4all-chat") #W
 if (GPT4ALL_LOCALHOST)
     cpack_ifw_add_repository("GPT4AllRepository" URL "http://localhost/repository")
 elseif(GPT4ALL_OFFLINE_INSTALLER)
-  add_compile_definitions(GPT4ALL_OFFLINE_INSTALLER)
+  # noop
 else()
   if(${CMAKE_SYSTEM_NAME} MATCHES Linux)
       cpack_ifw_add_repository("GPT4AllRepository" URL "https://gpt4all.io/installer_repos/linux/repository")

gpt4all-chat/README.md

@@ -32,8 +32,13 @@ One click installers for macOS, Linux, and Windows at https://gpt4all.io
 * Multi-chat - a list of current and past chats and the ability to save/delete/export and switch between
 * Text to speech - have the AI response with voice
 * Speech to text - give the prompt with your voice
+* Python bindings
+* Typescript bindings
 * Plugin support for langchain other developer tools
-* chat gui headless operation mode
+* Save your prompt/responses to disk
+* Upload prompt/response manually/automatically to nomic.ai to aid future training runs
+* Syntax highlighting support for programming languages, etc.
+* REST API with a built-in webserver in the chat gui itself with a headless operation mode as well
 * Advanced settings for changing temperature, topk, etc. (DONE)
 * * Improve the accessibility of the installer for screen reader users
 * YOUR IDEA HERE

gpt4all-chat/build_and_run.md

@@ -1,102 +1,57 @@
-# Building gpt4all-chat from source
+# Install Qt 6.x and setup/build gpt4all-chat from source
 
 Depending upon your operating system, there are many ways that Qt is distributed. 
 Here is the recommended method for getting the Qt dependency installed to setup and build 
 gpt4all-chat from source.
 
-## Prerequisites
+## Create a [Qt account](https://login.qt.io/register) 
 
-On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/d1e44cab-4245-4144-a91c-7b02267df2b2)
 
-macOS users do not need Vulkan, as GPT4All will use Metal instead.
+## Go to the Qt open source [download page](https://www.qt.io/download-qt-installer-oss)
 
-## Note for Linux users
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/d68f5f45-cca3-4fe9-acf4-cabdcb95f669)
 
-Linux users may install Qt via their distro's official packages instead of using the Qt installer. You need at least Qt 6.5, with support for QPdf and the Qt HTTP Server. It should be straightforward to build with just cmake and make, but you may continue to follow these instructions to build with Qt Creator.
+## Start the installer and sign in
 
-On Arch Linux, this looks like:
-```
-sudo pacman -S --needed base-devel qt6-base qt6-httpserver qtcreator cmake ninja
-```
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/899b1422-51ae-4bb5-acc9-b9027a8e9b19)
 
-On Ubuntu 23.04, this looks like:
-```
-sudo apt install build-essential libqt6gui6 qt6-base-dev libqt6httpserver6 qt6-httpserver-dev qtcreator cmake ninja-build
-```
+## After some screens about license, select custom
 
-## Download Qt
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/2290031a-fdb0-4f47-a7f1-d77ad5451068)
 
-- Go to https://login.qt.io/register to create a free Qt account.
-- Download the Qt Online Installer for your OS from here: https://www.qt.io/download-qt-installer-oss
-- Sign into the installer.
-- Agree to the terms of the (L)GPL 3 license.
-- Select whether you would like to send anonymous usage statistics to Qt.
-- On the Installation Folder page, leave the default installation path, and select "Custom Installation".
-
-## Customize the installation
+## Select the following
 
 ![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/c6e999e5-cc8a-4dfc-8065-b59139e8c7ae)
 
-Under "Qt", find the latest Qt 6.x release.
+NOTE: This is for macOS. For Linux it is similar, but you need MSVC for Windows, not the mingw install
 
-Under this release (e.g. Qt 6.5.0), select the target platform:
-- On macOS, it is just called "macOS".
-- On Windows, it is called "MSVC 2019 64-bit" (for 64-bit x86 CPUs). MinGW has not been tested.
+## Open up QtCreator
 
-Under this release, select the following additional components:
-- Qt Quick 3D
-- Qt 5 Compatibility Module
-- Qt Shader Tools
-- Additional Libraries (clicking the checkbox to the left of this item enables all of them)
-- Qt Debug information Files
-- Qt Quick Timeline
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/a34978f4-a220-459c-af66-e901d7ccd7bb)
 
-Under Developer and Designer Tools, select the following components:
-- Qt Creator
-- Qt Creator CDB Debugger Support (for Windows only)
-- Debugging Tools for Windows (for Windows only)
-- CMake
-- Ninja
+## Clone the git repo for gpt4all-chat
 
-Agree to the license and complete the installation.
-
-## Download the source code
-
-You must use git to download the source code for gpt4all:
 ```
 git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
 ```
 
-Note the use of --recurse-submodules, which makes sure the necessary dependencies are downloaded inside the repo. This is why you cannot simply download a zip archive.
-
-Windows users: To install git for Windows, see https://git-scm.com/downloads. Once it is installed, you should be able to shift-right click in any folder, "Open PowerShell window here" (or similar, depending on the version of Windows), and run the above command.
-
-## Open gpt4all-chat in Qt Creator
-
-Open Qt Creator. Navigate to File > Open File or Project, find the "gpt4all-chat" folder inside the freshly cloned repository, and select CMakeLists.txt.
+## Open the gpt4all-chat project in QtCreator
 
 ![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/3d3e2743-2a1d-43d6-9e55-62f7f4306de7)
 
+NOTE: File->Open File or Project and navigate to the gpt4all-chat repo and choose the CMakeLists.txt
+
 ## Configure project
 
-You can now expand the "Details" section next to the build kit. It is best to uncheck all but one build configuration, e.g. "Release", which will produce optimized binaries that are not useful for debugging.
-
-Click "Configure Project", and wait for it to complete.
-
 ![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/44d5aafb-a95d-434b-ba2a-a3138c0e49a0)
 
 ## Build project
 
-Now that the project has been configured, click the hammer button on the left sidebar to build the project.
-
 ![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/43cd7b42-32f0-4efa-9612-d51f85637103)
 
 ## Run project
 
-Click the play button on the left sidebar to run the Chat UI.
-
 ![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/611ea795-bdcd-4feb-a466-eb1c2e936e7e)
 
-## Updating the downloaded source code
 
-You do not need to make a fresh clone of the source code every time. To update it, you may open a terminal/command prompt in the repository, run `git pull`, and then `git submodule update --init --recursive`.

gpt4all-chat/chat.cpp

@@ -57,7 +57,6 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::generatedNameChanged, this, &Chat::generatedNameChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportSpeed, this, &Chat::handleTokenSpeedChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportDevice, this, &Chat::handleDeviceChanged, Qt::QueuedConnection);
-    connect(m_llmodel, &ChatLLM::reportFallbackReason, this, &Chat::handleFallbackReasonChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::databaseResultsChanged, this, &Chat::handleDatabaseResultsChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelInfoChanged, this, &Chat::handleModelInfoChanged, Qt::QueuedConnection);
 
@@ -353,12 +352,6 @@ void Chat::handleDeviceChanged(const QString &device)
     emit deviceChanged();
 }
 
-void Chat::handleFallbackReasonChanged(const QString &fallbackReason)
-{
-    m_fallbackReason = fallbackReason;
-    emit fallbackReasonChanged();
-}
-
 void Chat::handleDatabaseResultsChanged(const QList<ResultInfo> &results)
 {
     m_databaseResults = results;
@@ -385,11 +378,7 @@ bool Chat::serialize(QDataStream &stream, int version) const
         stream << m_modelInfo.filename();
     if (version > 2)
         stream << m_collections;
-
-    const bool serializeKV = MySettings::globalInstance()->saveChatsContext();
-    if (version > 5)
-        stream << serializeKV;
-    if (!m_llmodel->serialize(stream, version, serializeKV))
+    if (!m_llmodel->serialize(stream, version))
         return false;
     if (!m_chatModel->serialize(stream, version))
         return false;
@@ -408,40 +397,29 @@ bool Chat::deserialize(QDataStream &stream, int version)
     QString modelId;
     stream >> modelId;
     if (version > 4) {
-        if (ModelList::globalInstance()->contains(modelId))
-            m_modelInfo = ModelList::globalInstance()->modelInfo(modelId);
+        if (!ModelList::globalInstance()->contains(modelId))
+            return false;
+        m_modelInfo = ModelList::globalInstance()->modelInfo(modelId);
     } else {
-        if (ModelList::globalInstance()->containsByFilename(modelId))
-            m_modelInfo = ModelList::globalInstance()->modelInfoByFilename(modelId);
+        if (!ModelList::globalInstance()->containsByFilename(modelId))
+            return false;
+        m_modelInfo = ModelList::globalInstance()->modelInfoByFilename(modelId);
     }
-    if (!m_modelInfo.id().isEmpty())
-        emit modelInfoChanged();
-
-    bool discardKV = m_modelInfo.id().isEmpty();
+    emit modelInfoChanged();
 
     // Prior to version 2 gptj models had a bug that fixed the kv_cache to F32 instead of F16 so
     // unfortunately, we cannot deserialize these
     if (version < 2 && m_modelInfo.filename().contains("gpt4all-j"))
-        discardKV = true;
-
+        return false;
     if (version > 2) {
         stream >> m_collections;
         emit collectionListChanged(m_collections);
     }
-
-    bool deserializeKV = true;
-    if (version > 5)
-        stream >> deserializeKV;
-
     m_llmodel->setModelInfo(m_modelInfo);
-    if (!m_llmodel->deserialize(stream, version, deserializeKV, discardKV))
+    if (!m_llmodel->deserialize(stream, version))
         return false;
     if (!m_chatModel->deserialize(stream, version))
         return false;
-
-    if (!deserializeKV || discardKV)
-        m_llmodel->setStateFromText(m_chatModel->text());
-
     emit chatModelChanged();
     return stream.status() == QDataStream::Ok;
 }

gpt4all-chat/chat.h

@@ -26,7 +26,6 @@ class Chat : public QObject
     Q_PROPERTY(QString modelLoadingError READ modelLoadingError NOTIFY modelLoadingErrorChanged)
     Q_PROPERTY(QString tokenSpeed READ tokenSpeed NOTIFY tokenSpeedChanged);
     Q_PROPERTY(QString device READ device NOTIFY deviceChanged);
-    Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY fallbackReasonChanged);
     QML_ELEMENT
     QML_UNCREATABLE("Only creatable from c++!")
 
@@ -54,8 +53,6 @@ public:
     }
     ChatModel *chatModel() { return m_chatModel; }
 
-    bool isNewChat() const { return m_name == tr("New Chat") && !m_chatModel->count(); }
-
     Q_INVOKABLE void reset();
     Q_INVOKABLE void processSystemPrompt();
     Q_INVOKABLE bool isModelLoaded() const;
@@ -93,7 +90,6 @@ public:
 
     QString tokenSpeed() const { return m_tokenSpeed; }
     QString device() const { return m_device; }
-    QString fallbackReason() const { return m_fallbackReason; }
 
 public Q_SLOTS:
     void serverNewPromptResponsePair(const QString &prompt);
@@ -122,7 +118,6 @@ Q_SIGNALS:
     void collectionListChanged(const QList<QString> &collectionList);
     void tokenSpeedChanged();
     void deviceChanged();
-    void fallbackReasonChanged();
 
 private Q_SLOTS:
     void handleResponseChanged(const QString &response);
@@ -134,7 +129,6 @@ private Q_SLOTS:
     void handleModelLoadingError(const QString &error);
     void handleTokenSpeedChanged(const QString &tokenSpeed);
     void handleDeviceChanged(const QString &device);
-    void handleFallbackReasonChanged(const QString &device);
     void handleDatabaseResultsChanged(const QList<ResultInfo> &results);
     void handleModelInfoChanged(const ModelInfo &modelInfo);
     void handleModelInstalled();
@@ -148,7 +142,6 @@ private:
     QString m_modelLoadingError;
     QString m_tokenSpeed;
     QString m_device;
-    QString m_fallbackReason;
     QString m_response;
     QList<QString> m_collections;
     ChatModel *m_chatModel;

gpt4all-chat/chatlistmodel.cpp

@@ -5,7 +5,7 @@
 #include <QDataStream>
 
 #define CHAT_FORMAT_MAGIC 0xF5D553CC
-#define CHAT_FORMAT_VERSION 6
+#define CHAT_FORMAT_VERSION 5
 
 class MyChatListModel: public ChatListModel { };
 Q_GLOBAL_STATIC(MyChatListModel, chatListModelInstance)
@@ -17,10 +17,11 @@ ChatListModel *ChatListModel::globalInstance()
 ChatListModel::ChatListModel()
     : QAbstractListModel(nullptr)
     , m_newChat(nullptr)
+    , m_dummyChat(nullptr)
     , m_serverChat(nullptr)
     , m_currentChat(nullptr)
 {
-    addChat();
+    addDummyChat();
 
     ChatsRestoreThread *thread = new ChatsRestoreThread;
     connect(thread, &ChatsRestoreThread::chatRestored, this, &ChatListModel::restoreChat);
@@ -58,7 +59,10 @@ void ChatListModel::saveChats()
     for (Chat *chat : m_chats) {
         if (chat == m_serverChat)
             continue;
-        if (chat->isNewChat())
+        const bool isChatGPT = chat->modelInfo().isChatGPT;
+        if (!isChatGPT && !MySettings::globalInstance()->saveChats())
+            continue;
+        if (isChatGPT && !MySettings::globalInstance()->saveChatGPTChats())
             continue;
         toSave.append(chat);
     }
@@ -80,16 +84,13 @@ void ChatSaver::saveChats(const QVector<Chat *> &chats)
     const QString savePath = MySettings::globalInstance()->modelPath();
     for (Chat *chat : chats) {
         QString fileName = "gpt4all-" + chat->id() + ".chat";
-        QString filePath = savePath + "/" + fileName;
-        QFile originalFile(filePath);
-        QFile tempFile(filePath + ".tmp"); // Temporary file
-
-        bool success = tempFile.open(QIODevice::WriteOnly);
+        QFile file(savePath + "/" + fileName);
+        bool success = file.open(QIODevice::WriteOnly);
         if (!success) {
-            qWarning() << "ERROR: Couldn't save chat to temporary file:" << tempFile.fileName();
+            qWarning() << "ERROR: Couldn't save chat to file:" << file.fileName();
             continue;
         }
-        QDataStream out(&tempFile);
+        QDataStream out(&file);
 
         out << (quint32)CHAT_FORMAT_MAGIC;
         out << (qint32)CHAT_FORMAT_VERSION;
@@ -97,16 +98,11 @@ void ChatSaver::saveChats(const QVector<Chat *> &chats)
 
         qDebug() << "serializing chat" << fileName;
         if (!chat->serialize(out, CHAT_FORMAT_VERSION)) {
-            qWarning() << "ERROR: Couldn't serialize chat to file:" << tempFile.fileName();
-            tempFile.remove();
-            continue;
+            qWarning() << "ERROR: Couldn't serialize chat to file:" << file.fileName();
+            file.remove();
         }
-
-        if (originalFile.exists())
-            originalFile.remove();
-        tempFile.rename(filePath);
+        file.close();
     }
-
     qint64 elapsedTime = timer.elapsed();
     qDebug() << "serializing chats took:" << elapsedTime << "ms";
     emit saveChatsFinished();
@@ -193,47 +189,48 @@ void ChatsRestoreThread::run()
     });
 
     for (FileInfo &f : files) {
-        QFile file(f.file);
-        bool success = file.open(QIODevice::ReadOnly);
-        if (!success) {
-            qWarning() << "ERROR: Couldn't restore chat from file:" << file.fileName();
-            continue;
-        }
-        QDataStream in(&file);
-
-        qint32 version = 0;
-        if (!f.oldFile) {
-            // Read and check the header
-            quint32 magic;
-            in >> magic;
-            if (magic != CHAT_FORMAT_MAGIC) {
-                qWarning() << "ERROR: Chat file has bad magic:" << file.fileName();
+            QFile file(f.file);
+            bool success = file.open(QIODevice::ReadOnly);
+            if (!success) {
+                qWarning() << "ERROR: Couldn't restore chat from file:" << file.fileName();
                 continue;
             }
+            QDataStream in(&file);
 
-            // Read the version
-            in >> version;
-            if (version < 1) {
-                qWarning() << "ERROR: Chat file has non supported version:" << file.fileName();
-                continue;
+            qint32 version = 0;
+            if (!f.oldFile) {
+                // Read and check the header
+                quint32 magic;
+                in >> magic;
+                if (magic != CHAT_FORMAT_MAGIC) {
+                    qWarning() << "ERROR: Chat file has bad magic:" << file.fileName();
+                    continue;
+                }
+
+                // Read the version
+                in >> version;
+                if (version < 1) {
+                    qWarning() << "ERROR: Chat file has non supported version:" << file.fileName();
+                    continue;
+                }
+
+                if (version <= 1)
+                    in.setVersion(QDataStream::Qt_6_2);
             }
 
-            if (version <= 1)
-                in.setVersion(QDataStream::Qt_6_2);
-        }
+            qDebug() << "deserializing chat" << f.file;
 
-        qDebug() << "deserializing chat" << f.file;
-
-        Chat *chat = new Chat;
-        chat->moveToThread(qApp->thread());
-        if (!chat->deserialize(in, version)) {
-            qWarning() << "ERROR: Couldn't deserialize chat from file:" << file.fileName();
-        } else {
-            emit chatRestored(chat);
-        }
-        if (f.oldFile)
-           file.remove(); // No longer storing in this directory
-        file.close();
+            Chat *chat = new Chat;
+            chat->moveToThread(qApp->thread());
+            if (!chat->deserialize(in, version)) {
+                qWarning() << "ERROR: Couldn't deserialize chat from file:" << file.fileName();
+                file.remove();
+            } else {
+                emit chatRestored(chat);
+            }
+            if (f.oldFile)
+               file.remove(); // No longer storing in this directory
+            file.close();
     }
 
     qint64 elapsedTime = timer.elapsed();
@@ -245,13 +242,35 @@ void ChatListModel::restoreChat(Chat *chat)
     chat->setParent(this);
     connect(chat, &Chat::nameChanged, this, &ChatListModel::nameChanged);
 
-    beginInsertRows(QModelIndex(), m_chats.size(), m_chats.size());
-    m_chats.append(chat);
-    endInsertRows();
+    if (m_dummyChat) {
+        beginResetModel();
+        m_chats = QList<Chat*>({chat});
+        setCurrentChat(chat);
+        delete m_dummyChat;
+        m_dummyChat = nullptr;
+        endResetModel();
+    } else {
+        beginInsertRows(QModelIndex(), m_chats.size(), m_chats.size());
+        m_chats.append(chat);
+        endInsertRows();
+    }
 }
 
 void ChatListModel::chatsRestoredFinished()
 {
+    if (m_dummyChat) {
+        beginResetModel();
+        Chat *dummy = m_dummyChat;
+        m_dummyChat = nullptr;
+        m_chats.clear();
+        addChat();
+        delete dummy;
+        endResetModel();
+    }
+
+    if (m_chats.isEmpty())
+        addChat();
+
     addServerChat();
 }
 

gpt4all-chat/chatlistmodel.h

@@ -84,7 +84,7 @@ public:
     Q_INVOKABLE void addChat()
     {
         // Don't add a new chat if we already have one
-        if (m_newChat)
+        if (m_newChat || m_dummyChat)
             return;
 
         // Create a new chat pointer and connect it to determine when it is populated
@@ -101,6 +101,18 @@ public:
         setCurrentChat(m_newChat);
     }
 
+    Q_INVOKABLE void addDummyChat()
+    {
+        // Create a new dummy chat pointer and don't connect it
+        m_dummyChat = new Chat(this);
+        beginInsertRows(QModelIndex(), 0, 0);
+        m_chats.prepend(m_dummyChat);
+        endInsertRows();
+        emit countChanged();
+        m_currentChat = m_dummyChat;
+        emit currentChatChanged();
+    }
+
     Q_INVOKABLE void addServerChat()
     {
         // Create a new dummy chat pointer and don't connect it
@@ -240,6 +252,7 @@ private Q_SLOTS:
 
 private:
     Chat* m_newChat;
+    Chat* m_dummyChat;
     Chat* m_serverChat;
     Chat* m_currentChat;
     QList<Chat*> m_chats;

gpt4all-chat/chatllm.cpp

@@ -11,8 +11,11 @@
 
 #define MPT_INTERNAL_STATE_VERSION 0
 #define GPTJ_INTERNAL_STATE_VERSION 0
+#define REPLIT_INTERNAL_STATE_VERSION 0
 #define LLAMA_INTERNAL_STATE_VERSION 0
+#define FALCON_INTERNAL_STATE_VERSION 0
 #define BERT_INTERNAL_STATE_VERSION 0
+#define STARCODER_INTERNAL_STATE_VERSION 0
 
 class LLModelStore {
 public:
@@ -69,7 +72,6 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_forceMetal(MySettings::globalInstance()->forceMetal())
     , m_reloadingToChangeVariant(false)
     , m_processedSystemPrompt(false)
-    , m_restoreStateFromText(false)
 {
     moveToThread(&m_llmThread);
     connect(this, &ChatLLM::sendStartup, Network::globalInstance(), &Network::sendStartup);
@@ -228,7 +230,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         if (!m_isServer)
             LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
         m_llModelInfo = LLModelInfo();
-        emit modelLoadingError(QString("Previous attempt to load model resulted in crash for `%1` most likely due to insufficient memory. You should either remove this model or decrease your system RAM by closing other applications.").arg(modelInfo.filename()));
+        emit modelLoadingError(QString("Previous attempt to load model resulted in crash for `%1` most likely due to out of memory. You should either remove this model or decrease your system RAM by closing other applications.").arg(modelInfo.filename()));
     }
 
     if (fileInfo.exists()) {
@@ -265,52 +267,30 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                 // Pick the best match for the device
                 QString actualDevice = m_llModelInfo.model->implementation().buildVariant() == "metal" ? "Metal" : "CPU";
                 const QString requestedDevice = MySettings::globalInstance()->device();
-                if (requestedDevice == "CPU") {
-                    emit reportFallbackReason(""); // fallback not applicable
-                } else {
+                if (requestedDevice != "CPU") {
                     const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString());
                     std::vector<LLModel::GPUDevice> availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
-                    LLModel::GPUDevice *device = nullptr;
-
                     if (!availableDevices.empty() && requestedDevice == "Auto" && availableDevices.front().type == 2 /*a discrete gpu*/) {
-                        device = &availableDevices.front();
+                        m_llModelInfo.model->initializeGPUDevice(availableDevices.front());
+                        actualDevice = QString::fromStdString(availableDevices.front().name);
                     } else {
                         for (LLModel::GPUDevice &d : availableDevices) {
                             if (QString::fromStdString(d.name) == requestedDevice) {
-                                device = &d;
+                                m_llModelInfo.model->initializeGPUDevice(d);
+                                actualDevice = QString::fromStdString(d.name);
                                 break;
                             }
                         }
                     }
-
-                    emit reportFallbackReason(""); // no fallback yet
-                    std::string unavail_reason;
-                    if (!device) {
-                        // GPU not available
-                    } else if (!m_llModelInfo.model->initializeGPUDevice(*device, &unavail_reason)) {
-                        emit reportFallbackReason(QString::fromStdString("<br>" + unavail_reason));
-                    } else {
-                        actualDevice = QString::fromStdString(device->name);
-                    }
                 }
 
                 // Report which device we're actually using
                 emit reportDevice(actualDevice);
 
                 bool success = m_llModelInfo.model->loadModel(filePath.toStdString());
-                if (actualDevice == "CPU") {
-                    // we asked llama.cpp to use the CPU
-                } else if (!success) {
-                    // llama_init_from_file returned nullptr
+                if (!success && actualDevice != "CPU") {
                     emit reportDevice("CPU");
-                    emit reportFallbackReason("<br>GPU loading failed (out of VRAM?)");
                     success = m_llModelInfo.model->loadModel(filePath.toStdString());
-                } else if (!m_llModelInfo.model->usingGPUDevice()) {
-                    // ggml_vk_init was not called in llama.cpp
-                    // We might have had to fallback to CPU after load if the model is not possible to accelerate
-                    // for instance if the quantization method is not supported on Vulkan yet
-                    emit reportDevice("CPU");
-                    emit reportFallbackReason("<br>model or quant has no GPU support");
                 }
 
                 MySettings::globalInstance()->setAttemptModelLoad(QString());
@@ -322,11 +302,19 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     m_llModelInfo = LLModelInfo();
                     emit modelLoadingError(QString("Could not load model due to invalid model file for %1").arg(modelInfo.filename()));
                 } else {
+                    // We might have had to fallback to CPU after load if the model is not possible to accelerate
+                    // for instance if the quantization method is not supported on Vulkan yet
+                    if (actualDevice != "CPU" && !m_llModelInfo.model->usingGPUDevice())
+                        emit reportDevice("CPU");
+
                     switch (m_llModelInfo.model->implementation().modelType()[0]) {
                     case 'L': m_llModelType = LLModelType::LLAMA_; break;
                     case 'G': m_llModelType = LLModelType::GPTJ_; break;
                     case 'M': m_llModelType = LLModelType::MPT_; break;
+                    case 'R': m_llModelType = LLModelType::REPLIT_; break;
+                    case 'F': m_llModelType = LLModelType::FALCON_; break;
                     case 'B': m_llModelType = LLModelType::BERT_; break;
+                    case 'S': m_llModelType = LLModelType::STARCODER_; break;
                     default:
                         {
                             delete m_llModelInfo.model;
@@ -368,10 +356,10 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         emit modelLoadingError(QString("Could not find file for model %1").arg(modelInfo.filename()));
     }
 
-    if (m_llModelInfo.model) {
+    if (m_llModelInfo.model)
         setModelInfo(modelInfo);
-        processSystemPrompt();
-    }
+
+    processSystemPrompt();
     return m_llModelInfo.model;
 }
 
@@ -389,7 +377,7 @@ void ChatLLM::regenerateResponse()
     else
         m_ctx.n_past -= m_promptResponseTokens;
     m_ctx.n_past = std::max(0, m_ctx.n_past);
-    m_ctx.tokens.erase(m_ctx.tokens.end() - m_promptResponseTokens, m_ctx.tokens.end());
+    m_ctx.tokens.erase(m_ctx.tokens.end() -= m_promptResponseTokens, m_ctx.tokens.end());
     m_promptResponseTokens = 0;
     m_promptTokens = 0;
     m_response = std::string();
@@ -424,9 +412,6 @@ std::string trim_whitespace(const std::string& input) {
         return !std::isspace(c);
     });
 
-    if (first_non_whitespace == input.end())
-        return std::string();
-
     auto last_non_whitespace = std::find_if(input.rbegin(), input.rend(), [](unsigned char c) {
         return !std::isspace(c);
     }).base();
@@ -727,59 +712,26 @@ bool ChatLLM::handleSystemRecalculate(bool isRecalc)
     return false;
 }
 
-bool ChatLLM::handleRestoreStateFromTextPrompt(int32_t token)
-{
-#if defined(DEBUG)
-    qDebug() << "restore state from text prompt" << m_llmThread.objectName() << token << m_stopGenerating;
-#endif
-    Q_UNUSED(token);
-    return !m_stopGenerating;
-}
-
-bool ChatLLM::handleRestoreStateFromTextResponse(int32_t token, const std::string &response)
-{
-#if defined(DEBUG)
-    qDebug() << "restore state from text response" << m_llmThread.objectName() << token << response << m_stopGenerating;
-#endif
-    Q_UNUSED(token);
-    Q_UNUSED(response);
-    return false;
-}
-
-bool ChatLLM::handleRestoreStateFromTextRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "restore state from text recalc" << m_llmThread.objectName() << isRecalc;
-#endif
-    Q_UNUSED(isRecalc);
-    return false;
-}
-
-bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
+bool ChatLLM::serialize(QDataStream &stream, int version)
 {
     if (version > 1) {
         stream << m_llModelType;
         switch (m_llModelType) {
+        case REPLIT_: stream << REPLIT_INTERNAL_STATE_VERSION; break;
         case MPT_: stream << MPT_INTERNAL_STATE_VERSION; break;
         case GPTJ_: stream << GPTJ_INTERNAL_STATE_VERSION; break;
         case LLAMA_: stream << LLAMA_INTERNAL_STATE_VERSION; break;
+        case FALCON_: stream << FALCON_INTERNAL_STATE_VERSION; break;
         case BERT_: stream << BERT_INTERNAL_STATE_VERSION; break;
+        case STARCODER_: stream << STARCODER_INTERNAL_STATE_VERSION; break;
         default: Q_UNREACHABLE();
         }
     }
     stream << response();
     stream << generatedName();
     stream << m_promptResponseTokens;
-
-    if (!serializeKV) {
-#if defined(DEBUG)
-        qDebug() << "serialize" << m_llmThread.objectName() << m_state.size();
-#endif
-        return stream.status() == QDataStream::Ok;
-    }
-
     if (version <= 3) {
-        int responseLogits = 0;
+        int responseLogits;
         stream << responseLogits;
     }
     stream << m_ctx.n_past;
@@ -796,7 +748,7 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
     return stream.status() == QDataStream::Ok;
 }
 
-bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV, bool discardKV)
+bool ChatLLM::deserialize(QDataStream &stream, int version)
 {
     if (version > 1) {
         int internalStateVersion;
@@ -810,60 +762,26 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
     stream >> nameResponse;
     m_nameResponse = nameResponse.toStdString();
     stream >> m_promptResponseTokens;
-
-    // If we do not deserialize the KV or it is discarded, then we need to restore the state from the
-    // text only. This will be a costly operation, but the chat has to be restored from the text archive
-    // alone.
-    m_restoreStateFromText = !deserializeKV || discardKV;
-
-    if (!deserializeKV) {
-#if defined(DEBUG)
-        qDebug() << "deserialize" << m_llmThread.objectName();
-#endif
-        return stream.status() == QDataStream::Ok;
-    }
-
     if (version <= 3) {
         int responseLogits;
         stream >> responseLogits;
     }
-
-    int32_t n_past;
-    stream >> n_past;
-    if (!discardKV) m_ctx.n_past = n_past;
-
+    stream >> m_ctx.n_past;
     quint64 logitsSize;
     stream >> logitsSize;
-    if (!discardKV) {
-        m_ctx.logits.resize(logitsSize);
-        stream.readRawData(reinterpret_cast<char*>(m_ctx.logits.data()), logitsSize * sizeof(float));
-    } else {
-        stream.skipRawData(logitsSize * sizeof(float));
-    }
-
+    m_ctx.logits.resize(logitsSize);
+    stream.readRawData(reinterpret_cast<char*>(m_ctx.logits.data()), logitsSize * sizeof(float));
     quint64 tokensSize;
     stream >> tokensSize;
-    if (!discardKV) {
-        m_ctx.tokens.resize(tokensSize);
-        stream.readRawData(reinterpret_cast<char*>(m_ctx.tokens.data()), tokensSize * sizeof(int));
-    } else {
-        stream.skipRawData(tokensSize * sizeof(int));
-    }
-
+    m_ctx.tokens.resize(tokensSize);
+    stream.readRawData(reinterpret_cast<char*>(m_ctx.tokens.data()), tokensSize * sizeof(int));
     if (version > 0) {
         QByteArray compressed;
         stream >> compressed;
-        if (!discardKV)
-            m_state = qUncompress(compressed);
+        m_state = qUncompress(compressed);
     } else {
-        if (!discardKV)
-            stream >> m_state;
-        else {
-            QByteArray state;
-            stream >> m_state;
-        }
+        stream >> m_state;
     }
-
 #if defined(DEBUG)
     qDebug() << "deserialize" << m_llmThread.objectName();
 #endif
@@ -894,7 +812,7 @@ void ChatLLM::saveState()
 
 void ChatLLM::restoreState()
 {
-    if (!isModelLoaded())
+    if (!isModelLoaded() || m_state.isEmpty())
         return;
 
     if (m_llModelType == LLModelType::CHATGPT_) {
@@ -909,19 +827,10 @@ void ChatLLM::restoreState()
         return;
     }
 
-    if (m_restoreStateFromText) {
-        Q_ASSERT(m_state.isEmpty());
-        processRestoreStateFromText();
-    }
-
 #if defined(DEBUG)
     qDebug() << "restoreState" << m_llmThread.objectName() << "size:" << m_state.size();
 #endif
     m_processedSystemPrompt = true;
-
-    if (m_state.isEmpty())
-        return;
-
     m_llModelInfo.model->restoreState(static_cast<const uint8_t*>(reinterpret_cast<void*>(m_state.data())));
     m_state.clear();
     m_state.resize(0);
@@ -939,10 +848,7 @@ void ChatLLM::processSystemPrompt()
         return;
     }
 
-    // Start with a whole new context
     m_stopGenerating = false;
-    m_ctx = LLModel::PromptContext();
-
     auto promptFunc = std::bind(&ChatLLM::handleSystemPrompt, this, std::placeholders::_1);
     auto responseFunc = std::bind(&ChatLLM::handleSystemResponse, this, std::placeholders::_1,
         std::placeholders::_2);
@@ -973,54 +879,5 @@ void ChatLLM::processSystemPrompt()
     printf("\n");
     fflush(stdout);
 #endif
-
-    m_processedSystemPrompt = !m_stopGenerating;
-}
-
-void ChatLLM::processRestoreStateFromText()
-{
-    Q_ASSERT(isModelLoaded());
-    if (!isModelLoaded() || !m_restoreStateFromText || m_isServer)
-        return;
-
-    m_isRecalc = true;
-    emit recalcChanged();
-
-    m_stopGenerating = false;
-    m_ctx = LLModel::PromptContext();
-
-    auto promptFunc = std::bind(&ChatLLM::handleRestoreStateFromTextPrompt, this, std::placeholders::_1);
-    auto responseFunc = std::bind(&ChatLLM::handleRestoreStateFromTextResponse, this, std::placeholders::_1,
-        std::placeholders::_2);
-    auto recalcFunc = std::bind(&ChatLLM::handleRestoreStateFromTextRecalculate, this, std::placeholders::_1);
-
-    const QString promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
-    const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
-    const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
-    const float top_p = MySettings::globalInstance()->modelTopP(m_modelInfo);
-    const float temp = MySettings::globalInstance()->modelTemperature(m_modelInfo);
-    const int32_t n_batch = MySettings::globalInstance()->modelPromptBatchSize(m_modelInfo);
-    const float repeat_penalty = MySettings::globalInstance()->modelRepeatPenalty(m_modelInfo);
-    const int32_t repeat_penalty_tokens = MySettings::globalInstance()->modelRepeatPenaltyTokens(m_modelInfo);
-    int n_threads = MySettings::globalInstance()->threadCount();
-    m_ctx.n_predict = n_predict;
-    m_ctx.top_k = top_k;
-    m_ctx.top_p = top_p;
-    m_ctx.temp = temp;
-    m_ctx.n_batch = n_batch;
-    m_ctx.repeat_penalty = repeat_penalty;
-    m_ctx.repeat_last_n = repeat_penalty_tokens;
-    m_llModelInfo.model->setThreadCount(n_threads);
-    for (auto pair : m_stateFromText) {
-        const QString str = pair.first == "Prompt: " ? promptTemplate.arg(pair.second) : pair.second;
-        m_llModelInfo.model->prompt(str.toStdString(), promptFunc, responseFunc, recalcFunc, m_ctx);
-    }
-
-    if (!m_stopGenerating) {
-        m_restoreStateFromText = false;
-        m_stateFromText.clear();
-    }
-
-    m_isRecalc = false;
-    emit recalcChanged();
+    m_processedSystemPrompt = true;
 }

gpt4all-chat/chatllm.h

@@ -14,7 +14,10 @@ enum LLModelType {
     GPTJ_,
     LLAMA_,
     CHATGPT_,
+    REPLIT_,
+    FALCON_,
     BERT_,
+    STARCODER_
 };
 
 struct LLModelInfo {
@@ -92,9 +95,8 @@ public:
 
     QString generatedName() const { return QString::fromStdString(m_nameResponse); }
 
-    bool serialize(QDataStream &stream, int version, bool serializeKV);
-    bool deserialize(QDataStream &stream, int version, bool deserializeKV, bool discardKV);
-    void setStateFromText(const QVector<QPair<QString, QString>> &stateFromText) { m_stateFromText = stateFromText; }
+    bool serialize(QDataStream &stream, int version);
+    bool deserialize(QDataStream &stream, int version);
 
 public Q_SLOTS:
     bool prompt(const QList<QString> &collectionList, const QString &prompt);
@@ -111,7 +113,6 @@ public Q_SLOTS:
     void handleForceMetalChanged(bool forceMetal);
     void handleDeviceChanged();
     void processSystemPrompt();
-    void processRestoreStateFromText();
 
 Q_SIGNALS:
     void recalcChanged();
@@ -129,7 +130,6 @@ Q_SIGNALS:
     void requestRetrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
     void reportSpeed(const QString &speed);
     void reportDevice(const QString &device);
-    void reportFallbackReason(const QString &fallbackReason);
     void databaseResultsChanged(const QList<ResultInfo>&);
     void modelInfoChanged(const ModelInfo &modelInfo);
 
@@ -146,9 +146,6 @@ protected:
     bool handleSystemPrompt(int32_t token);
     bool handleSystemResponse(int32_t token, const std::string &response);
     bool handleSystemRecalculate(bool isRecalc);
-    bool handleRestoreStateFromTextPrompt(int32_t token);
-    bool handleRestoreStateFromTextResponse(int32_t token, const std::string &response);
-    bool handleRestoreStateFromTextRecalculate(bool isRecalc);
     void saveState();
     void restoreState();
 
@@ -173,8 +170,6 @@ private:
     bool m_forceMetal;
     bool m_reloadingToChangeVariant;
     bool m_processedSystemPrompt;
-    bool m_restoreStateFromText;
-    QVector<QPair<QString, QString>> m_stateFromText;
 };
 
 #endif // CHATLLM_H

gpt4all-chat/chatmodel.h

@@ -285,14 +285,6 @@ public:
         return stream.status() == QDataStream::Ok;
     }
 
-    QVector<QPair<QString, QString>> text() const
-    {
-        QVector<QPair<QString, QString>> result;
-        for (const auto &c : m_chatItems)
-            result << qMakePair(c.name, c.value);
-        return result;
-    }
-
 Q_SIGNALS:
     void countChanged();
 

gpt4all-chat/cmake/deploy-qt-mac.cmake.in

@@ -5,7 +5,10 @@ execute_process(COMMAND ${MACDEPLOYQT} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/pack
 file(GLOB MYGPTJLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libgptj*)
 file(GLOB MYMPTLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libmpt*)
 file(GLOB MYLLAMALIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllama*)
+file(GLOB MYREPLITLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libreplit*)
+file(GLOB MYFALCONLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libfalcon*)
 file(GLOB MYBERTLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libbert*)
+file(GLOB MYSTARCODERLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libstarcoder*)
 file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllmodel.*)
 file(COPY ${MYGPTJLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
@@ -13,8 +16,14 @@ file(COPY ${MYMPTLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLAMALIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
+file(COPY ${MYREPLITLIBS}
+     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
+file(COPY ${MYFALCONLLIBS}
+     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYBERTLLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
+file(COPY ${MYSTARCODERLLIBS}
+     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLAMALIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLMODELLIBS}

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.appdata.xml

@@ -1,48 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<component type="desktop">
-    <id>io.gpt4all.gpt4all</id>
-    <metadata_license>CC0-1.0</metadata_license>
-    <project_license>MIT License</project_license>
-    <name>GPT4ALL</name>
-    <summary>Open-source assistant-style large language models that run locally on your CPU and GPU</summary>
-    <description>
-        <p>Cross platform Qt based GUI for GPT4All</p>
-        <ul>
-            <li>Fast CPU and GPU based inference using ggml for open source LLM's</li>
-            <li>The UI is made to look and feel like you've come to expect from a chatty gpt</li>
-            <li>Check for updates so you can always stay fresh with latest models</li>
-            <li>Easy to install with precompiled binaries available for all three major desktop platforms</li>
-            <li>Multi-model - Ability to load more than one model and switch between them</li>
-            <li>Supports both llama.cpp and gptj.cpp style models</li>
-            <li>Model downloader in GUI featuring many popular open source models</li>
-            <li>Settings dialog to change temp, top_p, top_k, threads, etc</li>
-            <li>Copy your conversation to clipboard</li>
-        </ul>
-    </description>
-    <screenshots>
-        <screenshot type="default">
-            <caption>Main Window</caption>
-            <image>https://user-images.githubusercontent.com/50458173/231464085-da9edff6-a593-410e-8f38-7513f75c8aab.png</image>
-        </screenshot>
-    </screenshots>
-    <url type="homepage">https://gpt4all.io</url>
-    <url type="bugtracker">https://github.com/nomic-ai/gpt4all/issues</url>
-    <url type="vcs-browser">https://github.com/nomic-ai/gpt4all</url>
-    <releases>
-      <release version="2.4.19" date="2023-09-16">
-        <description>
-          <p>
-            <ul>
-              <li>A bugfix for crashes on systems that have a corrupted Vulkan driver or a corrupted version of the Vulkan shared library</li>
-            </ul>
-          </p>
-        </description>
-        </release>              
-    </releases>
-    <launchable type="desktop-id">io.gpt4all.gpt4all.desktop</launchable>
-    <content_rating type="oars-1.0">
-        <content_attribute id="language-profanity">mild</content_attribute>
-        <content_attribute id="language-humor">moderate</content_attribute>
-        <content_attribute id="language-discrimination">mild</content_attribute>
-    </content_rating>
-</component>

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.desktop

@@ -1,9 +0,0 @@
-[Desktop Entry]
-Name=GPT4ALL
-GenericName=Open-source assistant-style large language models that run locally on your CPU
-Comment=Run any GPT4All model natively on your home desktop with the auto-updating desktop chat client. See GPT4All Website for a full list of open-source models you can run with this powerful desktop application.
-Exec=chat
-Icon=io.gpt4all.gpt4all
-Type=Application
-Categories=Utility;Office;
-Keywords=GPT,Chat;AI

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.yml

@@ -1,166 +0,0 @@
-app-id: io.gpt4all.gpt4all
-default-branch: stable
-runtime: org.kde.Platform
-runtime-version: '6.5'
-sdk: org.kde.Sdk
-sdk-extensions:
-  - org.freedesktop.Sdk.Extension.node14
-finish-args:
-  - --share=ipc
-  - --socket=wayland
-  - --socket=x11
-  - --share=network
-  - --device=dri
-  - --env=LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/lib/x86_64-linux-gnu/
-  - --filesystem=xdg-documents:ro
-command: chat
-cleanup:
-  - /include
-
-modules:
-  - name: qthttpserver
-    buildsystem: cmake
-    sources:
-      - type: archive
-        url: https://invent.kde.org/qt/qt/qthttpserver/-/archive/6.5.2/qthttpserver-6.5.2.zip
-        sha256: 9fb7b14774b4ed62fe9097e13fa593af06ba75537783fc62f34652bada26abee
-
-  - name: python-html5lib
-    buildsystem: simple
-    build-commands:
-    - 'pip3 install --verbose --exists-action=i --no-index --find-links="file://${PWD}" --prefix=${FLATPAK_DEST} "html5lib" --no-build-isolation'
-    sources:
-      - type: file
-        url: https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl
-        sha256: a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78
-        x-checker-data:
-            type: pypi
-            name: webencodings
-            packagetype: bdist_wheel
-      - type: file
-        url: https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl
-        sha256: 0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d
-        x-checker-data:
-            type: pypi
-            name: html5lib
-            packagetype: bdist_wheel
-    cleanup:
-    - '*'
-
-  - name: qtwebengine
-    buildsystem: cmake
-    builddir: true
-    config-opts:
-      - -DQT_FEATURE_qtwebengine_build=OFF
-      - -DQT_FEATURE_qtpdf_build=ON
-    build-options:
-      append-path: /usr/lib/sdk/node14/bin
-      env:
-        - npm_config_nodedir=/usr/lib/sdk/node14
-    sources:
-      - type: git
-        url: https://invent.kde.org/qt/qt/qtwebengine.git
-        tag: v6.5.2
-        commit: ac887518e8243828333e923b5a1e61a007babde5
-  
-  - name: vulkan-headers
-    buildsystem: cmake
-    builddir: true
-    sources:
-      - type: git
-        url: https://github.com/KhronosGroup/Vulkan-Headers.git
-        tag: v1.3.224
-        commit: 2b55157592bf4c639b76cc16d64acaef565cc4b5
-
-  - name: fmt
-    buildsystem: cmake
-    builddir: true
-    sources:
-      - type: git
-        url: https://github.com/fmtlib/fmt.git
-        tag: 10.1.1
-        commit: f5e54359df4c26b6230fc61d38aa294581393084
-
-  - name: vulkan-tools
-    buildsystem: cmake
-    builddir: true
-    sources:
-      - type: git
-        url: https://github.com/KhronosGroup/Vulkan-Tools.git
-        tag: v1.3.224
-        commit: 497f232680b046db34ba9e9da065e6303a125851
-
-    
-
-    modules:
-      - name: shaderc
-        buildsystem: cmake-ninja
-        builddir: true
-        config-opts:
-          - -DSHADERC_SKIP_COPYRIGHT_CHECK=ON
-          - -DSHADERC_SKIP_EXAMPLES=ON
-          - -DSHADERC_SKIP_TESTS=ON
-          - -DSPIRV_SKIP_EXECUTABLES=ON
-          - -DENABLE_GLSLANG_BINARIES=OFF
-        cleanup:
-          - /bin
-          - /include
-          - /lib/cmake
-          - /lib/pkgconfig
-        sources:
-          - type: git
-            url: https://github.com/google/shaderc.git
-            tag: v2023.4
-            commit: 45b735dfddefe26a99b77e5a74e30d860713ac64
-#           x-checker-data:
-#             type: git
-#             tag-pattern: ^v(\d{4}\.\d{1,2})$
-          - type: git
-            url: https://github.com/KhronosGroup/SPIRV-Tools.git
-            tag: v2023.2
-            commit: 44d72a9b36702f093dd20815561a56778b2d181e
-            dest: third_party/spirv-tools
-            x-checker-data:
-              type: git
-              tag-pattern: ^v(\d{4}\.\d{1})$
-          - type: git
-            url: https://github.com/KhronosGroup/SPIRV-Headers.git
-            tag: sdk-1.3.250.1
-            commit: 268a061764ee69f09a477a695bf6a11ffe311b8d
-            dest: third_party/spirv-headers
-            x-checker-data:
-              type: git
-              tag-pattern: ^sdk-([\d.]+)$
-          - type: git
-            url: https://github.com/KhronosGroup/glslang.git
-            tag: 12.2.0
-            commit: d1517d64cfca91f573af1bf7341dc3a5113349c0
-            dest: third_party/glslang
-          
-          
-
-
-  - name: gpt4all
-    buildsystem: simple
-    build-commands:
-      - git submodule update --init --recursive
-      - mkdir build
-      - cmake  -S ./gpt4all-chat -B build -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF -DKOMPUTE_OPT_USE_BUILT_IN_FMT=OFF -DCMAKE_INSTALL_PREFIX=/app
-      - cmake --build build --config Release -- -j
-      - cmake --install build --prefix "/app"
-      - install -Dm644 logo.svg /app/share/icons/hicolor/scalable/apps/io.gpt4all.gpt4all.svg
-      - install -Dm644 io.gpt4all.gpt4all.desktop /app/share/applications/io.gpt4all.gpt4all.desktop
-      - install -Dm644 io.gpt4all.gpt4all.appdata.xml  /app/share/appdata/io.gpt4all.gpt4all.appdata.xml
-    sources:
-      - type: git
-        url: https://github.com/nomic-ai/gpt4all
-        tag: v2.4.19
-        commit: 84905aa28171545542fc653dbeca501ae5af383e
-      - type: file
-        url: https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/icons/logo.svg
-        sha256: 4c4e8476d0e2020585b69c6e2fc9e7d0cb12cbb36aa7b83c3a2e48ed4a9a424c
-      - type: file
-        path: io.gpt4all.gpt4all.desktop
-      - type: file
-        path: io.gpt4all.gpt4all.appdata.xml
-

gpt4all-chat/llm.cpp

@@ -9,7 +9,6 @@
 #include <QProcess>
 #include <QResource>
 #include <QSettings>
-#include <QDesktopServices>
 #include <fstream>
 
 class MyLLM: public LLM { };
@@ -61,10 +60,6 @@ bool LLM::hasSettingsAccess() const
 
 bool LLM::checkForUpdates() const
 {
-    #ifdef GPT4ALL_OFFLINE_INSTALLER
-    #pragma message "offline installer build will not check for updates!"
-    return QDesktopServices::openUrl(QUrl("https://gpt4all.io/"));
-    #else
     Network::globalInstance()->sendCheckForUpdates();
 
 #if defined(Q_OS_LINUX)
@@ -83,7 +78,6 @@ bool LLM::checkForUpdates() const
     }
 
     return QProcess::startDetached(fileName);
-    #endif
 }
 
 bool LLM::directoryExists(const QString &path) const

gpt4all-chat/main.qml

@@ -189,7 +189,7 @@ Window {
             + "causes include a bad file format, an incomplete or corrupted download, the wrong file "
             + "type, not enough system RAM or an incompatible model type. Here are some suggestions for resolving the problem:"
             + "<br><ul>"
-            + "<li>Ensure the model file has a compatible format and type"
+            + "<li>Ensure the model file has a compatible ggml format and type"
             + "<li>Check the model file is complete in the download folder"
             + "<li>You can find the download folder in the settings dialog"
             + "<li>If you've sideloaded the model ensure the file is not corrupt by checking md5sum"
@@ -1013,7 +1013,7 @@ Window {
             anchors.rightMargin: 30
             color: theme.mutedTextColor
             visible: currentChat.tokenSpeed !== ""
-            text: qsTr("Speed: ") + currentChat.tokenSpeed + "<br>" + qsTr("Device: ") + currentChat.device + currentChat.fallbackReason
+            text: qsTr("Speed: ") + currentChat.tokenSpeed + "<br>" + qsTr("Device: ") + currentChat.device
             font.pixelSize: theme.fontSizeLarge
         }
 

gpt4all-chat/metadata/models2.json

@@ -1,210 +0,0 @@
-[
-  {
-    "order": "a",
-    "md5sum": "48de9538c774188eb25a7e9ee024bbd3",
-    "name": "Mistral OpenOrca",
-    "filename": "mistral-7b-openorca.Q4_0.gguf",
-    "filesize": "4108927744",
-    "requires": "2.5.0",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "Mistral",
-    "systemPrompt": " ",
-    "description": "<strong>Best overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
-    "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.Q4_0.gguf"
-  },
-  {
-    "order": "b",
-    "md5sum": "97463be739b50525df56d33b26b00852",
-    "name": "Mistral Instruct",
-    "filename": "mistral-7b-instruct-v0.1.Q4_0.gguf",
-    "filesize": "4108916384",
-    "requires": "2.5.0",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "Mistral",
-    "systemPrompt": " ",
-    "description": "<strong>Best overall fast instruction following model</strong><br><ul><li>Fast responses</li><li>Trained by Mistral AI<li>Uncensored</li><li>Licensed for commercial use</li></ul>",
-    "url": "https://gpt4all.io/models/gguf/mistral-7b-instruct-v0.1.Q4_0.gguf",
-    "promptTemplate": "[INST] %1 [/INST]"
-  },
-  {
-    "order": "c",
-    "md5sum": "31cb6d527bd3bfb5e73c2e9dfbc75033",
-    "name": "GPT4All Falcon",
-    "filename": "gpt4all-falcon-q4_0.gguf",
-    "filesize": "4210419040",
-    "requires": "2.5.0",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "Falcon",
-    "systemPrompt": " ",
-    "description": "<strong>Very fast model with good quality</strong><br><ul><li>Fastest responses</li><li>Instruction based</li><li>Trained by TII<li>Finetuned by Nomic AI<li>Licensed for commercial use</ul>",
-    "url": "https://gpt4all.io/models/gguf/gpt4all-falcon-q4_0.gguf",
-    "promptTemplate": "### Instruction:\n%1\n### Response:\n"
-  },
-  {
-    "order": "e",
-    "md5sum": "5aff90007499bce5c64b1c0760c0b186",
-    "name": "Wizard v1.2",
-    "filename": "wizardlm-13b-v1.2.Q4_0.gguf",
-    "filesize": "7365834624",
-    "requires": "2.5.0",
-    "ramrequired": "16",
-    "parameters": "13 billion",
-    "quant": "q4_0",
-    "type": "LLaMA2",
-    "systemPrompt": " ",
-    "description": "<strong>Best overall larger model</strong><br><ul><li>Instruction based<li>Gives very long responses<li>Finetuned with only 1k of high-quality data<li>Trained by Microsoft and Peking University<li>Cannot be used commercially</ul",
-    "url": "https://gpt4all.io/models/gguf/wizardlm-13b-v1.2.Q4_0.gguf"
-  },
-  {
-    "order": "f",
-    "md5sum": "3d12810391d04d1153b692626c0c6e16",
-    "name": "Hermes",
-    "filename": "nous-hermes-llama2-13b.Q4_0.gguf",
-    "filesize": "7366062080",
-    "requires": "2.5.0",
-    "ramrequired": "16",
-    "parameters": "13 billion",
-    "quant": "q4_0",
-    "type": "LLaMA2",
-    "systemPrompt": " ",
-    "description": "<strong>Extremely good model</strong><br><ul><li>Instruction based<li>Gives long responses<li>Curated with 300,000 uncensored instructions<li>Trained by Nous Research<li>Cannot be used commercially</ul>",
-    "url": "https://gpt4all.io/models/gguf/nous-hermes-llama2-13b.Q4_0.gguf",
-    "promptTemplate": "### Instruction:\n%1\n### Response:\n"
-  },
-  {
-    "order": "g",
-    "md5sum": "40388eb2f8d16bb5d08c96fdfaac6b2c",
-    "name": "Snoozy",
-    "filename": "gpt4all-13b-snoozy-q4_0.gguf",
-    "filesize": "7365834624",
-    "requires": "2.5.0",
-    "ramrequired": "16",
-    "parameters": "13 billion",
-    "quant": "q4_0",
-    "type": "LLaMA",
-    "systemPrompt": " ",
-    "description": "<strong>Very good overall model</strong><br><ul><li>Instruction based<li>Based on the same dataset as Groovy<li>Slower than Groovy, with higher quality responses<li>Trained by Nomic AI<li>Cannot be used commercially</ul>",
-    "url": "https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf"
-  },
-  {
-    "order": "h",
-    "md5sum": "f5bc6a52f72efd9128efb2eeed802c86",
-    "name": "MPT Chat",
-    "filename": "mpt-7b-chat-q4_0.gguf",
-    "filesize": "3911522272",
-    "requires": "2.5.0",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "MPT",
-    "description": "<strong>Good model with novel architecture</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
-    "url": "https://gpt4all.io/models/gguf/mpt-7b-chat-q4_0.gguf",
-    "promptTemplate": "<|im_start|>user\n%1<|im_end|><|im_start|>assistant\n",
-    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
-  },
-  {
-    "order": "i",
-    "md5sum": "0e769317b90ac30d6e09486d61fefa26",
-    "name": "Mini Orca (Small)",
-    "filename": "orca-mini-3b-gguf2-q4_0.gguf",
-    "filesize": "1979946720",
-    "requires": "2.5.0",
-    "ramrequired": "4",
-    "parameters": "3 billion",
-    "quant": "q4_0",
-    "type": "OpenLLaMa",
-    "description": "<strong>Small version of new model with novel dataset</strong><br><ul><li>Instruction based<li>Explain tuned datasets<li>Orca Research Paper dataset construction approaches<li>Cannot be used commercially</ul>",
-    "url": "https://gpt4all.io/models/gguf/orca-mini-3b-gguf2-q4_0.gguf",
-    "promptTemplate": "### User:\n%1\n### Response:\n",
-    "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
-  },
-  {
-    "order": "j",
-    "md5sum": "51c627fac9062e208f9b386f105cbd48",
-    "disableGUI": "true",
-    "name": "Replit",
-    "filename": "replit-code-v1-3b-q4_0.gguf",
-    "filesize": "1532949760",
-    "requires": "2.5.0",
-    "ramrequired": "4",
-    "parameters": "3 billion",
-    "quant": "f16",
-    "type": "Replit",
-    "systemPrompt": " ",
-    "promptTemplate": "%1",
-    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>Licensed for commercial use</ul>",
-    "url": "https://gpt4all.io/models/gguf/replit-code-v1-3b-q4_0.gguf"
-  },
-  {
-    "order": "k",
-    "md5sum": "556fc3e13df42286997fb58e6f4c639f",
-    "disableGUI": "true",
-    "name": "Starcoder",
-    "filename": "starcoder-q4_0.gguf",
-    "filesize": "8987166880",
-    "requires": "2.5.0",
-    "ramrequired": "4",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "Starcoder",
-    "systemPrompt": " ",
-    "promptTemplate": "%1",
-    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based</ul>",
-    "url": "https://gpt4all.io/models/gguf/starcoder-q4_0.gguf"
-  },
-  {
-    "order": "l",
-    "md5sum": "e973dd26f0ffa6e46783feaea8f08c83",
-    "disableGUI": "true",
-    "name": "Rift coder",
-    "filename": "rift-coder-v0-7b-q4_0.gguf",
-    "filesize": "3825903776",
-    "requires": "2.5.0",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "LLaMA",
-    "systemPrompt": " ",
-    "promptTemplate": "%1",
-    "description": "Code completion model",
-    "url": "https://gpt4all.io/models/gguf/rift-coder-v0-7b-q4_0.gguf"
-  },
-  {
-    "order": "m",
-    "md5sum": "e479e6f38b59afc51a470d1953a6bfc7",
-    "disableGUI": "true",
-    "name": "SBert",
-    "filename": "all-MiniLM-L6-v2-f16.gguf",
-    "filesize": "45887744",
-    "requires": "2.5.0",
-    "ramrequired": "1",
-    "parameters": "1 million",
-    "quant": "f16",
-    "type": "Bert",
-    "systemPrompt": " ",
-    "description": "<strong>Sbert</strong><br><ul><li>For embeddings",
-    "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf"
-  },
-  {
-    "order": "n",
-    "md5sum": "919de4dd6f25351bcb0223790db1932d",
-    "name": "EM German Mistral",
-    "filename": "em_german_mistral_v01.Q4_0.gguf",
-    "filesize": "4108916352",
-    "requires": "2.5.0",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "Mistral",
-    "description": "<strong>Mistral-based model for German-language applications</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by ellamind<li>Finetuned on German instruction and chat data</a><li>Licensed for commercial use</ul>",
-    "url": "https://huggingface.co/TheBloke/em_german_mistral_v01-GGUF/resolve/main/em_german_mistral_v01.Q4_0.gguf",
-    "promptTemplate": "USER: %1 ASSISTANT: ",
-    "systemPrompt": "Du bist ein hilfreicher Assistent. "
-  }
-]

gpt4all-chat/metadata/release.json

@@ -75,7 +75,7 @@
 * resumable downloads for models
 * chat list in the drawer drop down
 * add/remove/rename chats
-* persist chats to disk and restore them with full context (WARNING: the average size of each chat on disk is ~1.5GB)
+* perist chats to disk and restore them with full context (WARNING: the average size of each chat on disk is ~1.5GB)
 * NOTE: to turn on the persistent chats feature you need to do so via the settings dialog as it is off by default
 * automatically rename chats using the AI after the first prompt/response pair
 * new usage statistics including more detailed hardware info to help debug problems on older hardware
@@ -524,7 +524,7 @@
     "version": "2.4.19",
     "notes":
 "
-* Fix a crash on systems with corrupted vulkan drivers or corrupted vulkan dlls
+* Fix a crasher on systems with corrupted vulkan drivers or corrupted vulkan dlls
 ",
     "contributors":
 "

gpt4all-chat/modellist.cpp

@@ -796,7 +796,7 @@ void ModelList::updateModelsFromDirectory()
                 QString filename = it.fileName();
 
                 // All files that end with .bin and have 'ggml' somewhere in the name
-                if (((filename.endsWith(".bin") || filename.endsWith(".gguf")) && (/*filename.contains("ggml") ||*/ filename.contains("gguf")) && !filename.startsWith("incomplete"))
+                if ((filename.endsWith(".bin") && filename.contains("ggml") && !filename.startsWith("incomplete"))
                     || (filename.endsWith(".txt") && filename.startsWith("chatgpt-"))) {
 
                     QString filePath = it.filePath();
@@ -834,14 +834,12 @@ void ModelList::updateModelsFromDirectory()
         processDirectory(localPath);
 }
 
-#define MODELS_VERSION 2
-
 void ModelList::updateModelsFromJson()
 {
 #if defined(USE_LOCAL_MODELSJSON)
-    QUrl jsonUrl("file://" + QDir::homePath() + QString("/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models%1.json").arg(MODELS_VERSION));
+    QUrl jsonUrl("file://" + QDir::homePath() + "/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models.json");
 #else
-    QUrl jsonUrl(QString("http://gpt4all.io/models/models%1.json").arg(MODELS_VERSION));
+    QUrl jsonUrl("http://gpt4all.io/models/models.json");
 #endif
     QNetworkRequest request(jsonUrl);
     QSslConfiguration conf = request.sslConfiguration();
@@ -883,9 +881,9 @@ void ModelList::updateModelsFromJsonAsync()
     emit asyncModelRequestOngoingChanged();
 
 #if defined(USE_LOCAL_MODELSJSON)
-    QUrl jsonUrl("file://" + QDir::homePath() + QString("/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models%1.json").arg(MODELS_VERSION));
+    QUrl jsonUrl("file://" + QDir::homePath() + "/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models.json");
 #else
-    QUrl jsonUrl(QString("http://gpt4all.io/models/models%1.json").arg(MODELS_VERSION));
+    QUrl jsonUrl("http://gpt4all.io/models/models.json");
 #endif
     QNetworkRequest request(jsonUrl);
     QSslConfiguration conf = request.sslConfiguration();

gpt4all-chat/modellist.h

@@ -115,7 +115,7 @@ private:
     double  m_repeatPenalty       = 1.18;
     int     m_repeatPenaltyTokens = 64;
     QString m_promptTemplate      = "### Human:\n%1\n### Assistant:\n";
-    QString m_systemPrompt        = "### System:\nYou are an AI assistant who gives a quality response to whatever humans ask of you.\n";
+    QString m_systemPrompt        = "### System:\nYou are an AI assistant who gives quality response to whatever humans ask of you.\n";
     friend class MySettings;
 };
 Q_DECLARE_METATYPE(ModelInfo)

gpt4all-chat/mysettings.cpp

@@ -10,7 +10,8 @@
 #include <QUrl>
 
 static int      default_threadCount         = std::min(4, (int32_t) std::thread::hardware_concurrency());
-static bool     default_saveChatsContext    = false;
+static bool     default_saveChats           = false;
+static bool     default_saveChatGPTChats    = true;
 static bool     default_serverChat          = false;
 static QString  default_userDefaultModel    = "Application default";
 static bool     default_forceMetal          = false;
@@ -102,7 +103,8 @@ void MySettings::restoreApplicationDefaults()
     setFontSize(default_fontSize);
     setDevice(default_device);
     setThreadCount(default_threadCount);
-    setSaveChatsContext(default_saveChatsContext);
+    setSaveChats(default_saveChats);
+    setSaveChatGPTChats(default_saveChatGPTChats);
     setServerChat(default_serverChat);
     setModelPath(defaultLocalModelsPath());
     setUserDefaultModel(default_userDefaultModel);
@@ -395,22 +397,40 @@ void MySettings::setThreadCount(int c)
     emit threadCountChanged();
 }
 
-bool MySettings::saveChatsContext() const
+bool MySettings::saveChats() const
 {
     QSettings setting;
     setting.sync();
-    return setting.value("saveChatsContext", default_saveChatsContext).toBool();
+    return setting.value("saveChats", default_saveChats).toBool();
 }
 
-void MySettings::setSaveChatsContext(bool b)
+void MySettings::setSaveChats(bool b)
 {
-    if (saveChatsContext() == b)
+    if (saveChats() == b)
         return;
 
     QSettings setting;
-    setting.setValue("saveChatsContext", b);
+    setting.setValue("saveChats", b);
     setting.sync();
-    emit saveChatsContextChanged();
+    emit saveChatsChanged();
+}
+
+bool MySettings::saveChatGPTChats() const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value("saveChatGPTChats", default_saveChatGPTChats).toBool();
+}
+
+void MySettings::setSaveChatGPTChats(bool b)
+{
+    if (saveChatGPTChats() == b)
+        return;
+
+    QSettings setting;
+    setting.setValue("saveChatGPTChats", b);
+    setting.sync();
+    emit saveChatGPTChatsChanged();
 }
 
 bool MySettings::serverChat() const

gpt4all-chat/mysettings.h

@@ -10,7 +10,8 @@ class MySettings : public QObject
 {
     Q_OBJECT
     Q_PROPERTY(int threadCount READ threadCount WRITE setThreadCount NOTIFY threadCountChanged)
-    Q_PROPERTY(bool saveChatsContext READ saveChatsContext WRITE setSaveChatsContext NOTIFY saveChatsContextChanged)
+    Q_PROPERTY(bool saveChats READ saveChats WRITE setSaveChats NOTIFY saveChatsChanged)
+    Q_PROPERTY(bool saveChatGPTChats READ saveChatGPTChats WRITE setSaveChatGPTChats NOTIFY saveChatGPTChatsChanged)
     Q_PROPERTY(bool serverChat READ serverChat WRITE setServerChat NOTIFY serverChatChanged)
     Q_PROPERTY(QString modelPath READ modelPath WRITE setModelPath NOTIFY modelPathChanged)
     Q_PROPERTY(QString userDefaultModel READ userDefaultModel WRITE setUserDefaultModel NOTIFY userDefaultModelChanged)
@@ -63,8 +64,10 @@ public:
     // Application settings
     int threadCount() const;
     void setThreadCount(int c);
-    bool saveChatsContext() const;
-    void setSaveChatsContext(bool b);
+    bool saveChats() const;
+    void setSaveChats(bool b);
+    bool saveChatGPTChats() const;
+    void setSaveChatGPTChats(bool b);
     bool serverChat() const;
     void setServerChat(bool b);
     QString modelPath() const;
@@ -119,7 +122,8 @@ Q_SIGNALS:
     void promptTemplateChanged(const ModelInfo &model);
     void systemPromptChanged(const ModelInfo &model);
     void threadCountChanged();
-    void saveChatsContextChanged();
+    void saveChatsChanged();
+    void saveChatGPTChatsChanged();
     void serverChatChanged();
     void modelPathChanged();
     void userDefaultModelChanged();

gpt4all-chat/network.cpp

@@ -317,6 +317,16 @@ void Network::sendNetworkToggled(bool isActive)
     sendMixpanelEvent("network_toggled", QVector<KeyValue>{kv});
 }
 
+void Network::sendSaveChatsToggled(bool isActive)
+{
+    if (!MySettings::globalInstance()->networkUsageStatsActive())
+        return;
+    KeyValue kv;
+    kv.key = QString("isActive");
+    kv.value = QJsonValue(isActive);
+    sendMixpanelEvent("savechats_toggled", QVector<KeyValue>{kv});
+}
+
 void Network::sendNewChat(int count)
 {
     if (!MySettings::globalInstance()->networkUsageStatsActive())

gpt4all-chat/network.h

@@ -38,6 +38,7 @@ public Q_SLOTS:
     void sendDownloadFinished(const QString &model, bool success);
     Q_INVOKABLE void sendSettingsDialog();
     Q_INVOKABLE void sendNetworkToggled(bool active);
+    Q_INVOKABLE void sendSaveChatsToggled(bool active);
     Q_INVOKABLE void sendNewChat(int count);
     Q_INVOKABLE void sendRemoveChat();
     Q_INVOKABLE void sendRenameChat();

gpt4all-chat/qml/ApplicationSettings.qml

@@ -234,35 +234,53 @@ MySettingsTab {
             Accessible.description: ToolTip.text
         }
         Label {
-            id: saveChatsContextLabel
-            text: qsTr("Save chats context to disk:")
+            id: saveChatsLabel
+            text: qsTr("Save chats to disk:")
             color: theme.textColor
             font.pixelSize: theme.fontSizeLarge
             Layout.row: 7
             Layout.column: 0
         }
         MyCheckBox {
-            id: saveChatsContextBox
+            id: saveChatsBox
             Layout.row: 7
             Layout.column: 1
-            checked: MySettings.saveChatsContext
+            checked: MySettings.saveChats
             onClicked: {
-                MySettings.saveChatsContext = !MySettings.saveChatsContext
+                Network.sendSaveChatsToggled(saveChatsBox.checked);
+                MySettings.saveChats = !MySettings.saveChats
             }
             ToolTip.text: qsTr("WARNING: Saving chats to disk can be ~2GB per chat")
             ToolTip.visible: hovered
         }
+        Label {
+            id: saveChatGPTChatsLabel
+            text: qsTr("Save ChatGPT chats to disk:")
+            color: theme.textColor
+            font.pixelSize: theme.fontSizeLarge
+            Layout.row: 8
+            Layout.column: 0
+        }
+        MyCheckBox {
+            id: saveChatGPTChatsBox
+            Layout.row: 8
+            Layout.column: 1
+            checked: MySettings.saveChatGPTChats
+            onClicked: {
+                MySettings.saveChatGPTChats = !MySettings.saveChatGPTChats
+            }
+        }
         Label {
             id: serverChatLabel
             text: qsTr("Enable API server:")
             color: theme.textColor
             font.pixelSize: theme.fontSizeLarge
-            Layout.row: 8
+            Layout.row: 9
             Layout.column: 0
         }
         MyCheckBox {
             id: serverChatBox
-            Layout.row: 8
+            Layout.row: 9
             Layout.column: 1
             checked: MySettings.serverChat
             onClicked: {
@@ -272,7 +290,7 @@ MySettingsTab {
             ToolTip.visible: hovered
         }
         Rectangle {
-            Layout.row: 9
+            Layout.row: 10
             Layout.column: 0
             Layout.columnSpan: 3
             Layout.fillWidth: true

gpt4all-chat/qml/LocalDocsSettings.qml

@@ -242,13 +242,13 @@ MySettingsTab {
             Layout.column: 0
             color: theme.textColor
             font.pixelSize: theme.fontSizeLarge
-            text: qsTr("Max document snippets per prompt:")
+            text: qsTr("Document snippets per prompt:")
         }
 
         MyTextField {
             Layout.row: 2
             Layout.column: 1
-            ToolTip.text: qsTr("Max best N matches of retrieved document snippets to add to the context for prompt.\nNOTE: larger numbers increase likelihood of factual responses, but also result in slower generation.")
+            ToolTip.text: qsTr("Best N matches of retrieved document snippets to add to the context for prompt.\nNOTE: larger numbers increase likelihood of factual responses, but also result in slower generation.")
             ToolTip.visible: hovered
             text: MySettings.localDocsRetrievalSize
             validator: IntValidator {

gpt4all-chat/qml/ModelDownloaderDialog.qml

@@ -47,7 +47,7 @@ MyDialog {
             Layout.fillHeight: true
             horizontalAlignment: Qt.AlignHCenter
             verticalAlignment: Qt.AlignVCenter
-            text: qsTr("Network error: could not retrieve http://gpt4all.io/models/models2.json")
+            text: qsTr("Network error: could not retrieve http://gpt4all.io/models/models.json")
             font.pixelSize: theme.fontSizeLarge
             color: theme.mutedTextColor
         }

gpt4all-training/train.py

@@ -75,7 +75,7 @@ def train(accelerator, config):
         else DummyOptim
     )
 
-    # karpathy doesn't decay embedding, maybe we should exclude
+    # karpathy doesn't decay embeddding, maybe we should exclude
     # https://github.com/karpathy/minGPT/commit/bbbdac74fa9b2e55574d70056163ffbae42310c1#diff-2075fa9c224b395be5bda85544dd36572b59c76c54562819eadadbf268602834R157s
     optimizer = optimizer_cls(model.parameters(), lr=config["lr"], weight_decay=config["weight_decay"])