Speculative fix for build on mac.

Signed-off-by: Aaron Miller <apage43@ninjawhale.com>

.circleci/continue_config.yml

@@ -27,7 +27,176 @@ jobs:
       - image: circleci/python:3.7
     steps:
       - run: echo "CircleCI pipeline triggered"
-
+  build-offline-chat-installer-macos:
+    macos:
+      xcode: 14.0.0
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - macos-qt-cache_v2
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
+              hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: macos-qt-cache_v2
+          paths:
+            - ~/Qt
+      - run:
+          name: Build
+          command: |
+            mkdir build
+            cd build
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
+              -DCMAKE_GENERATOR:STRING=Ninja \
+              -DBUILD_UNIVERSAL=ON \
+              -DMACDEPLOYQT=~/Qt/6.5.1/macos/bin/macdeployqt \
+              -DGPT4ALL_OFFLINE_INSTALLER=ON \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
+              -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
+              -S ../gpt4all-chat \
+              -B .
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package
+            mkdir upload
+            cp gpt4all-installer-* upload
+      - store_artifacts:
+          path: build/upload
+  build-offline-chat-installer-linux:
+    machine:
+      image: ubuntu-2204:2023.04.2
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - linux-qt-cache
+      - run:
+          name: Setup Linux and Dependencies
+          command: |
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
+              chmod +x qt-unified-linux-x64-4.6.0-online.run
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: linux-qt-cache
+          paths:
+            - ~/Qt
+      - run:
+          name: Build linuxdeployqt
+          command: |
+            git clone https://github.com/nomic-ai/linuxdeployqt
+            cd linuxdeployqt && qmake && sudo make install
+      - run:
+          name: Build
+          command: |
+            set -eo pipefail
+            export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
+            mkdir build
+            cd build
+            mkdir upload
+            ~/Qt/Tools/CMake/bin/cmake -DGPT4ALL_OFFLINE_INSTALLER=ON -DCMAKE_BUILD_TYPE=Release -S ../gpt4all-chat -B .
+            ~/Qt/Tools/CMake/bin/cmake --build . --target all
+            ~/Qt/Tools/CMake/bin/cmake --build . --target install
+            ~/Qt/Tools/CMake/bin/cmake --build . --target package
+            cp gpt4all-installer-* upload
+      - store_artifacts:
+          path: build/upload
+  build-offline-chat-installer-windows:
+    machine:
+      image: 'windows-server-2019-vs2019:2022.08.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - windows-qt-cache
+      - run:
+          name: Installing Qt
+          command: |
+            if (-not (Test-Path C:\Qt)) {
+              Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+            }
+      - save_cache:  # this is the new step to save cache
+          key: windows-qt-cache
+          paths:
+            - C:\Qt
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Build
+          command: |
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
+            $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
+            $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.6\bin"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
+            mkdir build
+            cd build
+            & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
+              "-DCMAKE_GENERATOR:STRING=Ninja" `
+              "-DCMAKE_BUILD_TYPE=Release" `
+              "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
+              "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
+              "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
+              "-DGPT4ALL_OFFLINE_INSTALLER=ON" `
+              "-S ..\gpt4all-chat" `
+              "-B ."
+            & "C:\Qt\Tools\Ninja\ninja.exe"
+            & "C:\Qt\Tools\Ninja\ninja.exe" install
+            & "C:\Qt\Tools\Ninja\ninja.exe" package
+            mkdir upload
+            copy gpt4all-installer-win64.exe upload
+      - store_artifacts:
+          path: build/upload
   build-gpt4all-chat-linux:
     machine:
       image: ubuntu-2204:2023.04.2
@@ -163,6 +332,7 @@ jobs:
             cd build
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
               -DCMAKE_GENERATOR:STRING=Ninja \
+              -DBUILD_UNIVERSAL=ON \
               -DCMAKE_BUILD_TYPE=Release \
               -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
               -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
@@ -828,6 +998,20 @@ workflows:
     when: << pipeline.parameters.run-default-workflow >>
     jobs:
       - default-job
+  build-chat-offline-installers:
+    when: << pipeline.parameters.run-chat-workflow >>
+    jobs:
+      - hold:
+          type: approval
+      - build-offline-chat-installer-macos:
+          requires:
+            - hold
+      - build-offline-chat-installer-windows:
+          requires:
+            - hold
+      - build-offline-chat-installer-linux:
+          requires:
+            - hold
   build-and-test-gpt4all-chat:
     when: << pipeline.parameters.run-chat-workflow >>
     jobs:

.gitmodules

@@ -1,9 +1,4 @@
-[submodule "llama.cpp-230519"]
-	path = gpt4all-backend/llama.cpp-230519
-	url = https://github.com/ggerganov/llama.cpp.git
-[submodule "llama.cpp-230511"]
-	path = gpt4all-backend/llama.cpp-230511
-	url = https://github.com/nomic-ai/llama.cpp
 [submodule "llama.cpp-mainline"]
 	path = gpt4all-backend/llama.cpp-mainline
 	url = https://github.com/nomic-ai/llama.cpp.git
+	branch = gguf

README.md

@@ -2,6 +2,8 @@
 
 <p align="center">Open-source assistant-style large language models that run locally on your CPU</p>
 
+<p align="center"><strong>New</strong>: Now with Nomic Vulkan Universal GPU support. <a href="https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan">Learn more</a>.</p>
+
 <p align="center">
 <a href="https://gpt4all.io">GPT4All Website</a>
 </p>

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -26,7 +26,7 @@ router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
 async def list_engines():
     '''
     List all available GPT4All models from
-    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json
+    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json
     '''
     raise NotImplementedError()
     return ListEnginesResponse(data=[])

gpt4all-backend/CMakeLists.txt

@@ -20,7 +20,7 @@ endif()
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 
 set(LLMODEL_VERSION_MAJOR 0)
-set(LLMODEL_VERSION_MINOR 4)
+set(LLMODEL_VERSION_MINOR 5)
 set(LLMODEL_VERSION_PATCH 0)
 set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
 project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
@@ -97,35 +97,19 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    add_library(replit-mainline-${BUILD_VARIANT} SHARED
-    replit.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-    target_compile_definitions(replit-mainline-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
-    prepare_target(replit-mainline llama-mainline)
-
     if (NOT LLAMA_METAL)
-# FIXME: These need to be forward ported to latest ggml
-#        add_library(gptj-${BUILD_VARIANT} SHARED
-#            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-#        prepare_target(gptj ggml-230511)
+        add_library(gptj-${BUILD_VARIANT} SHARED
+            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        prepare_target(gptj llama-mainline)
 
-        add_library(falcon-${BUILD_VARIANT} SHARED
-            falcon.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        target_compile_definitions(falcon-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
-        prepare_target(falcon llama-mainline)
-# FIXME: These need to be forward ported to latest ggml
-#        add_library(mpt-${BUILD_VARIANT} SHARED
-#            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-#        prepare_target(mpt ggml-230511)
+        add_library(mpt-${BUILD_VARIANT} SHARED
+            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        prepare_target(mpt llama-mainline)
 
         add_library(bert-${BUILD_VARIANT} SHARED
             bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
         target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
         prepare_target(bert llama-mainline)
-
-        add_library(starcoder-${BUILD_VARIANT} SHARED
-            starcoder.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        target_compile_definitions(starcoder-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
-        prepare_target(starcoder llama-mainline)
     endif()
 endforeach()
 
@@ -134,6 +118,8 @@ add_library(llmodel
     llmodel_c.h llmodel_c.cpp
     dlhandle.h
 )
+target_link_libraries(llmodel PRIVATE ggml-mainline-default)
+target_compile_definitions(llmodel PRIVATE GGML_BUILD_VARIANT="default")
 target_compile_definitions(llmodel PRIVATE LIB_FILE_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}")
 
 set_target_properties(llmodel PROPERTIES

gpt4all-backend/bert.cpp

@@ -4,10 +4,10 @@
 #include "ggml.h"
 
 #include <cassert>
+#include <cinttypes>
 #include <cmath>
 #include <cstdio>
 #include <cstring>
-#include <fstream>
 #include <map>
 #include <string>
 #include <vector>
@@ -34,7 +34,6 @@ struct bert_hparams
     int32_t n_intermediate = 1536;
     int32_t n_head = 12;
     int32_t n_layer = 6;
-    int32_t f16 = 1;
 };
 
 struct bert_layer
@@ -88,7 +87,6 @@ struct bert_model
     std::vector<bert_layer> layers;
 
     struct ggml_context *ctx;
-    std::map<std::string, struct ggml_tensor *> tensors;
 };
 
 // Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
@@ -345,7 +343,7 @@ void bert_eval(
 
     // embd norm
     {
-        inpL = ggml_norm(ctx0, inpL);
+        inpL = ggml_norm(ctx0, inpL, 1e-5f);
 
         inpL = ggml_add(ctx0,
                         ggml_mul(ctx0,
@@ -406,7 +404,7 @@ void bert_eval(
 
         // attention norm
         {
-            cur = ggml_norm(ctx0, cur);
+            cur = ggml_norm(ctx0, cur, 1e-5f);
 
             cur = ggml_add(ctx0,
                            ggml_mul(ctx0,
@@ -432,7 +430,7 @@ void bert_eval(
 
         // output norm
         {
-            cur = ggml_norm(ctx0, cur);
+            cur = ggml_norm(ctx0, cur, 1e-5f);
 
             cur = ggml_add(ctx0,
                            ggml_mul(ctx0,
@@ -482,7 +480,6 @@ void bert_eval(
 //
 
 void bert_free(bert_ctx * ctx) {
-    ggml_free(ctx->model.ctx);
     delete ctx;
 }
 
@@ -492,63 +489,130 @@ struct bert_ctx * bert_load_from_file(const char *fname)
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname);
 #endif
 
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin)
-    {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname);
-        return nullptr;
-    }
-
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *)&magic, sizeof(magic));
-        if (magic != 0x62657274)
-        {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname);
-            return nullptr;
-        }
-    }
-
     bert_ctx * new_bert = new bert_ctx;
     bert_model & model = new_bert->model;
     bert_vocab & vocab = new_bert->vocab;
 
+    struct gguf_init_params params = {
+        /*.no_alloc = */ false,
+        /*.ctx      = */ &model.ctx,
+    };
+    gguf_context *ggufctx = gguf_init_from_file(fname, params);
+    if (!ggufctx) {
+        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
+        return nullptr;
+    }
+
+    printf("%s: gguf version     = %d\n", __func__, gguf_get_version(ggufctx));
+    printf("%s: gguf alignment   = %zu\n", __func__, gguf_get_alignment(ggufctx));
+    printf("%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx));
+
+    // print some standard metadata
+    {
+        int keyidx;
+
+        keyidx = gguf_find_key(ggufctx, "general.name");
+        if (keyidx != -1) { printf("%s: model name           = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.description");
+        if (keyidx != -1) { printf("%s: model description    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.author");
+        if (keyidx != -1) { printf("%s: model author         = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.license");
+        if (keyidx != -1) { printf("%s: model license        = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.architecture");
+        if (keyidx != -1) { printf("%s: model architecture   = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.file_type");
+        if (keyidx != -1) { printf("%s: model file type      = %" PRIu32 "\n", __func__, gguf_get_val_u32(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "gptneox.tensor_data_layout");
+        if (keyidx != -1) { printf("%s: model data layout    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.source.huggingface.repository");
+        if (keyidx != -1) { printf("%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+    }
+
+    // check required metadata
+    {
+        // check model architecture kv
+        int keyidx = gguf_find_key(ggufctx, "general.architecture");
+        if (keyidx == -1) {
+            fprintf(stderr, "%s: gguf model architecture not found!\n", __func__);
+            return nullptr;
+        }
+        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
+            fprintf(stderr, "%s: model architecture not supported!\n", __func__);
+            return nullptr;
+        }
+    }
+
     // load hparams
     {
         auto &hparams = model.hparams;
 
-        fin.read((char *)&hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *)&hparams.n_max_tokens, sizeof(hparams.n_max_tokens));
-        fin.read((char *)&hparams.n_embd, sizeof(hparams.n_embd));
-        fin.read((char *)&hparams.n_intermediate, sizeof(hparams.n_intermediate));
-        fin.read((char *)&hparams.n_head, sizeof(hparams.n_head));
-        fin.read((char *)&hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *)&hparams.f16, sizeof(hparams.f16));
+        bool ok = false;
+        int keyidx;
+
+        do {
+            keyidx = gguf_find_key(ggufctx, "bert.context_length");
+            if (keyidx == -1) { break; }
+            hparams.n_max_tokens = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.embedding_length");
+            if (keyidx == -1) { break; }
+            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.feed_forward_length");
+            if (keyidx == -1) { break; }
+            hparams.n_intermediate = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.attention.head_count");
+            if (keyidx == -1) { break; }
+            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.block_count");
+            if (keyidx == -1) { break; }
+            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
+
+            ok = true;
+        } while (false);
+
+        if (!ok) {
+            fprintf(stderr, "%s: required hparam missing!\n", __func__);
+            return nullptr;
+        }
 
 #if defined(DEBUG_BERT)
-        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
         printf("%s: n_max_tokens   = %d\n", __func__, hparams.n_max_tokens);
-        printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_intermediate  = %d\n", __func__, hparams.n_intermediate);
-        printf("%s: n_head  = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
-        printf("%s: f16     = %d\n", __func__, hparams.f16);
+        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
+        printf("%s: n_intermediate = %d\n", __func__, hparams.n_intermediate);
+        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
+        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
 #endif
     }
 
     // load vocab
     {
-        int32_t n_vocab = model.hparams.n_vocab;
+        auto & hparams = model.hparams;
 
-        std::string word;
-        for (int i = 0; i < n_vocab; i++)
-        {
-            uint32_t len;
-            fin.read((char *)&len, sizeof(len));
+        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
+        if (keyidx == -1) {
+            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
+            return nullptr;
+        }
+        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
+            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
+            return nullptr;
+        }
 
-            word.resize(len);
-            fin.read((char *)word.data(), len);
+        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
+        if (tokens_keyidx == -1) {
+            fprintf(stderr, "%s: bert tokenizer vocab not found!\n", __func__);
+            return nullptr;
+        }
+
+        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
+        printf("%s: bert tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
+
+        for (int i = 0; i < hparams.n_vocab; i++) {
+            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
 
             if (word[0] == '#' && word[1] == '#')
             {
@@ -564,290 +628,52 @@ struct bert_ctx * bert_load_from_file(const char *fname)
         }
     }
 
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = GGML_TYPE_COUNT;
-    switch (model.hparams.f16)
-    {
-    case 0:
-        wtype = GGML_TYPE_F32;
-        break;
-    case 1:
-        wtype = GGML_TYPE_F16;
-        break;
-    case 2:
-        wtype = GGML_TYPE_Q4_0;
-        break;
-    case 3:
-        wtype = GGML_TYPE_Q4_1;
-        break;
-    default:
-    {
-        fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
-                __func__, fname, model.hparams.f16);
-        bert_free(new_bert);
-        return nullptr;
-    }
-    }
-
     auto &ctx = model.ctx;
 
-    size_t model_mem_req = 0;
-
-    {
-        const auto &hparams = model.hparams;
-
-        const int n_embd = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_max_tokens = hparams.n_max_tokens;
-        const int n_intermediate = hparams.n_intermediate;
-        const int n_vocab = hparams.n_vocab;
-
-        // Calculate size requirements
-
-        model_mem_req += n_embd * n_vocab * ggml_type_sizef(wtype); // word_embeddings
-        model_mem_req += n_embd * 2 * ggml_type_sizef(wtype); // token_type_embeddings
-        model_mem_req += n_embd * n_max_tokens * ggml_type_sizef(wtype); // position_embeddings
-
-        model_mem_req += 2 * n_embd * ggml_type_sizef(GGML_TYPE_F32); // ln_e_*
-
-        model_mem_req += 4 * n_layer * (n_embd * ggml_type_sizef(GGML_TYPE_F32)); // ln_*
-
-        model_mem_req += 4 * n_layer * (n_embd * n_embd * ggml_type_sizef(wtype)); // kqvo weights
-        model_mem_req += 4 * n_layer * (n_embd * ggml_type_sizef(GGML_TYPE_F32)); // kqvo bias
-
-        model_mem_req += 2 * n_layer * (n_embd * n_intermediate * ggml_type_sizef(wtype)); // ff_*_w
-        model_mem_req += n_layer * (n_intermediate * ggml_type_sizef(GGML_TYPE_F32)); // ff_i_b
-        model_mem_req += n_layer * (n_embd * ggml_type_sizef(GGML_TYPE_F32)); // ff_o_b
-
-        model_mem_req += (5 + 16 * n_layer) * ggml_tensor_overhead(); // object overhead
-
 #if defined(DEBUG_BERT)
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, model_mem_req / (1024.0 * 1024.0));
+    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ggml_get_mem_size(ctx) / (1024.0 * 1024.0));
 #endif
-    }
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size = model_mem_req,
-            .mem_buffer = NULL,
-            .no_alloc = false,
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx)
-        {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            bert_free(new_bert);
-            return nullptr;
-        }
-    }
 
     // prepare memory for the weights
     {
-        const auto &hparams = model.hparams;
-
-        const int n_embd = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_intermediate = hparams.n_intermediate;
-        const int n_max_tokens = hparams.n_max_tokens;
-        const int n_vocab = hparams.n_vocab;
-
+        const int n_layer = model.hparams.n_layer;
         model.layers.resize(n_layer);
 
-        model.word_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
-        model.token_type_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, 2);
-        model.position_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_max_tokens);
+        model.word_embeddings       = ggml_get_tensor(ctx, "token_embd.weight");
+        model.token_type_embeddings = ggml_get_tensor(ctx, "token_types.weight");
+        model.position_embeddings   = ggml_get_tensor(ctx, "position_embd.weight");
+        model.ln_e_w                = ggml_get_tensor(ctx, "output_norm.weight");
+        model.ln_e_b                = ggml_get_tensor(ctx, "output_norm.bias");
 
-        model.ln_e_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.ln_e_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-        // map by name
-        model.tensors["embeddings.word_embeddings.weight"] = model.word_embeddings;
-        model.tensors["embeddings.token_type_embeddings.weight"] = model.token_type_embeddings;
-        model.tensors["embeddings.position_embeddings.weight"] = model.position_embeddings;
-
-        model.tensors["embeddings.LayerNorm.weight"] = model.ln_e_w;
-        model.tensors["embeddings.LayerNorm.bias"] = model.ln_e_b;
+        auto name = [](int i, std::string n) {
+            static std::string key;
+            key = "blk." + std::to_string(i) + "." + n;
+            return key.c_str();
+        };
 
         for (int i = 0; i < n_layer; ++i)
         {
             auto &layer = model.layers[i];
 
-            layer.ln_att_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.ln_att_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.ln_out_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.ln_out_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            layer.q_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
-            layer.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.k_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
-            layer.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.v_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
-            layer.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.o_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
-            layer.o_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            layer.ff_i_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_intermediate);
-            layer.ff_i_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_intermediate);
-
-            layer.ff_o_w = ggml_new_tensor_2d(ctx, wtype, n_intermediate, n_embd);
-            layer.ff_o_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            // map by name
-
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.query.weight"] = layer.q_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.query.bias"] = layer.q_b;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.key.weight"] = layer.k_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.key.bias"] = layer.k_b;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.value.weight"] = layer.v_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.self.value.bias"] = layer.v_b;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.LayerNorm.weight"] = layer.ln_att_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.LayerNorm.bias"] = layer.ln_att_b;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.dense.weight"] = layer.o_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".attention.output.dense.bias"] = layer.o_b;
-
-            model.tensors["encoder.layer." + std::to_string(i) + ".intermediate.dense.weight"] = layer.ff_i_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".intermediate.dense.bias"] = layer.ff_i_b;
-
-            model.tensors["encoder.layer." + std::to_string(i) + ".output.LayerNorm.weight"] = layer.ln_out_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".output.LayerNorm.bias"] = layer.ln_out_b;
-            model.tensors["encoder.layer." + std::to_string(i) + ".output.dense.weight"] = layer.ff_o_w;
-            model.tensors["encoder.layer." + std::to_string(i) + ".output.dense.bias"] = layer.ff_o_b;
+            layer.ln_att_w = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
+            layer.ln_att_b = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
+            layer.ln_out_w = ggml_get_tensor(ctx, name(i, "ffn_norm.weight"));
+            layer.ln_out_b = ggml_get_tensor(ctx, name(i, "ffn_norm.bias"));
+            layer.q_w      = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
+            layer.q_b      = ggml_get_tensor(ctx, name(i, "attn_q.bias"));
+            layer.k_w      = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
+            layer.k_b      = ggml_get_tensor(ctx, name(i, "attn_k.bias"));
+            layer.v_w      = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
+            layer.v_b      = ggml_get_tensor(ctx, name(i, "attn_v.bias"));
+            layer.o_w      = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
+            layer.o_b      = ggml_get_tensor(ctx, name(i, "attn_output.bias"));
+            layer.ff_i_w   = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
+            layer.ff_i_b   = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
+            layer.ff_o_w   = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
+            layer.ff_o_b   = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
         }
     }
 
-    // load weights
-    {
-        int n_tensors = 0;
-#if defined(DEBUG_BERT)
-        size_t total_size = 0;
-#endif
-
-#if defined(DEBUG_BERT)
-        printf("%s: ", __func__);
-#endif
-
-        while (true)
-        {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ftype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ftype), sizeof(ftype));
-
-            if (fin.eof())
-            {
-                break;
-            }
-
-            int64_t nelements = 1;
-            int64_t ne[2] = {1, 1};
-            for (int i = 0; i < n_dims; ++i)
-            {
-                int32_t ne_cur;
-                fin.read(reinterpret_cast<char *>(&ne_cur), sizeof(ne_cur));
-                ne[i] = ne_cur;
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-
-            if (model.tensors.find(name.data()) == model.tensors.end())
-            {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                bert_free(new_bert);
-                return nullptr;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements)
-            {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                bert_free(new_bert);
-                return nullptr;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1])
-            {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%ld, %ld], expected [%ld, %ld]\n",
-                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
-                bert_free(new_bert);
-                return nullptr;
-            }
-
-#if defined(DEBUG_BERT)
-            static const char *ftype_str[] = {
-                "f32",
-                "f16",
-                "q4_0",
-                "q4_1",
-            };
-            printf("%24s - [%5ld, %5ld], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ftype_str[ftype], ggml_nbytes(tensor) / 1024.0 / 1024.0, ggml_nbytes(tensor));
-#endif
-
-            size_t bpe = 0;
-
-            switch (ftype)
-            {
-            case 0:
-                bpe = ggml_type_size(GGML_TYPE_F32);
-                break;
-            case 1:
-                bpe = ggml_type_size(GGML_TYPE_F16);
-                break;
-            case 2:
-                bpe = ggml_type_size(GGML_TYPE_Q4_0);
-                assert(ne[0] % 64 == 0);
-                break;
-            case 3:
-                bpe = ggml_type_size(GGML_TYPE_Q4_1);
-                assert(ne[0] % 64 == 0);
-                break;
-            default:
-            {
-                fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype);
-                bert_free(new_bert);
-                return nullptr;
-            }
-            };
-
-            if ((nelements * bpe) / ggml_blck_size(tensor->type) != ggml_nbytes(tensor))
-            {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %lu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements * bpe);
-                bert_free(new_bert);
-                return nullptr;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-#if defined(DEBUG_BERT)
-            // printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
-            total_size += ggml_nbytes(tensor);
-#endif
-
-            if (++n_tensors % 8 == 0)
-            {
-#if defined(DEBUG_BERT)
-                printf(".");
-                fflush(stdout);
-#endif
-            }
-        }
-
-#if defined(DEBUG_BERT)
-        printf(" done\n");
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size / 1024.0 / 1024.0, n_tensors);
-#endif
-    }
-
-    fin.close();
-
     // Calculate space requirements for setting up context buffers later
     {
         bert_vocab_id tokens[] = {0, 1, 2, 3};
@@ -1019,6 +845,16 @@ const std::vector<LLModel::Token> &Bert::endTokens() const
     return out;
 }
 
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != GGUF_TYPE_STRING) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -1038,13 +874,21 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    if (magic != 0x62657274) {
-         return false;
-    }
-    return true;
+DLL_EXPORT bool magic_match(const char * fname) {
+    struct ggml_context * ctx_meta = NULL;
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ &ctx_meta,
+    };
+    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
+    if (!ctx_gguf)
+        return false;
+
+    bool isValid = gguf_get_version(ctx_gguf) <= 2;
+    isValid = isValid && get_arch_name(ctx_gguf) == "bert";
+
+    gguf_free(ctx_gguf);
+    return isValid;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/falcon.cpp

@@ -1,985 +0,0 @@
-#include "ggml.h"
-#define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "falcon_impl.h"
-#include "llama.h"
-#include "llama-util.h"
-#include "utils.h"
-#include "llmodel_shared.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <iostream>
-#include <sstream>
-
-namespace {
-const char *modelType_ = "Falcon";
-}
-
-// commented out 40B support as it presently would require forking ggml/llama.cpp
-// can re-add once mainline ggml supports it
-
-#define FALCON_MAGIC 0x67676a74
-
-// default hparams (Falcon 7B)
-struct falcon_hparams {
-    int32_t n_vocab = 65024;
-    int32_t n_embd  = 4544;
-    int32_t n_head  = 71;
-    int32_t n_head_kv = 1;
-    int32_t n_layer = 32;
-    int32_t falcon_version = 7; // 7 for Falcon-7B, 40 for Falcon-40B
-    int32_t ftype   = 1;
-    int32_t n_ctx   = 2048;
-};
-
-struct falcon_layer {
-    // normalization
-    struct ggml_tensor* input_layernorm;
-    struct ggml_tensor* input_layernorm_b;
-    //struct ggml_tensor* attention_norm;    // Falcon-40B only
-    //struct ggml_tensor* attention_norm_b;  // Falcon-40B only
-
-    // attention
-    struct ggml_tensor* query_key_value;
-    struct ggml_tensor* wo;
-
-    // ff
-    struct ggml_tensor* ffn_up;
-    struct ggml_tensor* ffn_down;
-};
-
-struct falcon_model {
-    falcon_hparams hparams;
-
-    struct ggml_tensor* tok_embeddings;
-    struct ggml_tensor* output_norm;
-    struct ggml_tensor* output_norm_b;
-    struct ggml_tensor* lm_head;
-
-    std::vector<falcon_layer> layers;
-
-    // key + value memory
-    llm_kv_cache kv_self;
-
-    struct ggml_context* ctx;
-    std::map<std::string, struct ggml_tensor*> tensors;
-
-    llm_buffer eval_buf;
-    llm_buffer work_buf;
-    llm_buffer scr0_buf;
-    llm_buffer scr1_buf;
-};
-
-static bool kv_cache_init(
-        const struct falcon_hparams & hparams,
-              struct llm_kv_cache & cache,
-                         ggml_type   wtype,
-                               int   n_ctx) {
-    const int n_embd  = hparams.n_embd;
-    const int dim_head  = n_embd / hparams.n_head;
-    const int dim_kv = dim_head * hparams.n_head_kv;
-    const int n_layer = hparams.n_layer;
-
-    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
-    const int64_t n_elements = dim_kv * n_mem;
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
-    struct ggml_init_params params;
-    params.mem_size   = cache.buf.size;
-    params.mem_buffer = cache.buf.addr;
-    params.no_alloc   = false;
-
-    cache.ctx = ggml_init(params);
-    if (!cache.ctx) {
-        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
-        return false;
-    }
-
-    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    return true;
-}
-
-// load the model's weights from a file
-bool falcon_model_load(const std::string & fname, falcon_model & model, gpt_vocab & vocab, size_t *mem_req) {
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
-    if (mem_req) {
-        *mem_req = 0;
-    }
-
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin) {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
-        return false;
-    }
-
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *) &magic, sizeof(magic));
-        if (magic != FALCON_MAGIC) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
-            return false;
-        }
-    }
-
-    uint32_t format_version;
-    fin.read((char *) &format_version, sizeof(format_version));
-
-    // load hparams
-    {
-        auto & hparams = model.hparams;
-
-        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
-        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
-        fin.read((char *) &hparams.n_head_kv, sizeof(hparams.n_head_kv));
-        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *) &hparams.falcon_version, sizeof(hparams.falcon_version));
-        fin.read((char *) &hparams.ftype,   sizeof(hparams.ftype));
-
-        if (hparams.falcon_version != 7) { // && hparams.falcon_version != 40) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad Falcon version: %d)\n", __func__, fname.c_str(), hparams.falcon_version);
-            return false;
-        }
-
-        const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
-
-        printf("%s: n_vocab   = %d\n", __func__, hparams.n_vocab);
-        printf("%s: n_embd    = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_head    = %d\n", __func__, hparams.n_head);
-        printf("%s: n_head_kv = %d\n", __func__, hparams.n_head_kv);
-        printf("%s: n_layer   = %d\n", __func__, hparams.n_layer);
-        printf("%s: ftype     = %d\n", __func__, hparams.ftype);
-        printf("%s: qntvr     = %d\n", __func__, qntvr);
-
-        hparams.ftype %= GGML_QNT_VERSION_FACTOR;
-    }
-
-    // load vocab
-    {
-        const int32_t n_vocab = model.hparams.n_vocab;
-
-        std::string word;
-        std::vector<char> buf(128);
-
-        for (int i = 0; i < n_vocab; i++) {
-            uint32_t len;
-            fin.read((char *) &len, sizeof(len));
-
-            buf.resize(len);
-            fin.read((char *) buf.data(), len);
-            word.assign(buf.data(), len);
-
-            uint32_t dummy;
-            fin.read((char *) &dummy, sizeof(dummy));
-
-            vocab.token_to_id[word] = i;
-            vocab.id_to_token[i] = word;
-        }
-    }
-
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
-    if (wtype == GGML_TYPE_COUNT) {
-        fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
-                __func__, fname.c_str(), model.hparams.ftype);
-        return false;
-    }
-
-    auto & ctx = model.ctx;
-
-    size_t ctx_size = 0;
-
-    {
-        const auto& hparams = model.hparams;
-
-        const int n_embd = hparams.n_embd;
-        const int n_head = hparams.n_head;
-        const int n_head_kv = hparams.n_head_kv;
-        const int n_layer = hparams.n_layer;
-        const int n_ctx = hparams.n_ctx;
-        const int n_ff = 4 * model.hparams.n_embd;
-        const int n_vocab = hparams.n_vocab;
-        const int head_dim = hparams.n_embd / hparams.n_head;
-
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // tok_embeddings
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm_b
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // lm_head
-
-        // if (hparams.version == 40) { // Falcon-40B
-        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm
-        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm_b
-        // }
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm_b
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * (n_head_kv * 2 + n_head) * head_dim);  // query_key_value
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_embd);  // wo
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_ff);  // ffn_up
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_ff * n_embd);  // ffn_down
- 
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
-    }
-
-    if (mem_req) {
-        const int n_embd  = model.hparams.n_embd;
-        const int dim_head  = n_embd / model.hparams.n_head;
-        const int dim_kv = dim_head * model.hparams.n_head_kv;
-        const int n_layer = model.hparams.n_layer;
-
-        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
-        const int64_t n_elements = dim_kv * n_mem;
-        size_t kv_cache_size = 2u*n_elements*ggml_type_size(wtype) + 2_MiB;
-        *mem_req = ctx_size + kv_cache_size;
-        return false;
-    }
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size   = ctx_size,
-            .mem_buffer = NULL,
-            .no_alloc   = false,
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx) {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            return false;
-        }
-    }
-
-    // prepare memory for the weights
-    {
-        const auto& hparams = model.hparams;
-
-        const int n_embd = hparams.n_embd;
-        const int n_head = hparams.n_head;
-        const int n_head_kv = hparams.n_head_kv;
-        const int n_layer = hparams.n_layer;
-        const int n_ff = 4 * model.hparams.n_embd;
-        const int n_vocab = hparams.n_vocab;
-        const int head_dim = hparams.n_embd / hparams.n_head;
-
-        model.layers.resize(n_layer);
-
-        model.tok_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
-
-        model.output_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.output_norm_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
-
-        // map by name
-        model.tensors["transformer.word_embeddings.weight"] =
-            model.tok_embeddings;
-
-        model.tensors["transformer.ln_f.weight"] = model.output_norm;
-        model.tensors["transformer.ln_f.bias"] = model.output_norm_b;
-        model.tensors["lm_head.weight"] = model.lm_head;
-
-        for (int i = 0; i < n_layer; ++i) {
-            auto& layer = model.layers[i];
-
-            layer.input_layernorm =
-                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.input_layernorm_b =
-                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            // if (hparams.version == 40) { // for Falcon-40B only
-            //     layer.attention_norm =
-            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            //     layer.attention_norm_b =
-            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            // }
-
-            // query_key_value shape for config.multi_query == True:
-            layer.query_key_value = ggml_new_tensor_2d(
-                ctx, wtype, n_embd, (n_head_kv * 2 + n_head) * head_dim);
-            layer.wo = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
-
-            layer.ffn_up = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);
-            layer.ffn_down = ggml_new_tensor_2d(ctx, wtype, n_ff, n_embd);
-
-            // map by name
-            // if (hparams.version == 40) {
-            //     // Falcon-40B:
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_mlp.weight"] = layer.input_layernorm;
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_mlp.bias"] = layer.input_layernorm_b;
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_attn.weight"] = layer.attention_norm;
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_attn.bias"] = layer.attention_norm_b;
-            // } else {
-            // Falcon-7B:
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".input_layernorm.weight"] = layer.input_layernorm;
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".input_layernorm.bias"] = layer.input_layernorm_b;
-            //}
-
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".self_attention.query_key_value.weight"] =
-                layer.query_key_value;
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".self_attention.dense.weight"] = layer.wo;
-
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".mlp.dense_h_to_4h.weight"] = layer.ffn_up;
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".mlp.dense_4h_to_h.weight"] = layer.ffn_down;
-        }
-    }
-
-    // key + value memory
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_layer = hparams.n_layer;
-        const int n_ctx   = hparams.n_ctx;
-        const int n_head_kv = hparams.n_head_kv;
-        const int head_dim = hparams.n_embd / hparams.n_head;
-
-        const int64_t n_mem      = n_layer*n_ctx;
-        const int64_t n_elements = head_dim*n_mem;
-
-        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F32, model.hparams.n_ctx)) {
-            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
-            ggml_free(ctx);
-            return false;
-        }
-        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
-
-        printf("%s: memory_size = %8.2f MB, n_mem = %" PRId64 "\n", __func__, memory_size/1024.0/1024.0, n_mem);
-    }
-
-    // load weights
-    {
-        int n_tensors = 0;
-        size_t total_size = 0;
-
-        printf("%s: ", __func__);
-
-        while (true) {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ttype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
-
-            if (fin.eof()) {
-                break;
-            }
-
-            int32_t nelements = 1;
-            int32_t ne[2] = { 1, 1 };
-            for (int i = 0; i < n_dims; ++i) {
-                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-            fin.seekg(-static_cast<ptrdiff_t>(fin.tellg()) & 31, std::ios_base::cur);
-
-            if (model.tensors.find(name.data()) == model.tensors.end()) {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                return false;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                return false;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%5d, %5d], expected [%5d, %5d]\n",
-                        __func__, name.data(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
-                return false;
-            }
-
-            // for debugging
-            if (0) {
-                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
-            }
-
-            const size_t bpe = ggml_type_size(ggml_type(ttype));
-
-            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
-                return false;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-            total_size += ggml_nbytes(tensor);
-            if (++n_tensors % 8 == 0) {
-                printf(".");
-                fflush(stdout);
-            }
-        }
-
-        printf(" done\n");
-
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
-    }
-
-    fin.close();
-
-    model.eval_buf.resize(1280u * 1024 * 1024);
-    model.scr0_buf.resize(256u * 1024 * 1024);
-    model.scr1_buf.resize(256u * 1024 * 1024);
-    return true;
-}
-
-
-// evaluate the transformer
-//
-//   - model:     the model
-//   - n_threads: number of threads to use
-//   - n_past:    the context size so far
-//   - embd_inp:  the embeddings of the tokens in the context
-//   - embd_w:    the predicted logits for the next token
-//
-bool falcon_eval(
-        falcon_model & model,
-        const int n_threads,
-        const int n_past,
-        const std::vector<gpt_vocab::id> & embd_inp,
-              std::vector<float>         & embd_w,
-              size_t                     & mem_per_token) {
-    const int N = embd_inp.size();
-
-    const auto & hparams = model.hparams;
-
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_ctx   = hparams.n_ctx;
-    const int n_head  = hparams.n_head;
-    const int n_head_kv = hparams.n_head_kv;
-    const int n_vocab = hparams.n_vocab;
-    const int version = hparams.falcon_version;
-    const size_t head_dim = n_embd / n_head;
-
-   struct ggml_init_params eval_ctx_params = {
-        .mem_size = model.eval_buf.size,
-        .mem_buffer = model.eval_buf.addr,
-        .no_alloc = false,
-    };
-
-    struct ggml_context * ctx0 = ggml_init(eval_ctx_params);
-    struct ggml_cgraph gf = {};
-
-    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
-
-    // wte
-    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.tok_embeddings, embd);
-    struct ggml_tensor* repeat_dummy = ggml_new_tensor_3d(ctx0, inpL->type, head_dim, N + n_past, n_head);
-
-    ggml_type wtype = GGML_TYPE_F32;
-    const int sizeof_wtype = ggml_type_sizef(wtype);
-
-    for (int il = 0; il < n_layer; ++il) {
-        struct ggml_tensor * cur;
-        struct ggml_tensor * layernorm_output;
-
-        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-        // self-attention
-        {
-            layernorm_output = ggml_norm(ctx0, inpL);
-
-            layernorm_output = ggml_add(ctx0,
-                    ggml_mul(ctx0,
-                        ggml_repeat(ctx0, model.layers[il].input_layernorm, layernorm_output),
-                        layernorm_output),
-                    ggml_repeat(ctx0, model.layers[il].input_layernorm_b, layernorm_output));
-
-            // if (version == 40) { // Falcon-40B only
-            //     cur = ggml_norm(ctx0, inpL);
-
-            //     cur = ggml_add(ctx0,
-            //             ggml_mul(ctx0,
-            //                 ggml_repeat(ctx0, model.layers[il].attention_norm, cur),
-            //                 cur),
-            //             ggml_repeat(ctx0, model.layers[il].attention_norm_b, cur));
-            // }
-            // else {
-            cur = layernorm_output;
-            // }
-
-            // compute QKV
-
-            cur = ggml_mul_mat(ctx0, model.layers[il].query_key_value, cur);
-
-            // Note that the strides for Kcur, Vcur are set up so that the
-            // resulting views are misaligned with the tensor's storage
-            // (by applying the K/V offset we shift the tensor's original
-            // view to stick out behind the viewed QKV tensor's allocated
-            // memory, so to say). This is ok because no actual accesses
-            // happen to that out-of-range memory, but it can require some
-            // trickery when trying to accurately dump these views for
-            // debugging.
-
-            struct ggml_tensor * Qcur = ggml_view_3d(
-                ctx0, cur, head_dim, n_head, N,
-                head_dim * sizeof_wtype,
-                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
-                0);
-
-            struct ggml_tensor * Kcur = ggml_view_3d(
-                ctx0, cur, head_dim, n_head_kv, N,
-                head_dim * sizeof_wtype,
-                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
-                head_dim * n_head * sizeof_wtype);
-
-            struct ggml_tensor * Vcur = ggml_view_3d(
-                ctx0, cur, head_dim, n_head_kv, N,
-                head_dim * sizeof_wtype,
-                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
-                head_dim * (n_head + n_head_kv) * sizeof_wtype);
-
-            // using mode = 2 for neox mode
-            Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, head_dim, 2, n_ctx);
-            Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, head_dim, 2, n_ctx);
-
-            // store key and value to memory
-            {
-                struct ggml_tensor* k = ggml_view_1d(
-                    ctx0, model.kv_self.k, N * n_head_kv * head_dim,
-                    (ggml_element_size(model.kv_self.k) * n_head_kv * head_dim) *
-                        (il * n_ctx + n_past));
-                struct ggml_tensor* v = ggml_view_1d(
-                    ctx0, model.kv_self.v, N * n_head_kv * head_dim,
-                    (ggml_element_size(model.kv_self.v) * n_head_kv * head_dim) *
-                        (il * n_ctx + n_past));
-
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
-            }
-
-            struct ggml_tensor * K = ggml_permute(
-                ctx0,
-                ggml_view_3d(
-                    ctx0,
-                    model.kv_self.k,
-                    head_dim, n_head_kv, n_past + N,
-                    head_dim * sizeof_wtype,
-                    head_dim * n_head_kv * sizeof_wtype,
-                    il * n_ctx * ggml_element_size(model.kv_self.k) * n_head_kv * head_dim),
-                0, 2, 1, 3);
-
-            // K * Q
-
-            // changed from repeat2 back to repeat, will not support 40B!
-            K = ggml_cont(ctx0, ggml_repeat(ctx0, K, repeat_dummy));
-
-            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
-            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
-
-            // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled =
-                ggml_scale_inplace(ctx0,
-                        KQ,
-                        ggml_new_f32(ctx0, 1.0f/sqrt(float(head_dim)))
-                        );
-
-            // KQ_masked = mask_past(KQ_scaled)
-            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
-
-            // KQ = soft_max(KQ_masked)
-            struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
-
-            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor* V = ggml_permute(
-                ctx0,
-                ggml_view_3d(
-                    ctx0,
-                    model.kv_self.v,
-                    head_dim, n_head_kv, n_past + N,
-                    head_dim * sizeof_wtype,
-                    head_dim * n_head_kv * sizeof_wtype,
-                    il * n_ctx * ggml_element_size(model.kv_self.v) * n_head_kv * head_dim),
-                0, 2, 1, 3);
-
-            // changed from repeat2 back to repeat, will not support 40B!
-            V = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_repeat(ctx0, V, repeat_dummy)));
-
-            // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
-
-            // KQV_merged = KQV.permute(0, 2, 1, 3)
-            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            // cur = KQV_merged.contiguous().view(n_embd, N)
-            cur = ggml_cpy(ctx0,
-                    KQV_merged,
-                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-            // projection
-            {
-                cur = ggml_mul_mat(ctx0,
-                        model.layers[il].wo,
-                        cur);
-            }
-        }
-
-        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
-
-        struct ggml_tensor* inpFF = layernorm_output;
-        struct ggml_tensor* attn_out = ggml_cpy(
-            ctx0, cur, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-        {
-            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_up, inpFF);
-            cur = ggml_gelu(ctx0, cur);
-            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_down, cur);
-        }
-
-        cur = ggml_add(ctx0, cur, attn_out);
-        cur = ggml_add(ctx0, cur, inpL);
-        // input for next layer
-        inpL = cur;
-    }
-
-    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-    // norm
-    {
-        inpL = ggml_norm(ctx0, inpL);
-
-        // inpL = ln_f_g*inpL + ln_f_b
-        inpL = ggml_add(ctx0,
-                ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.output_norm, inpL),
-                    inpL),
-                ggml_repeat(ctx0, model.output_norm_b, inpL));
-    }
-
-    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
-
-    // lm_head
-    {
-        inpL = ggml_mul_mat(ctx0, model.lm_head, inpL);
-
-        //inpL = ggml_add(ctx0,
-        //        ggml_repeat(ctx0, model.lmh_b, inpL),
-        //        inpL);
-    }
-
-    // logits -> probs
-    //inpL = ggml_soft_max_inplace(ctx0, inpL);
-
-    // run the computation
-    ggml_build_forward_expand(&gf, inpL);
-    ggml_graph_compute_g4a(model.work_buf, &gf, n_threads);
-  
-
-    //if (n_past%100 == 0) {
-    //    ggml_graph_print   (&gf);
-    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
-    //}
-
-    //embd_w.resize(n_vocab*N);
-    //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
-
-    // return result for just the last token
-    embd_w.resize(n_vocab);
-    memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
-
-    if (mem_per_token == 0) {
-        mem_per_token = ggml_used_mem(ctx0)/N;
-    }
-    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
-
-    ggml_free(ctx0);
-
-    return true;
-}
-
-
-#define MAX_RNG_STATE 64*1024
-size_t falcon_get_state_size(const falcon_model &model) {
-    const size_t s_rng_size        = sizeof(size_t);
-    const size_t s_rng             = MAX_RNG_STATE;
-    const size_t s_kv_size         = sizeof(size_t);
-    const size_t s_kv_ntok         = sizeof(int);
-    const size_t s_kv              = model.kv_self.buf.size;
-    const size_t s_total = (
-        + s_rng_size
-        + s_rng
-        + s_kv_size
-        + s_kv_ntok
-        + s_kv
-    );
-    return s_total;
-}
-
-size_t falcon_copy_state_data(const falcon_model &model, const std::mt19937 &rng, uint8_t *dest)
-{
-    uint8_t * out = dest;
-    // copy rng
-    {
-        std::stringstream rng_ss;
-        rng_ss << rng;
-
-        const size_t rng_size = rng_ss.str().size();
-        char rng_buf[MAX_RNG_STATE];
-
-        memset(&rng_buf[0], 0, MAX_RNG_STATE);
-        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
-
-        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
-        memcpy(out, &rng_buf[0], MAX_RNG_STATE); out += MAX_RNG_STATE;
-    }
-
-    // copy kv cache
-    {
-        const size_t kv_size = model.kv_self.buf.size;
-        const int    kv_ntok = model.kv_self.n;
-
-        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
-        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
-
-        if (kv_size) {
-            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
-        }
-    }
-
-    const size_t written  = out - dest;
-    assert(written == falcon_get_state_size(model));
-    fflush(stdout);
-    return written;
-}
-
-size_t falcon_set_state_data(falcon_model *model, std::mt19937 *rng, const uint8_t *src)
-{
-    const uint8_t * in = src;
-
-    // set rng
-    {
-        size_t rng_size;
-        char   rng_buf[MAX_RNG_STATE];
-
-        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
-        memcpy(&rng_buf[0], in, MAX_RNG_STATE); in += MAX_RNG_STATE;
-
-        std::stringstream rng_ss;
-        rng_ss.str(std::string(&rng_buf[0], rng_size));
-        rng_ss >> *rng;
-
-        assert(rng_ss.fail() == false);
-    }
-
-    // set kv cache
-    {
-        size_t kv_size;
-        int kv_ntok;
-
-        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
-        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
-
-        if (kv_size) {
-            assert(model->kv_self.buf.size == kv_size);
-
-            void * k_data = model->kv_self.k->data; // remember data pointers
-            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
-
-            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
-
-            model->kv_self.k->data = k_data; // restore correct data pointers
-            model->kv_self.v->data = v_data;
-
-        }
-
-        model->kv_self.n = kv_ntok;
-    }
-
-    const size_t nread    = in - src;
-    assert(nread == falcon_get_state_size(*model));
-    fflush(stdout);
-    return nread;
-}
-
-struct FalconPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    gpt_vocab vocab;
-    falcon_model *model = nullptr;
-    int64_t n_threads = 0;
-    size_t mem_per_token = 0;
-    std::mt19937 rng;
-};
-
-Falcon::Falcon() : d_ptr(new FalconPrivate) {
-    d_ptr->model = new falcon_model;
-    d_ptr->model->ctx = nullptr;
-    d_ptr->modelLoaded = false;
-}
-
-Falcon::~Falcon() {
-    if(d_ptr->model->ctx) {
-        ggml_free(d_ptr->model->ctx);
-        d_ptr->model->ctx = nullptr;
-    }
-    delete d_ptr->model;
-}
-
-bool Falcon::loadModel(const std::string &modelPath)
-{
-    std::mt19937 rng(time(NULL));
-    d_ptr->rng = rng;
-
-    // load the model
-    if (!falcon_model_load(modelPath, *d_ptr->model, d_ptr->vocab, nullptr)) {
-        std::cerr << "FALCON ERROR: failed to load model from " <<  modelPath;
-        return false;
-    }
-
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    fflush(stdout);
-    return true;
-}
-
-bool Falcon::isModelLoaded() const
-{
-    return d_ptr -> modelLoaded;
-}
-
-size_t Falcon::requiredMem(const std::string &modelPath)
-{
-    falcon_model dummy_model;
-    gpt_vocab dummy_vocab;
-    size_t mem_req;
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-    falcon_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
-    return mem_req;
-}
-
-size_t Falcon::stateSize() const
-{
-    return falcon_get_state_size(*d_ptr->model);
-}
-
-size_t Falcon::saveState(uint8_t *dest) const
-{
-    return falcon_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
-}
-
-size_t Falcon::restoreState(const uint8_t *src)
-{
-    return falcon_set_state_data(d_ptr->model, &d_ptr->rng, src);
-}
-
-void Falcon::setThreadCount(int32_t n_threads)
-{
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t Falcon::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-std::vector<LLModel::Token> Falcon::tokenize(PromptContext &, const std::string &str) const
-{
-    return ::gpt_tokenize(d_ptr->vocab, str);
-}
-
-LLModel::Token Falcon::sampleToken(PromptContext &promptCtx) const
-{
-    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
-    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
-        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks,
-        promptCtx.logits,
-        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty,
-        d_ptr->rng);
-}
-
-std::string Falcon::tokenToString(Token id) const
-{
-    return d_ptr->vocab.id_to_token[id];
-}
-
-bool Falcon::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    // determine the required inference memory per token:
-    static bool initialized = false;
-    if (!initialized) {
-        falcon_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
-            d_ptr->mem_per_token);
-        initialized = true;
-    }
-
-    return falcon_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
-}
-
-int32_t Falcon::contextLength() const
-{
-    return d_ptr->model->hparams.n_ctx;
-}
-
-const std::vector<LLModel::Token> &Falcon::endTokens() const
-{
-    static const std::vector<LLModel::Token> out = { 11 };
-    return out;
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    uint32_t version = 0;
-    f.read(reinterpret_cast<char*>(&version), sizeof(version));
-    if (magic != FALCON_MAGIC) {
-         return false; 
-    }
-    falcon_hparams hparams;
-    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
-    // we're matching the file format of existing pre-converted models
-    // compatible with ctransformers llama.cpp based format, which also
-    // unfortunately shares its magic number what llama uses, so we now
-    // differentiate by n_vocab
-    // give some wiggle room over the max to allow for finetunes that expand the
-    // vocabulary
-    if (!(hparams.n_vocab >= 65024 && hparams.n_vocab <= 65100)) {
-        return false;
-    }
-    if (hparams.falcon_version != 7) {
-        return false;
-    }
-    return true;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new Falcon;
-}
-}

gpt4all-backend/falcon_impl.h

@@ -1,42 +0,0 @@
-#ifndef FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of falcon.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef FALCON_H
-#define FALCON_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include <memory>
-#include "llmodel.h"
-
-struct FalconPrivate;
-class Falcon : public LLModel {
-public:
-    Falcon();
-    ~Falcon();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    std::unique_ptr<FalconPrivate> d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // Falcon_H

gpt4all-backend/gptj.cpp

@@ -9,7 +9,6 @@
 #include <cmath>
 #include <cstdio>
 #include <cstring>
-#include <fstream>
 #include <map>
 #include <string>
 #include <vector>
@@ -42,7 +41,7 @@ struct gptj_hparams {
     int32_t n_head  = 16;
     int32_t n_layer = 28;
     int32_t n_rot   = 64;
-    int32_t f16     = 1;
+    float norm_eps  = 1e-5;
 };
 
 struct gptj_layer {
@@ -128,216 +127,149 @@ static bool kv_cache_init(
     return true;
 }
 
-// load the model's weights from a stream
-bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
+// load the model's weights from a file path
+bool gptj_model_load(const std::string &fname, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
     if(mem_req != nullptr) {
         *mem_req = 0;
     }
 
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *) &magic, sizeof(magic));
-        if (magic != 0x67676d6c) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
-            return false;
-        }
+    // create the ggml context
+    struct gguf_init_params params = {
+        /*.no_alloc = */ false,
+        /*.ctx      = */ &model.ctx,
+    };
+
+    gguf_context *ggufctx = gguf_init_from_file(fname.c_str(), params);
+    if (!ggufctx) {
+        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
+        return false;
     }
 
     // load hparams
     {
         auto & hparams = model.hparams;
 
-        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
-        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
-        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
-        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *) &hparams.n_rot,   sizeof(hparams.n_rot));
-        fin.read((char *) &hparams.f16,     sizeof(hparams.f16));
+        bool ok = false;
+        int keyidx;
+
+        do {
+            keyidx = gguf_find_key(ggufctx, "gptj.context_length");
+            if (keyidx == -1) { break; }
+            hparams.n_ctx = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.embedding_length");
+            if (keyidx == -1) { break; }
+            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.attention.head_count");
+            if (keyidx == -1) { break; }
+            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.block_count");
+            if (keyidx == -1) { break; }
+            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.rope.dimension_count");
+            if (keyidx == -1) { break; }
+            hparams.n_rot = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.attention.layer_norm_epsilon");
+            if (keyidx == -1) { break; }
+            hparams.norm_eps = gguf_get_val_f32(ggufctx, keyidx);
+
+            ok = true;
+        } while (false);
+
+        if (!ok) {
+            fprintf(stderr, "%s: required hparam missing!\n", __func__);
+            return false;
+        }
 
-        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
         printf("%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
         printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
         printf("%s: n_head  = %d\n", __func__, hparams.n_head);
         printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
         printf("%s: n_rot   = %d\n", __func__, hparams.n_rot);
-        printf("%s: f16     = %d\n", __func__, hparams.f16);
     }
 
     // load vocab
     {
-        int32_t n_vocab = 0;
-        fin.read((char *) &n_vocab, sizeof(n_vocab));
+        auto & hparams = model.hparams;
 
-        if (n_vocab != model.hparams.n_vocab) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
-                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
+        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
+        if (keyidx == -1) {
+            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
+            return false;
+        }
+        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "gpt2") != 0) {
+            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
             return false;
         }
 
-        std::string word;
-        for (int i = 0; i < n_vocab; i++) {
-            uint32_t len;
-            fin.read((char *) &len, sizeof(len));
+        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
+        if (tokens_keyidx == -1) {
+            fprintf(stderr, "%s: gpt2 tokenizer vocab not found!\n", __func__);
+            return false;
+        }
 
-            word.resize(len);
-            fin.read((char *) word.data(), len);
+        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
+        printf("%s: gpt2 tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
 
+        for (int i = 0; i < hparams.n_vocab; i++) {
+            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
             vocab.token_to_id[word] = i;
             vocab.id_to_token[i] = word;
         }
     }
 
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = GGML_TYPE_COUNT;
-    switch (model.hparams.f16) {
-        case 0: wtype = GGML_TYPE_F32;  break;
-        case 1: wtype = GGML_TYPE_F16;  break;
-        case 2: wtype = GGML_TYPE_Q4_0; break;
-        case 3: wtype = GGML_TYPE_Q4_1; break;
-        case 5: wtype = GGML_TYPE_Q4_2; break;
-        default:
-                {
-                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
-                            __func__, fname.c_str(), model.hparams.f16);
-                    return false;
-                }
-    }
-
     auto & ctx = model.ctx;
 
-    size_t ctx_size = 0;
-
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_ctx   = hparams.n_ctx;
-        const int n_vocab = hparams.n_vocab;
-
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_g
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_b
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype); // wte
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype);         // lmh_g
-        ctx_size +=        n_vocab*ggml_type_sizef(GGML_TYPE_F32); // lmh_b
-
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_g
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_b
-
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_q_proj_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_k_proj_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_v_proj_w
-
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_proj_w
-
-        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_fc_w
-        ctx_size += n_layer*(       4*n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_fc_b
-
-        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_proj_w
-        ctx_size += n_layer*(         n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_proj_b
-
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_k
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_v
-
-        ctx_size += (5 + 10*n_layer)*256; // object overhead
-
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
-    }
+    size_t ctx_size = ggml_get_mem_size(ctx);
+    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size / (1024.0 * 1024.0));
 
     if (mem_req != nullptr) {
-        *mem_req += ctx_size;
-        const int n_embd  = model.hparams.n_embd;
-        const int n_layer = model.hparams.n_layer;
-
-        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
-        const int64_t n_elements = n_embd*n_mem;
-
-        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+        *mem_req = ctx_size;
+        gguf_free(ggufctx);
         return false;
     }
 
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size   = ctx_size,
-            .mem_buffer = NULL,
-            .no_alloc = false
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx) {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            return false;
-        }
-    }
-
     // prepare memory for the weights
     {
         const auto & hparams = model.hparams;
+        model.layers.resize(hparams.n_layer);
 
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_vocab = hparams.n_vocab;
+        model.wte    = ggml_get_tensor(ctx, "token_embd.weight");
 
-        model.layers.resize(n_layer);
+        model.ln_f_g = ggml_get_tensor(ctx, "output_norm.weight");
+        model.ln_f_b = ggml_get_tensor(ctx, "output_norm.bias");
 
-        model.wte    = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
+        model.lmh_g  = ggml_get_tensor(ctx, "output.weight");
+        model.lmh_b  = ggml_get_tensor(ctx, "output.bias");
 
-        model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        auto name = [](int i, std::string n) {
+            static std::string key;
+            key = "blk." + std::to_string(i) + "." + n;
+            return key.c_str();
+        };
 
-        model.lmh_g  = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
-        model.lmh_b  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab);
-
-        // map by name
-        model.tensors["transformer.wte.weight"] = model.wte;
-
-        model.tensors["transformer.ln_f.weight"] = model.ln_f_g;
-        model.tensors["transformer.ln_f.bias"]   = model.ln_f_b;
-
-        model.tensors["lm_head.weight"] = model.lmh_g;
-        model.tensors["lm_head.bias"]   = model.lmh_b;
-
-        for (int i = 0; i < n_layer; ++i) {
+        for (int i = 0; i < hparams.n_layer; ++i) {
             auto & layer = model.layers[i];
 
-            layer.ln_1_g          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
-            layer.ln_1_b          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
+            layer.ln_1_g          = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
+            layer.ln_1_b          = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
 
-            layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
-            layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
-            layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_q_proj_w = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
+            layer.c_attn_k_proj_w = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
+            layer.c_attn_v_proj_w = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
 
-            layer.c_attn_proj_w   = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_proj_w   = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
 
-            layer.c_mlp_fc_w      = ggml_new_tensor_2d(ctx, wtype,           n_embd, 4*n_embd);
-            layer.c_mlp_fc_b      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
+            layer.c_mlp_fc_w      = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
+            layer.c_mlp_fc_b      = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
 
-            layer.c_mlp_proj_w    = ggml_new_tensor_2d(ctx, wtype,         4*n_embd,   n_embd);
-            layer.c_mlp_proj_b    = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
-
-            // map by name
-            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.weight"]          = layer.ln_1_g;
-            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.bias"]            = layer.ln_1_b;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.q_proj.weight"]   = layer.c_attn_q_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.k_proj.weight"]   = layer.c_attn_k_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.v_proj.weight"]   = layer.c_attn_v_proj_w;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.out_proj.weight"] = layer.c_attn_proj_w;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.weight"]     = layer.c_mlp_fc_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.bias"]       = layer.c_mlp_fc_b;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.weight"]    = layer.c_mlp_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.bias"]      = layer.c_mlp_proj_b;
+            layer.c_mlp_proj_w    = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
+            layer.c_mlp_proj_b    = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
         }
     }
 
@@ -354,113 +286,12 @@ bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & m
         printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
     }
 
-    // load weights
-    {
-        int n_tensors = 0;
-        size_t total_size = 0;
-
-        printf("%s: ", __func__);
-
-        while (true) {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ftype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ftype),  sizeof(ftype));
-
-            if (fin.eof()) {
-                break;
-            }
-
-            int32_t nelements = 1;
-            int32_t ne[2] = { 1, 1 };
-            for (int i = 0; i < n_dims; ++i) {
-                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-
-            if (model.tensors.find(name.data()) == model.tensors.end()) {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                return false;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                return false;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
-                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
-                return false;
-            }
-
-            if (0) {
-                static const char * ftype_str[] = { "f32", "f16", "q4_0", "q4_1", };
-                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ftype_str[ftype], ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
-            }
-
-            size_t bpe = 0;
-
-            switch (ftype) {
-                case 0: bpe = ggml_type_size(GGML_TYPE_F32);  break;
-                case 1: bpe = ggml_type_size(GGML_TYPE_F16);  break;
-                case 2: bpe = ggml_type_size(GGML_TYPE_Q4_0); assert(ne[0] % 64 == 0); break;
-                case 3: bpe = ggml_type_size(GGML_TYPE_Q4_1); assert(ne[0] % 64 == 0); break;
-                default:
-                        {
-                            fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype);
-                            return false;
-                        }
-            };
-
-            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
-                return false;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
-            total_size += ggml_nbytes(tensor);
-            if (++n_tensors % 8 == 0) {
-                printf(".");
-                fflush(stdout);
-            }
-        }
-
-        printf(" done\n");
-
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
-    }
-
     model.scr0_buf.resize(256u * 1024 * 1024);
     model.scr1_buf.resize(256u * 1024 * 1024);
 
     return true;
 }
 
-// load the model's weights from a file path
-bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) {
-
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin) {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
-        return false;
-    }
-
-    bool loaded = gptj_model_load(fname, fin, model, vocab);
-    fin.close();
-    return loaded;
-}
-
 // evaluate the transformer
 //
 //   - model:     the model
@@ -513,7 +344,6 @@ bool gptj_eval(
 
     struct ggml_context * ctx0 = ggml_init(params);
     struct ggml_cgraph gf = {};
-    gf.n_threads = n_threads;
 
     struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
     memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -526,7 +356,7 @@ bool gptj_eval(
         ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
         // norm
         {
-            cur = ggml_norm(ctx0, inpL);
+            cur = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
 
             // cur = ln_1_g*cur + ln_1_b
             cur = ggml_add(ctx0,
@@ -540,37 +370,31 @@ bool gptj_eval(
 
         // self-attention
         {
-            struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur);
-            struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur);
-            struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur);
+            struct ggml_tensor * Qcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
+            struct ggml_tensor * Kcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
 
             // store key and value to memory
             {
+                struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur));
+
                 struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_1d(ctx0, model.kv_self.v, N*n_embd, (ggml_element_size(model.kv_self.v)*n_embd)*(il*n_ctx + n_past));
+                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
+                        (   n_ctx)*ggml_element_size(model.kv_self.v),
+                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
 
                 ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
                 ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
             }
 
             // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q =
-                ggml_permute(ctx0,
-                        ggml_rope(ctx0,
-                            ggml_cpy(ctx0,
-                                Qcur,
-                                ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),
-                            n_past, n_rot, 0),
-                        0, 2, 1, 3);
+            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
 
             // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
             struct ggml_tensor * K =
                 ggml_permute(ctx0,
-                        ggml_rope(ctx0,
-                            ggml_reshape_3d(ctx0,
-                                ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                                n_embd/n_head, n_head, n_past + N),
-                            n_past, n_rot, 1),
+                        ggml_reshape_3d(ctx0,
+                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
+                            n_embd/n_head, n_head, n_past + N),
                         0, 2, 1, 3);
 
             // K * Q
@@ -590,17 +414,15 @@ bool gptj_eval(
             struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
 
             // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V_trans =
-                ggml_cpy(ctx0,
-                        ggml_permute(ctx0,
-                            ggml_reshape_3d(ctx0,
-                                ggml_view_1d(ctx0, model.kv_self.v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd),
-                                n_embd/n_head, n_head, n_past + N),
-                            1, 2, 0, 3),
-                        ggml_new_tensor_3d(ctx0, model.kv_self.v->type, n_past + N, n_embd/n_head, n_head));
+            struct ggml_tensor * V =
+                ggml_view_3d(ctx0, model.kv_self.v,
+                        n_past + N, n_embd/n_head, n_head,
+                        n_ctx*ggml_element_size(model.kv_self.v),
+                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
+                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
 
             // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
 
             // KQV_merged = KQV.permute(0, 2, 1, 3)
             struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
@@ -656,7 +478,7 @@ bool gptj_eval(
 
     // norm
     {
-        inpL = ggml_norm(ctx0, inpL);
+        inpL = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
 
         // inpL = ln_f_g*inpL + ln_f_b
         inpL = ggml_add(ctx0,
@@ -680,9 +502,18 @@ bool gptj_eval(
     // logits -> probs
     //inpL = ggml_soft_max(ctx0, inpL);
 
-    // run the computation
     ggml_build_forward_expand(&gf, inpL);
-    ggml_graph_compute       (ctx0, &gf);
+
+    // run the computation
+    {
+        std::unique_ptr<uint8_t []> data;
+        auto plan = ggml_graph_plan(&gf, n_threads);
+        if (plan.work_size > 0) {
+            data.reset(new uint8_t[plan.work_size]);
+            plan.work_data = data.get();
+        }
+        ggml_graph_compute(&gf, &plan);
+    }
 
     //if (n_past%100 == 0) {
     //    ggml_graph_print   (&gf);
@@ -836,8 +667,7 @@ size_t GPTJ::requiredMem(const std::string &modelPath) {
     gptj_model dummy_model;
     gpt_vocab dummy_vocab;
     size_t mem_req;
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-    gptj_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
+    gptj_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
     return mem_req;
 }
 
@@ -845,10 +675,8 @@ bool GPTJ::loadModel(const std::string &modelPath) {
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
 
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-
     // load the model
-    if (!gptj_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab)) {
+    if (!gptj_model_load(modelPath, *d_ptr->model, d_ptr->vocab)) {
         std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
         return false;
     }
@@ -939,6 +767,16 @@ const std::vector<LLModel::Token> &GPTJ::endTokens() const
     return fres;
 }
 
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != GGUF_TYPE_STRING) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -958,15 +796,21 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    gptj_hparams hparams;
-    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
-    if (!(hparams.n_vocab >= 50300 && hparams.n_vocab <= 50400)) {
-        return false; // not a gptj.
-    }
-    return magic == 0x67676d6c;
+DLL_EXPORT bool magic_match(const char * fname) {
+    struct ggml_context * ctx_meta = NULL;
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ &ctx_meta,
+    };
+    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
+    if (!ctx_gguf)
+        return false;
+
+    bool isValid = gguf_get_version(ctx_gguf) <= 2;
+    isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
+
+    gguf_free(ctx_gguf);
+    return isValid;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llama.cpp.cmake

@@ -174,6 +174,9 @@ if (LLAMA_KOMPUTE)
         add_custom_command(
             OUTPUT ${spv_file}
             DEPENDS ${LLAMA_DIR}/${source}
+              ${LLAMA_DIR}/kompute/common.comp
+              ${LLAMA_DIR}/kompute/op_getrows.comp
+              ${LLAMA_DIR}/kompute/op_mul_mv_q_n.comp
             COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
             COMMENT "Compiling ${source} to ${source}.spv"
         )
@@ -185,19 +188,35 @@ if (LLAMA_KOMPUTE)
         string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}")
         set(OUTPUT_HEADER_FILE "${HEADER_FILE}")
         message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}")
-        add_custom_command(
-          OUTPUT ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
-          COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-          DEPENDS ${spv_file} xxd
-          COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
-        )
+        if(CMAKE_GENERATOR MATCHES "Visual Studio")
+            add_custom_command(
+              OUTPUT ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              DEPENDS ${spv_file} xxd
+              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd"
+            )
+        else()
+            add_custom_command(
+              OUTPUT ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              DEPENDS ${spv_file} xxd
+              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
+            )
+        endif()
       endforeach()
     endfunction()
 
@@ -220,11 +239,14 @@ if (LLAMA_KOMPUTE)
           kompute/op_rmsnorm.comp
           kompute/op_diagmask.comp
           kompute/op_mul_mat_f16.comp
+          kompute/op_mul_mat_q8_0.comp
           kompute/op_mul_mat_q4_0.comp
           kompute/op_mul_mat_q4_1.comp
+          kompute/op_mul_mat_q6_k.comp
           kompute/op_getrows_f16.comp
           kompute/op_getrows_q4_0.comp
           kompute/op_getrows_q4_1.comp
+          kompute/op_getrows_q6_k.comp
           kompute/op_rope.comp
           kompute/op_cpy_f16_f16.comp
           kompute/op_cpy_f16_f32.comp
@@ -247,11 +269,14 @@ if (LLAMA_KOMPUTE)
           shaderop_rmsnorm.h
           shaderop_diagmask.h
           shaderop_mul_mat_f16.h
+          shaderop_mul_mat_q8_0.h
           shaderop_mul_mat_q4_0.h
           shaderop_mul_mat_q4_1.h
+          shaderop_mul_mat_q6_k.h
           shaderop_getrows_f16.h
           shaderop_getrows_q4_0.h
           shaderop_getrows_q4_1.h
+          shaderop_getrows_q6_k.h
           shaderop_rope.h
           shaderop_cpy_f16_f16.h
           shaderop_cpy_f16_f32.h
@@ -330,6 +355,13 @@ endif()
 # TODO: probably these flags need to be tweaked on some architectures
 #       feel free to update the Makefile for your architecture and send a pull request or issue
 message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
+if (MSVC)
+  string(TOLOWER "${CMAKE_GENERATOR_PLATFORM}" CMAKE_GENERATOR_PLATFORM_LWR)
+  message(STATUS "CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}")
+else ()
+  set(CMAKE_GENERATOR_PLATFORM_LWR "")
+endif ()
+
 if (NOT MSVC)
     if (LLAMA_STATIC)
         add_link_options(-static)
@@ -345,6 +377,139 @@ if (NOT MSVC)
     endif()
 endif()
 
+if ((${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm") OR (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") OR ("${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "arm64"))
+    message(STATUS "ARM detected")
+    if (MSVC)
+        add_compile_definitions(__ARM_NEON)
+        add_compile_definitions(__ARM_FEATURE_FMA)
+        add_compile_definitions(__ARM_FEATURE_DOTPROD)
+        # add_compile_definitions(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) # MSVC doesn't support vdupq_n_f16, vld1q_f16, vst1q_f16
+        add_compile_definitions(__aarch64__) # MSVC defines _M_ARM64 instead
+    else()
+        include(CheckCXXCompilerFlag)
+        check_cxx_compiler_flag(-mfp16-format=ieee COMPILER_SUPPORTS_FP16_FORMAT_I3E)
+        if (NOT "${COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
+            add_compile_options(-mfp16-format=ieee)
+        endif()
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv6")
+            # Raspberry Pi 1, Zero
+            add_compile_options(-mfpu=neon-fp-armv8 -mno-unaligned-access)
+        endif()
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7")
+            # Raspberry Pi 2
+            add_compile_options(-mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations)
+        endif()
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv8")
+            # Raspberry Pi 3, 4, Zero 2 (32-bit)
+            add_compile_options(-mno-unaligned-access)
+        endif()
+    endif()
+elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$" OR "${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "^(x86_64|i686|amd64|x64)$" )
+    message(STATUS "x86 detected")
+    if (MSVC)
+        if (LLAMA_AVX512)
+            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX512>)
+            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
+            # MSVC has no compile-time flags enabling specific
+            # AVX512 extensions, neither it defines the
+            # macros corresponding to the extensions.
+            # Do it manually.
+            if (LLAMA_AVX512_VBMI)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
+            endif()
+            if (LLAMA_AVX512_VNNI)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
+            endif()
+        elseif (LLAMA_AVX2)
+            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX2>)
+            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
+        elseif (LLAMA_AVX)
+            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX>)
+            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
+        endif()
+    else()
+        if (LLAMA_F16C)
+            add_compile_options(-mf16c)
+        endif()
+        if (LLAMA_FMA)
+            add_compile_options(-mfma)
+        endif()
+        if (LLAMA_AVX)
+            add_compile_options(-mavx)
+        endif()
+        if (LLAMA_AVX2)
+            add_compile_options(-mavx2)
+        endif()
+        if (LLAMA_AVX512)
+            add_compile_options(-mavx512f)
+            add_compile_options(-mavx512bw)
+        endif()
+        if (LLAMA_AVX512_VBMI)
+            add_compile_options(-mavx512vbmi)
+        endif()
+        if (LLAMA_AVX512_VNNI)
+            add_compile_options(-mavx512vnni)
+        endif()
+    endif()
+elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
+    message(STATUS "PowerPC detected")
+    add_compile_options(-mcpu=native -mtune=native)
+    #TODO: Add  targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be)
+else()
+    message(STATUS "Unknown architecture")
+endif()
+
+#
+# POSIX conformance
+#
+
+# clock_gettime came in POSIX.1b (1993)
+# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional
+# posix_memalign came in POSIX.1-2001 / SUSv3
+# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985)
+add_compile_definitions(_XOPEN_SOURCE=600)
+
+# Somehow in OpenBSD whenever POSIX conformance is specified
+# some string functions rely on locale_t availability,
+# which was introduced in POSIX.1-2008, forcing us to go higher
+if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
+    remove_definitions(-D_XOPEN_SOURCE=600)
+    add_compile_definitions(_XOPEN_SOURCE=700)
+endif()
+
+# Data types, macros and functions related to controlling CPU affinity and
+# some memory allocation are available on Linux through GNU extensions in libc
+if (CMAKE_SYSTEM_NAME MATCHES "Linux")
+    add_compile_definitions(_GNU_SOURCE)
+endif()
+
+# RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1,
+# and on macOS its availability depends on enabling Darwin extensions
+# similarly on DragonFly, enabling BSD extensions is necessary
+if (
+    CMAKE_SYSTEM_NAME MATCHES "Darwin" OR
+    CMAKE_SYSTEM_NAME MATCHES "iOS" OR
+    CMAKE_SYSTEM_NAME MATCHES "tvOS" OR
+    CMAKE_SYSTEM_NAME MATCHES "DragonFly"
+)
+    add_compile_definitions(_DARWIN_C_SOURCE)
+endif()
+
+# alloca is a non-standard interface that is not visible on BSDs when
+# POSIX conformance is specified, but not all of them provide a clean way
+# to enable it in such cases
+if (CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+    add_compile_definitions(__BSD_VISIBLE)
+endif()
+if (CMAKE_SYSTEM_NAME MATCHES "NetBSD")
+    add_compile_definitions(_NETBSD_SOURCE)
+endif()
+if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
+    add_compile_definitions(_BSD_SOURCE)
+endif()
+
 function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
     message(STATUS "Configuring ggml implementation target llama${SUFFIX} in ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}")
 
@@ -452,15 +617,14 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
 
     if (WITH_LLAMA)
         # Backwards compatibility with old llama.cpp versions
-        set(LLAMA_UTIL_SOURCE_FILE llama-util.h)
+#        set(LLAMA_UTIL_SOURCE_FILE llama-util.h)
         if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
             set(LLAMA_UTIL_SOURCE_FILE llama_util.h)
         endif()
 
         add_library(llama${SUFFIX} STATIC
                     ${DIRECTORY}/llama.cpp
-                    ${DIRECTORY}/llama.h
-                    ${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
+                    ${DIRECTORY}/llama.h)
 
         if (LLAMA_METAL AND GGML_METAL_SOURCES)
             target_compile_definitions(llama${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)

gpt4all-backend/llamamodel.cpp

@@ -39,15 +39,10 @@ const char *modelType_ = "LLaMA";
 struct gpt_params {
     int32_t seed          = -1;   // RNG seed
     int32_t n_keep        = 0;    // number of tokens to keep from initial prompt
-#if LLAMA_DATE <= 230511
-    int32_t n_parts       = -1;   // amount of model parts (-1 = determine from model dimensions)
-#endif
 
-#if LLAMA_DATE >= 230519
     // sampling parameters
     float   tfs_z         = 1.0f; // 1.0 = disabled
     float   typical_p     = 1.0f; // 1.0 = disabled
-#endif
 
     std::string prompt = "";
 
@@ -57,7 +52,6 @@ struct gpt_params {
     bool use_mlock         = false; // use mlock to keep model in memory
 };
 
-#if LLAMA_DATE >= 230519
 static int llama_sample_top_p_top_k(
         llama_context *ctx,
         const llama_token *last_n_tokens_data,
@@ -85,7 +79,6 @@ static int llama_sample_top_p_top_k(
     llama_sample_temperature(ctx, &candidates_p, temp);
     return llama_sample_token(ctx, &candidates_p);
 }
-#endif
 
 struct LLamaPrivate {
     const std::string modelPath;
@@ -93,6 +86,7 @@ struct LLamaPrivate {
     llama_context *ctx = nullptr;
     llama_context_params params;
     int64_t n_threads = 0;
+    std::vector<LLModel::Token> end_tokens;
 };
 
 LLamaModel::LLamaModel()
@@ -149,9 +143,6 @@ bool LLamaModel::loadModel(const std::string &modelPath)
 #else
     d_ptr->params.use_mlock  = params.use_mlock;
 #endif
-#if LLAMA_DATE <= 230511
-    d_ptr->params.n_parts  = params.n_parts;
-#endif
 #ifdef GGML_USE_METAL
     std::cerr << "llama.cpp: using Metal" << std::endl;
     // metal always runs the whole model if n_gpu_layers is not 0, at least
@@ -176,6 +167,8 @@ bool LLamaModel::loadModel(const std::string &modelPath)
         return false;
     }
 
+    d_ptr->end_tokens = {llama_token_eos(d_ptr->ctx)};
+
 #ifdef GGML_USE_KOMPUTE
     if (ggml_vk_has_device()) {
         std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
@@ -226,9 +219,9 @@ size_t LLamaModel::restoreState(const uint8_t *src)
 
 std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str) const
 {
-    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
+    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos(d_ptr->ctx));
     std::vector<LLModel::Token> fres(str.size()+4);
-    auto fres_len = llama_tokenize(d_ptr->ctx, str.c_str(), fres.data(), fres.size(), useBOS);
+    auto fres_len = llama_tokenize(d_ptr->ctx, str.c_str(), str.length(), fres.data(), fres.size(), useBOS);
     fres.resize(fres_len);
     return fres;
 }
@@ -249,16 +242,7 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
 
 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
 {
-    // When we recalculate context we could have erased the original BOS token... we need to replace it
-    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
-    if (useBOS) {
-        std::vector<int32_t> myTokens;
-        myTokens.push_back(llama_token_bos());
-        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
-        ctx.n_past += 1;
-        return llama_eval(d_ptr->ctx, myTokens.data(), myTokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
-    } else
-        return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+    return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
 }
 
 int32_t LLamaModel::contextLength() const
@@ -268,8 +252,7 @@ int32_t LLamaModel::contextLength() const
 
 const std::vector<LLModel::Token> &LLamaModel::endTokens() const
 {
-    static const std::vector<LLModel::Token> fres = {llama_token_eos()};
-    return fres;
+    return d_ptr->end_tokens;
 }
 
 #if defined(GGML_USE_KOMPUTE)
@@ -308,8 +291,9 @@ bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string& d
 #endif
 }
 
-bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device)
+bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device, std::string *unavail_reason)
 {
+    bool result = false;
 #if defined(GGML_USE_KOMPUTE)
     ggml_vk_device vkDevice;
     vkDevice.index = device.index;
@@ -317,10 +301,16 @@ bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device)
     vkDevice.heapSize = device.heapSize;
     vkDevice.name = device.name;
     vkDevice.vendor = device.vendor;
-    return ggml_vk_init_device(vkDevice);
+    result = ggml_vk_init_device(vkDevice);
+    if (!result && unavail_reason) {
+        *unavail_reason = "failed to init device";
+    }
 #else
-    return false;
+    if (unavail_reason) {
+        *unavail_reason = "built without kompute";
+    }
 #endif
+    return result;
 }
 
 bool LLamaModel::initializeGPUDevice(int device)
@@ -351,6 +341,16 @@ bool LLamaModel::usingGPUDevice()
     return false;
 }
 
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != (GGUF_TYPE_STRING)) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -370,39 +370,23 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(std::istream& f) {
-    // Check magic
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    if (magic != 0x67676a74) return false;
-    // Check version
-    uint32_t version = 0;
-    f.read(reinterpret_cast<char*>(&version), sizeof(version));
-    if (!(version LLAMA_VERSIONS)) {
+DLL_EXPORT bool magic_match(const char * fname) {
+
+    struct ggml_context * ctx_meta = NULL;
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ &ctx_meta,
+    };
+    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
+    if (!ctx_gguf)
         return false;
-    }
-    llama_file_hparams hparams;
-    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
-    if (!(hparams.n_vocab >= 32000 && hparams.n_vocab <= 32100)) {
-        return false; // not a llama.
-    }
-#ifdef GGML_USE_METAL
-    // Check quant supported on metal
-    // skip fields
-    switch(hparams.ftype) {
-        // currently supported on Metal https://github.com/ggerganov/llama.cpp/blob/ae9663f1887513e152839e91f61c513075a19422/ggml-metal.m#L51-L55
-        case LLAMA_FTYPE_MOSTLY_F16:
-        case LLAMA_FTYPE_MOSTLY_Q2_K:
-        case LLAMA_FTYPE_MOSTLY_Q4_0:
-        case LLAMA_FTYPE_MOSTLY_Q6_K:
-        case LLAMA_FTYPE_MOSTLY_Q4_K_S:
-        case LLAMA_FTYPE_MOSTLY_Q4_K_M:
-            return true;
-        default: // unsupported quant-type for Metal
-            return false;
-    }
-#endif
-    return true;
+
+    bool isValid = gguf_get_version(ctx_gguf) <= 2;
+    auto arch = get_arch_name(ctx_gguf);
+    isValid = isValid && (arch == "llama" || arch == "starcoder" || arch == "falcon");
+
+    gguf_free(ctx_gguf);
+    return isValid;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llamamodel_impl.h

@@ -27,7 +27,7 @@ public:
     int32_t threadCount() const override;
     std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) override;
     bool initializeGPUDevice(size_t memoryRequired, const std::string& device) override;
-    bool initializeGPUDevice(const GPUDevice &device) override;
+    bool initializeGPUDevice(const GPUDevice &device, std::string *unavail_reason) override;
     bool initializeGPUDevice(int device) override;
     bool hasGPUDevice() override;
     bool usingGPUDevice() override;

gpt4all-backend/llmodel.cpp

@@ -52,7 +52,7 @@ LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
     m_buildVariant = get_build_variant();
-    m_magicMatch = m_dlhandle->get<bool(std::ifstream&)>("magic_match");
+    m_magicMatch = m_dlhandle->get<bool(const char*)>("magic_match");
     assert(m_magicMatch);
     m_construct = m_dlhandle->get<LLModel *()>("construct");
     assert(m_construct);
@@ -111,10 +111,9 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     return *libs;
 }
 
-const LLModel::Implementation* LLModel::Implementation::implementation(std::ifstream& f, const std::string& buildVariant) {
+const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
     for (const auto& i : implementationList()) {
-        f.seekg(0);
-        if (!i.m_magicMatch(f)) continue;
+        if (!i.m_magicMatch(fname)) continue;
         if (buildVariant != i.m_buildVariant) continue;
         return &i;
     }
@@ -126,16 +125,13 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
     if (!has_at_least_minimal_hardware())
         return nullptr;
 
-    // Read magic
-    std::ifstream f(modelPath, std::ios::binary);
-    if (!f) return nullptr;
     // Get correct implementation
     const Implementation* impl = nullptr;
 
     #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
         if (buildVariant == "auto") {
             size_t total_mem = getSystemTotalRAMInBytes();
-            impl = implementation(f, "metal");
+            impl = implementation(modelPath.c_str(), "metal");
             if(impl) {
                 LLModel* metalimpl = impl->m_construct();
                 metalimpl->m_implementation = impl;
@@ -161,10 +157,9 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
                 buildVariant = "default";
             }
         }
-        impl = implementation(f, buildVariant);
+        impl = implementation(modelPath.c_str(), buildVariant);
         if (!impl) return nullptr;
     }
-    f.close();
 
     // Construct and return llmodel implementation
     auto fres = impl->m_construct();

gpt4all-backend/llmodel.h

@@ -27,13 +27,13 @@ public:
 
         static bool isImplementation(const Dlhandle&);
         static const std::vector<Implementation>& implementationList();
-        static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
+        static const Implementation *implementation(const char *fname, const std::string& buildVariant);
         static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
         static void setImplementationsSearchPath(const std::string& path);
         static const std::string& implementationsSearchPath();
 
     private:
-        bool (*m_magicMatch)(std::ifstream& f);
+        bool (*m_magicMatch)(const char *fname);
         LLModel *(*m_construct)();
 
     private:
@@ -97,10 +97,16 @@ public:
 
     virtual std::vector<GPUDevice> availableGPUDevices(size_t /*memoryRequired*/) { return std::vector<GPUDevice>(); }
     virtual bool initializeGPUDevice(size_t /*memoryRequired*/, const std::string& /*device*/) { return false; }
-    virtual bool initializeGPUDevice(const GPUDevice &/*device*/) { return false; }
+    virtual bool initializeGPUDevice(const GPUDevice &/*device*/, std::string *unavail_reason = nullptr) {
+        if (unavail_reason) {
+            *unavail_reason = "unsupported model type";
+        }
+        return false;
+    }
     virtual bool initializeGPUDevice(int /*device*/) { return false; }
     virtual bool hasGPUDevice() { return false; }
     virtual bool usingGPUDevice() { return false; }
+    static std::vector<GPUDevice> availableGPUDevices();
 
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of

gpt4all-backend/llmodel_shared.cpp

@@ -4,6 +4,10 @@
 #include <iostream>
 #include <unordered_set>
 
+#ifdef GGML_USE_KOMPUTE
+#include "ggml-vulkan.h"
+#endif
+
 void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
     size_t i = 0;
     promptCtx.n_past = 0;
@@ -88,10 +92,10 @@ void LLModel::prompt(const std::string &prompt,
             if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
                 promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(batch.at(t));
+            promptCtx.n_past += 1;
             if (!promptCallback(batch.at(t)))
                 return;
         }
-        promptCtx.n_past += batch.size();
         i = batch_end;
     }
 
@@ -122,8 +126,6 @@ void LLModel::prompt(const std::string &prompt,
             return;
         }
 
-        promptCtx.n_past += 1;
-
         // display text
         for (const auto token : endTokens()) {
             if (id == token) return;
@@ -158,6 +160,7 @@ void LLModel::prompt(const std::string &prompt,
             if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
                 promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(t);
+            promptCtx.n_past += 1;
             //TODO: Conversion to std::string can be avoided here...
             if (!responseCallback(t, std::string(tokenToString(t))))
                 return;
@@ -174,3 +177,26 @@ std::vector<float> LLModel::embedding(const std::string &/*text*/)
     }
     return std::vector<float>();
 }
+
+std::vector<LLModel::GPUDevice> LLModel::availableGPUDevices()
+{
+#if defined(GGML_USE_KOMPUTE)
+    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(0);
+
+    std::vector<LLModel::GPUDevice> devices;
+    for(const auto& vkDevice : vkDevices) {
+        LLModel::GPUDevice device;
+        device.index = vkDevice.index;
+        device.type = vkDevice.type;
+        device.heapSize = vkDevice.heapSize;
+        device.name = vkDevice.name;
+        device.vendor = vkDevice.vendor;
+
+        devices.push_back(device);
+    }
+
+    return devices;
+#else
+    return std::vector<LLModel::GPUDevice>();
+#endif
+}

gpt4all-backend/llmodel_shared.h

@@ -80,7 +80,6 @@ struct llm_kv_cache {
     }
 };
 
-#if LLAMA_DATE >= 230519
 inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads) {
     struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
     if (plan.work_size > 0) {
@@ -89,4 +88,3 @@ inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_t
     }
     ggml_graph_compute(graph, &plan);
 }
-#endif

gpt4all-backend/mpt.cpp

@@ -9,7 +9,6 @@
 #include <cmath>
 #include <cstdio>
 #include <cstring>
-#include <fstream>
 #include <map>
 #include <random>
 #include <string>
@@ -29,6 +28,7 @@
 #include <sstream>
 #include <thread>
 #include <unordered_set>
+#include <unordered_map>
 #include <regex>
 #include <ggml.h>
 
@@ -46,8 +46,8 @@ struct mpt_hparams {
     int32_t n_layer      = 32;
     float alibi_bias_max = 8;
     float clip_qkv       = 0;
+    float norm_eps       = 1e-5;
     int32_t expand       = 4;
-    int32_t f16          = 1;
 };
 
 struct mpt_layer {
@@ -78,7 +78,6 @@ struct mpt_model {
 
     struct llm_kv_cache kv_self;
     struct ggml_context * ctx;
-    std::map<std::string, struct ggml_tensor *> tensors;
 
 
     llm_buffer eval_buf;
@@ -92,6 +91,121 @@ struct mpt_model {
     }
 };
 
+enum mpt_token_type {
+    MPT_TOKEN_TYPE_NORMAL  = 1,
+    MPT_TOKEN_TYPE_CONTROL = 3,
+};
+
+using replit_piece_t = std::pair<std::size_t, float>;
+using replit_piece_map_t = std::unordered_map<std::string, replit_piece_t>;
+
+static const std::string replit_ws_symbol = "\342\226\201";
+
+struct mpt_vocab {
+    bool is_replit = false;
+    gpt_vocab raw;
+    replit_piece_map_t piece_map;
+    std::vector<std::string> vocab;
+
+    const char * end_of_text() const {
+        return is_replit ? "<|endoftext|>" : "<|im_end|>";
+    }
+};
+
+std::pair<std::vector<LLModel::Token>, float> encode_word(const std::string & word, const replit_piece_map_t & model) {
+    std::vector<int> best_segmentations_starts(word.length() + 1, -1);
+    best_segmentations_starts[0] = 0;
+
+    std::vector<float> best_segmentations_scores(word.length() + 1, -std::numeric_limits<float>::infinity());
+    best_segmentations_scores[0] = 1.0;
+
+    for (size_t start_idx = 0; start_idx < word.length(); ++start_idx) {
+        float best_score_at_start = best_segmentations_scores[start_idx];
+        for (size_t end_idx = start_idx + 1; end_idx <= word.length(); ++end_idx) {
+            std::string token = word.substr(start_idx, end_idx - start_idx);
+            if (model.count(token) && best_score_at_start != -std::numeric_limits<float>::infinity()) {
+                float token_score = model.at(token).second;
+                float score = token_score + best_score_at_start;
+                if (best_segmentations_scores[end_idx] == -std::numeric_limits<float>::infinity() ||
+                    best_segmentations_scores[end_idx] > score) {
+                    best_segmentations_starts[end_idx] = start_idx;
+                    best_segmentations_scores[end_idx] = score;
+                }
+            }
+        }
+    }
+
+    if (best_segmentations_scores.back() == -std::numeric_limits<float>::infinity()) {
+        return std::make_pair(std::vector<LLModel::Token>{0}, 0.0f);
+    }
+
+    float score = best_segmentations_scores.back();
+    int start = best_segmentations_starts.back();
+    int end = word.length();
+    std::vector<LLModel::Token> tokens;
+    while (start != 0) {
+        const auto token_id = model.at(word.substr(start, end - start)).first;
+        tokens.insert(tokens.begin(), token_id);
+        int next_start = best_segmentations_starts[start];
+        end = start;
+        start = next_start;
+    }
+    const auto token_id = model.at(word.substr(start, end - start)).first;
+    tokens.insert(tokens.begin(), token_id);
+    return std::make_pair(tokens, score);
+}
+
+bool replit_tokenizer_load(mpt_vocab & tokenizer, gguf_context * ggufctx, int tokens_keyidx, int max_vocab_size) {
+    int scores_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.scores");
+    if (scores_keyidx == -1) {
+        fprintf(stderr, "%s: llama token scores not found!\n", __func__);
+        return false;
+    }
+    const auto *scores = reinterpret_cast<const float *>(gguf_get_arr_data(ggufctx, scores_keyidx));
+
+    for (LLModel::Token i = 0; i < max_vocab_size; i++) {
+        std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
+        tokenizer.piece_map[word] = std::make_pair(i, -scores[i]);
+        tokenizer.raw.id_to_token[i] = word;
+        tokenizer.raw.token_to_id[word] = i;
+    }
+
+    return true;
+}
+
+std::string replace_all(const std::string & str,    // where to work
+                        const std::string & find,   // substitute 'find'
+                        const std::string & replace //      by 'replace'
+) {
+    std::string result;
+    size_t find_len = find.size();
+    size_t pos, from = 0;
+    while (std::string::npos != (pos = str.find(find, from))) {
+        result.append(str, from, pos - from);
+        result.append(replace);
+        from = pos + find_len;
+    }
+    result.append(str, from, std::string::npos);
+    return result;
+}
+
+std::vector<LLModel::Token> replit_tokenizer_tokenize(mpt_vocab & tokenizer, const std::string & text) {
+    std::vector<LLModel::Token> tokens;
+    auto normalized_text = replace_all(text, " ", replit_ws_symbol);
+    auto tokenized = encode_word(normalized_text, tokenizer.piece_map);
+
+    return tokenized.first;
+}
+
+std::string replit_tokenizer_detokenize(mpt_vocab & tokenizer, const std::vector<LLModel::Token> & tokens) {
+    std::string text;
+    for (auto token : tokens) {
+        text += tokenizer.raw.id_to_token[token];
+    }
+    return replace_all(text, replit_ws_symbol, " ");
+}
+
+
 static bool kv_cache_init(
         const struct mpt_hparams & hparams,
              struct llm_kv_cache & cache,
@@ -123,20 +237,62 @@ static bool kv_cache_init(
     return true;
 }
 
-// load the model's weights from a stream. if mem_req ptr is passed the model is
+// load the model's weights from a file path. if mem_req ptr is passed the model is
 // only partially parsed to estimate required memory
-bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & model, gpt_vocab & vocab, size_t * mem_req) {
+bool mpt_model_load(const std::string &fname, mpt_model & model, mpt_vocab & vocab, size_t * mem_req) {
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
     if (mem_req != nullptr) {
         *mem_req = 0;
     }
 
-    // verify magic
+    // create the ggml context
+    struct gguf_init_params params = {
+        /*.no_alloc = */ false,
+        /*.ctx      = */ &model.ctx,
+    };
+
+    gguf_context *ggufctx = gguf_init_from_file(fname.c_str(), params);
+    if (!ggufctx) {
+        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
+        return false;
+    }
+
+    printf("%s: gguf version     = %d\n", __func__, gguf_get_version(ggufctx));
+    printf("%s: gguf alignment   = %zu\n", __func__, gguf_get_alignment(ggufctx));
+    printf("%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx));
+
+    // print some standard metadata
     {
-        uint32_t magic;
-        fin.read((char *) &magic, sizeof(magic));
-        if (magic != 0x67676d6d) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
+        int keyidx;
+
+        keyidx = gguf_find_key(ggufctx, "general.name");
+        if (keyidx != -1) { printf("%s: model name           = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.description");
+        if (keyidx != -1) { printf("%s: model description    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.author");
+        if (keyidx != -1) { printf("%s: model author         = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.license");
+        if (keyidx != -1) { printf("%s: model license        = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.architecture");
+        if (keyidx != -1) { printf("%s: model architecture   = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.file_type");
+        if (keyidx != -1) { printf("%s: model file type      = %" PRIu32 "\n", __func__, gguf_get_val_u32(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "gptneox.tensor_data_layout");
+        if (keyidx != -1) { printf("%s: model data layout    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.source.huggingface.repository");
+        if (keyidx != -1) { printf("%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+    }
+
+    // check required metadata
+    {
+        // check model architecture kv
+        int keyidx = gguf_find_key(ggufctx, "general.architecture");
+        if (keyidx == -1) {
+            fprintf(stderr, "%s: gguf model architecture not found!\n", __func__);
+            return false;
+        }
+        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "mpt") != 0) {
+            fprintf(stderr, "%s: model architecture not supported!\n", __func__);
             return false;
         }
     }
@@ -145,182 +301,153 @@ bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & mod
     {
         auto & hparams = model.hparams;
 
-        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
-        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
-        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
-        fin.read((char *) &hparams.alibi_bias_max,  sizeof(hparams.alibi_bias_max));
-        fin.read((char *) &hparams.clip_qkv,  sizeof(hparams.clip_qkv));
-        fin.read((char *) &hparams.f16,   sizeof(hparams.f16));
+        bool ok = false;
+        int keyidx;
+
+        do {
+            keyidx = gguf_find_key(ggufctx, "mpt.context_length");
+            if (keyidx == -1) { break; }
+            hparams.n_ctx = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "mpt.embedding_length");
+            if (keyidx == -1) { break; }
+            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "mpt.attention.head_count");
+            if (keyidx == -1) { break; }
+            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "mpt.block_count");
+            if (keyidx == -1) { break; }
+            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "mpt.attention.max_alibi_bias");
+            if (keyidx == -1) { break; }
+            hparams.alibi_bias_max = gguf_get_val_f32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "mpt.attention.clamp_kqv");
+            if (keyidx != -1) { // optional
+                hparams.clip_qkv = gguf_get_val_f32(ggufctx, keyidx);
+            }
+
+            keyidx = gguf_find_key(ggufctx, "mpt.attention.layer_norm_epsilon");
+            if (keyidx == -1) { break; }
+            hparams.norm_eps = gguf_get_val_f32(ggufctx, keyidx);
+
+            ok = true;
+        } while (false);
+
+        if (!ok) {
+            fprintf(stderr, "%s: required hparam missing!\n", __func__);
+            return false;
+        }
 
-        printf("%s: n_vocab        = %d\n", __func__, hparams.n_vocab);
         printf("%s: n_ctx          = %d\n", __func__, hparams.n_ctx);
         printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
         printf("%s: n_head         = %d\n", __func__, hparams.n_head);
         printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
         printf("%s: alibi_bias_max = %f\n", __func__, hparams.alibi_bias_max);
         printf("%s: clip_qkv       = %f\n", __func__, hparams.clip_qkv);
-        printf("%s: ftype          = %d\n", __func__, hparams.f16);
     }
 
     // load vocab
     {
-        int32_t n_vocab = model.hparams.n_vocab;
-        fin.read((char *) &n_vocab, sizeof(n_vocab));
+        auto & hparams = model.hparams;
 
-        if (n_vocab != model.hparams.n_vocab) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
-                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
+        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
+        if (tokens_keyidx == -1) {
+            fprintf(stderr, "%s: tokenizer vocab not found!\n", __func__);
             return false;
         }
 
-        std::string word;
-        for (int i = 0; i < n_vocab; i++) {
-            uint32_t len;
-            fin.read((char *) &len, sizeof(len));
-            bool special = false;
-            if (len & (1<<31)) {
-                len = len &~ (1<<31);
-                special = true;
-            }
-
-            if (len > 0) {
-                word.resize(len);
-                fin.read((char *) word.data(), len);
-                vocab.token_to_id[word] = i;
-                vocab.id_to_token[i] = word;
-            }
-
-            if(special) {
-                vocab.add_special_token(word);
-            }
+        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
+        if (keyidx == -1) {
+            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
+            return false;
         }
-    }
+        std::string tokenizer_model(gguf_get_val_str(ggufctx, keyidx));
 
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = GGML_TYPE_COUNT;
-    switch (model.hparams.f16) {
-        case 0: wtype = GGML_TYPE_F32;  break;
-        case 1: wtype = GGML_TYPE_F16;  break;
-        case 2: wtype = GGML_TYPE_Q4_0; break;
-        case 3: wtype = GGML_TYPE_Q4_1; break;
-        case 5: wtype = GGML_TYPE_Q4_2; break;
-        default:
-                {
-                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
-                            __func__, fname.c_str(), model.hparams.f16);
+        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
+        printf("%s: %s tokenizer vocab = %d\n", __func__, tokenizer_model.c_str(), int(hparams.n_vocab));
+
+        if (tokenizer_model == "llama") { // Replit
+            vocab.is_replit = true;
+            if (!replit_tokenizer_load(vocab, ggufctx, tokens_keyidx, hparams.n_vocab)) {
+                return false;
+            }
+        } else if (tokenizer_model == "gpt2") {
+            int toktypes_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.token_type");
+            if (toktypes_keyidx == -1) {
+                fprintf(stderr, "%s: gpt2 token types not found!\n", __func__);
+                return false;
+            }
+            const auto *toktypes = reinterpret_cast<const uint32_t *>(gguf_get_arr_data(ggufctx, toktypes_keyidx));
+
+            for (int i = 0; i < hparams.n_vocab; i++) {
+                std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
+
+                bool special = false;
+                if (toktypes[i] == MPT_TOKEN_TYPE_CONTROL) {
+                    special = true;
+                } else if (toktypes[i] != MPT_TOKEN_TYPE_NORMAL) {
+                    fprintf(stderr, "%s: unknown token type: %d\n", __func__, int(toktypes[i]));
                     return false;
                 }
+
+                vocab.raw.token_to_id[word] = i;
+                vocab.raw.id_to_token[i] = word;
+
+                if (special) {
+                    vocab.raw.add_special_token(word);
+                }
+            }
+        } else {
+            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
+            return false;
+        }
     }
 
     auto & ctx = model.ctx;
 
-    size_t ctx_size = 0;
-
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_ctx   = hparams.n_ctx;
-        const int n_vocab = hparams.n_vocab;
-        const int expand  = hparams.expand;
-
-
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_w
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(GGML_TYPE_F32); // wte
-
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_1_w
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_2_w
-
-        ctx_size += n_layer*(3*n_embd*n_embd*ggml_type_sizef(wtype)); // attn_Wqkv_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // attn_out_proj_w
-
-        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype));  // ffn_up_proj_w
-        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype)); // ffn_down_proj_w
-
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_k
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_v
-
-        // TODO probably less now?
-        ctx_size += (5 + 10*n_layer)*256; // object overhead
-
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
-    }
+    size_t ctx_size = ggml_get_mem_size(ctx);
+    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size / (1024.0 * 1024.0));
 
     if (mem_req != nullptr) {
-        *mem_req += ctx_size;
-        const int n_embd  = model.hparams.n_embd;
-        const int n_layer = model.hparams.n_layer;
-
-        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
-        const int64_t n_elements = n_embd*n_mem;
-
-        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+        *mem_req = ctx_size;
+        gguf_free(ggufctx);
         return false;
     }
 
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size   = ctx_size,
-            .mem_buffer = NULL,
-            .no_alloc   = false,
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx) {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            return false;
-        }
-    }
-
     // prepare memory for the weights
     {
         const auto & hparams = model.hparams;
+        model.layers.resize(hparams.n_layer);
 
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_vocab = hparams.n_vocab;
-        const int expand  = hparams.expand;
+        model.wte      = ggml_get_tensor(ctx, "token_embd.weight");
+        model.norm_f_w = ggml_get_tensor(ctx, "output_norm.weight");
 
-        model.layers.resize(n_layer);
+        auto name = [](int i, std::string n) {
+            static std::string key;
+            key = "blk." + std::to_string(i) + "." + n;
+            return key.c_str();
+        };
 
-        model.wte    = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab);
-        model.norm_f_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        for (int i = 0; i < hparams.n_layer; ++i) {
+            auto &layer = model.layers[i];
 
-        // map by name
-        model.tensors["transformer.wte.weight"] = model.wte;
-        model.tensors["transformer.norm_f.weight"] = model.norm_f_w;
+            layer.norm_1_w        = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
+            layer.norm_2_w        = ggml_get_tensor(ctx, name(i, "ffn_norm.weight"));
 
-        for (int i = 0; i < n_layer; ++i) {
-            auto & layer = model.layers[i];
-
-            layer.norm_1_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.norm_2_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            layer.attn_Wqkv_w     = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd * 3);
-            layer.attn_out_proj_w = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd);
-            layer.ffn_up_proj_w   = ggml_new_tensor_2d(ctx, wtype,        n_embd, expand*n_embd);
-            layer.ffn_down_proj_w = ggml_new_tensor_2d(ctx, wtype, expand*n_embd, n_embd);
-
-            // map by name
-            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_1.weight"]        = layer.norm_1_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_2.weight"]        = layer.norm_2_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.Wqkv.weight"]     = layer.attn_Wqkv_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.out_proj.weight"] = layer.attn_out_proj_w;
-
-            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.up_proj.weight"]   = layer.ffn_up_proj_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.down_proj.weight"] = layer.ffn_down_proj_w;
+            layer.attn_Wqkv_w     = ggml_get_tensor(ctx, name(i, "attn_qkv.weight"));
+            layer.attn_out_proj_w = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
+            layer.ffn_up_proj_w   = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
+            layer.ffn_down_proj_w = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
         }
     }
 
     // key + value memory
     {
-        const auto & hparams = model.hparams;
+        const auto &hparams = model.hparams;
         if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
             fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
             ggml_free(ctx);
@@ -331,101 +458,12 @@ bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & mod
         printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
     }
 
-    // load weights
-    {
-        int n_tensors = 0;
-        size_t total_size = 0;
-
-        printf("%s: ", __func__);
-
-        while (true) {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ttype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
-
-            if (fin.eof()) {
-                break;
-            }
-
-            int32_t nelements = 1;
-            int32_t ne[2] = { 1, 1 };
-            for (int i = 0; i < n_dims; ++i) {
-                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-
-            if (model.tensors.find(name.data()) == model.tensors.end()) {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                return false;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                return false;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
-                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
-                return false;
-            }
-
-            // for debugging
-            if (0) {
-                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
-            }
-
-            const size_t bpe = ggml_type_size(ggml_type(ttype));
-
-            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
-                return false;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ttype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
-            total_size += ggml_nbytes(tensor);
-            if (++n_tensors % 8 == 0) {
-                printf(".");
-                fflush(stdout);
-            }
-        }
-
-        printf(" done\n");
-
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
-    }
-
     model.scr0_buf.resize(256u * 1024 * 1024);
     model.scr1_buf.resize(256u * 1024 * 1024);
 
     return true;
 }
 
-// load the model's weights from a file path
-bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vocab) {
-
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin) {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
-        return false;
-    }
-
-    bool loaded = mpt_model_load(fname, fin, model, vocab, nullptr);
-    fin.close();
-    return loaded;
-}
-
 bool mpt_eval(
         mpt_model & model,
         const int n_threads,
@@ -467,7 +505,6 @@ bool mpt_eval(
 
     struct ggml_context * ctx0 = ggml_init(params);
     struct ggml_cgraph gf = {};
-    gf.n_threads = n_threads;
 
     struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
     memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -484,7 +521,7 @@ bool mpt_eval(
         {
 
             // norm1
-            cur = ggml_norm(ctx0, cur);
+            cur = ggml_norm(ctx0, cur, model.hparams.norm_eps);
             cur = ggml_mul(ctx0,
                     ggml_repeat(ctx0, model.layers[il].norm_1_w, cur),
                     cur);
@@ -535,7 +572,9 @@ bool mpt_eval(
 
 
             // Alibi
-            struct ggml_tensor * KQ_scaled_biased = ggml_alibi(ctx0, ggml_cont(ctx0, KQ_scaled), n_past, n_head);
+            struct ggml_tensor * KQ_scaled_biased = ggml_alibi(
+                ctx0, ggml_cont(ctx0, KQ_scaled), n_past, n_head, model.hparams.alibi_bias_max
+            );
 
             // KQ_masked = mask_past(KQ_scaled)
             struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled_biased, n_past);
@@ -575,7 +614,7 @@ bool mpt_eval(
         {
             cur = resSA;
             // norm2
-            cur = ggml_norm(ctx0, cur);
+            cur = ggml_norm(ctx0, cur, model.hparams.norm_eps);
             cur = ggml_mul(ctx0,
                     ggml_repeat(ctx0, model.layers[il].norm_2_w, cur),
                     cur);
@@ -598,7 +637,7 @@ bool mpt_eval(
     struct ggml_tensor * out = inpL;
     // -> logits
     {
-        out = ggml_norm(ctx0, out);
+        out = ggml_norm(ctx0, out, model.hparams.norm_eps);
         out = ggml_mul(ctx0,
                     ggml_repeat(ctx0, model.norm_f_w, out),
                     out);
@@ -606,10 +645,19 @@ bool mpt_eval(
         out = ggml_mul_mat(ctx0, model.wte, out);
     }
 
+    ggml_build_forward_expand(&gf, out);
+
 
     // run the computation
-    ggml_build_forward_expand(&gf, out);
-    ggml_graph_compute       (ctx0, &gf);
+    {
+        std::unique_ptr<uint8_t []> data;
+        auto plan = ggml_graph_plan(&gf, n_threads);
+        if (plan.work_size > 0) {
+            data.reset(new uint8_t[plan.work_size]);
+            plan.work_data = data.get();
+        }
+        ggml_graph_compute(&gf, &plan);
+    }
 
 
     // return result for just the last token
@@ -739,12 +787,12 @@ size_t mpt_set_state_data(mpt_model *model, std::mt19937 *rng, const uint8_t *sr
 struct MPTPrivate {
     const std::string modelPath;
     bool modelLoaded;
-    gpt_vocab vocab;
+    mpt_vocab vocab;
     mpt_model *model = nullptr;
     int64_t n_threads = 0;
     size_t mem_per_token = 0;
     std::mt19937 rng;
-    bool has_im_end = false;
+    bool has_end_of_text = false;
 };
 
 MPT::MPT()
@@ -756,10 +804,9 @@ MPT::MPT()
 
 size_t MPT::requiredMem(const std::string &modelPath) {
     mpt_model dummy_model;
-    gpt_vocab dummy_vocab;
+    mpt_vocab dummy_vocab;
     size_t mem_req;
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-    mpt_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
+    mpt_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
     return mem_req;
 }
 
@@ -767,17 +814,16 @@ bool MPT::loadModel(const std::string &modelPath) {
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
 
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-
     // load the model
-    if (!mpt_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab, nullptr)) {
+    if (!mpt_model_load(modelPath, *d_ptr->model, d_ptr->vocab, nullptr)) {
         std::cerr << "MPT ERROR: failed to load model from " <<  modelPath;
         return false;
     }
 
     d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
     d_ptr->modelLoaded = true;
-    d_ptr->has_im_end = d_ptr->vocab.token_to_id.find("<|im_end|>") != d_ptr->vocab.token_to_id.end();
+    const auto & vocab = d_ptr->vocab;
+    d_ptr->has_end_of_text = vocab.raw.token_to_id.find(vocab.end_of_text()) != vocab.raw.token_to_id.end();
     fflush(stdout);
     return true;
 }
@@ -818,12 +864,18 @@ size_t MPT::restoreState(const uint8_t *src)
 
 std::vector<LLModel::Token> MPT::tokenize(PromptContext &, const std::string &str) const
 {
-    return ::gpt_tokenize(d_ptr->vocab, str);
+    if (d_ptr->vocab.is_replit) {
+        return replit_tokenizer_tokenize(d_ptr->vocab, str);
+    }
+    return ::gpt_tokenize(d_ptr->vocab.raw, str);
 }
 
 std::string MPT::tokenToString(Token id) const
 {
-    return d_ptr->vocab.id_to_token[id];
+    if (d_ptr->vocab.is_replit) {
+        return replit_tokenizer_detokenize(d_ptr->vocab, {id});
+    }
+    return d_ptr->vocab.raw.id_to_token[id];
 }
 
 LLModel::Token MPT::sampleToken(PromptContext &promptCtx) const
@@ -858,10 +910,23 @@ int32_t MPT::contextLength() const
 
 const std::vector<LLModel::Token> &MPT::endTokens() const
 {
-    static const std::vector<LLModel::Token> fres = {0, d_ptr->vocab.token_to_id["<|im_end|>"]};
+    static std::vector<LLModel::Token> fres;
+    if (fres.empty()) {
+        fres = {0, d_ptr->vocab.raw.token_to_id[d_ptr->vocab.end_of_text()]};
+    }
     return fres;
 }
 
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != GGUF_TYPE_STRING) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -881,10 +946,21 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    return magic == 0x67676d6d;
+DLL_EXPORT bool magic_match(const char * fname) {
+    struct ggml_context * ctx_meta = NULL;
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ &ctx_meta,
+    };
+    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
+    if (!ctx_gguf)
+        return false;
+
+    bool isValid = gguf_get_version(ctx_gguf) <= 2;
+    isValid = isValid && get_arch_name(ctx_gguf) == "mpt";
+
+    gguf_free(ctx_gguf);
+    return isValid;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/replit_impl.h

@@ -1,44 +0,0 @@
-#ifndef REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of replit.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef REPLIT_H
-#define REPLIT_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include "llmodel.h"
-
-#define GGML_QNT_VERSION_FACTOR 1000 // do not change this
-
-struct ReplitPrivate;
-class Replit : public LLModel {
-public:
-    Replit();
-    ~Replit();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string & modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-    bool usingGPUDevice() override;
-
-private:
-    ReplitPrivate *d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string tokenToString(Token) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // REPLIT_H

gpt4all-backend/scripts/convert_bert_hf_to_ggml.py

@@ -1,102 +0,0 @@
-import sys
-import struct
-import json
-import torch
-import numpy as np
-
-from transformers import AutoModel, AutoTokenizer
-
-if len(sys.argv) < 3:
-    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = sys.argv[1]
-fname_out = sys.argv[1] + "/ggml-model.bin"
-
-with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f:
-    encoder = json.load(f)
-
-with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
-    hparams = json.load(f)
-
-with open(dir_model + "/vocab.txt", "r", encoding="utf-8") as f:
-    vocab = f.readlines()
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-    fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
-
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-model = AutoModel.from_pretrained(dir_model, low_cpu_mem_usage=True)
-print (model)
-
-print(tokenizer.encode('I believe the meaning of life is'))
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-fout = open(fname_out, "wb")
-
-print(hparams)
-
-fout.write(struct.pack("i", 0x62657274)) # magic: ggml in hex
-fout.write(struct.pack("i", hparams["vocab_size"]))
-fout.write(struct.pack("i", hparams["max_position_embeddings"]))
-fout.write(struct.pack("i", hparams["hidden_size"]))
-fout.write(struct.pack("i", hparams["intermediate_size"]))
-fout.write(struct.pack("i", hparams["num_attention_heads"]))
-fout.write(struct.pack("i", hparams["num_hidden_layers"]))
-fout.write(struct.pack("i", ftype))
-
-for i in range(hparams["vocab_size"]):
-    text = vocab[i][:-1] # strips newline at the end
-    #print(f"{i}:{text}")
-    data = bytes(text, 'utf-8')
-    fout.write(struct.pack("i", len(data)))
-    fout.write(data)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
-        continue
-    print("Processing variable: " + name + " with shape: ", data.shape)
-
-    n_dims = len(data.shape);
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-            print("  Converting to float16")
-            data = data.astype(np.float16)
-            l_type = 1
-    else:
-        l_type = 0
-
-    # header
-    str = name.encode('utf-8')
-    fout.write(struct.pack("iii", n_dims, len(str), l_type))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str);
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-backend/scripts/convert_bert_hf_to_gguf.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import struct
+import sys
+from pathlib import Path
+
+import gguf
+import numpy as np
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+
+
+if not 2 <= len(sys.argv) < 4:
+    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = Path(sys.argv[1])
+
+with open(dir_model / "vocab.txt", encoding="utf-8") as f:
+    vocab = f.readlines()
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+
+fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
+
+
+ARCH = gguf.MODEL_ARCH.BERT
+gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
+
+print("gguf: get model metadata")
+
+config = AutoConfig(dir_model)
+
+block_count = config.num_hidden_layers
+gguf_writer.add_name("BERT")
+gguf_writer.add_context_length(config.max_position_embeddings)
+gguf_writer.add_embedding_length(config.hidden_size)
+gguf_writer.add_feed_forward_length(config.intermediate_size)
+gguf_writer.add_block_count(block_count)
+gguf_writer.add_head_count(config.num_attention_heads)
+gguf_writer.add_file_type(ftype)
+
+print("gguf: get tokenizer metadata")
+
+try:
+    with open(dir_model / "tokenizer.json", encoding="utf-8") as f:
+        tokenizer_json = json.load(f)
+except FileNotFoundError as e:
+    print(f'Error: Missing {e.filename!r}', file=sys.stderr)
+    sys.exit(1)
+
+print("gguf: get wordpiece tokenizer vocab")
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+print(tokenizer.encode('I believe the meaning of life is'))
+
+tokens: list[bytearray] = []
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+
+# The number of tokens in tokenizer.json can differ from the expected vocab size.
+# This causes downstream issues with mismatched tensor sizes when running the inference
+for i in range(config.vocab_size):
+    try:
+        text = reverse_vocab[i]
+    except KeyError:
+        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
+        pad_token = f"[PAD{i}]".encode("utf8")
+        text = bytearray(pad_token)
+
+    tokens.append(text)
+
+gguf_writer.add_tokenizer_model("bert")  # wordpiece
+gguf_writer.add_token_list(tokens)
+
+special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
+special_vocab.add_to_gguf(gguf_writer)
+
+print("gguf: get tensor metadata")
+
+model = AutoModel.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
+print(model)
+
+tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
+        continue
+    print("Processing variable:", name, "with shape:", data.shape)
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+        print("  Converting to float16")
+        data = data.astype(np.float16)
+        l_type = 1
+    else:
+        l_type = 0
+
+    # map tensor names
+    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
+    if new_name is None:
+        print("Can not map tensor '" + name + "'")
+        sys.exit()
+
+    gguf_writer.add_tensor(new_name, data)
+
+
+print("gguf: write header")
+gguf_writer.write_header_to_file()
+print("gguf: write metadata")
+gguf_writer.write_kv_data_to_file()
+print("gguf: write tensors")
+gguf_writer.write_tensors_to_file()
+
+gguf_writer.close()
+
+print(f"gguf: model successfully exported to '{fname_out}'")
+print()

gpt4all-backend/scripts/convert_falcon_hf_to_ggml.py

@@ -1,143 +0,0 @@
-# Based on: https://github.com/KerfuffleV2/ggml-falcon/blob/feat-improve-falcon-convert-hf/examples/falcon/convert-hf-to-ggml.py
-# Convert Hugging Face fine-tuned bloom-like models to ggml format
-#
-# Usage:
-#
-#   python3 convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]
-#
-# This script is similar to "convert-pt-to-ggml.py"
-#
-
-import io
-import os
-import sys
-import struct
-import json
-import code
-import torch
-import numpy as np
-import gc
-
-from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
-
-# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
-def bytes_to_unicode():
-    """
-    Returns list of utf-8 byte and a corresponding list of unicode strings.
-    The reversible bpe codes work on unicode strings.
-    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
-    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
-    This is a significant percentage of your normal, say, 32K bpe vocab.
-    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
-    And avoids mapping to whitespace/control characters the bpe code barfs on.
-    """
-    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
-    cs = bs[:]
-    n = 0
-    for b in range(2**8):
-        if b not in bs:
-            bs.append(b)
-            cs.append(2**8+n)
-            n += 1
-    cs = [chr(n) for n in cs]
-    return dict(zip(bs, cs))
-
-if len(sys.argv) < 3:
-    print("INFO: GGML V1 files produced are meant to be finalized through examples/falcon_quantize which will bring them to latest version and precision of choice");
-    print("Usage: python convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]")
-    print("  model_directory: name of the directory and model you convert (it should be a subdirectory)")
-    print("  output-directory: directory where the output file will be written")
-    print("  use-f32:    if present, use float32 instead of float16 (f32 is recommended)")
-    sys.exit(1)
-
-# num_parts = int(sys.argv[1])
-dir_model = sys.argv[1] # name and dir of model
-dir_out = sys.argv[2]   # output directory
-
-# make sure the output directory exists
-os.makedirs(dir_out, exist_ok=True)
-
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-ftype = 1
-if len(sys.argv) > 3:
-    ftype = 0
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-# print(tokenizer)
-config = AutoConfig.from_pretrained(dir_model, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(dir_model, trust_remote_code=True, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)
-hparams = config.to_dict()
-
-n_head = hparams["n_head"]
-n_head_kv = hparams["n_head_kv"] if "n_head_kv" in hparams else 1
-head_dim = hparams["hidden_size"] // n_head
-print("* Loading model from: ", dir_model)
-
-fname_out = dir_out + f"/ggml-model-{dir_model.split('/')[-1]}-{ftype_str[ftype]}.bin"
-fout = open(fname_out, "wb")
-fout.write(struct.pack("i", 0x67676a74)) # magic: ggmf in hex (version 1) - possibly change to ggfc ?
-fout.write(struct.pack("i", 1)) # version
-fout.write(struct.pack("i", hparams["vocab_size"]))
-fout.write(struct.pack("i", hparams["hidden_size"]))
-fout.write(struct.pack("i", n_head))
-fout.write(struct.pack("i", n_head_kv))
-fout.write(struct.pack("i", hparams["n_layer"]))
-fout.write(struct.pack("i", 40 if "n_head_kv" in hparams else 7)) # obsolete field that breaks ggml compatibility - todo again remove one day
-fout.write(struct.pack("i", ftype))
-
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-byte_encoder = bytes_to_unicode()
-byte_decoder = {v:k for k, v in byte_encoder.items()}
-
-for i in range(hparams["vocab_size"]):
-    text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
-    fout.write(struct.pack("i", len(text)))
-    fout.write(text)
-    fout.write(struct.pack("f", 0.0)) # falcon uses bpe on RefinedWeb - no probability scores used
-
-model = model.state_dict()
-for name in model.keys():
-    src = name
-    # The original query_key_value tensor contains n_head_kv "kv groups",
-    # each consisting of n_head/n_head_kv query weights followed by one key
-    # and one value weight (shared by all query heads in the kv group).
-    # This layout makes it a big pain to work with in GGML.
-    # So we rearrange them here,, so that we have n_head query weights
-    # followed by n_head_kv key weights followed by n_head_kv value weights,
-    # in contiguous fashion.
-
-    if "query_key_value" in src:
-        qkv = model[src].view(
-            n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)
-
-        q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head)
-        k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)
-        v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)
-
-        model[src] = torch.cat((q,k,v)).reshape_as(model[src])
-    data = model[src].squeeze()
-    n_dims = len(data.shape)
-    # default type is fp32
-    ftype_cur = 1 if ftype == 1 and n_dims > 1 else 0
-    data = data.to(dtype = torch.float16 if ftype_cur == 1 else torch.float32).numpy()
-    print(f'  |', name, data.shape, '->', data.dtype)
-    # header
-    str = name.encode('utf-8')
-    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str)
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-backend/scripts/convert_gptj_to_gguf.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+# Convert GPT-J-6B h5 transformer model to ggml format
+#
+# Load the model using GPTJForCausalLM.
+# Iterate over all variables and write them to a binary file.
+#
+# For each variable, write the following:
+#   - Number of dimensions (int)
+#   - Name length (int)
+#   - Dimensions (int[n_dims])
+#   - Name (char[name_length])
+#   - Data (float[n_dims])
+#
+# By default, the bigger matrices are converted to 16-bit floats.
+# This can be disabled by adding the "ftype" CLI argument.
+#
+# At the start of the ggml file we write the model parameters
+# and vocabulary.
+#
+
+from __future__ import annotations
+
+import sys
+import struct
+import json
+from pathlib import Path
+
+import gguf
+import numpy as np
+from transformers import AutoTokenizer, GPTJConfig, GPTJForCausalLM
+from transformers.models.gpt2 import tokenization_gpt2
+
+
+if not 2 <= len(sys.argv) < 4:
+    print("Usage: python {} dir-model [ftype]\n".format(Path(__file__).name))
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = Path(sys.argv[1])
+fname_out = dir_model / "ggml-model.gguf"
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+
+fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
+
+
+ARCH = gguf.MODEL_ARCH.GPTJ
+gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
+
+print("gguf: get model metadata")
+
+config = GPTJConfig(dir_model)
+
+block_count = config.n_layer
+gguf_writer.add_name("GPT-J")
+gguf_writer.add_context_length(config.n_positions)
+gguf_writer.add_embedding_length(config.n_embd)
+gguf_writer.add_block_count(block_count)
+gguf_writer.add_feed_forward_length(4 * config.n_embd)
+gguf_writer.add_head_count(config.n_head)
+gguf_writer.add_rope_dimension_count(config.rotary_dim)
+gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
+gguf_writer.add_file_type(ftype)
+
+print("gguf: get gpt2 tokenizer vocab")
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+byte_encoder = tokenization_gpt2.bytes_to_unicode()
+byte_decoder = {v: k for k, v in byte_encoder.items()}
+
+tokens: list[bytearray] = []
+
+for i in range(config.vocab_size):
+    if i in reverse_vocab:
+        try:
+            text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+        except KeyError:
+            text = bytearray()
+            for c in reverse_vocab[i]:
+                if ord(c) < 256:  # single byte character
+                    text.append(byte_decoder[c])
+                else:  # multibyte special token character
+                    text.extend(c.encode('utf-8'))
+    else:
+        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
+        pad_token = f"[PAD{i}]".encode("utf8")
+        text = bytearray(pad_token)
+
+    tokens.append(text)
+
+
+gguf_writer.add_tokenizer_model("gpt2")
+gguf_writer.add_token_list(tokens)
+
+special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
+special_vocab.add_to_gguf(gguf_writer)
+
+print("gguf: get tensor metadata")
+
+model = GPTJForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
+#print (model)
+
+tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
+
+list_vars = model.state_dict()
+#print (list_vars)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable:", name, "with shape:", data.shape)
+
+    # we don't need these
+    if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
+        print("  Skipping variable:", name)
+        continue
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+        print("  Converting to float16")
+        data = data.astype(np.float16)
+        ftype_cur = 1
+    elif ftype == 1 or data.dtype != np.float32:
+        print("  Converting to float32")
+        data = data.astype(np.float32)
+        ftype_cur = 0
+
+    # map tensor names
+    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
+    if new_name is None:
+        print("Can not map tensor '" + name + "'")
+        sys.exit()
+
+    gguf_writer.add_tensor(new_name, data)
+
+
+print("gguf: write header")
+gguf_writer.write_header_to_file()
+print("gguf: write metadata")
+gguf_writer.write_kv_data_to_file()
+print("gguf: write tensors")
+gguf_writer.write_tensors_to_file()
+
+gguf_writer.close()
+
+print(f"gguf: model successfully exported to '{fname_out}'")
+print()

gpt4all-backend/scripts/convert_mpt_hf_to_ggml.py

@@ -1,145 +0,0 @@
-# Convert Hugging Face fine-tuned bloom-like models to ggml format
-#
-# Usage:
-#
-#   python3 models/convert-h5-to-ggml.py 
-#
-# This script is similar to "convert-pt-to-ggml.py"
-#
-
-import io
-import os
-import sys
-import struct
-import json
-import code
-import torch
-import numpy as np
-
-from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BloomForCausalLM
-
-# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
-def bytes_to_unicode():
-    """
-    Returns list of utf-8 byte and a corresponding list of unicode strings.
-    The reversible bpe codes work on unicode strings.
-    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
-    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
-    This is a significant percentage of your normal, say, 32K bpe vocab.
-    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
-    And avoids mapping to whitespace/control characters the bpe code barfs on.
-    """
-    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
-    cs = bs[:]
-    n = 0
-    for b in range(2**8):
-        if b not in bs:
-            bs.append(b)
-            cs.append(2**8+n)
-            n += 1
-    cs = [chr(n) for n in cs]
-    return dict(zip(bs, cs))
-
-if len(sys.argv) < 3:
-    print("Usage: python convert-hf-to-ggml.py model_name dir-output [use-f32]")
-    print("  model_name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
-    print("  dir-output: directory where the output file will be written")
-    print("  use-f32:    if present, use float32 instead of float16")
-    sys.exit(1)
-
-model_name = sys.argv[1]
-dir_out = sys.argv[2]
-
-# make sure the output directory exists
-os.makedirs(dir_out, exist_ok=True)
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-ftype = 1
-if len(sys.argv) > 3:
-    ftype = 0
-
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
-hparams = config.to_dict()
-print("Loading model: ", model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32, low_cpu_mem_usage=True)
-print("Model loaded: ", model_name)
-
-
-fname_out = dir_out + f"/ggml-model-{model_name.split('/')[-1]}-{ftype_str[ftype]}.bin"
-fout = open(fname_out, "wb")
-vocab = tokenizer.vocab
-
-hparams["multiple_of"] = 1
-fout.write(struct.pack("I", 0x67676d6d)) # magic: ggml in hex
-fout.write(struct.pack("I", model.config.vocab_size))
-fout.write(struct.pack("I", model.config.max_seq_len))
-fout.write(struct.pack("I", model.config.n_layers))
-fout.write(struct.pack("I", model.config.n_heads))
-fout.write(struct.pack("I", model.config.d_model))
-fout.write(struct.pack("f", model.config.attn_config['alibi_bias_max']))
-clip_qkv = model.config.attn_config['clip_qkv']
-fout.write(struct.pack("f",  clip_qkv if clip_qkv is not None else 0))
-fout.write(struct.pack("I", ftype))
-
-# # Is this correct??
-# dot_token = tokenizer.encode(".")[0]
-# write tokens to ggml file 
-dot_token = tokenizer.encode('.')[0]
-fout.write(struct.pack("I", model.config.vocab_size))
-
-for i in range(model.config.vocab_size):
-    text = tokenizer.decode([dot_token, i]).encode('utf-8')
-    # remove the first byte (it's always '.')
-    text = text[1:]
-    enclen = len(text)
-    if i in tokenizer.all_special_ids:
-        print(f"special token: {text}")
-        enclen = enclen | 1<<31
-    fout.write(struct.pack("I", enclen))
-    fout.write(text)
-    
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable: " + name + " with shape: ", data.shape)
-
-    n_dims = len(data.shape);
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0;
-    if ftype != 0:
-        # Keep token embeddings in fp32
-        if name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
-            print("  Converting to float16")
-            data = data.astype(np.float16)
-            ftype_cur = 1
-        else:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-    else:
-        if data.dtype != np.float32:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-
-    # header
-    str = name.encode('utf-8')
-    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str);
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-backend/scripts/convert_mpt_hf_to_gguf.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# Convert Hugging Face fine-tuned bloom-like models to ggml format
+#
+# Usage:
+#
+#   python3 models/convert-h5-to-ggml.py 
+#
+# This script is similar to "convert-pt-to-ggml.py"
+#
+
+from __future__ import annotations
+
+import json
+import struct
+import sys
+from pathlib import Path
+
+import gguf
+import numpy as np
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
+from transformers.models.gpt2 import tokenization_gpt2
+
+
+if not 3 <= len(sys.argv) < 5:
+    print("Usage: {} model-name dir-output [ftype]".format(Path(__file__).name))
+    print("  model-name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
+    print("  dir-output: directory where the output file will be written")
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+model_name = sys.argv[1]
+dir_out = Path(sys.argv[2])
+
+# make sure the output directory exists
+dir_out.mkdir(exist_ok=True)
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 3:
+    ftype = int(sys.argv[3])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+
+fname_out = dir_out / f"ggml-model-{Path(model_name).name}-{ftype_str[ftype]}.gguf"
+
+
+ARCH = gguf.MODEL_ARCH.MPT
+gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
+
+print("gguf: get model metadata")
+
+config = AutoConfig.from_pretrained(model_name)
+
+block_count = config.n_layers
+gguf_writer.add_name("MPT")
+gguf_writer.add_context_length(config.max_seq_len)
+gguf_writer.add_embedding_length(config.d_model)
+gguf_writer.add_block_count(block_count)
+gguf_writer.add_feed_forward_length(4 * config.d_model)
+gguf_writer.add_head_count(config.n_heads)
+gguf_writer.add_max_alibi_bias(config.attn_config.alibi_bias_max)
+gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
+gguf_writer.add_file_type(ftype)
+
+clip_qkv = config.attn_config.clip_qkv
+if clip_qkv is not None:
+    gguf_writer.add_clamp_kqv(clip_qkv)
+
+print("gguf: get gpt2 tokenizer vocab")
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+special_ids = tokenizer.all_special_ids
+
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+added_tokens = tokenizer.get_added_vocab().values()
+byte_encoder = tokenization_gpt2.bytes_to_unicode()
+byte_decoder = {v: k for k, v in byte_encoder.items()}
+
+tokens: list[bytearray] = []
+toktypes: list[gguf.TokenType] = []
+
+# The number of tokens in tokenizer.json can differ from the expected vocab size.
+# This causes downstream issues with mismatched tensor sizes when running the inference
+for i in range(config.vocab_size):
+    if i not in reverse_vocab:
+        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
+        pad_token = f"[PAD{i}]".encode("utf8")
+        text = bytearray(pad_token)
+    elif i in added_tokens:
+        # these tokens are not encoded, for some reason
+        text = bytearray(reverse_vocab[i].encode('utf-8'))
+    else:
+        text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+
+    tokens.append(text)
+
+    # TODO(cebtenzzre): is there a better way to do this?
+    toktypes.append(gguf.TokenType.CONTROL if i in special_ids else gguf.TokenType.NORMAL)
+
+gguf_writer.add_tokenizer_model("gpt2")
+gguf_writer.add_token_list(tokens)
+gguf_writer.add_token_types(toktypes)
+
+print("gguf: get tensor metadata")
+
+print("Loading model:", model_name)
+model = AutoModelForCausalLM.from_pretrained(
+    model_name, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32, low_cpu_mem_usage=True,
+)
+print("Model loaded:", model_name)
+
+tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable:", name, "with shape:", data.shape)
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0
+    # Keep token embeddings in fp32
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
+        print("  Converting to float16")
+        data = data.astype(np.float16)
+        ftype_cur = 1
+    elif ftype == 1 or data.dtype != np.float32:
+        print("  Converting to float32")
+        data = data.astype(np.float32)
+        ftype_cur = 0
+
+    # map tensor names
+    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
+    if new_name is None:
+        print("Can not map tensor '" + name + "'")
+        sys.exit()
+
+    gguf_writer.add_tensor(new_name, data)
+
+
+print("gguf: write header")
+gguf_writer.write_header_to_file()
+print("gguf: write metadata")
+gguf_writer.write_kv_data_to_file()
+print("gguf: write tensors")
+gguf_writer.write_tensors_to_file()
+
+gguf_writer.close()
+
+print(f"gguf: model successfully exported to '{fname_out}'")
+print()

gpt4all-backend/scripts/convert_replit_hf_to_ggml.py

@@ -1,113 +0,0 @@
-from pathlib import Path
-import sys
-import struct
-import json
-import numpy as np
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import sentencepiece.sentencepiece_model_pb2 as model
-
-if len(sys.argv) < 3:
-    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-
-# output in the same directory as the model
-dir_model = sys.argv[1]
-fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b.bin"
-
-
-with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
-    hparams = json.load(f)
-
-sp_proto = model.ModelProto()
-sp_proto.ParseFromString(open(Path(sys.argv[1]) / "spiece.model", "rb").read())
-
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-    fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".bin"
-
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(
-    dir_model, low_cpu_mem_usage=True, trust_remote_code=True
-)
-# print (model)
-
-# print(tokenizer.encode('I believe the meaning of life is'))
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-fout = open(fname_out, "wb")
-
-print(hparams)
-
-fout.write(struct.pack("i", 0x7265706c))  # magic: repl in hex
-fout.write(struct.pack("i", hparams["vocab_size"]))
-fout.write(struct.pack("i", hparams["max_seq_len"]))
-fout.write(struct.pack("i", hparams["d_model"]))
-fout.write(struct.pack("i", hparams["n_heads"]))
-fout.write(struct.pack("i", hparams["n_layers"]))
-fout.write(struct.pack("i", ftype))
-
-
-# TODO: temporary hack to not deal with implementing the tokenizer
-for piece in sp_proto.pieces:
-    encoded_piece = piece.piece.encode("utf-8")
-    fout.write(struct.pack("i", len(encoded_piece)))
-    fout.write(encoded_piece)
-    fout.write(struct.pack("f", piece.score))
-
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable: " + name + " with shape: ", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    if ftype != 0:
-        if name[-7:] == ".weight" and n_dims == 2:
-            print("  Converting to float16")
-            data = data.astype(np.float16)
-            ftype_cur = 1
-        else:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-    else:
-        if data.dtype != np.float32:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-
-    # header
-    str = name.encode("utf-8")
-    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str)
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-backend/scripts/convert_replit_hf_to_gguf.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import struct
+import sys
+from pathlib import Path
+
+import gguf
+import numpy as np
+from sentencepiece import SentencePieceProcessor
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+
+if not 2 <= len(sys.argv) < 4:
+    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = Path(sys.argv[1])
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+
+fname_out = dir_model / ("ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".gguf")
+
+
+ARCH = gguf.MODEL_ARCH.MPT
+gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
+
+print("gguf: get model metadata")
+
+config = AutoConfig(dir_model)
+
+block_count = config.n_layers
+gguf_writer.add_name("Replit")
+gguf_writer.add_context_length(config.max_seq_len)
+gguf_writer.add_embedding_length(config.d_model)
+gguf_writer.add_block_count(block_count)
+gguf_writer.add_feed_forward_length(4 * config.d_model)
+gguf_writer.add_head_count(config.n_heads)
+gguf_writer.add_max_alibi_bias(config.attn_config.alibi_bias_max)
+gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
+gguf_writer.add_file_type(ftype)
+
+clip_qkv = config.attn_config.clip_qkv
+if clip_qkv is not None:
+    gguf_writer.add_clamp_kqv(clip_qkv)
+
+print("gguf: get sentencepiece tokenizer vocab")
+
+tokenizer = SentencePieceProcessor(str(dir_model / "spiece.model"))
+#print(tokenizer.encode('I believe the meaning of life is'))
+
+tokens: list[bytearray] = []
+scores: list[float] = []
+toktypes: list[int] = []
+
+for i in range(tokenizer.vocab_size()):
+    tokens.append(tokenizer.id_to_piece(i).encode('utf-8'))
+    scores.append(tokenizer.get_score(i))
+
+    toktype = gguf.TokenType.NORMAL
+    if tokenizer.is_unknown(i):
+        toktype = gguf.TokenType.UNKNOWN
+    elif tokenizer.is_control(i):
+        toktype = gguf.TokenType.CONTROL
+    elif tokenizer.is_unused(i):
+        toktype = gguf.TokenType.UNUSED
+    elif tokenizer.is_byte(i):
+        toktype = gguf.TokenType.BYTE
+
+    toktypes.append(toktype)
+
+gguf_writer.add_tokenizer_model("llama")  # sentencepiece
+gguf_writer.add_token_list(tokens)
+gguf_writer.add_token_scores(scores)
+gguf_writer.add_token_types(toktypes)
+
+special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
+special_vocab.add_to_gguf(gguf_writer)
+
+print("gguf: get tensor metadata")
+
+model = AutoModelForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
+#print(model)
+
+tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+print(config)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable:", name, "with shape:", data.shape)
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+        print("  Converting to float16")
+        data = data.astype(np.float16)
+        ftype_cur = 1
+    elif ftype == 1 or data.dtype != np.float32:
+        print("  Converting to float32")
+        data = data.astype(np.float32)
+        ftype_cur = 0
+
+    # map tensor names
+    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
+    if new_name is None:
+        print("Can not map tensor '" + name + "'")
+        sys.exit()
+
+    gguf_writer.add_tensor(new_name, data)
+
+
+print("gguf: write header")
+gguf_writer.write_header_to_file()
+print("gguf: write metadata")
+gguf_writer.write_kv_data_to_file()
+print("gguf: write tensors")
+gguf_writer.write_tensors_to_file()
+
+gguf_writer.close()
+
+print(f"gguf: model successfully exported to '{fname_out}'")
+print()

gpt4all-backend/starcoder_impl.h

@@ -1,42 +0,0 @@
-#ifndef STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of starcoder.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef STARCODER_H
-#define STARCODER_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include <memory>
-#include "llmodel.h"
-
-struct StarcoderPrivate;
-class Starcoder : public LLModel {
-public:
-    Starcoder();
-    ~Starcoder();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    std::unique_ptr<StarcoderPrivate> d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // STARCODER_H

gpt4all-bindings/python/README.md

@@ -20,7 +20,7 @@ pip install gpt4all
 1. Setup `llmodel`
 
 ```
-git clone --recurse-submodules git@github.com:nomic-ai/gpt4all.git
+git clone --recurse-submodules https://github.com/nomic-ai/gpt4all.git
 cd gpt4all/gpt4all-backend/
 mkdir build
 cd build

gpt4all-bindings/python/docs/gpt4all_chat.md

@@ -7,7 +7,7 @@ It is optimized to run 7-13B parameter LLMs on the CPU's of any computer running
 ## Running LLMs on CPU
 The GPT4All Chat UI supports models from all newer versions of `GGML`, `llama.cpp` including the `LLaMA`, `MPT`, `replit`,  `GPT-J` and `falcon` architectures
 
-GPT4All maintains an official list of recommended models located in [models.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
+GPT4All maintains an official list of recommended models located in [models2.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
 
 #### Sideloading any GGML model
 If a model is compatible with the gpt4all-backend, you can sideload it into GPT4All Chat by:

gpt4all-bindings/python/docs/gpt4all_faq.md

@@ -61,12 +61,12 @@ or `allowDownload=true` (default), a model is automatically downloaded into `.ca
 unless it already exists.
 
 In case of connection issues or errors during the download, you might want to manually verify the model file's MD5
-checksum by comparing it with the one listed in [models.json].
+checksum by comparing it with the one listed in [models2.json].
 
 As an alternative to the basic downloader built into the bindings, you can choose to download from the 
 <https://gpt4all.io/> website instead. Scroll down to 'Model Explorer' and pick your preferred model.
 
-[models.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json
+[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
 
 #### I need the chat GUI and bindings to behave the same
 
@@ -93,7 +93,7 @@ The chat GUI and bindings are based on the same backend. You can make them behav
 - Next you'll have to compare the templates, adjusting them as necessary, based on how you're using the bindings.
     - Specifically, in Python:
         - With simple `generate()` calls, the input has to be surrounded with system and prompt templates.
-        - When using a chat session, it depends on whether the bindings are allowed to download [models.json]. If yes,
+        - When using a chat session, it depends on whether the bindings are allowed to download [models2.json]. If yes,
           and in the chat GUI the default templates are used, it'll be handled automatically. If no, use
           `chat_session()` template parameters to customize them.
 

gpt4all-bindings/python/docs/gpt4all_modal.md

@@ -8,7 +8,7 @@ import modal
 
 def download_model():
     import gpt4all
-    #you can use any model from https://gpt4all.io/models/models.json
+    #you can use any model from https://gpt4all.io/models/models2.json
     return gpt4all.GPT4All("ggml-gpt4all-j-v1.3-groovy.bin")
 
 image=modal.Image.debian_slim().pip_install("gpt4all").run_function(download_model)
@@ -31,4 +31,4 @@ def main():
     model = GPT4All()
     for i in range(10):
         model.generate.call()
-```
+```

gpt4all-bindings/python/docs/gpt4all_python.md

@@ -77,10 +77,10 @@ When using GPT4All models in the `chat_session` context:
 - Consecutive chat exchanges are taken into account and not discarded until the session ends; as long as the model has capacity.
 - Internal K/V caches are preserved from previous conversation history, speeding up inference.
 - The model is given a system and prompt template which make it chatty. Depending on `allow_download=True` (default),
-  it will obtain the latest version of [models.json] from the repository, which contains specifically tailored templates
+  it will obtain the latest version of [models2.json] from the repository, which contains specifically tailored templates
   for models. Conversely, if it is not allowed to download, it falls back to default templates instead.
 
-[models.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json
+[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
 
 
 ### Streaming Generations
@@ -379,7 +379,7 @@ logging infrastructure offers [many more customization options][py-logging-cookb
 
 ### Without Online Connectivity
 To prevent GPT4All from accessing online resources, instantiate it with `allow_download=False`. This will disable both
-downloading missing models and [models.json], which contains information about them. As a result, predefined templates
+downloading missing models and [models2.json], which contains information about them. As a result, predefined templates
 are used instead of model-specific system and prompt templates:
 
 === "GPT4All Default Templates Example"

gpt4all-bindings/python/docs/index.md

@@ -38,7 +38,7 @@ The GPT4All software ecosystem is compatible with the following Transformer arch
 - `MPT` (including `Replit`)
 - `GPT-J`
 
-You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json)
+You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json)
 
 
 GPT4All models are artifacts produced through a process known as neural network quantization.

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -2,6 +2,7 @@
 Python only API for running all GPT4All models.
 """
 import os
+import sys
 import time
 from contextlib import contextmanager
 from pathlib import Path
@@ -29,17 +30,14 @@ class Embed4All:
     Python class that handles embeddings for GPT4All.
     """
 
-    def __init__(
-        self,
-        n_threads: Optional[int] = None,
-    ):
+    def __init__(self, model_name: Optional[str] = None, n_threads: Optional[int] = None, **kwargs):
         """
         Constructor
 
         Args:
             n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
         """
-        self.gpt4all = GPT4All(model_name='ggml-all-MiniLM-L6-v2-f16.bin', n_threads=n_threads)
+        self.gpt4all = GPT4All(model_name or 'ggml-all-MiniLM-L6-v2-f16.gguf', n_threads=n_threads, **kwargs)
 
     def embed(self, text: str) -> List[float]:
         """
@@ -107,12 +105,12 @@ class GPT4All:
     @staticmethod
     def list_models() -> List[ConfigType]:
         """
-        Fetch model list from https://gpt4all.io/models/models.json.
+        Fetch model list from https://gpt4all.io/models/models2.json.
 
         Returns:
             Model list in JSON format.
         """
-        return requests.get("https://gpt4all.io/models/models.json").json()
+        return requests.get("https://gpt4all.io/models/models2.json").json()
 
     @staticmethod
     def retrieve_model(
@@ -172,7 +170,7 @@ class GPT4All:
             config.pop("url", None)
             config["path"] = model_dest
             if verbose:
-                print("Found model file at ", model_dest)
+                print("Found model file at", model_dest, file=sys.stderr)
 
         # If model file does not exist, download
         elif allow_download:
@@ -314,7 +312,6 @@ class GPT4All:
             callback: pyllmodel.ResponseCallbackType,
             output_collector: List[MessageType],
         ) -> pyllmodel.ResponseCallbackType:
-
             def _callback(token_id: int, response: str) -> bool:
                 nonlocal callback, output_collector
 
@@ -423,6 +420,6 @@ def empty_chat_session(system_prompt: str = "") -> List[MessageType]:
 
 
 def append_bin_suffix_if_missing(model_name):
-    if not model_name.endswith(".bin"):
+    if not model_name.endswith((".bin", ".gguf")):
         model_name += ".bin"
     return model_name

gpt4all-bindings/python/gpt4all/pyllmodel.py

@@ -259,12 +259,13 @@ class LLModel:
         True if model loaded successfully, False otherwise
         """
         model_path_enc = model_path.encode("utf-8")
-        self.model = llmodel.llmodel_model_create(model_path_enc)
+        err = LLModelError()
+        self.model = llmodel.llmodel_model_create2(model_path_enc, b"auto", ctypes.byref(err))
 
-        if self.model is not None:
-            llmodel.llmodel_loadModel(self.model, model_path_enc)
-        else:
-            raise ValueError("Unable to instantiate model")
+        if self.model is None:
+            raise ValueError(f"Unable to instantiate model: code={err.code}, {err.message.decode()}")
+
+        llmodel.llmodel_loadModel(self.model, model_path_enc)
 
         filename = os.path.basename(model_path)
         self.model_name = os.path.splitext(filename)[0]

gpt4all-bindings/typescript/src/config.js

@@ -21,7 +21,7 @@ const DEFAULT_MODEL_CONFIG = {
     promptTemplate: "### Human: \n%1\n### Assistant:\n",
 }
 
-const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models.json";
+const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models2.json";
 
 const DEFAULT_PROMPT_CONTEXT = {
     temp: 0.7,

gpt4all-bindings/typescript/src/util.js

@@ -236,7 +236,7 @@ async function retrieveModel(modelName, options = {}) {
         file: retrieveOptions.modelConfigFile,
         url:
             retrieveOptions.allowDownload &&
-            "https://gpt4all.io/models/models.json",
+            "https://gpt4all.io/models/models2.json",
     });
 
     const loadedModelConfig = availableModels.find(

gpt4all-chat/CMakeLists.txt

@@ -17,8 +17,8 @@ if(APPLE)
 endif()
 
 set(APP_VERSION_MAJOR 2)
-set(APP_VERSION_MINOR 4)
-set(APP_VERSION_PATCH 17)
+set(APP_VERSION_MINOR 5)
+set(APP_VERSION_PATCH 0)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file
@@ -189,30 +189,21 @@ install(TARGETS llamamodel-mainline-default DESTINATION lib COMPONENT ${COMPONEN
 if(APPLE)
 install(TARGETS llamamodel-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 endif()
-install(TARGETS falcon-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS falcon-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-#install(TARGETS mpt-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-#install(TARGETS mpt-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS replit-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS replit-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-if(APPLE)
-install(TARGETS replit-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-endif()
+install(TARGETS mpt-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS mpt-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS bert-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS bert-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS starcoder-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS starcoder-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 
 set(CPACK_GENERATOR "IFW")
 set(CPACK_VERBATIM_VARIABLES YES)
 set(CPACK_IFW_VERBOSE ON)
 
 if(${CMAKE_SYSTEM_NAME} MATCHES Linux)
-    set(LINUXDEPLOYQT "$ENV{HOME}/dev/linuxdeployqt/build/tools/linuxdeployqt/linuxdeployqt")
+    find_program(LINUXDEPLOYQT linuxdeployqt HINTS "$ENV{HOME}/dev/linuxdeployqt/build/tools/linuxdeployqt" "$ENV{HOME}/project/linuxdeployqt/bin")
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/deploy-qt-linux.cmake.in"
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-linux.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-linux.cmake)
-    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.5")
+    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.6")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-linux")
     set(CPACK_IFW_TARGET_DIRECTORY "@HomeDir@/${COMPONENT_NAME_MAIN}")
 elseif(${CMAKE_SYSTEM_NAME} MATCHES Windows)
@@ -220,7 +211,7 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES Windows)
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/deploy-qt-windows.cmake.in"
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-windows.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-windows.cmake)
-    set(CPACK_IFW_ROOT "C:/Qt/Tools/QtInstallerFramework/4.5")
+    set(CPACK_IFW_ROOT "C:/Qt/Tools/QtInstallerFramework/4.6")
     set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.ico")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-win64")
     set(CPACK_IFW_TARGET_DIRECTORY "@HomeDir@\\${COMPONENT_NAME_MAIN}")
@@ -229,7 +220,7 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/deploy-qt-mac.cmake.in"
                    "${CMAKE_BINARY_DIR}/cmake/deploy-qt-mac.cmake" @ONLY)
     set(CPACK_PRE_BUILD_SCRIPTS ${CMAKE_BINARY_DIR}/cmake/deploy-qt-mac.cmake)
-    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.5")
+    set(CPACK_IFW_ROOT "~/Qt/Tools/QtInstallerFramework/4.6")
     set(CPACK_IFW_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns")
     set(CPACK_PACKAGE_FILE_NAME "${COMPONENT_NAME_MAIN}-installer-darwin")
     set(CPACK_IFW_TARGET_DIRECTORY "@ApplicationsDir@/${COMPONENT_NAME_MAIN}")
@@ -259,7 +250,11 @@ set(CPACK_IFW_PACKAGE_WIZARD_SHOW_PAGE_LIST OFF)
 include(InstallRequiredSystemLibraries)
 include(CPack)
 include(CPackIFW)
-cpack_add_component(${COMPONENT_NAME_MAIN} DOWNLOADED)
+if(GPT4ALL_OFFLINE_INSTALLER)
+  cpack_add_component(${COMPONENT_NAME_MAIN})
+else()
+  cpack_add_component(${COMPONENT_NAME_MAIN} DOWNLOADED)
+endif()
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} ESSENTIAL FORCED_INSTALLATION)
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} VERSION ${APP_VERSION})
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} LICENSES "MIT LICENSE" ${CPACK_RESOURCE_FILE_LICENSE})
@@ -269,7 +264,7 @@ cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} REPLACES "gpt4all-chat") #W
 if (GPT4ALL_LOCALHOST)
     cpack_ifw_add_repository("GPT4AllRepository" URL "http://localhost/repository")
 elseif(GPT4ALL_OFFLINE_INSTALLER)
-    cpack_ifw_add_repository("GPT4AllRepository" URL "file://${CMAKE_BINARY_DIR}/packages")
+  add_compile_definitions(GPT4ALL_OFFLINE_INSTALLER)
 else()
   if(${CMAKE_SYSTEM_NAME} MATCHES Linux)
       cpack_ifw_add_repository("GPT4AllRepository" URL "https://gpt4all.io/installer_repos/linux/repository")

gpt4all-chat/chat.cpp

@@ -57,6 +57,7 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::generatedNameChanged, this, &Chat::generatedNameChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportSpeed, this, &Chat::handleTokenSpeedChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportDevice, this, &Chat::handleDeviceChanged, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::reportFallbackReason, this, &Chat::handleFallbackReasonChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::databaseResultsChanged, this, &Chat::handleDatabaseResultsChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelInfoChanged, this, &Chat::handleModelInfoChanged, Qt::QueuedConnection);
 
@@ -352,6 +353,12 @@ void Chat::handleDeviceChanged(const QString &device)
     emit deviceChanged();
 }
 
+void Chat::handleFallbackReasonChanged(const QString &fallbackReason)
+{
+    m_fallbackReason = fallbackReason;
+    emit fallbackReasonChanged();
+}
+
 void Chat::handleDatabaseResultsChanged(const QList<ResultInfo> &results)
 {
     m_databaseResults = results;

gpt4all-chat/chat.h

@@ -26,6 +26,7 @@ class Chat : public QObject
     Q_PROPERTY(QString modelLoadingError READ modelLoadingError NOTIFY modelLoadingErrorChanged)
     Q_PROPERTY(QString tokenSpeed READ tokenSpeed NOTIFY tokenSpeedChanged);
     Q_PROPERTY(QString device READ device NOTIFY deviceChanged);
+    Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY fallbackReasonChanged);
     QML_ELEMENT
     QML_UNCREATABLE("Only creatable from c++!")
 
@@ -90,6 +91,7 @@ public:
 
     QString tokenSpeed() const { return m_tokenSpeed; }
     QString device() const { return m_device; }
+    QString fallbackReason() const { return m_fallbackReason; }
 
 public Q_SLOTS:
     void serverNewPromptResponsePair(const QString &prompt);
@@ -118,6 +120,7 @@ Q_SIGNALS:
     void collectionListChanged(const QList<QString> &collectionList);
     void tokenSpeedChanged();
     void deviceChanged();
+    void fallbackReasonChanged();
 
 private Q_SLOTS:
     void handleResponseChanged(const QString &response);
@@ -129,6 +132,7 @@ private Q_SLOTS:
     void handleModelLoadingError(const QString &error);
     void handleTokenSpeedChanged(const QString &tokenSpeed);
     void handleDeviceChanged(const QString &device);
+    void handleFallbackReasonChanged(const QString &device);
     void handleDatabaseResultsChanged(const QList<ResultInfo> &results);
     void handleModelInfoChanged(const ModelInfo &modelInfo);
     void handleModelInstalled();
@@ -142,6 +146,7 @@ private:
     QString m_modelLoadingError;
     QString m_tokenSpeed;
     QString m_device;
+    QString m_fallbackReason;
     QString m_response;
     QList<QString> m_collections;
     ChatModel *m_chatModel;

gpt4all-chat/chatllm.cpp

@@ -11,11 +11,8 @@
 
 #define MPT_INTERNAL_STATE_VERSION 0
 #define GPTJ_INTERNAL_STATE_VERSION 0
-#define REPLIT_INTERNAL_STATE_VERSION 0
 #define LLAMA_INTERNAL_STATE_VERSION 0
-#define FALCON_INTERNAL_STATE_VERSION 0
 #define BERT_INTERNAL_STATE_VERSION 0
-#define STARCODER_INTERNAL_STATE_VERSION 0
 
 class LLModelStore {
 public:
@@ -230,7 +227,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         if (!m_isServer)
             LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
         m_llModelInfo = LLModelInfo();
-        emit modelLoadingError(QString("Previous attempt to load model resulted in crash for `%1` most likely due to out of memory. You should either remove this model or decrease your system RAM by closing other applications.").arg(modelInfo.filename()));
+        emit modelLoadingError(QString("Previous attempt to load model resulted in crash for `%1` most likely due to insufficient memory. You should either remove this model or decrease your system RAM by closing other applications.").arg(modelInfo.filename()));
     }
 
     if (fileInfo.exists()) {
@@ -263,40 +260,56 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             if (m_llModelInfo.model) {
                 // Update the settings that a model is being loaded and update the device list
                 MySettings::globalInstance()->setAttemptModelLoad(filePath);
-                std::vector<LLModel::GPUDevice> devices = m_llModelInfo.model->availableGPUDevices(0);
-                QVector<QString> deviceList{ "Auto" };
-                for (LLModel::GPUDevice &d : devices)
-                    deviceList << QString::fromStdString(d.name);
-                deviceList << "CPU";
-                MySettings::globalInstance()->setDeviceList(deviceList);
 
                 // Pick the best match for the device
                 QString actualDevice = m_llModelInfo.model->implementation().buildVariant() == "metal" ? "Metal" : "CPU";
                 const QString requestedDevice = MySettings::globalInstance()->device();
-                if (requestedDevice != "CPU") {
+                if (requestedDevice == "CPU") {
+                    emit reportFallbackReason(""); // fallback not applicable
+                } else {
                     const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString());
                     std::vector<LLModel::GPUDevice> availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
+                    LLModel::GPUDevice *device = nullptr;
+
                     if (!availableDevices.empty() && requestedDevice == "Auto" && availableDevices.front().type == 2 /*a discrete gpu*/) {
-                        m_llModelInfo.model->initializeGPUDevice(availableDevices.front());
-                        actualDevice = QString::fromStdString(availableDevices.front().name);
+                        device = &availableDevices.front();
                     } else {
                         for (LLModel::GPUDevice &d : availableDevices) {
                             if (QString::fromStdString(d.name) == requestedDevice) {
-                                m_llModelInfo.model->initializeGPUDevice(d);
-                                actualDevice = QString::fromStdString(d.name);
+                                device = &d;
                                 break;
                             }
                         }
                     }
+
+                    emit reportFallbackReason(""); // no fallback yet
+                    std::string unavail_reason;
+                    if (!device) {
+                        // GPU not available
+                    } else if (!m_llModelInfo.model->initializeGPUDevice(*device, &unavail_reason)) {
+                        emit reportFallbackReason(QString::fromStdString("<br>Using CPU: " + unavail_reason));
+                    } else {
+                        actualDevice = QString::fromStdString(device->name);
+                    }
                 }
 
                 // Report which device we're actually using
                 emit reportDevice(actualDevice);
 
                 bool success = m_llModelInfo.model->loadModel(filePath.toStdString());
-                if (!success && actualDevice != "CPU") {
+                if (actualDevice == "CPU") {
+                    // we asked llama.cpp to use the CPU
+                } else if (!success) {
+                    // llama_init_from_file returned nullptr
                     emit reportDevice("CPU");
+                    emit reportFallbackReason("<br>Using CPU: loading failed (out of VRAM?)");
                     success = m_llModelInfo.model->loadModel(filePath.toStdString());
+                } else if (!m_llModelInfo.model->usingGPUDevice()) {
+                    // ggml_vk_init was not called in llama.cpp
+                    // We might have had to fallback to CPU after load if the model is not possible to accelerate
+                    // for instance if the quantization method is not supported on Vulkan yet
+                    emit reportDevice("CPU");
+                    emit reportFallbackReason("<br>Using CPU: unsupported model or quant");
                 }
 
                 MySettings::globalInstance()->setAttemptModelLoad(QString());
@@ -308,19 +321,11 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     m_llModelInfo = LLModelInfo();
                     emit modelLoadingError(QString("Could not load model due to invalid model file for %1").arg(modelInfo.filename()));
                 } else {
-                    // We might have had to fallback to CPU after load if the model is not possible to accelerate
-                    // for instance if the quantization method is not supported on Vulkan yet
-                    if (actualDevice != "CPU" && !m_llModelInfo.model->usingGPUDevice())
-                        emit reportDevice("CPU");
-
                     switch (m_llModelInfo.model->implementation().modelType()[0]) {
                     case 'L': m_llModelType = LLModelType::LLAMA_; break;
                     case 'G': m_llModelType = LLModelType::GPTJ_; break;
                     case 'M': m_llModelType = LLModelType::MPT_; break;
-                    case 'R': m_llModelType = LLModelType::REPLIT_; break;
-                    case 'F': m_llModelType = LLModelType::FALCON_; break;
                     case 'B': m_llModelType = LLModelType::BERT_; break;
-                    case 'S': m_llModelType = LLModelType::STARCODER_; break;
                     default:
                         {
                             delete m_llModelInfo.model;
@@ -362,10 +367,10 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         emit modelLoadingError(QString("Could not find file for model %1").arg(modelInfo.filename()));
     }
 
-    if (m_llModelInfo.model)
+    if (m_llModelInfo.model) {
         setModelInfo(modelInfo);
-
-    processSystemPrompt();
+        processSystemPrompt();
+    }
     return m_llModelInfo.model;
 }
 
@@ -383,7 +388,7 @@ void ChatLLM::regenerateResponse()
     else
         m_ctx.n_past -= m_promptResponseTokens;
     m_ctx.n_past = std::max(0, m_ctx.n_past);
-    m_ctx.tokens.erase(m_ctx.tokens.end() -= m_promptResponseTokens, m_ctx.tokens.end());
+    m_ctx.tokens.erase(m_ctx.tokens.end() - m_promptResponseTokens, m_ctx.tokens.end());
     m_promptResponseTokens = 0;
     m_promptTokens = 0;
     m_response = std::string();
@@ -723,13 +728,10 @@ bool ChatLLM::serialize(QDataStream &stream, int version)
     if (version > 1) {
         stream << m_llModelType;
         switch (m_llModelType) {
-        case REPLIT_: stream << REPLIT_INTERNAL_STATE_VERSION; break;
         case MPT_: stream << MPT_INTERNAL_STATE_VERSION; break;
         case GPTJ_: stream << GPTJ_INTERNAL_STATE_VERSION; break;
         case LLAMA_: stream << LLAMA_INTERNAL_STATE_VERSION; break;
-        case FALCON_: stream << FALCON_INTERNAL_STATE_VERSION; break;
         case BERT_: stream << BERT_INTERNAL_STATE_VERSION; break;
-        case STARCODER_: stream << STARCODER_INTERNAL_STATE_VERSION; break;
         default: Q_UNREACHABLE();
         }
     }
@@ -886,4 +888,4 @@ void ChatLLM::processSystemPrompt()
     fflush(stdout);
 #endif
     m_processedSystemPrompt = true;
-}
+}

gpt4all-chat/chatllm.h

@@ -14,10 +14,7 @@ enum LLModelType {
     GPTJ_,
     LLAMA_,
     CHATGPT_,
-    REPLIT_,
-    FALCON_,
     BERT_,
-    STARCODER_
 };
 
 struct LLModelInfo {
@@ -130,6 +127,7 @@ Q_SIGNALS:
     void requestRetrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
     void reportSpeed(const QString &speed);
     void reportDevice(const QString &device);
+    void reportFallbackReason(const QString &fallbackReason);
     void databaseResultsChanged(const QList<ResultInfo>&);
     void modelInfoChanged(const ModelInfo &modelInfo);
 

gpt4all-chat/cmake/deploy-qt-mac.cmake.in

@@ -5,10 +5,7 @@ execute_process(COMMAND ${MACDEPLOYQT} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/pack
 file(GLOB MYGPTJLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libgptj*)
 file(GLOB MYMPTLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libmpt*)
 file(GLOB MYLLAMALIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllama*)
-file(GLOB MYREPLITLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libreplit*)
-file(GLOB MYFALCONLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libfalcon*)
 file(GLOB MYBERTLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libbert*)
-file(GLOB MYSTARCODERLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libstarcoder*)
 file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllmodel.*)
 file(COPY ${MYGPTJLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
@@ -16,14 +13,8 @@ file(COPY ${MYMPTLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLAMALIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYREPLITLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYFALCONLLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYBERTLLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYSTARCODERLLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLAMALIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLMODELLIBS}

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.appdata.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<component type="desktop">
+    <id>io.gpt4all.gpt4all</id>
+    <metadata_license>CC0-1.0</metadata_license>
+    <project_license>MIT License</project_license>
+    <name>GPT4ALL</name>
+    <summary>Open-source assistant-style large language models that run locally on your CPU and GPU</summary>
+    <description>
+        <p>Cross platform Qt based GUI for GPT4All</p>
+        <ul>
+            <li>Fast CPU and GPU based inference using ggml for open source LLM's</li>
+            <li>The UI is made to look and feel like you've come to expect from a chatty gpt</li>
+            <li>Check for updates so you can always stay fresh with latest models</li>
+            <li>Easy to install with precompiled binaries available for all three major desktop platforms</li>
+            <li>Multi-model - Ability to load more than one model and switch between them</li>
+            <li>Supports both llama.cpp and gptj.cpp style models</li>
+            <li>Model downloader in GUI featuring many popular open source models</li>
+            <li>Settings dialog to change temp, top_p, top_k, threads, etc</li>
+            <li>Copy your conversation to clipboard</li>
+        </ul>
+    </description>
+    <screenshots>
+        <screenshot type="default">
+            <caption>Main Window</caption>
+            <image>https://user-images.githubusercontent.com/50458173/231464085-da9edff6-a593-410e-8f38-7513f75c8aab.png</image>
+        </screenshot>
+    </screenshots>
+    <url type="homepage">https://gpt4all.io</url>
+    <url type="bugtracker">https://github.com/nomic-ai/gpt4all/issues</url>
+    <url type="vcs-browser">https://github.com/nomic-ai/gpt4all</url>
+    <releases>
+      <release version="2.4.19" date="2023-09-16">
+        <description>
+          <p>
+            <ul>
+              <li>A bugfix for crashes on systems that have a corrupted Vulkan driver or a corrupted version of the Vulkan shared library</li>
+            </ul>
+          </p>
+        </description>
+        </release>              
+    </releases>
+    <launchable type="desktop-id">io.gpt4all.gpt4all.desktop</launchable>
+    <content_rating type="oars-1.0">
+        <content_attribute id="language-profanity">mild</content_attribute>
+        <content_attribute id="language-humor">moderate</content_attribute>
+        <content_attribute id="language-discrimination">mild</content_attribute>
+    </content_rating>
+</component>

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.desktop

@@ -0,0 +1,9 @@
+[Desktop Entry]
+Name=GPT4ALL
+GenericName=Open-source assistant-style large language models that run locally on your CPU
+Comment=Run any GPT4All model natively on your home desktop with the auto-updating desktop chat client. See GPT4All Website for a full list of open-source models you can run with this powerful desktop application.
+Exec=chat
+Icon=io.gpt4all.gpt4all
+Type=Application
+Categories=Utility;Office;
+Keywords=GPT,Chat;AI

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.yml

@@ -0,0 +1,166 @@
+app-id: io.gpt4all.gpt4all
+default-branch: stable
+runtime: org.kde.Platform
+runtime-version: '6.5'
+sdk: org.kde.Sdk
+sdk-extensions:
+  - org.freedesktop.Sdk.Extension.node14
+finish-args:
+  - --share=ipc
+  - --socket=wayland
+  - --socket=x11
+  - --share=network
+  - --device=dri
+  - --env=LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/lib/x86_64-linux-gnu/
+  - --filesystem=xdg-documents:ro
+command: chat
+cleanup:
+  - /include
+
+modules:
+  - name: qthttpserver
+    buildsystem: cmake
+    sources:
+      - type: archive
+        url: https://invent.kde.org/qt/qt/qthttpserver/-/archive/6.5.2/qthttpserver-6.5.2.zip
+        sha256: 9fb7b14774b4ed62fe9097e13fa593af06ba75537783fc62f34652bada26abee
+
+  - name: python-html5lib
+    buildsystem: simple
+    build-commands:
+    - 'pip3 install --verbose --exists-action=i --no-index --find-links="file://${PWD}" --prefix=${FLATPAK_DEST} "html5lib" --no-build-isolation'
+    sources:
+      - type: file
+        url: https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl
+        sha256: a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78
+        x-checker-data:
+            type: pypi
+            name: webencodings
+            packagetype: bdist_wheel
+      - type: file
+        url: https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl
+        sha256: 0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d
+        x-checker-data:
+            type: pypi
+            name: html5lib
+            packagetype: bdist_wheel
+    cleanup:
+    - '*'
+
+  - name: qtwebengine
+    buildsystem: cmake
+    builddir: true
+    config-opts:
+      - -DQT_FEATURE_qtwebengine_build=OFF
+      - -DQT_FEATURE_qtpdf_build=ON
+    build-options:
+      append-path: /usr/lib/sdk/node14/bin
+      env:
+        - npm_config_nodedir=/usr/lib/sdk/node14
+    sources:
+      - type: git
+        url: https://invent.kde.org/qt/qt/qtwebengine.git
+        tag: v6.5.2
+        commit: ac887518e8243828333e923b5a1e61a007babde5
+  
+  - name: vulkan-headers
+    buildsystem: cmake
+    builddir: true
+    sources:
+      - type: git
+        url: https://github.com/KhronosGroup/Vulkan-Headers.git
+        tag: v1.3.224
+        commit: 2b55157592bf4c639b76cc16d64acaef565cc4b5
+
+  - name: fmt
+    buildsystem: cmake
+    builddir: true
+    sources:
+      - type: git
+        url: https://github.com/fmtlib/fmt.git
+        tag: 10.1.1
+        commit: f5e54359df4c26b6230fc61d38aa294581393084
+
+  - name: vulkan-tools
+    buildsystem: cmake
+    builddir: true
+    sources:
+      - type: git
+        url: https://github.com/KhronosGroup/Vulkan-Tools.git
+        tag: v1.3.224
+        commit: 497f232680b046db34ba9e9da065e6303a125851
+
+    
+
+    modules:
+      - name: shaderc
+        buildsystem: cmake-ninja
+        builddir: true
+        config-opts:
+          - -DSHADERC_SKIP_COPYRIGHT_CHECK=ON
+          - -DSHADERC_SKIP_EXAMPLES=ON
+          - -DSHADERC_SKIP_TESTS=ON
+          - -DSPIRV_SKIP_EXECUTABLES=ON
+          - -DENABLE_GLSLANG_BINARIES=OFF
+        cleanup:
+          - /bin
+          - /include
+          - /lib/cmake
+          - /lib/pkgconfig
+        sources:
+          - type: git
+            url: https://github.com/google/shaderc.git
+            tag: v2023.4
+            commit: 45b735dfddefe26a99b77e5a74e30d860713ac64
+#           x-checker-data:
+#             type: git
+#             tag-pattern: ^v(\d{4}\.\d{1,2})$
+          - type: git
+            url: https://github.com/KhronosGroup/SPIRV-Tools.git
+            tag: v2023.2
+            commit: 44d72a9b36702f093dd20815561a56778b2d181e
+            dest: third_party/spirv-tools
+            x-checker-data:
+              type: git
+              tag-pattern: ^v(\d{4}\.\d{1})$
+          - type: git
+            url: https://github.com/KhronosGroup/SPIRV-Headers.git
+            tag: sdk-1.3.250.1
+            commit: 268a061764ee69f09a477a695bf6a11ffe311b8d
+            dest: third_party/spirv-headers
+            x-checker-data:
+              type: git
+              tag-pattern: ^sdk-([\d.]+)$
+          - type: git
+            url: https://github.com/KhronosGroup/glslang.git
+            tag: 12.2.0
+            commit: d1517d64cfca91f573af1bf7341dc3a5113349c0
+            dest: third_party/glslang
+          
+          
+
+
+  - name: gpt4all
+    buildsystem: simple
+    build-commands:
+      - git submodule update --init --recursive
+      - mkdir build
+      - cmake  -S ./gpt4all-chat -B build -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF -DKOMPUTE_OPT_USE_BUILT_IN_FMT=OFF -DCMAKE_INSTALL_PREFIX=/app
+      - cmake --build build --config Release -- -j
+      - cmake --install build --prefix "/app"
+      - install -Dm644 logo.svg /app/share/icons/hicolor/scalable/apps/io.gpt4all.gpt4all.svg
+      - install -Dm644 io.gpt4all.gpt4all.desktop /app/share/applications/io.gpt4all.gpt4all.desktop
+      - install -Dm644 io.gpt4all.gpt4all.appdata.xml  /app/share/appdata/io.gpt4all.gpt4all.appdata.xml
+    sources:
+      - type: git
+        url: https://github.com/nomic-ai/gpt4all
+        tag: v2.4.19
+        commit: 84905aa28171545542fc653dbeca501ae5af383e
+      - type: file
+        url: https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/icons/logo.svg
+        sha256: 4c4e8476d0e2020585b69c6e2fc9e7d0cb12cbb36aa7b83c3a2e48ed4a9a424c
+      - type: file
+        path: io.gpt4all.gpt4all.desktop
+      - type: file
+        path: io.gpt4all.gpt4all.appdata.xml
+

gpt4all-chat/llm.cpp

@@ -9,6 +9,7 @@
 #include <QProcess>
 #include <QResource>
 #include <QSettings>
+#include <QDesktopServices>
 #include <fstream>
 
 class MyLLM: public LLM { };
@@ -60,6 +61,10 @@ bool LLM::hasSettingsAccess() const
 
 bool LLM::checkForUpdates() const
 {
+    #ifdef GPT4ALL_OFFLINE_INSTALLER
+    #pragma message "offline installer build will not check for updates!"
+    return QDesktopServices::openUrl(QUrl("https://gpt4all.io/"));
+    #else
     Network::globalInstance()->sendCheckForUpdates();
 
 #if defined(Q_OS_LINUX)
@@ -78,6 +83,7 @@ bool LLM::checkForUpdates() const
     }
 
     return QProcess::startDetached(fileName);
+    #endif
 }
 
 bool LLM::directoryExists(const QString &path) const

gpt4all-chat/main.qml

@@ -189,7 +189,7 @@ Window {
             + "causes include a bad file format, an incomplete or corrupted download, the wrong file "
             + "type, not enough system RAM or an incompatible model type. Here are some suggestions for resolving the problem:"
             + "<br><ul>"
-            + "<li>Ensure the model file has a compatible ggml format and type"
+            + "<li>Ensure the model file has a compatible format and type"
             + "<li>Check the model file is complete in the download folder"
             + "<li>You can find the download folder in the settings dialog"
             + "<li>If you've sideloaded the model ensure the file is not corrupt by checking md5sum"
@@ -1013,7 +1013,7 @@ Window {
             anchors.rightMargin: 30
             color: theme.mutedTextColor
             visible: currentChat.tokenSpeed !== ""
-            text: qsTr("Speed: ") + currentChat.tokenSpeed + "<br>" + qsTr("Device: ") + currentChat.device
+            text: qsTr("Speed: ") + currentChat.tokenSpeed + "<br>" + qsTr("Device: ") + currentChat.device + currentChat.fallbackReason
             font.pixelSize: theme.fontSizeLarge
         }
 

gpt4all-chat/metadata/models2.json

@@ -0,0 +1,194 @@
+[
+  {
+    "order": "a",
+    "md5sum": "48de9538c774188eb25a7e9ee024bbd3",
+    "name": "Mistral OpenOrca",
+    "filename": "mistral-7b-openorca.Q4_0.gguf",
+    "filesize": "4108927744",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "systemPrompt": " ",
+    "description": "<strong>Best overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
+    "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.Q4_0.gguf"
+  },
+  {
+    "order": "b",
+    "md5sum": "97463be739b50525df56d33b26b00852",
+    "name": "Mistral Instruct",
+    "filename": "mistral-7b-instruct-v0.1.Q4_0.gguf",
+    "filesize": "4108916384",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "systemPrompt": " ",
+    "description": "<strong>Best overall fast instruction following model</strong><br><ul><li>Fast responses</li><li>Trained by Mistral AI<li>Uncensored</li><li>Licensed for commercial use</li></ul>",
+    "url": "https://gpt4all.io/models/gguf/mistral-7b-instruct-v0.1.Q4_0.gguf",
+    "promptTemplate": "[INST] %1 [/INST]"
+  },
+  {
+    "order": "c",
+    "md5sum": "31cb6d527bd3bfb5e73c2e9dfbc75033",
+    "name": "GPT4All Falcon",
+    "filename": "gpt4all-falcon-q4_0.gguf",
+    "filesize": "4210419040",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Falcon",
+    "systemPrompt": " ",
+    "description": "<strong>Very fast model with good quality</strong><br><ul><li>Fastest responses</li><li>Instruction based</li><li>Trained by TII<li>Finetuned by Nomic AI<li>Licensed for commercial use</ul>",
+    "url": "https://gpt4all.io/models/gguf/gpt4all-falcon-q4_0.gguf",
+    "promptTemplate": "### Instruction:\n%1\n### Response:\n"
+  },
+  {
+    "order": "e",
+    "md5sum": "5aff90007499bce5c64b1c0760c0b186",
+    "name": "Wizard v1.2",
+    "filename": "wizardlm-13b-v1.2.Q4_0.gguf",
+    "filesize": "7365834624",
+    "requires": "2.5.0",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_0",
+    "type": "LLaMA2",
+    "systemPrompt": " ",
+    "description": "<strong>Best overall larger model</strong><br><ul><li>Instruction based<li>Gives very long responses<li>Finetuned with only 1k of high-quality data<li>Trained by Microsoft and Peking University<li>Cannot be used commercially</ul",
+    "url": "https://gpt4all.io/models/gguf/wizardlm-13b-v1.2.Q4_0.gguf"
+  },
+  {
+    "order": "f",
+    "md5sum": "3d12810391d04d1153b692626c0c6e16",
+    "name": "Hermes",
+    "filename": "nous-hermes-llama2-13b.Q4_0.gguf",
+    "filesize": "7366062080",
+    "requires": "2.5.0",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_0",
+    "type": "LLaMA2",
+    "systemPrompt": " ",
+    "description": "<strong>Extremely good model</strong><br><ul><li>Instruction based<li>Gives long responses<li>Curated with 300,000 uncensored instructions<li>Trained by Nous Research<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/nous-hermes-llama2-13b.Q4_0.gguf",
+    "promptTemplate": "### Instruction:\n%1\n### Response:\n"
+  },
+  {
+    "order": "g",
+    "md5sum": "40388eb2f8d16bb5d08c96fdfaac6b2c",
+    "name": "Snoozy",
+    "filename": "gpt4all-13b-snoozy-q4_0.gguf",
+    "filesize": "7365834624",
+    "requires": "2.5.0",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_0",
+    "type": "LLaMA",
+    "systemPrompt": " ",
+    "description": "<strong>Very good overall model</strong><br><ul><li>Instruction based<li>Based on the same dataset as Groovy<li>Slower than Groovy, with higher quality responses<li>Trained by Nomic AI<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf"
+  },
+  {
+    "order": "h",
+    "md5sum": "f5bc6a52f72efd9128efb2eeed802c86",
+    "name": "MPT Chat",
+    "filename": "mpt-7b-chat-q4_0.gguf",
+    "filesize": "3911522272",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "MPT",
+    "description": "<strong>Good model with novel architecture</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/mpt-7b-chat-q4_0.gguf",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|><|im_start|>assistant\n",
+    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
+  },
+  {
+    "order": "i",
+    "md5sum": "aae346fe095e60139ca39b3fda4ac7ae",
+    "name": "Mini Orca (Small)",
+    "filename": "orca-mini-3b.q4_0.gguf",
+    "filesize": "1928648352",
+    "requires": "2.5.0",
+    "ramrequired": "4",
+    "parameters": "3 billion",
+    "quant": "q4_0",
+    "type": "OpenLLaMa",
+    "description": "<strong>Small version of new model with novel dataset</strong><br><ul><li>Instruction based<li>Explain tuned datasets<li>Orca Research Paper dataset construction approaches<li>Licensed for commercial use</ul>",
+    "url": "https://gpt4all.io/models/gguf/orca-mini-3b.q4_0.gguf",
+    "promptTemplate": "### User:\n%1\n### Response:\n",
+    "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
+  },
+  {
+    "order": "j",
+    "md5sum": "51c627fac9062e208f9b386f105cbd48",
+    "disableGUI": "true",
+    "name": "Replit",
+    "filename": "replit-code-v1-3b-q4_0.gguf",
+    "filesize": "1532949760",
+    "requires": "2.5.0",
+    "ramrequired": "4",
+    "parameters": "3 billion",
+    "quant": "f16",
+    "type": "Replit",
+    "systemPrompt": " ",
+    "promptTemplate": "%1",
+    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>Licensed for commercial use</ul>",
+    "url": "https://gpt4all.io/models/gguf/replit-code-v1-3b-q4_0.gguf"
+  },
+  {
+    "order": "k",
+    "md5sum": "556fc3e13df42286997fb58e6f4c639f",
+    "disableGUI": "true",
+    "name": "Starcoder",
+    "filename": "starcoder-q4_0.gguf",
+    "filesize": "8987166880",
+    "requires": "2.5.0",
+    "ramrequired": "4",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Starcoder",
+    "systemPrompt": " ",
+    "promptTemplate": "%1",
+    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based</ul>",
+    "url": "https://gpt4all.io/models/gguf/starcoder-q4_0.gguf"
+  },
+  {
+    "order": "l",
+    "md5sum": "e973dd26f0ffa6e46783feaea8f08c83",
+    "disableGUI": "true",
+    "name": "Rift coder",
+    "filename": "rift-coder-v0-7b-q4_0.gguf",
+    "filesize": "3825903776",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "LLaMA",
+    "systemPrompt": " ",
+    "promptTemplate": "%1",
+    "description": "Code completion model",
+    "url": "https://gpt4all.io/models/gguf/rift-coder-v0-7b-q4_0.gguf"
+  },
+  {
+    "order": "m",
+    "md5sum": "e479e6f38b59afc51a470d1953a6bfc7",
+    "disableGUI": "true",
+    "name": "SBert",
+    "filename": "all-MiniLM-L6-v2-f16.gguf",
+    "filesize": "45887744",
+    "requires": "2.5.0",
+    "ramrequired": "1",
+    "parameters": "1 million",
+    "quant": "f16",
+    "type": "Bert",
+    "systemPrompt": " ",
+    "description": "<strong>Sbert</strong><br><ul><li>For embeddings",
+    "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf"
+  }
+]

gpt4all-chat/metadata/release.json

@@ -506,6 +506,29 @@
 "
 * Adam Treat (Nomic AI)
 * Aaron Miller (Nomic AI)
+"
+  },
+  {
+    "version": "2.4.18",
+    "notes":
+"
+* Bugfix for devices to show up in the settings combobox on application start and not just on model load
+* Send information on requested device and actual device on model load to help assess which model/gpu/os combos are working
+",
+    "contributors":
+"
+* Adam Treat (Nomic AI)
+"
+  },
+  {
+    "version": "2.4.19",
+    "notes":
+"
+* Fix a crasher on systems with corrupted vulkan drivers or corrupted vulkan dlls
+",
+    "contributors":
+"
+* Adam Treat (Nomic AI)
 "
   }
 ]

gpt4all-chat/modellist.cpp

@@ -796,7 +796,7 @@ void ModelList::updateModelsFromDirectory()
                 QString filename = it.fileName();
 
                 // All files that end with .bin and have 'ggml' somewhere in the name
-                if ((filename.endsWith(".bin") && filename.contains("ggml") && !filename.startsWith("incomplete"))
+                if (((filename.endsWith(".bin") || filename.endsWith(".gguf")) && (/*filename.contains("ggml") ||*/ filename.contains("gguf")) && !filename.startsWith("incomplete"))
                     || (filename.endsWith(".txt") && filename.startsWith("chatgpt-"))) {
 
                     QString filePath = it.filePath();
@@ -834,12 +834,14 @@ void ModelList::updateModelsFromDirectory()
         processDirectory(localPath);
 }
 
+#define MODELS_VERSION 2
+
 void ModelList::updateModelsFromJson()
 {
 #if defined(USE_LOCAL_MODELSJSON)
-    QUrl jsonUrl("file://" + QDir::homePath() + "/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models.json");
+    QUrl jsonUrl("file://" + QDir::homePath() + QString("/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models%1.json").arg(MODELS_VERSION));
 #else
-    QUrl jsonUrl("http://gpt4all.io/models/models.json");
+    QUrl jsonUrl(QString("http://gpt4all.io/models/models%1.json").arg(MODELS_VERSION));
 #endif
     QNetworkRequest request(jsonUrl);
     QSslConfiguration conf = request.sslConfiguration();
@@ -881,9 +883,9 @@ void ModelList::updateModelsFromJsonAsync()
     emit asyncModelRequestOngoingChanged();
 
 #if defined(USE_LOCAL_MODELSJSON)
-    QUrl jsonUrl("file://" + QDir::homePath() + "/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models.json");
+    QUrl jsonUrl("file://" + QDir::homePath() + QString("/dev/large_language_models/gpt4all/gpt4all-chat/metadata/models%1.json").arg(MODELS_VERSION));
 #else
-    QUrl jsonUrl("http://gpt4all.io/models/models.json");
+    QUrl jsonUrl(QString("http://gpt4all.io/models/models%1.json").arg(MODELS_VERSION));
 #endif
     QNetworkRequest request(jsonUrl);
     QSslConfiguration conf = request.sslConfiguration();

gpt4all-chat/modellist.h

@@ -115,7 +115,7 @@ private:
     double  m_repeatPenalty       = 1.18;
     int     m_repeatPenaltyTokens = 64;
     QString m_promptTemplate      = "### Human:\n%1\n### Assistant:\n";
-    QString m_systemPrompt        = "### System:\nYou are an AI assistant who gives quality response to whatever humans ask of you.\n";
+    QString m_systemPrompt        = "### System:\nYou are an AI assistant who gives a quality response to whatever humans ask of you.\n";
     friend class MySettings;
 };
 Q_DECLARE_METATYPE(ModelInfo)

gpt4all-chat/mysettings.cpp

@@ -1,5 +1,6 @@
 #include "mysettings.h"
 #include "modellist.h"
+#include "../gpt4all-backend/llmodel.h"
 
 #include <QDir>
 #include <QFile>
@@ -63,6 +64,13 @@ MySettings::MySettings()
     : QObject{nullptr}
 {
     QSettings::setDefaultFormat(QSettings::IniFormat);
+
+    std::vector<LLModel::GPUDevice> devices = LLModel::availableGPUDevices();
+    QVector<QString> deviceList{ "Auto" };
+    for (LLModel::GPUDevice &d : devices)
+        deviceList << QString::fromStdString(d.name);
+    deviceList << "CPU";
+    setDeviceList(deviceList);
 }
 
 Q_INVOKABLE QVector<QString> MySettings::deviceList() const

gpt4all-chat/network.cpp

@@ -393,6 +393,8 @@ void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &valu
     properties.insert("name", QCoreApplication::applicationName() + " v"
         + QCoreApplication::applicationVersion());
     properties.insert("model", ChatListModel::globalInstance()->currentChat()->modelInfo().filename());
+    properties.insert("requestedDevice", MySettings::globalInstance()->device());
+    properties.insert("actualDevice", ChatListModel::globalInstance()->currentChat()->device());
 
     // Some additional startup information
     if (ev == "startup") {

gpt4all-chat/qml/ModelDownloaderDialog.qml

@@ -47,7 +47,7 @@ MyDialog {
             Layout.fillHeight: true
             horizontalAlignment: Qt.AlignHCenter
             verticalAlignment: Qt.AlignVCenter
-            text: qsTr("Network error: could not retrieve http://gpt4all.io/models/models.json")
+            text: qsTr("Network error: could not retrieve http://gpt4all.io/models/models2.json")
             font.pixelSize: theme.fontSizeLarge
             color: theme.mutedTextColor
         }