vulkan python bindings on windows fixes

Signed-off-by: Andriy Mulyar <andriy.mulyar@gmail.com>

.circleci/config.yml

@@ -1,194 +1,19 @@
 version: 2.1
+setup: true
 orbs:
-  win: circleci/windows@5.0
-  python: circleci/python@1.2
-
-jobs:
-  build-py-docs:
-    docker:
-      - image: circleci/python:3.8
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          # TODO: eventually this will be cleaned up so we aren't building
-          # new dependencies each time unnecessarily.
-          # This will be introduced once we setup branch and path filtering
-          command: |
-            sudo apt-get update
-            sudo apt-get -y install python3 python3-pip
-            sudo pip3 install awscli --upgrade
-            sudo pip3 install mkdocs mkdocs-material mkautodoc 'mkdocstrings[python]'
-      - run:
-          name: Make Documentation
-          command: |
-            cd gpt4all-bindings/python/
-            mkdocs build
-      - run:
-          name: Deploy Documentation
-          command: |
-            cd gpt4all-bindings/python/
-            aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat
-      - run:
-          name: Invalidate docs.gpt4all.io cloudfront
-          command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"
-
-            
-
-  build-py-linux:
-    docker:
-      - image: circleci/python:3.8
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y cmake build-essential
-            pip install setuptools wheel cmake
-      - run:
-          name: Build C library
-          command: |
-            git submodule init
-            git submodule update
-            cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake ..
-            cmake --build . --parallel
-      - run:
-          name: Build wheel
-          command: |
-            cd gpt4all-bindings/python/
-            python setup.py bdist_wheel --plat-name=manylinux1_x86_64
-      - persist_to_workspace:
-          root: gpt4all-bindings/python/dist
-          paths:
-            - "*.whl"
-
-  build-py-macos:
-    macos:
-      xcode: "14.2.0"
-    resource_class: macos.m1.large.gen1
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            brew install cmake
-            pip install setuptools wheel cmake
-      - run:
-          name: Build C library
-          command: |
-            git submodule init
-            git submodule update
-            cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake .. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
-            cmake --build . --parallel
-      - run:
-          name: Build wheel
-          command: |
-            cd gpt4all-bindings/python
-            python setup.py bdist_wheel --plat-name=macosx_10_9_universal2
-      - persist_to_workspace:
-          root: gpt4all-bindings/python/dist
-          paths:
-            - "*.whl"
-
-  build-py-windows:
-    executor:
-      name: win/default
-    steps:
-      - checkout
-      - run:
-          name: Install MinGW64
-          command: choco install -y mingw --force --no-progress
-      - run:
-          name: Add MinGW64 to PATH
-          command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
-      - run:
-          name: Install dependencies
-          command: choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
-      - run:
-          name: Install Python dependencies
-          command: pip install setuptools wheel cmake
-      - run:
-          name: Build C library
-          command: |
-            git submodule init
-            git submodule update
-            cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake -G "MinGW Makefiles" .. 
-            cmake --build . --parallel
-      - run:
-          name: Build wheel
-          # TODO: As part of this task, we need to move mingw64 binaries into package.
-          # This is terrible and needs a more robust solution eventually.
-          command: |
-            cd gpt4all-bindings/python
-            cd gpt4all
-            mkdir llmodel_DO_NOT_MODIFY
-            mkdir llmodel_DO_NOT_MODIFY/build/
-            cp 'C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
-            cd ..
-            python setup.py bdist_wheel --plat-name=win_amd64
-      - persist_to_workspace:
-          root: gpt4all-bindings/python/dist
-          paths:
-            - "*.whl"
-
-  store-and-upload-wheels:
-    docker:
-      - image: circleci/python:3.8
-    steps:
-      - setup_remote_docker
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Install dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y cmake build-essential
-            pip install setuptools wheel twine
-      - run:
-          name: Upload Python package
-          command: |
-            twine upload /tmp/workspace/*.whl --username __token__ --password $PYPI_CRED
-      - store_artifacts:
-          path: /tmp/workspace
+  path-filtering: circleci/path-filtering@0.0.1
 
 workflows:
-  version: 2
-  deploy-docs:
+  version: 2.1
+  generate-config:
     jobs:
-      - build-py-docs:
-          filters:
-            branches:
-              only:
-                - main
-  # build-py-deploy:
-  #   jobs:
-  #     - build-py-linux:
-  #         filters:
-  #           branches:
-  #             only:
-  #     - build-py-macos:
-  #         filters:
-  #           branches:
-  #             only:
-  #     - build-py-windows:
-  #         filters:
-  #           branches:
-  #             only:
-  #     - store-and-upload-wheels:
-  #         filters:
-  #           branches:
-  #             only:
-  #         requires:
-  #           - build-py-windows
-  #           - build-py-linux
-  #           - build-py-macos
+      - path-filtering/filter:
+          base-revision: main
+          config-path: .circleci/continue_config.yml
+          mapping: |
+            gpt4all-bindings/python/.* run-python-workflow true
+            gpt4all-bindings/typescript/.* run-ts-workflow true
+            gpt4all-bindings/csharp/.* run-csharp-workflow true
+            gpt4all-backend/.* run-chat-workflow true
+            gpt4all-chat/.* run-chat-workflow true
+            .* run-default-workflow true

.circleci/continue_config.yml

@@ -0,0 +1,993 @@
+version: 2.1
+orbs:
+  win: circleci/windows@5.0
+  python: circleci/python@1.2
+  node: circleci/node@5.1
+
+parameters:
+  run-default-workflow:
+    type: boolean
+    default: false
+  run-python-workflow:
+    type: boolean
+    default: false
+  run-chat-workflow:
+    type: boolean
+    default: false
+  run-ts-workflow:
+    type: boolean
+    default: false
+  run-csharp-workflow:
+    type: boolean
+    default: false
+
+jobs:
+  default-job:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - run: echo "CircleCI pipeline triggered"
+
+  build-gpt4all-chat-linux:
+    machine:
+      image: ubuntu-2204:2023.04.2
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - linux-qt-cache
+      - run:
+          name: Setup Linux and Dependencies
+          command: |
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
+              chmod +x qt-unified-linux-x64-4.6.0-online.run
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: linux-qt-cache
+          paths:
+            - ~/Qt
+      - run:
+          name: Build
+          command: |
+            export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
+            mkdir build
+            cd build
+            ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S ../gpt4all-chat -B .
+            ~/Qt/Tools/CMake/bin/cmake --build . --target all
+
+  build-gpt4all-chat-windows:
+    machine:
+      image: 'windows-server-2019-vs2019:2022.08.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - windows-qt-cache
+      - run:
+          name: Installing Qt
+          command: |
+            if (-not (Test-Path C:\Qt)) {
+              Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+            }
+      - save_cache:  # this is the new step to save cache
+          key: windows-qt-cache
+          paths:
+            - C:\Qt
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Build
+          command: |
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
+            $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
+            mkdir build
+            cd build
+            & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
+              "-DCMAKE_GENERATOR:STRING=Ninja" `
+              "-DCMAKE_BUILD_TYPE=Release" `
+              "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
+              "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
+              "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
+              "-S ..\gpt4all-chat" `
+              "-B ."
+            & "C:\Qt\Tools\Ninja\ninja.exe"
+
+  build-gpt4all-chat-macos:
+    macos:
+      xcode: 14.0.0
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - macos-qt-cache_v2
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
+              hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: macos-qt-cache_v2
+          paths:
+            - ~/Qt
+      - run:
+          name: Build
+          command: |
+            mkdir build
+            cd build
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
+              -DCMAKE_GENERATOR:STRING=Ninja \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
+              -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
+              -S ../gpt4all-chat \
+              -B .
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
+  build-ts-docs: 
+    docker: 
+      - image: cimg/base:stable
+    steps:
+      - checkout
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - node/install-packages:
+          pkg-manager: yarn
+          app-dir: gpt4all-bindings/typescript
+          override-ci-command: yarn install
+      - run:
+          name: build docs ts yo
+          command: |
+            cd gpt4all-bindings/typescript
+            yarn docs:build
+  build-py-docs:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - checkout 
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get -y install python3 python3-pip
+            sudo pip3 install awscli --upgrade
+            sudo pip3 install mkdocs mkdocs-material mkautodoc 'mkdocstrings[python]'
+      - run:
+          name: Make Documentation
+          command: |
+            cd gpt4all-bindings/python/
+            mkdocs build
+      - run:
+          name: Deploy Documentation
+          command: |
+            cd gpt4all-bindings/python/
+            aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat
+      - run:
+          name: Invalidate docs.gpt4all.io cloudfront
+          command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"
+
+  build-py-linux:
+    machine:
+      image: ubuntu-2204:2023.04.2
+    steps:
+      - checkout
+      - run:
+          name: Set Python Version
+          command: pyenv global 3.11.2
+      - run:
+          name: Install dependencies
+          command: |
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            sudo apt-get update
+            sudo apt-get install -y cmake build-essential vulkan-sdk
+            pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            cmake ..
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          command: |
+            cd gpt4all-bindings/python/
+            python setup.py bdist_wheel --plat-name=manylinux1_x86_64
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  build-py-macos:
+    macos:
+      xcode: "14.2.0"
+    resource_class: macos.m1.large.gen1
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            brew install cmake
+            pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            cmake .. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          command: |
+            cd gpt4all-bindings/python
+            python setup.py bdist_wheel --plat-name=macosx_10_9_universal2
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  build-py-windows:
+    executor:
+      name: win/default
+    steps:
+      - checkout
+      - run:
+          name: Install MinGW64
+          command: choco install -y mingw --force --no-progress
+      - run:
+          name: Add MinGW64 to PATH
+          command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install dependencies
+          command:
+            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+      - run:
+          name: Install Python dependencies
+          command: pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            $env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            cmake -G "MinGW Makefiles" .. -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          # TODO: As part of this task, we need to move mingw64 binaries into package.
+          # This is terrible and needs a more robust solution eventually.
+          command: |
+            cd gpt4all-bindings/python
+            cd gpt4all
+            mkdir llmodel_DO_NOT_MODIFY
+            mkdir llmodel_DO_NOT_MODIFY/build/
+            cp 'C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
+            cd ..
+            python setup.py bdist_wheel --plat-name=win_amd64
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  store-and-upload-wheels:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - setup_remote_docker
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y cmake build-essential
+            pip install setuptools wheel twine
+      - run:
+          name: Upload Python package
+          command: |
+            twine upload /tmp/workspace/*.whl --username __token__ --password $PYPI_CRED
+      - store_artifacts:
+          path: /tmp/workspace
+
+  build-bindings-backend-linux:
+    machine:
+      image: ubuntu-2204:2023.04.2
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install dependencies
+          command: |
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list            
+            sudo apt-get update
+            sudo apt-get install -y cmake build-essential vulkan-sdk
+      - run:
+          name: Build Libraries
+          command: |
+            cd gpt4all-backend
+            mkdir -p runtimes/build
+            cd runtimes/build
+            cmake ../..
+            cmake --build . --parallel --config Release
+            mkdir ../linux-x64
+            cp -L *.so ../linux-x64 # otherwise persist_to_workspace seems to mess symlinks
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/linux-x64/*.so
+
+  build-bindings-backend-macos:
+    macos:
+      xcode: "14.0.0"
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install dependencies
+          command: |
+            brew install cmake
+      - run:
+          name: Build Libraries
+          command: |
+            cd gpt4all-backend
+            mkdir -p runtimes/build
+            cd runtimes/build
+            cmake ../.. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake --build . --parallel --config Release
+            mkdir ../osx-x64
+            cp -L *.dylib ../osx-x64
+            cp ../../llama.cpp-mainline/*.metal ../osx-x64
+            ls ../osx-x64
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/osx-x64/*.dylib
+            - runtimes/osx-x64/*.metal
+
+  build-bindings-backend-windows:
+    executor:
+      name: win/default
+      size: large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install MinGW64
+          command: choco install -y mingw --force --no-progress
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install dependencies
+          command: |
+            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+      - run:
+          name: Build Libraries
+          command: |
+            $MinGWBin = "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
+            $Env:Path += ";$MinGwBin"
+            $Env:Path += ";C:\Program Files\CMake\bin"
+            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            cd gpt4all-backend
+            mkdir runtimes/win-x64
+            cd runtimes/win-x64
+            cmake -G "MinGW Makefiles" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON ../..
+            cmake --build . --parallel --config Release
+            cp "$MinGWBin\libgcc*.dll" .
+            cp "$MinGWBin\libstdc++*.dll" .
+            cp "$MinGWBin\libwinpthread*.dll" .
+            cp bin/*.dll .
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/win-x64/*.dll
+
+  build-bindings-backend-windows-msvc:
+    machine:
+      image: 'windows-server-2022-gui:2023.03.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install dependencies
+          command: |
+            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'  
+      - run:
+          name: Build Libraries
+          command: |
+            $Env:Path += ";C:\Program Files\CMake\bin"
+            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            cd gpt4all-backend
+            mkdir runtimes/win-x64_msvc
+            cd runtimes/win-x64_msvc
+            cmake -G "Visual Studio 17 2022" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -A X64 ../..
+            cmake --build . --parallel --config Release
+            cp bin/Release/*.dll .
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/win-x64_msvc/*.dll
+
+  build-csharp-linux:
+    docker:
+      - image: mcr.microsoft.com/dotnet/sdk:7.0-jammy # Ubuntu 22.04
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: "Prepare Native Libs"
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes/linux-x64/native
+            cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
+            ls -R runtimes
+      - restore_cache:
+          keys:
+            - gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: "Install project dependencies"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet restore Gpt4All
+      - save_cache:
+          paths:
+            - ~/.nuget/packages
+          key: gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: Build C# Project
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet build Gpt4All --configuration Release --nologo
+      - run:
+          name: "Run C# Tests"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+      - run:
+          name: Test results
+          command: |
+              cd gpt4all-bindings/csharp/Gpt4All.Tests
+              dotnet tool install -g trx2junit
+              export PATH="$PATH:$HOME/.dotnet/tools"
+              trx2junit TestResults/*.trx
+      - store_test_results:
+          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+
+  build-csharp-windows:
+    executor:
+      name: win/default
+      size: large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - gpt4all-csharp-nuget-packages-win
+      - attach_workspace:
+          at: C:\Users\circleci\workspace
+      - run:
+          name: "Prepare Native Libs"
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes\win-x64\native
+            cp C:\Users\circleci\workspace\runtimes\win-x64\*.dll runtimes\win-x64\native\
+            ls -R runtimes
+      - run:
+          name: "Install project dependencies"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet.exe restore Gpt4All
+      - save_cache:
+          paths:
+            - C:\Users\circleci\.nuget\packages
+          key: gpt4all-csharp-nuget-packages-win
+      - run:
+          name: Build C# Project
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet.exe build Gpt4All --configuration Release --nologo
+      - run:
+          name: "Run C# Tests"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet.exe test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+      - run:
+          name: Test results
+          command: |
+              cd gpt4all-bindings/csharp/Gpt4All.Tests
+              dotnet tool install -g trx2junit
+              $Env:Path += ";$Env:USERPROFILE\.dotnet\tools"
+              trx2junit TestResults/*.trx
+      - store_test_results:
+          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+
+  build-csharp-macos:
+    macos:
+      xcode: "14.0.0"
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: Install dependencies
+          command: |
+            brew install --cask dotnet-sdk
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: "Prepare Native Libs"
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes/osx/native
+            cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
+            cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
+            ls -R runtimes
+      - run:
+          name: "Install project dependencies"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet restore Gpt4All
+      - save_cache:
+          paths:
+            - ~/.nuget/packages
+          key: gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: Build C# Project
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet build Gpt4All --configuration Release --nologo
+      - run:
+          name: "Run C# Tests"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+      - run:
+          name: Test results
+          command: |
+              cd gpt4all-bindings/csharp/Gpt4All.Tests
+              dotnet tool install -g trx2junit
+              export PATH="$PATH:$HOME/.dotnet/tools"
+              trx2junit TestResults/*.trx
+      - store_test_results:
+          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+
+  store-and-upload-nupkgs:
+    docker:
+      - image: mcr.microsoft.com/dotnet/sdk:6.0-jammy # Ubuntu 22.04
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - checkout
+      - restore_cache:
+           keys:
+             - gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: NuGet Pack
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes/linux-x64/native
+            cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
+            mkdir -p runtimes/win-x64/native
+            cp /tmp/workspace/runtimes/win-x64/*.dll runtimes/win-x64/native/
+            mkdir -p runtimes/osx/native
+            cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
+            cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
+            dotnet pack ./Gpt4All/Gpt4All.csproj -p:IncludeSymbols=true -p:SymbolPackageFormat=snupkg -c Release
+            dotnet nuget push ./Gpt4All/bin/Release/Gpt4All.*.nupkg -s $NUGET_URL -k $NUGET_TOKEN --skip-duplicate
+      - store_artifacts:
+          path: gpt4all-bindings/csharp/Gpt4All/bin/Release
+
+  build-nodejs-linux: 
+    docker: 
+      - image: cimg/base:stable
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - node/install-packages:
+          app-dir: gpt4all-bindings/typescript
+          pkg-manager: yarn
+      - run:
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn prebuildify -t 18.16.0 --napi
+      - run: 
+          command: |
+            mkdir -p gpt4all-backend/prebuilds/linux-x64
+            mkdir -p gpt4all-backend/runtimes/linux-x64
+            cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so gpt4all-backend/runtimes/linux-x64
+            cp gpt4all-bindings/typescript/prebuilds/linux-x64/*.node gpt4all-backend/prebuilds/linux-x64
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - prebuilds/linux-x64/*.node 
+            - runtimes/linux-x64/*-*.so
+  build-nodejs-macos: 
+    macos:
+      xcode: "14.0.0"
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - node/install-packages:
+          app-dir: gpt4all-bindings/typescript
+          pkg-manager: yarn
+      - run:
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn prebuildify -t 18.16.0 --napi
+      - run: 
+          name: "Persisting all necessary things to workspace"
+          command: |  
+            mkdir -p gpt4all-backend/prebuilds/darwin-x64
+            mkdir -p gpt4all-backend/runtimes/darwin-x64
+            cp /tmp/gpt4all-backend/runtimes/osx-x64/*-*.* gpt4all-backend/runtimes/darwin-x64
+            cp gpt4all-bindings/typescript/prebuilds/darwin-x64/*.node gpt4all-backend/prebuilds/darwin-x64
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - prebuilds/darwin-x64/*.node
+            - runtimes/darwin-x64/*-*.*
+
+  build-nodejs-windows: 
+    executor:
+      name: win/default
+      size: large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - run: choco install wget -y
+      - run:
+         command: wget https://nodejs.org/dist/v18.16.0/node-v18.16.0-x86.msi -P C:\Users\circleci\Downloads\
+         shell: cmd.exe
+      - run: MsiExec.exe /i C:\Users\circleci\Downloads\node-v18.16.0-x86.msi /qn
+      - run: 
+          command: |
+              Start-Process powershell -verb runAs -Args "-start GeneralProfile"
+              nvm install 18.16.0
+              nvm use 18.16.0
+      - run: node --version 
+      - run:           
+          command: |
+            npm install -g yarn
+            cd gpt4all-bindings/typescript
+            yarn install
+      - run:
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn prebuildify -t 18.16.0 --napi 
+      - run: 
+          command: |
+            mkdir -p gpt4all-backend/prebuilds/win32-x64
+            mkdir -p gpt4all-backend/runtimes/win32-x64
+            cp /tmp/gpt4all-backend/runtimes/win-x64_msvc/*-*.dll gpt4all-backend/runtimes/win32-x64
+            cp gpt4all-bindings/typescript/prebuilds/win32-x64/*.node gpt4all-backend/prebuilds/win32-x64
+        
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - prebuilds/win32-x64/*.node
+            - runtimes/win32-x64/*-*.dll
+
+  prepare-npm-pkg:
+    docker: 
+      - image: cimg/base:stable
+    steps:
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - checkout
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - run: 
+          command: |
+            cd gpt4all-bindings/typescript
+            # excluding llmodel. nodejs bindings dont need llmodel.dll
+            mkdir -p runtimes/win32-x64/native
+            mkdir -p prebuilds/win32-x64/
+            cp /tmp/gpt4all-backend/runtimes/win-x64_msvc/*-*.dll runtimes/win32-x64/native/ 
+            cp /tmp/gpt4all-backend/prebuilds/win32-x64/*.node prebuilds/win32-x64/    
+
+            mkdir -p runtimes/linux-x64/native 
+            mkdir -p prebuilds/linux-x64/
+            cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so runtimes/linux-x64/native/ 
+            cp /tmp/gpt4all-backend/prebuilds/linux-x64/*.node prebuilds/linux-x64/    
+
+            mkdir -p runtimes/darwin-x64/native
+            mkdir -p prebuilds/darwin-x64/
+            cp /tmp/gpt4all-backend/runtimes/darwin-x64/*-*.* runtimes/darwin-x64/native/
+            cp /tmp/gpt4all-backend/prebuilds/darwin-x64/*.node prebuilds/darwin-x64/    
+            
+            # Fallback build if user is not on above prebuilds
+            mv -f binding.ci.gyp binding.gyp
+
+            mkdir gpt4all-backend
+            cd ../../gpt4all-backend
+            mv llmodel.h llmodel.cpp llmodel_c.cpp llmodel_c.h sysinfo.h dlhandle.h ../gpt4all-bindings/typescript/gpt4all-backend/
+            
+      # Test install
+      - node/install-packages:
+          app-dir: gpt4all-bindings/typescript
+          pkg-manager: yarn
+          override-ci-command: yarn install
+      - run: 
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn run test
+      - run:
+          command: |
+            cd gpt4all-bindings/typescript
+            npm set //registry.npmjs.org/:_authToken=$NPM_TOKEN
+            npm publish --access public --tag alpha
+
+workflows:
+  version: 2
+  default:
+    when: << pipeline.parameters.run-default-workflow >>
+    jobs:
+      - default-job
+  build-and-test-gpt4all-chat:
+    when: << pipeline.parameters.run-chat-workflow >>
+    jobs:
+      - hold:
+          type: approval
+      - build-gpt4all-chat-linux:
+          requires:
+            - hold
+      - build-gpt4all-chat-windows:
+          requires:
+            - hold
+      - build-gpt4all-chat-macos:
+          requires:
+            - hold
+  deploy-docs:
+    when: << pipeline.parameters.run-python-workflow >>
+    jobs:
+      - build-ts-docs:
+          filters:
+            branches:
+              only:
+                - main
+      - build-py-docs:
+          filters:
+            branches:
+              only:
+                - main
+  build-py-deploy:
+    when: << pipeline.parameters.run-python-workflow >>
+    jobs:
+      - pypi-hold:
+          type: approval
+      - hold:
+          type: approval
+      - build-py-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-py-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-py-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - store-and-upload-wheels:
+          filters:
+            branches:
+              only:
+          requires:
+            - pypi-hold
+            - build-py-windows
+            - build-py-linux
+            - build-py-macos
+  build-bindings:
+    when: 
+        or:
+         - << pipeline.parameters.run-python-workflow >>
+         - << pipeline.parameters.run-csharp-workflow >>
+         - <<   pipeline.parameters.run-ts-workflow   >>
+    jobs:
+      - hold:
+          type: approval
+      - nuget-hold:
+          type: approval
+      - npm-hold:
+          type: approval
+      - build-bindings-backend-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-bindings-backend-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-bindings-backend-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-bindings-backend-windows-msvc:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+
+      # NodeJs Jobs 
+      - prepare-npm-pkg: 
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-nodejs-linux
+            - build-nodejs-windows
+            - build-nodejs-macos
+      - build-nodejs-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-bindings-backend-linux
+      - build-nodejs-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-bindings-backend-windows-msvc
+      - build-nodejs-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-bindings-backend-macos
+
+
+      # CSharp Jobs
+      - build-csharp-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - nuget-hold
+            - build-bindings-backend-linux
+      - build-csharp-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - nuget-hold
+            - build-bindings-backend-windows
+      - build-csharp-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - nuget-hold
+            - build-bindings-backend-macos
+      - store-and-upload-nupkgs:
+          filters:
+            branches:
+              only:
+          requires:
+            - nuget-hold
+            - build-csharp-windows
+            - build-csharp-linux
+            - build-csharp-macos

.codespellrc

@@ -1,4 +1,3 @@
 [codespell]
-skip = .git,*.pdf,*.svg
-#
-# ignore-words-list =
+ignore-words-list = blong, belong, afterall, som
+skip = .git,*.pdf,*.svg,*.lock

.gitignore

@@ -1,3 +1,6 @@
+*.arrow
+squad_*
+*sbert_embedded*
 *.pkl
 ckpts*
 .deepspeed_env
@@ -178,3 +181,6 @@ CMakeLists.txt.user
 gpt4all-chat/models/*
 build_*
 build-*
+
+# IntelliJ
+.idea/

.gitmodules

@@ -3,7 +3,7 @@
 	url = https://github.com/ggerganov/llama.cpp.git
 [submodule "llama.cpp-230511"]
 	path = gpt4all-backend/llama.cpp-230511
-	url = https://github.com/manyoso/llama.cpp.git
+	url = https://github.com/nomic-ai/llama.cpp
 [submodule "llama.cpp-mainline"]
 	path = gpt4all-backend/llama.cpp-mainline
-	url = https://github.com/ggerganov/llama.cpp.git
+	url = https://github.com/nomic-ai/llama.cpp.git

LICENSE_SOM.txt

@@ -0,0 +1,30 @@
+Software for Open Models License (SOM)
+Version 1.0 dated August 30th, 2023
+
+This license governs use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
+
+This license is intended to encourage open release of models created, modified, processed, or otherwise used via the Software under open licensing terms, and should be interpreted in light of that intent.
+
+1. Definitions
+The “Licensor” is the person or entity who is making the Software available under this license. “Software” is the software made available by Licensor under this license.
+A “Model” is the output of a machine learning algorithm, and excludes the Software.
+“Model Source Materials” must include the Model and model weights, and may include any input data, input data descriptions, documentation or training descriptions for the Model.
+“Open Licensing Terms” means: (a) any open source license approved by the Open Source Initiative, or (b) any other terms that make the Model Source Materials publicly available free of charge, and allow recipients to use, modify and distribute the Model Source Materials. Terms described in (b) may include reasonable restrictions such as non-commercial or non-production limitations, or require use in compliance with law.
+
+2. Grant of Rights. Subject to the conditions and limitations in section 3:
+(A) Copyright Grant. Licensor grants you a non-exclusive, worldwide, royalty-free copyright license to copy, modify, and distribute the Software and any modifications of the Software you create under this license. The foregoing license includes without limitation the right to create, modify, and use Models using this Software.
+
+(B) Patent Grant. Licensor grants you a non-exclusive, worldwide, royalty-free license, under any patents owned or controlled by Licensor, to make, have made, use, sell, offer for sale, import, or otherwise exploit the Software.  No license is granted to patent rights that are not embodied in the operation of the Software in the form provided by Licensor.
+
+3. Conditions and Limitations
+(A) Model Licensing and Access. If you use the Software to create, modify, process, or otherwise use any Model, including usage to create inferences with a Model, whether or not you make the Model available to others, you must make that Model Source Materials publicly available under Open Licensing Terms. 
+
+(B) No Re-Licensing. If you redistribute the Software, or modifications to the Software made under the license granted above, you must make it available only under the terms of this license. You may offer additional terms such as warranties, maintenance and support, but You, and not Licensor, are responsible for performing such terms.
+
+(C) No Trademark License. This license does not grant you rights to use the Licensor’s name, logo, or trademarks.
+
+(D) If you assert in writing a claim against any person or entity alleging that the use of the Software infringes any patent, all of your licenses to the Software under Section 2 end automatically as of the date you asserted the claim.
+
+(E) If you distribute any portion of the Software, you must retain all copyright, patent, trademark, and attribution notices that are present in the Software, and you must include a copy of this license.
+
+(F) The Software is licensed “as-is.” You bear the entire risk of using it. Licensor gives You no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws that this license cannot change. To the extent permitted under your local laws, the Licensor disclaims and excludes the implied warranties of merchantability, fitness for a particular purpose and non-infringement. To the extent this disclaimer is unlawful, you, and not Licensor, are responsible for any liability.

README.md

@@ -1,4 +1,5 @@
 <h1 align="center">GPT4All</h1>
+
 <p align="center">Open-source assistant-style large language models that run locally on your CPU</p>
 
 <p align="center">
@@ -25,11 +26,11 @@ GPT4All is made possible by our compute partner <a href="https://www.paperspace.
   <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
 </p>
 <p align="center">
-Run on an M1 Mac (not sped up!)
+Run on an M1 macOS Device (not sped up!)
 </p>
 
 ## GPT4All: An ecosystem of open-source on-edge large language models.
-GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs.
+GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
 
 Learn more in the [documentation](https://docs.gpt4all.io).
 
@@ -43,20 +44,12 @@ Run any GPT4All model natively on your home desktop with the auto-updating deskt
 
 Direct Installer Links:
 
-* [Mac/OSX](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
+* [macOS](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
 
 * [Windows](https://gpt4all.io/installers/gpt4all-installer-win64.exe)
 
 * [Ubuntu](https://gpt4all.io/installers/gpt4all-installer-linux.run)
 
-If you have older hardware that only supports avx and not avx2 you can use these.
-
-* [Mac/OSX - avx-only](https://gpt4all.io/installers/gpt4all-installer-darwin-avx-only.dmg)
-
-* [Windows - avx-only](https://gpt4all.io/installers/gpt4all-installer-win64-avx-only.exe)
-
-* [Ubuntu - avx-only](https://gpt4all.io/installers/gpt4all-installer-linux-avx-only.run)
-
 Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io/)
 
 ### Chat Client building and running
@@ -65,11 +58,15 @@ Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io
 
 ### Bindings
 
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a>
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/java">:computer: Official Java Bindings</a>
 
+### Integrations
+
+* 🗃️ [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
 
 ## Contributing
 GPT4All welcomes contributions, involvement, and discussion from the open source community!

gpt4all-api/.gitignore

@@ -0,0 +1,112 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+app/__pycache__/
+gpt4all_api/__pycache__/
+gpt4all_api/app/api_v1/__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# VS Code
+.vscode/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+*.lock
+*.cache

gpt4all-api/.isort.cfg

@@ -0,0 +1,7 @@
+[settings]
+known_third_party=geopy,nltk,np,numpy,pandas,pysbd,fire,torch
+
+line_length=120
+include_trailing_comma=True
+multi_line_output=3
+use_parentheses=True

gpt4all-api/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2023 Nomic, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

gpt4all-api/README.md

@@ -1,2 +1,87 @@
-# GPT4All API
-This directory will contain code to build out a RESTful API for GPT4All models. Exact details TBD, but as an MVP, user should be able to send requests to list, download, and generate text with different models.
+# GPT4All REST API
+This directory contains the source code to run and build docker images that run a FastAPI app
+for serving inference from GPT4All models. The API matches the OpenAI API spec.
+
+## Tutorial
+
+The following tutorial assumes that you have checked out this repo and cd'd into it.
+
+### Starting the app
+
+First change your working directory to `gpt4all/gpt4all-api`.
+
+Now you can build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
+```bash
+DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
+```
+
+Then, start the backend with:
+
+```bash
+docker compose up --build
+```
+
+This will run both the API and locally hosted GPU inference server. If you want to run the API without the GPU inference server, you can run:
+
+```bash
+docker compose up --build gpt4all_api
+```
+
+To run the API with the GPU inference server, you will need to include environment variables (like the `MODEL_ID`). Edit the `.env` file and run
+```bash
+docker compose --env-file .env up --build
+```
+
+
+#### Spinning up your app
+Run `docker compose up` to spin up the backend. Monitor the logs for errors in-case you forgot to set an environment variable above.
+
+
+#### Development
+Run
+
+```bash
+docker compose up --build
+```
+and edit files in the `api` directory. The api will hot-reload on changes.
+
+You can run the unit tests with
+
+```bash
+make test
+```
+
+#### Viewing API documentation
+
+Once the FastAPI ap is started you can access its documentation and test the search endpoint by going to:
+```
+localhost:80/docs
+```
+
+This documentation should match the OpenAI OpenAPI spec located at https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+#### Running inference
+```python
+import openai
+openai.api_base = "http://localhost:4891/v1"
+
+openai.api_key = "not needed for a local LLM"
+
+
+def test_completion():
+    model = "gpt4all-j-v1.3-groovy"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Completion.create(
+        model=model,
+        prompt=prompt,
+        max_tokens=50,
+        temperature=0.28,
+        top_p=0.95,
+        n=1,
+        echo=True,
+        stream=False
+    )
+    assert len(response['choices'][0]['text']) > len(prompt)
+    print(response)
+```

gpt4all-api/docker-compose.gpu.yaml

@@ -0,0 +1,24 @@
+version: "3.8"
+
+services:
+  gpt4all_gpu:
+    image: ghcr.io/huggingface/text-generation-inference:0.9.3
+    container_name: gpt4all_gpu
+    restart: always #restart on error (usually code compilation from save during bad state)
+    environment:
+      - HUGGING_FACE_HUB_TOKEN=token
+      - USE_FLASH_ATTENTION=false
+      - MODEL_ID=''
+      - NUM_SHARD=1
+    command: --model-id $MODEL_ID --num-shard $NUM_SHARD
+    volumes:
+      - ./:/data
+    ports:
+      - "8080:80"
+    shm_size: 1g
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: [gpu]

gpt4all-api/docker-compose.yaml

@@ -0,0 +1,19 @@
+version: "3.8"
+
+services:
+  gpt4all_api:
+    image: gpt4all_api
+    container_name: gpt4all_api
+    restart: always #restart on error (usually code compilation from save during bad state)
+    ports:
+      - "4891:4891"
+    environment:
+      - APP_ENVIRONMENT=dev
+      - WEB_CONCURRENCY=2
+      - LOGLEVEL=debug
+      - PORT=4891
+      - model=ggml-mpt-7b-chat.bin
+      - inference_mode=cpu
+    volumes:
+      - './gpt4all_api/app:/app'
+    command: ["/start-reload.sh"]

gpt4all-api/gpt4all_api/Dockerfile.buildkit

@@ -0,0 +1,23 @@
+# syntax=docker/dockerfile:1.0.0-experimental
+FROM tiangolo/uvicorn-gunicorn:python3.11
+
+ARG MODEL_BIN=ggml-mpt-7b-chat.bin
+
+# Put first so anytime this file changes other cached layers are invalidated.
+COPY gpt4all_api/requirements.txt /requirements.txt
+
+RUN pip install --upgrade pip
+
+# Run various pip install commands with ssh keys from host machine.
+RUN --mount=type=ssh pip install -r /requirements.txt && \
+  rm -Rf /root/.cache && rm -Rf /tmp/pip-install*
+
+# Finally, copy app and client.
+COPY gpt4all_api/app /app
+
+RUN mkdir -p /models
+
+# Include the following line to bake a model into the image and not have to download it on API start.
+RUN wget -q --show-progress=off https://gpt4all.io/models/${MODEL_BIN} -P /models \
+  && md5sum /models/${MODEL_BIN}
+

gpt4all-api/gpt4all_api/README.md

@@ -0,0 +1 @@
+# FastAPI app for serving GPT4All models

gpt4all-api/gpt4all_api/app/api_v1/api.py

@@ -0,0 +1,9 @@
+from api_v1.routes import chat, completions, engines, health
+from fastapi import APIRouter
+
+router = APIRouter()
+
+router.include_router(chat.router)
+router.include_router(completions.router)
+router.include_router(engines.router)
+router.include_router(health.router)

gpt4all-api/gpt4all_api/app/api_v1/events.py

@@ -0,0 +1,29 @@
+import logging
+
+from api_v1.settings import settings
+from fastapi import HTTPException
+from fastapi.responses import JSONResponse
+from starlette.requests import Request
+
+log = logging.getLogger(__name__)
+
+
+startup_msg_fmt = """
+ Starting up GPT4All API
+"""
+
+
+async def on_http_error(request: Request, exc: HTTPException):
+    return JSONResponse({'detail': exc.detail}, status_code=exc.status_code)
+
+
+async def on_startup(app):
+    startup_msg = startup_msg_fmt.format(settings=settings)
+    log.info(startup_msg)
+
+
+def startup_event_handler(app):
+    async def start_app() -> None:
+        await on_startup(app)
+
+    return start_app

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -0,0 +1,61 @@
+import logging
+import time
+from typing import Dict, List
+
+from api_v1.settings import settings
+from fastapi import APIRouter, Depends, Response, Security, status
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+class ChatCompletionMessage(BaseModel):
+    role: str
+    content: str
+
+
+class ChatCompletionRequest(BaseModel):
+    model: str = Field(..., description='The model to generate a completion from.')
+    messages: List[ChatCompletionMessage] = Field(..., description='The model to generate a completion from.')
+
+
+class ChatCompletionChoice(BaseModel):
+    message: ChatCompletionMessage
+    index: int
+    finish_reason: str
+
+
+class ChatCompletionUsage(BaseModel):
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+
+
+class ChatCompletionResponse(BaseModel):
+    id: str
+    object: str = 'text_completion'
+    created: int
+    model: str
+    choices: List[ChatCompletionChoice]
+    usage: ChatCompletionUsage
+
+
+router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
+
+
+@router.post("/completions", response_model=ChatCompletionResponse)
+async def chat_completion(request: ChatCompletionRequest):
+    '''
+    Completes a GPT4All model response.
+    '''
+
+    return ChatCompletionResponse(
+        id='asdf',
+        created=time.time(),
+        model=request.model,
+        choices=[{}],
+        usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+    )

gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py

@@ -0,0 +1,215 @@
+import json
+from typing import List, Dict, Iterable, AsyncIterable
+import logging
+import time
+from typing import Dict, List, Union, Optional
+from uuid import uuid4
+import aiohttp
+import asyncio
+from api_v1.settings import settings
+from fastapi import APIRouter, Depends, Response, Security, status, HTTPException
+from fastapi.responses import StreamingResponse
+from gpt4all import GPT4All
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+class CompletionRequest(BaseModel):
+    model: str = Field(settings.model, description='The model to generate a completion from.')
+    prompt: Union[List[str], str] = Field(..., description='The prompt to begin completing from.')
+    max_tokens: int = Field(None, description='Max tokens to generate')
+    temperature: float = Field(settings.temp, description='Model temperature')
+    top_p: Optional[float] = Field(settings.top_p, description='top_p')
+    top_k: Optional[int] = Field(settings.top_k, description='top_k')
+    n: int = Field(1, description='How many completions to generate for each prompt')
+    stream: bool = Field(False, description='Stream responses')
+    repeat_penalty: float = Field(settings.repeat_penalty, description='Repeat penalty')
+
+
+class CompletionChoice(BaseModel):
+    text: str
+    index: int
+    logprobs: float
+    finish_reason: str
+
+
+class CompletionUsage(BaseModel):
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+
+
+class CompletionResponse(BaseModel):
+    id: str
+    object: str = 'text_completion'
+    created: int
+    model: str
+    choices: List[CompletionChoice]
+    usage: CompletionUsage
+
+
+class CompletionStreamResponse(BaseModel):
+    id: str
+    object: str = 'text_completion'
+    created: int
+    model: str
+    choices: List[CompletionChoice]
+
+
+router = APIRouter(prefix="/completions", tags=["Completion Endpoints"])
+
+def stream_completion(output: Iterable, base_response: CompletionStreamResponse):
+    """
+    Streams a GPT4All output to the client.
+
+    Args:
+        output: The output of GPT4All.generate(), which is an iterable of tokens.
+        base_response: The base response object, which is cloned and modified for each token.
+
+    Returns:
+        A Generator of CompletionStreamResponse objects, which are serialized to JSON Event Stream format.
+    """
+    for token in output:
+        chunk = base_response.copy()
+        chunk.choices = [dict(CompletionChoice(
+            text=token,
+            index=0,
+            logprobs=-1,
+            finish_reason=''
+        ))]
+        yield f"data: {json.dumps(dict(chunk))}\n\n"
+
+async def gpu_infer(payload, header):
+    async with aiohttp.ClientSession() as session:
+        try:
+            async with session.post(
+                settings.hf_inference_server_host, headers=header, data=json.dumps(payload)
+            ) as response:
+                resp = await response.json()
+            return resp
+
+        except aiohttp.ClientError as e:
+            # Handle client-side errors (e.g., connection error, invalid URL)
+            logger.error(f"Client error: {e}")
+        except aiohttp.ServerError as e:
+            # Handle server-side errors (e.g., internal server error)
+            logger.error(f"Server error: {e}")
+        except json.JSONDecodeError as e:
+            # Handle JSON decoding errors
+            logger.error(f"JSON decoding error: {e}")
+        except Exception as e:
+            # Handle other unexpected exceptions
+            logger.error(f"Unexpected error: {e}")
+
+@router.post("/", response_model=CompletionResponse)
+async def completions(request: CompletionRequest):
+    '''
+    Completes a GPT4All model response.
+    '''
+    if settings.inference_mode == "gpu":
+        params = request.dict(exclude={'model', 'prompt', 'max_tokens', 'n'})
+        params["max_new_tokens"] = request.max_tokens
+        params["num_return_sequences"] = request.n
+
+        header = {"Content-Type": "application/json"}
+        if isinstance(request.prompt, list):
+            tasks = []
+            for prompt in request.prompt:
+                payload = {"parameters": params}
+                payload["inputs"] = prompt
+                task = gpu_infer(payload, header)
+                tasks.append(task)
+            results = await asyncio.gather(*tasks)
+
+            choices = []
+            for response in results:
+                scores = response["scores"] if "scores" in response else -1.0
+                choices.append(
+                    dict(
+                        CompletionChoice(
+                            text=response["generated_text"], index=0, logprobs=scores, finish_reason='stop'
+                        )
+                    )
+                )
+
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=choices,
+                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+            )
+
+        else:
+            payload = {"parameters": params}
+            # If streaming, we need to return a StreamingResponse
+            payload["inputs"] = request.prompt
+
+            resp = await gpu_infer(payload, header)
+
+            output = resp["generated_text"]
+            # this returns all logprobs
+            scores = resp["scores"] if "scores" in resp else -1.0
+
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[dict(CompletionChoice(text=output, index=0, logprobs=scores, finish_reason='stop'))],
+                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+            )
+
+    else:
+
+        if request.model != settings.model:
+            raise HTTPException(status_code=400,
+                                detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
+
+        if isinstance(request.prompt, list):
+            if len(request.prompt) > 1:
+                raise HTTPException(status_code=400, detail="Can only infer one inference per request in CPU mode.")
+            else:
+                request.prompt = request.prompt[0]
+
+        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+        output = model.generate(prompt=request.prompt,
+                                max_tokens=request.max_tokens,
+                                streaming=request.stream,
+                                top_k=request.top_k,
+                                top_p=request.top_p,
+                                temp=request.temperature,
+                                )
+
+        # If streaming, we need to return a StreamingResponse
+        if request.stream:
+            base_chunk = CompletionStreamResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[]
+            )
+            return StreamingResponse((response for response in stream_completion(output, base_chunk)),
+                                     media_type="text/event-stream")
+        else:
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[dict(CompletionChoice(
+                    text=output,
+                    index=0,
+                    logprobs=-1,
+                    finish_reason='stop'
+                ))],
+                usage={
+                    'prompt_tokens': 0,  # TODO how to compute this?
+                    'completion_tokens': 0,
+                    'total_tokens': 0
+                }
+            )

gpt4all-api/gpt4all_api/app/api_v1/routes/embeddings.py

@@ -0,0 +1,65 @@
+from typing import List, Union
+from fastapi import APIRouter
+from api_v1.settings import settings
+from gpt4all import Embed4All
+from pydantic import BaseModel, Field
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+class EmbeddingRequest(BaseModel):
+    model: str = Field(
+        settings.model, description="The model to generate an embedding from."
+    )
+    input: Union[str, List[str], List[int], List[List[int]]] = Field(
+        ..., description="Input text to embed, encoded as a string or array of tokens."
+    )
+
+
+class EmbeddingUsage(BaseModel):
+    prompt_tokens: int = 0
+    total_tokens: int = 0
+
+
+class Embedding(BaseModel):
+    index: int = 0
+    object: str = "embedding"
+    embedding: List[float]
+
+
+class EmbeddingResponse(BaseModel):
+    object: str = "list"
+    model: str
+    data: List[Embedding]
+    usage: EmbeddingUsage
+
+
+router = APIRouter(prefix="/embeddings", tags=["Embedding Endpoints"])
+
+embedder = Embed4All()
+
+
+def get_embedding(data: EmbeddingRequest) -> EmbeddingResponse:
+    """
+    Calculates the embedding for the given input using a specified model.
+
+    Args:
+        data (EmbeddingRequest): An EmbeddingRequest object containing the input data
+        and model name.
+
+    Returns:
+        EmbeddingResponse: An EmbeddingResponse object encapsulating the calculated embedding,
+        usage info, and the model name.
+    """
+    embedding = embedder.embed(data.input)
+    return EmbeddingResponse(
+        data=[Embedding(embedding=embedding)], usage=EmbeddingUsage(), model=data.model
+    )
+
+
+@router.post("/", response_model=EmbeddingResponse)
+def embeddings(data: EmbeddingRequest):
+    """
+    Creates a GPT4All embedding
+    """
+    return get_embedding(data)

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -0,0 +1,40 @@
+import logging
+from typing import Dict, List
+
+from api_v1.settings import settings
+from fastapi import APIRouter, Depends, Response, Security, status
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+class ListEnginesResponse(BaseModel):
+    data: List[Dict] = Field(..., description="All available models.")
+
+
+class EngineResponse(BaseModel):
+    data: List[Dict] = Field(..., description="All available models.")
+
+
+router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
+
+
+@router.get("/", response_model=ListEnginesResponse)
+async def list_engines():
+    '''
+    List all available GPT4All models from
+    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json
+    '''
+    raise NotImplementedError()
+    return ListEnginesResponse(data=[])
+
+
+@router.get("/{engine_id}", response_model=EngineResponse)
+async def retrieve_engine(engine_id: str):
+    ''' '''
+
+    raise NotImplementedError()
+    return EngineResponse()

gpt4all-api/gpt4all_api/app/api_v1/routes/health.py

@@ -0,0 +1,13 @@
+import logging
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+
+log = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/health", tags=["Health"])
+
+
+@router.get('/', response_class=JSONResponse)
+async def health_check():
+    """Runs a health check on this instance of the API."""
+    return JSONResponse({'status': 'ok'}, headers={'Access-Control-Allow-Origin': '*'})

gpt4all-api/gpt4all_api/app/api_v1/settings.py

@@ -0,0 +1,19 @@
+from pydantic import BaseSettings
+
+
+class Settings(BaseSettings):
+    app_environment = 'dev'
+    model: str = 'ggml-mpt-7b-chat.bin'
+    gpt4all_path: str = '/models'
+    inference_mode: str = "cpu"
+    hf_inference_server_host: str = "http://gpt4all_gpu:80/generate"
+    sentry_dns: str = None
+
+    temp: float = 0.18
+    top_p: float = 1.0
+    top_k: int = 50
+    repeat_penalty: float = 1.18
+
+
+
+settings = Settings()

gpt4all-api/gpt4all_api/app/docs.py

@@ -0,0 +1,3 @@
+desc = 'GPT4All API'
+
+endpoint_paths = {'health': '/health'}

gpt4all-api/gpt4all_api/app/main.py

@@ -0,0 +1,84 @@
+import logging
+import os
+
+import docs
+from api_v1 import events
+from api_v1.api import router as v1_router
+from api_v1.settings import settings
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.logger import logger as fastapi_logger
+from starlette.middleware.cors import CORSMiddleware
+
+logger = logging.getLogger(__name__)
+
+app = FastAPI(title='GPT4All API', description=docs.desc)
+
+# CORS Configuration (in-case you want to deploy)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["GET", "POST", "OPTIONS"],
+    allow_headers=["*"],
+)
+
+logger.info('Adding v1 endpoints..')
+
+# add v1
+app.include_router(v1_router, prefix='/v1')
+app.add_event_handler('startup', events.startup_event_handler(app))
+app.add_exception_handler(HTTPException, events.on_http_error)
+
+
+@app.on_event("startup")
+async def startup():
+    global model
+    if settings.inference_mode == "cpu":
+        logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
+        from gpt4all import GPT4All
+
+        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
+
+    else:
+        # is it possible to do this once the server is up?
+        ## TODO block until HF inference server is up.
+        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
+
+
+
+@app.on_event("shutdown")
+async def shutdown():
+    logger.info("Shutting down API")
+
+
+if settings.sentry_dns is not None:
+    import sentry_sdk
+
+    def traces_sampler(sampling_context):
+        if 'health' in sampling_context['transaction_context']['name']:
+            return False
+
+    sentry_sdk.init(
+        dsn=settings.sentry_dns, traces_sample_rate=0.1, traces_sampler=traces_sampler, send_default_pii=False
+    )
+
+# This is needed to get logs to show up in the app
+if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
+    gunicorn_error_logger = logging.getLogger("gunicorn.error")
+    gunicorn_logger = logging.getLogger("gunicorn")
+
+    root_logger = logging.getLogger()
+    fastapi_logger.setLevel(gunicorn_logger.level)
+    fastapi_logger.handlers = gunicorn_error_logger.handlers
+    root_logger.setLevel(gunicorn_logger.level)
+
+    uvicorn_logger = logging.getLogger("uvicorn.access")
+    uvicorn_logger.handlers = gunicorn_error_logger.handlers
+else:
+    # https://github.com/tiangolo/fastapi/issues/2019
+    LOG_FORMAT2 = (
+        "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
+    )
+    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -0,0 +1,59 @@
+"""
+Use the OpenAI python API to test gpt4all models.
+"""
+from typing import List, get_args
+
+import openai
+
+openai.api_base = "http://localhost:4891/v1"
+
+openai.api_key = "not needed for a local LLM"
+
+
+def test_completion():
+    model = "ggml-mpt-7b-chat.bin"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Completion.create(
+        model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
+    )
+    assert len(response['choices'][0]['text']) > len(prompt)
+
+def test_streaming_completion():
+    model = "ggml-mpt-7b-chat.bin"
+    prompt = "Who is Michael Jordan?"
+    tokens = []
+    for resp in openai.Completion.create(
+            model=model,
+            prompt=prompt,
+            max_tokens=50,
+            temperature=0.28,
+            top_p=0.95,
+            n=1,
+            echo=True,
+            stream=True):
+        tokens.append(resp.choices[0].text)
+
+    assert (len(tokens) > 0)
+    assert (len("".join(tokens)) > len(prompt))
+
+
+def test_batched_completion():
+    model = "ggml-mpt-7b-chat.bin"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Completion.create(
+        model=model, prompt=[prompt] * 3, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
+    )
+    assert len(response['choices'][0]['text']) > len(prompt)
+    assert len(response['choices']) == 3
+
+
+def test_embedding():
+    model = "ggml-all-MiniLM-L6-v2-f16.bin"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Embedding.create(model=model, input=prompt)
+    output = response["data"][0]["embedding"]
+    args = get_args(List[float])
+
+    assert response["model"] == model
+    assert isinstance(output, list)
+    assert all(isinstance(x, args) for x in output)

gpt4all-api/gpt4all_api/requirements.txt

@@ -0,0 +1,12 @@
+aiohttp>=3.6.2
+aiofiles
+pydantic>=1.4.0,<2.0.0
+requests>=2.24.0
+ujson>=2.0.2
+fastapi>=0.95.0
+Jinja2>=3.0
+gpt4all>=1.0.0
+pytest
+openai
+black
+isort

gpt4all-api/makefile

@@ -0,0 +1,46 @@
+ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+APP_NAME:=gpt4all_api
+PYTHON:=python3.8
+SHELL := /bin/bash
+
+all: dependencies
+
+fresh: clean dependencies
+
+testenv: clean_testenv test_build
+	docker compose -f docker-compose.yaml up --build
+
+testenv_gpu: clean_testenv test_build
+	docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up --build
+
+testenv_d: clean_testenv test_build
+	docker compose up --build -d
+
+test:
+	docker compose exec $(APP_NAME) pytest -svv --disable-warnings -p no:cacheprovider /app/tests
+
+test_build:
+    DOCKER_BUILDKIT=1 docker build -t $(APP_NAME) --progress plain -f $(APP_NAME)/Dockerfile.buildkit .
+
+clean_testenv:
+	docker compose down -v
+
+fresh_testenv: clean_testenv testenv
+
+venv:
+	if [ ! -d $(ROOT_DIR)/env ]; then $(PYTHON) -m venv $(ROOT_DIR)/env; fi
+
+dependencies: venv
+	source $(ROOT_DIR)/env/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
+
+clean: clean_testenv
+	# Remove existing environment
+	rm -rf $(ROOT_DIR)/env;
+	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
+
+
+black:
+	source $(ROOT_DIR)/env/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
+
+isort:
+	source $(ROOT_DIR)/env/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)

gpt4all-backend/CMakeLists.txt

@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.16)
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if(APPLE)
   option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
@@ -9,7 +10,9 @@ if(APPLE)
     set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
   else()
     # Build for the host architecture on macOS
-    set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
+    if(NOT CMAKE_OSX_ARCHITECTURES)
+      set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
+    endif()
   endif()
 endif()
 
@@ -17,7 +20,7 @@ endif()
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 
 set(LLMODEL_VERSION_MAJOR 0)
-set(LLMODEL_VERSION_MINOR 2)
+set(LLMODEL_VERSION_MINOR 4)
 set(LLMODEL_VERSION_PATCH 0)
 set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
 project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
@@ -36,9 +39,16 @@ else()
     message(STATUS "Interprocedural optimization support detected")
 endif()
 
+if(NOT APPLE)
+  set(LLAMA_KOMPUTE YES)
+endif()
+
 include(llama.cpp.cmake)
 
 set(BUILD_VARIANTS default avxonly)
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    set(BUILD_VARIANTS ${BUILD_VARIANTS} metal)
+endif()
 
 set(CMAKE_VERBOSE_MAKEFILE ON)
 
@@ -54,10 +64,15 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     set(LLAMA_F16C ${GPT4ALL_ALLOW_NON_AVX})
     set(LLAMA_FMA  ${GPT4ALL_ALLOW_NON_AVX})
 
+    if (BUILD_VARIANT STREQUAL metal)
+        set(LLAMA_METAL YES)
+    else()
+        set(LLAMA_METAL NO)
+    endif()
+
     # Include GGML
+    set(LLAMA_K_QUANTS YES)
     include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
-    include_ggml(llama.cpp-230511 -230511-${BUILD_VARIANT} ON)
-    include_ggml(llama.cpp-230519 -230519-${BUILD_VARIANT} ON)
 
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
@@ -65,13 +80,14 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         message(STATUS "Configuring model implementation target ${TARGET_NAME}")
         # Link to ggml/llama
         target_link_libraries(${TARGET_NAME}
-            PUBLIC ${BASE_LIB}-${BUILD_VARIANT})
+            PRIVATE ${BASE_LIB}-${BUILD_VARIANT})
         # Let it know about its build variant
         target_compile_definitions(${TARGET_NAME}
             PRIVATE GGML_BUILD_VARIANT="${BUILD_VARIANT}")
         # Enable IPO if possible
-        set_property(TARGET ${TARGET_NAME}
-                     PROPERTY INTERPROCEDURAL_OPTIMIZATION ${IPO_SUPPORTED})
+# FIXME: Doesn't work with msvc reliably. See https://github.com/nomic-ai/gpt4all/issues/841
+#        set_property(TARGET ${TARGET_NAME}
+#                     PROPERTY INTERPROCEDURAL_OPTIMIZATION ${IPO_SUPPORTED})
     endfunction()
 
     # Add each individual implementations
@@ -81,25 +97,36 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    add_library(llamamodel-230519-${BUILD_VARIANT} SHARED
-        llamamodel.cpp llmodel_shared.cpp)
-    target_compile_definitions(llamamodel-230519-${BUILD_VARIANT} PRIVATE
-        LLAMA_VERSIONS===2 LLAMA_DATE=230519)
-    prepare_target(llamamodel-230519 llama-230519)
+    add_library(replit-mainline-${BUILD_VARIANT} SHARED
+    replit.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+    target_compile_definitions(replit-mainline-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+    prepare_target(replit-mainline llama-mainline)
 
-    add_library(llamamodel-230511-${BUILD_VARIANT} SHARED
-        llamamodel.cpp llmodel_shared.cpp)
-    target_compile_definitions(llamamodel-230511-${BUILD_VARIANT} PRIVATE
-        LLAMA_VERSIONS=<=1 LLAMA_DATE=230511)
-    prepare_target(llamamodel-230511 llama-230511)
+    if (NOT LLAMA_METAL)
+# FIXME: These need to be forward ported to latest ggml
+#        add_library(gptj-${BUILD_VARIANT} SHARED
+#            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+#        prepare_target(gptj ggml-230511)
 
-    add_library(gptj-${BUILD_VARIANT} SHARED
-        gptj.cpp utils.h utils.cpp llmodel_shared.cpp)
-    prepare_target(gptj ggml-230511)
+        add_library(falcon-${BUILD_VARIANT} SHARED
+            falcon.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(falcon-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+        prepare_target(falcon llama-mainline)
+# FIXME: These need to be forward ported to latest ggml
+#        add_library(mpt-${BUILD_VARIANT} SHARED
+#            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+#        prepare_target(mpt ggml-230511)
 
-    add_library(mpt-${BUILD_VARIANT} SHARED
-        mpt.cpp utils.h utils.cpp llmodel_shared.cpp)
-    prepare_target(mpt ggml-230511)
+        add_library(bert-${BUILD_VARIANT} SHARED
+            bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+        prepare_target(bert llama-mainline)
+
+        add_library(starcoder-${BUILD_VARIANT} SHARED
+            starcoder.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(starcoder-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+        prepare_target(starcoder llama-mainline)
+    endif()
 endforeach()
 
 add_library(llmodel

gpt4all-backend/bert_impl.h

@@ -0,0 +1,44 @@
+#ifndef BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of bert.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef BERT_H
+#define BERT_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <memory>
+#include "llmodel.h"
+
+struct BertPrivate;
+class Bert : public LLModel {
+public:
+    Bert();
+    ~Bert();
+
+    bool supportsEmbedding() const override { return true; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+    std::vector<float> embedding(const std::string &text) override;
+
+private:
+    std::unique_ptr<BertPrivate> d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // BERT_H

gpt4all-backend/dlhandle.h

@@ -18,7 +18,7 @@ public:
     };
 
     Dlhandle() : chandle(nullptr) {}
-    Dlhandle(const std::string& fpath, int flags = RTLD_LAZY) {
+    Dlhandle(const std::string& fpath, int flags = RTLD_LAZY | RTLD_LOCAL) {
         chandle = dlopen(fpath.c_str(), flags);
         if (!chandle) {
             throw Exception("dlopen(\""+fpath+"\"): "+dlerror());
@@ -75,7 +75,7 @@ public:
 
     Dlhandle() : chandle(nullptr) {}
     Dlhandle(const std::string& fpath) {
-        chandle = LoadLibraryA(fpath.c_str());
+        chandle = LoadLibraryExA(fpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
         if (!chandle) {
             throw Exception("dlopen(\""+fpath+"\"): Error");
         }

gpt4all-backend/falcon.cpp

@@ -0,0 +1,985 @@
+#include "ggml.h"
+#define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#include "falcon_impl.h"
+#include "llama.h"
+#include "llama-util.h"
+#include "utils.h"
+#include "llmodel_shared.h"
+
+#include <cassert>
+#include <cinttypes>
+#include <iostream>
+#include <sstream>
+
+namespace {
+const char *modelType_ = "Falcon";
+}
+
+// commented out 40B support as it presently would require forking ggml/llama.cpp
+// can re-add once mainline ggml supports it
+
+#define FALCON_MAGIC 0x67676a74
+
+// default hparams (Falcon 7B)
+struct falcon_hparams {
+    int32_t n_vocab = 65024;
+    int32_t n_embd  = 4544;
+    int32_t n_head  = 71;
+    int32_t n_head_kv = 1;
+    int32_t n_layer = 32;
+    int32_t falcon_version = 7; // 7 for Falcon-7B, 40 for Falcon-40B
+    int32_t ftype   = 1;
+    int32_t n_ctx   = 2048;
+};
+
+struct falcon_layer {
+    // normalization
+    struct ggml_tensor* input_layernorm;
+    struct ggml_tensor* input_layernorm_b;
+    //struct ggml_tensor* attention_norm;    // Falcon-40B only
+    //struct ggml_tensor* attention_norm_b;  // Falcon-40B only
+
+    // attention
+    struct ggml_tensor* query_key_value;
+    struct ggml_tensor* wo;
+
+    // ff
+    struct ggml_tensor* ffn_up;
+    struct ggml_tensor* ffn_down;
+};
+
+struct falcon_model {
+    falcon_hparams hparams;
+
+    struct ggml_tensor* tok_embeddings;
+    struct ggml_tensor* output_norm;
+    struct ggml_tensor* output_norm_b;
+    struct ggml_tensor* lm_head;
+
+    std::vector<falcon_layer> layers;
+
+    // key + value memory
+    llm_kv_cache kv_self;
+
+    struct ggml_context* ctx;
+    std::map<std::string, struct ggml_tensor*> tensors;
+
+    llm_buffer eval_buf;
+    llm_buffer work_buf;
+    llm_buffer scr0_buf;
+    llm_buffer scr1_buf;
+};
+
+static bool kv_cache_init(
+        const struct falcon_hparams & hparams,
+              struct llm_kv_cache & cache,
+                         ggml_type   wtype,
+                               int   n_ctx) {
+    const int n_embd  = hparams.n_embd;
+    const int dim_head  = n_embd / hparams.n_head;
+    const int dim_kv = dim_head * hparams.n_head_kv;
+    const int n_layer = hparams.n_layer;
+
+    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
+    const int64_t n_elements = dim_kv * n_mem;
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+    struct ggml_init_params params;
+    params.mem_size   = cache.buf.size;
+    params.mem_buffer = cache.buf.addr;
+    params.no_alloc   = false;
+
+    cache.ctx = ggml_init(params);
+    if (!cache.ctx) {
+        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
+        return false;
+    }
+
+    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+    return true;
+}
+
+// load the model's weights from a file
+bool falcon_model_load(const std::string & fname, falcon_model & model, gpt_vocab & vocab, size_t *mem_req) {
+    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+    if (mem_req) {
+        *mem_req = 0;
+    }
+
+    auto fin = std::ifstream(fname, std::ios::binary);
+    if (!fin) {
+        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
+        return false;
+    }
+
+    // verify magic
+    {
+        uint32_t magic;
+        fin.read((char *) &magic, sizeof(magic));
+        if (magic != FALCON_MAGIC) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
+            return false;
+        }
+    }
+
+    uint32_t format_version;
+    fin.read((char *) &format_version, sizeof(format_version));
+
+    // load hparams
+    {
+        auto & hparams = model.hparams;
+
+        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
+        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
+        fin.read((char *) &hparams.n_head_kv, sizeof(hparams.n_head_kv));
+        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
+        fin.read((char *) &hparams.falcon_version, sizeof(hparams.falcon_version));
+        fin.read((char *) &hparams.ftype,   sizeof(hparams.ftype));
+
+        if (hparams.falcon_version != 7) { // && hparams.falcon_version != 40) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad Falcon version: %d)\n", __func__, fname.c_str(), hparams.falcon_version);
+            return false;
+        }
+
+        const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
+
+        printf("%s: n_vocab   = %d\n", __func__, hparams.n_vocab);
+        printf("%s: n_embd    = %d\n", __func__, hparams.n_embd);
+        printf("%s: n_head    = %d\n", __func__, hparams.n_head);
+        printf("%s: n_head_kv = %d\n", __func__, hparams.n_head_kv);
+        printf("%s: n_layer   = %d\n", __func__, hparams.n_layer);
+        printf("%s: ftype     = %d\n", __func__, hparams.ftype);
+        printf("%s: qntvr     = %d\n", __func__, qntvr);
+
+        hparams.ftype %= GGML_QNT_VERSION_FACTOR;
+    }
+
+    // load vocab
+    {
+        const int32_t n_vocab = model.hparams.n_vocab;
+
+        std::string word;
+        std::vector<char> buf(128);
+
+        for (int i = 0; i < n_vocab; i++) {
+            uint32_t len;
+            fin.read((char *) &len, sizeof(len));
+
+            buf.resize(len);
+            fin.read((char *) buf.data(), len);
+            word.assign(buf.data(), len);
+
+            uint32_t dummy;
+            fin.read((char *) &dummy, sizeof(dummy));
+
+            vocab.token_to_id[word] = i;
+            vocab.id_to_token[i] = word;
+        }
+    }
+
+    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
+    // in order to save memory and also to speed up the computation
+    ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
+    if (wtype == GGML_TYPE_COUNT) {
+        fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
+                __func__, fname.c_str(), model.hparams.ftype);
+        return false;
+    }
+
+    auto & ctx = model.ctx;
+
+    size_t ctx_size = 0;
+
+    {
+        const auto& hparams = model.hparams;
+
+        const int n_embd = hparams.n_embd;
+        const int n_head = hparams.n_head;
+        const int n_head_kv = hparams.n_head_kv;
+        const int n_layer = hparams.n_layer;
+        const int n_ctx = hparams.n_ctx;
+        const int n_ff = 4 * model.hparams.n_embd;
+        const int n_vocab = hparams.n_vocab;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // tok_embeddings
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm_b
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // lm_head
+
+        // if (hparams.version == 40) { // Falcon-40B
+        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm
+        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm_b
+        // }
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm_b
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * (n_head_kv * 2 + n_head) * head_dim);  // query_key_value
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_embd);  // wo
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_ff);  // ffn_up
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_ff * n_embd);  // ffn_down
+ 
+        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
+    }
+
+    if (mem_req) {
+        const int n_embd  = model.hparams.n_embd;
+        const int dim_head  = n_embd / model.hparams.n_head;
+        const int dim_kv = dim_head * model.hparams.n_head_kv;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = dim_kv * n_mem;
+        size_t kv_cache_size = 2u*n_elements*ggml_type_size(wtype) + 2_MiB;
+        *mem_req = ctx_size + kv_cache_size;
+        return false;
+    }
+
+    // create the ggml context
+    {
+        struct ggml_init_params params = {
+            .mem_size   = ctx_size,
+            .mem_buffer = NULL,
+            .no_alloc   = false,
+        };
+
+        model.ctx = ggml_init(params);
+        if (!model.ctx) {
+            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
+            return false;
+        }
+    }
+
+    // prepare memory for the weights
+    {
+        const auto& hparams = model.hparams;
+
+        const int n_embd = hparams.n_embd;
+        const int n_head = hparams.n_head;
+        const int n_head_kv = hparams.n_head_kv;
+        const int n_layer = hparams.n_layer;
+        const int n_ff = 4 * model.hparams.n_embd;
+        const int n_vocab = hparams.n_vocab;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        model.layers.resize(n_layer);
+
+        model.tok_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
+
+        model.output_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.output_norm_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
+
+        // map by name
+        model.tensors["transformer.word_embeddings.weight"] =
+            model.tok_embeddings;
+
+        model.tensors["transformer.ln_f.weight"] = model.output_norm;
+        model.tensors["transformer.ln_f.bias"] = model.output_norm_b;
+        model.tensors["lm_head.weight"] = model.lm_head;
+
+        for (int i = 0; i < n_layer; ++i) {
+            auto& layer = model.layers[i];
+
+            layer.input_layernorm =
+                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.input_layernorm_b =
+                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+            // if (hparams.version == 40) { // for Falcon-40B only
+            //     layer.attention_norm =
+            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            //     layer.attention_norm_b =
+            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            // }
+
+            // query_key_value shape for config.multi_query == True:
+            layer.query_key_value = ggml_new_tensor_2d(
+                ctx, wtype, n_embd, (n_head_kv * 2 + n_head) * head_dim);
+            layer.wo = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
+
+            layer.ffn_up = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);
+            layer.ffn_down = ggml_new_tensor_2d(ctx, wtype, n_ff, n_embd);
+
+            // map by name
+            // if (hparams.version == 40) {
+            //     // Falcon-40B:
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_mlp.weight"] = layer.input_layernorm;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_mlp.bias"] = layer.input_layernorm_b;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_attn.weight"] = layer.attention_norm;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_attn.bias"] = layer.attention_norm_b;
+            // } else {
+            // Falcon-7B:
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".input_layernorm.weight"] = layer.input_layernorm;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".input_layernorm.bias"] = layer.input_layernorm_b;
+            //}
+
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".self_attention.query_key_value.weight"] =
+                layer.query_key_value;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".self_attention.dense.weight"] = layer.wo;
+
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".mlp.dense_h_to_4h.weight"] = layer.ffn_up;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".mlp.dense_4h_to_h.weight"] = layer.ffn_down;
+        }
+    }
+
+    // key + value memory
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_layer = hparams.n_layer;
+        const int n_ctx   = hparams.n_ctx;
+        const int n_head_kv = hparams.n_head_kv;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        const int64_t n_mem      = n_layer*n_ctx;
+        const int64_t n_elements = head_dim*n_mem;
+
+        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F32, model.hparams.n_ctx)) {
+            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
+            ggml_free(ctx);
+            return false;
+        }
+        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
+
+        printf("%s: memory_size = %8.2f MB, n_mem = %" PRId64 "\n", __func__, memory_size/1024.0/1024.0, n_mem);
+    }
+
+    // load weights
+    {
+        int n_tensors = 0;
+        size_t total_size = 0;
+
+        printf("%s: ", __func__);
+
+        while (true) {
+            int32_t n_dims;
+            int32_t length;
+            int32_t ttype;
+
+            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
+            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
+
+            if (fin.eof()) {
+                break;
+            }
+
+            int32_t nelements = 1;
+            int32_t ne[2] = { 1, 1 };
+            for (int i = 0; i < n_dims; ++i) {
+                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+                nelements *= ne[i];
+            }
+
+            std::string name(length, 0);
+            fin.read(&name[0], length);
+            fin.seekg(-static_cast<ptrdiff_t>(fin.tellg()) & 31, std::ios_base::cur);
+
+            if (model.tensors.find(name.data()) == model.tensors.end()) {
+                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
+                return false;
+            }
+
+            auto tensor = model.tensors[name.data()];
+            if (ggml_nelements(tensor) != nelements) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
+                return false;
+            }
+
+            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%5d, %5d], expected [%5d, %5d]\n",
+                        __func__, name.data(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
+                return false;
+            }
+
+            // for debugging
+            if (0) {
+                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
+            }
+
+            const size_t bpe = ggml_type_size(ggml_type(ttype));
+
+            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
+                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
+                return false;
+            }
+
+            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
+
+            total_size += ggml_nbytes(tensor);
+            if (++n_tensors % 8 == 0) {
+                printf(".");
+                fflush(stdout);
+            }
+        }
+
+        printf(" done\n");
+
+        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
+    }
+
+    fin.close();
+
+    model.eval_buf.resize(1280u * 1024 * 1024);
+    model.scr0_buf.resize(256u * 1024 * 1024);
+    model.scr1_buf.resize(256u * 1024 * 1024);
+    return true;
+}
+
+
+// evaluate the transformer
+//
+//   - model:     the model
+//   - n_threads: number of threads to use
+//   - n_past:    the context size so far
+//   - embd_inp:  the embeddings of the tokens in the context
+//   - embd_w:    the predicted logits for the next token
+//
+bool falcon_eval(
+        falcon_model & model,
+        const int n_threads,
+        const int n_past,
+        const std::vector<gpt_vocab::id> & embd_inp,
+              std::vector<float>         & embd_w,
+              size_t                     & mem_per_token) {
+    const int N = embd_inp.size();
+
+    const auto & hparams = model.hparams;
+
+    const int n_embd  = hparams.n_embd;
+    const int n_layer = hparams.n_layer;
+    const int n_ctx   = hparams.n_ctx;
+    const int n_head  = hparams.n_head;
+    const int n_head_kv = hparams.n_head_kv;
+    const int n_vocab = hparams.n_vocab;
+    const int version = hparams.falcon_version;
+    const size_t head_dim = n_embd / n_head;
+
+   struct ggml_init_params eval_ctx_params = {
+        .mem_size = model.eval_buf.size,
+        .mem_buffer = model.eval_buf.addr,
+        .no_alloc = false,
+    };
+
+    struct ggml_context * ctx0 = ggml_init(eval_ctx_params);
+    struct ggml_cgraph gf = {};
+
+    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
+
+    // wte
+    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.tok_embeddings, embd);
+    struct ggml_tensor* repeat_dummy = ggml_new_tensor_3d(ctx0, inpL->type, head_dim, N + n_past, n_head);
+
+    ggml_type wtype = GGML_TYPE_F32;
+    const int sizeof_wtype = ggml_type_sizef(wtype);
+
+    for (int il = 0; il < n_layer; ++il) {
+        struct ggml_tensor * cur;
+        struct ggml_tensor * layernorm_output;
+
+        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
+        // self-attention
+        {
+            layernorm_output = ggml_norm(ctx0, inpL);
+
+            layernorm_output = ggml_add(ctx0,
+                    ggml_mul(ctx0,
+                        ggml_repeat(ctx0, model.layers[il].input_layernorm, layernorm_output),
+                        layernorm_output),
+                    ggml_repeat(ctx0, model.layers[il].input_layernorm_b, layernorm_output));
+
+            // if (version == 40) { // Falcon-40B only
+            //     cur = ggml_norm(ctx0, inpL);
+
+            //     cur = ggml_add(ctx0,
+            //             ggml_mul(ctx0,
+            //                 ggml_repeat(ctx0, model.layers[il].attention_norm, cur),
+            //                 cur),
+            //             ggml_repeat(ctx0, model.layers[il].attention_norm_b, cur));
+            // }
+            // else {
+            cur = layernorm_output;
+            // }
+
+            // compute QKV
+
+            cur = ggml_mul_mat(ctx0, model.layers[il].query_key_value, cur);
+
+            // Note that the strides for Kcur, Vcur are set up so that the
+            // resulting views are misaligned with the tensor's storage
+            // (by applying the K/V offset we shift the tensor's original
+            // view to stick out behind the viewed QKV tensor's allocated
+            // memory, so to say). This is ok because no actual accesses
+            // happen to that out-of-range memory, but it can require some
+            // trickery when trying to accurately dump these views for
+            // debugging.
+
+            struct ggml_tensor * Qcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                0);
+
+            struct ggml_tensor * Kcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head_kv, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                head_dim * n_head * sizeof_wtype);
+
+            struct ggml_tensor * Vcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head_kv, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                head_dim * (n_head + n_head_kv) * sizeof_wtype);
+
+            // using mode = 2 for neox mode
+            Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, head_dim, 2, n_ctx);
+            Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, head_dim, 2, n_ctx);
+
+            // store key and value to memory
+            {
+                struct ggml_tensor* k = ggml_view_1d(
+                    ctx0, model.kv_self.k, N * n_head_kv * head_dim,
+                    (ggml_element_size(model.kv_self.k) * n_head_kv * head_dim) *
+                        (il * n_ctx + n_past));
+                struct ggml_tensor* v = ggml_view_1d(
+                    ctx0, model.kv_self.v, N * n_head_kv * head_dim,
+                    (ggml_element_size(model.kv_self.v) * n_head_kv * head_dim) *
+                        (il * n_ctx + n_past));
+
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
+            }
+
+            struct ggml_tensor * K = ggml_permute(
+                ctx0,
+                ggml_view_3d(
+                    ctx0,
+                    model.kv_self.k,
+                    head_dim, n_head_kv, n_past + N,
+                    head_dim * sizeof_wtype,
+                    head_dim * n_head_kv * sizeof_wtype,
+                    il * n_ctx * ggml_element_size(model.kv_self.k) * n_head_kv * head_dim),
+                0, 2, 1, 3);
+
+            // K * Q
+
+            // changed from repeat2 back to repeat, will not support 40B!
+            K = ggml_cont(ctx0, ggml_repeat(ctx0, K, repeat_dummy));
+
+            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
+
+            // KQ_scaled = KQ / sqrt(n_embd/n_head)
+            struct ggml_tensor * KQ_scaled =
+                ggml_scale_inplace(ctx0,
+                        KQ,
+                        ggml_new_f32(ctx0, 1.0f/sqrt(float(head_dim)))
+                        );
+
+            // KQ_masked = mask_past(KQ_scaled)
+            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
+
+            // KQ = soft_max(KQ_masked)
+            struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
+
+            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
+            struct ggml_tensor* V = ggml_permute(
+                ctx0,
+                ggml_view_3d(
+                    ctx0,
+                    model.kv_self.v,
+                    head_dim, n_head_kv, n_past + N,
+                    head_dim * sizeof_wtype,
+                    head_dim * n_head_kv * sizeof_wtype,
+                    il * n_ctx * ggml_element_size(model.kv_self.v) * n_head_kv * head_dim),
+                0, 2, 1, 3);
+
+            // changed from repeat2 back to repeat, will not support 40B!
+            V = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_repeat(ctx0, V, repeat_dummy)));
+
+            // KQV = transpose(V) * KQ_soft_max
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
+
+            // KQV_merged = KQV.permute(0, 2, 1, 3)
+            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
+
+            // cur = KQV_merged.contiguous().view(n_embd, N)
+            cur = ggml_cpy(ctx0,
+                    KQV_merged,
+                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+
+            // projection
+            {
+                cur = ggml_mul_mat(ctx0,
+                        model.layers[il].wo,
+                        cur);
+            }
+        }
+
+        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
+
+        struct ggml_tensor* inpFF = layernorm_output;
+        struct ggml_tensor* attn_out = ggml_cpy(
+            ctx0, cur, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+
+        {
+            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_up, inpFF);
+            cur = ggml_gelu(ctx0, cur);
+            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_down, cur);
+        }
+
+        cur = ggml_add(ctx0, cur, attn_out);
+        cur = ggml_add(ctx0, cur, inpL);
+        // input for next layer
+        inpL = cur;
+    }
+
+    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
+    // norm
+    {
+        inpL = ggml_norm(ctx0, inpL);
+
+        // inpL = ln_f_g*inpL + ln_f_b
+        inpL = ggml_add(ctx0,
+                ggml_mul(ctx0,
+                    ggml_repeat(ctx0, model.output_norm, inpL),
+                    inpL),
+                ggml_repeat(ctx0, model.output_norm_b, inpL));
+    }
+
+    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
+
+    // lm_head
+    {
+        inpL = ggml_mul_mat(ctx0, model.lm_head, inpL);
+
+        //inpL = ggml_add(ctx0,
+        //        ggml_repeat(ctx0, model.lmh_b, inpL),
+        //        inpL);
+    }
+
+    // logits -> probs
+    //inpL = ggml_soft_max_inplace(ctx0, inpL);
+
+    // run the computation
+    ggml_build_forward_expand(&gf, inpL);
+    ggml_graph_compute_g4a(model.work_buf, &gf, n_threads);
+  
+
+    //if (n_past%100 == 0) {
+    //    ggml_graph_print   (&gf);
+    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
+    //}
+
+    //embd_w.resize(n_vocab*N);
+    //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
+
+    // return result for just the last token
+    embd_w.resize(n_vocab);
+    memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
+
+    if (mem_per_token == 0) {
+        mem_per_token = ggml_used_mem(ctx0)/N;
+    }
+    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
+
+    ggml_free(ctx0);
+
+    return true;
+}
+
+
+#define MAX_RNG_STATE 64*1024
+size_t falcon_get_state_size(const falcon_model &model) {
+    const size_t s_rng_size        = sizeof(size_t);
+    const size_t s_rng             = MAX_RNG_STATE;
+    const size_t s_kv_size         = sizeof(size_t);
+    const size_t s_kv_ntok         = sizeof(int);
+    const size_t s_kv              = model.kv_self.buf.size;
+    const size_t s_total = (
+        + s_rng_size
+        + s_rng
+        + s_kv_size
+        + s_kv_ntok
+        + s_kv
+    );
+    return s_total;
+}
+
+size_t falcon_copy_state_data(const falcon_model &model, const std::mt19937 &rng, uint8_t *dest)
+{
+    uint8_t * out = dest;
+    // copy rng
+    {
+        std::stringstream rng_ss;
+        rng_ss << rng;
+
+        const size_t rng_size = rng_ss.str().size();
+        char rng_buf[MAX_RNG_STATE];
+
+        memset(&rng_buf[0], 0, MAX_RNG_STATE);
+        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
+
+        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
+        memcpy(out, &rng_buf[0], MAX_RNG_STATE); out += MAX_RNG_STATE;
+    }
+
+    // copy kv cache
+    {
+        const size_t kv_size = model.kv_self.buf.size;
+        const int    kv_ntok = model.kv_self.n;
+
+        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
+        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
+
+        if (kv_size) {
+            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
+        }
+    }
+
+    const size_t written  = out - dest;
+    assert(written == falcon_get_state_size(model));
+    fflush(stdout);
+    return written;
+}
+
+size_t falcon_set_state_data(falcon_model *model, std::mt19937 *rng, const uint8_t *src)
+{
+    const uint8_t * in = src;
+
+    // set rng
+    {
+        size_t rng_size;
+        char   rng_buf[MAX_RNG_STATE];
+
+        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
+        memcpy(&rng_buf[0], in, MAX_RNG_STATE); in += MAX_RNG_STATE;
+
+        std::stringstream rng_ss;
+        rng_ss.str(std::string(&rng_buf[0], rng_size));
+        rng_ss >> *rng;
+
+        assert(rng_ss.fail() == false);
+    }
+
+    // set kv cache
+    {
+        size_t kv_size;
+        int kv_ntok;
+
+        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
+        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
+
+        if (kv_size) {
+            assert(model->kv_self.buf.size == kv_size);
+
+            void * k_data = model->kv_self.k->data; // remember data pointers
+            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
+
+            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
+
+            model->kv_self.k->data = k_data; // restore correct data pointers
+            model->kv_self.v->data = v_data;
+
+        }
+
+        model->kv_self.n = kv_ntok;
+    }
+
+    const size_t nread    = in - src;
+    assert(nread == falcon_get_state_size(*model));
+    fflush(stdout);
+    return nread;
+}
+
+struct FalconPrivate {
+    const std::string modelPath;
+    bool modelLoaded;
+    gpt_vocab vocab;
+    falcon_model *model = nullptr;
+    int64_t n_threads = 0;
+    size_t mem_per_token = 0;
+    std::mt19937 rng;
+};
+
+Falcon::Falcon() : d_ptr(new FalconPrivate) {
+    d_ptr->model = new falcon_model;
+    d_ptr->model->ctx = nullptr;
+    d_ptr->modelLoaded = false;
+}
+
+Falcon::~Falcon() {
+    if(d_ptr->model->ctx) {
+        ggml_free(d_ptr->model->ctx);
+        d_ptr->model->ctx = nullptr;
+    }
+    delete d_ptr->model;
+}
+
+bool Falcon::loadModel(const std::string &modelPath)
+{
+    std::mt19937 rng(time(NULL));
+    d_ptr->rng = rng;
+
+    // load the model
+    if (!falcon_model_load(modelPath, *d_ptr->model, d_ptr->vocab, nullptr)) {
+        std::cerr << "FALCON ERROR: failed to load model from " <<  modelPath;
+        return false;
+    }
+
+    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    d_ptr->modelLoaded = true;
+    fflush(stdout);
+    return true;
+}
+
+bool Falcon::isModelLoaded() const
+{
+    return d_ptr -> modelLoaded;
+}
+
+size_t Falcon::requiredMem(const std::string &modelPath)
+{
+    falcon_model dummy_model;
+    gpt_vocab dummy_vocab;
+    size_t mem_req;
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    falcon_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
+    return mem_req;
+}
+
+size_t Falcon::stateSize() const
+{
+    return falcon_get_state_size(*d_ptr->model);
+}
+
+size_t Falcon::saveState(uint8_t *dest) const
+{
+    return falcon_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
+}
+
+size_t Falcon::restoreState(const uint8_t *src)
+{
+    return falcon_set_state_data(d_ptr->model, &d_ptr->rng, src);
+}
+
+void Falcon::setThreadCount(int32_t n_threads)
+{
+    d_ptr->n_threads = n_threads;
+}
+
+int32_t Falcon::threadCount() const
+{
+    return d_ptr->n_threads;
+}
+
+std::vector<LLModel::Token> Falcon::tokenize(PromptContext &, const std::string &str) const
+{
+    return ::gpt_tokenize(d_ptr->vocab, str);
+}
+
+LLModel::Token Falcon::sampleToken(PromptContext &promptCtx) const
+{
+    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
+    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
+        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
+        n_prev_toks,
+        promptCtx.logits,
+        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
+        promptCtx.repeat_penalty,
+        d_ptr->rng);
+}
+
+std::string Falcon::tokenToString(Token id) const
+{
+    return d_ptr->vocab.id_to_token[id];
+}
+
+bool Falcon::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
+{
+    // determine the required inference memory per token:
+    static bool initialized = false;
+    if (!initialized) {
+        falcon_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
+            d_ptr->mem_per_token);
+        initialized = true;
+    }
+
+    return falcon_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
+}
+
+int32_t Falcon::contextLength() const
+{
+    return d_ptr->model->hparams.n_ctx;
+}
+
+const std::vector<LLModel::Token> &Falcon::endTokens() const
+{
+    static const std::vector<LLModel::Token> out = { 11 };
+    return out;
+}
+
+#if defined(_WIN32)
+#define DLL_EXPORT __declspec(dllexport)
+#else
+#define DLL_EXPORT __attribute__ ((visibility ("default")))
+#endif
+
+extern "C" {
+DLL_EXPORT bool is_g4a_backend_model_implementation() {
+    return true;
+}
+
+DLL_EXPORT const char *get_model_type() {
+    return modelType_;
+}
+
+DLL_EXPORT const char *get_build_variant() {
+    return GGML_BUILD_VARIANT;
+}
+
+DLL_EXPORT bool magic_match(std::istream& f) {
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    uint32_t version = 0;
+    f.read(reinterpret_cast<char*>(&version), sizeof(version));
+    if (magic != FALCON_MAGIC) {
+         return false; 
+    }
+    falcon_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    // we're matching the file format of existing pre-converted models
+    // compatible with ctransformers llama.cpp based format, which also
+    // unfortunately shares its magic number what llama uses, so we now
+    // differentiate by n_vocab
+    // give some wiggle room over the max to allow for finetunes that expand the
+    // vocabulary
+    if (!(hparams.n_vocab >= 65024 && hparams.n_vocab <= 65100)) {
+        return false;
+    }
+    if (hparams.falcon_version != 7) {
+        return false;
+    }
+    return true;
+}
+
+DLL_EXPORT LLModel *construct() {
+    return new Falcon;
+}
+}

gpt4all-backend/falcon_impl.h

@@ -0,0 +1,42 @@
+#ifndef FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of falcon.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef FALCON_H
+#define FALCON_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <memory>
+#include "llmodel.h"
+
+struct FalconPrivate;
+class Falcon : public LLModel {
+public:
+    Falcon();
+    ~Falcon();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    std::unique_ptr<FalconPrivate> d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // Falcon_H

gpt4all-backend/gptj.cpp

@@ -2,8 +2,10 @@
 #include "gptj_impl.h"
 
 #include "utils.h"
+#include "llmodel_shared.h"
 
 #include <cassert>
+#include <cinttypes>
 #include <cmath>
 #include <cstdio>
 #include <cstring>
@@ -30,8 +32,6 @@
 
 namespace {
 const char *modelType_ = "GPT-J";
-
-static const size_t MB = 1024*1024;
 }
 
 // default hparams (GPT-J 6B)
@@ -65,39 +65,6 @@ struct gptj_layer {
     struct ggml_tensor * c_mlp_proj_b;
 };
 
-struct gptj_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-
-    void resize(size_t size) {
-        delete[] addr;
-        addr = new uint8_t[size];
-        this->size = size;
-    }
-
-    ~gptj_buffer() {
-        fflush(stdout);
-        delete[] addr;
-    }
-};
-
-struct gptj_kv_cache {
-    struct ggml_tensor * k;
-    struct ggml_tensor * v;
-
-    struct ggml_context * ctx = NULL;
-
-    gptj_buffer buf;
-
-    int n; // number of tokens currently in the cache
-
-    ~gptj_kv_cache() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
 struct gptj_model {
     gptj_hparams hparams;
 
@@ -113,13 +80,15 @@ struct gptj_model {
     std::vector<gptj_layer> layers;
 
     // key + value memory
-    struct gptj_kv_cache kv_self;
+    struct llm_kv_cache kv_self;
 
     //
     struct ggml_context * ctx;
     std::map<std::string, struct ggml_tensor *> tensors;
 
-    gptj_buffer buf;
+    llm_buffer eval_buf;
+    llm_buffer scr0_buf;
+    llm_buffer scr1_buf;
 
     ~gptj_model() {
         if (ctx) {
@@ -130,7 +99,7 @@ struct gptj_model {
 
 static bool kv_cache_init(
         const struct gptj_hparams & hparams,
-             struct gptj_kv_cache & cache,
+              struct llm_kv_cache & cache,
                          ggml_type   wtype,
                                int   n_ctx) {
     const int n_embd  = hparams.n_embd;
@@ -139,7 +108,7 @@ static bool kv_cache_init(
     const int64_t n_mem      = (int64_t)n_layer*n_ctx;
     const int64_t n_elements = n_embd*n_mem;
 
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB);
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
 
     struct ggml_init_params params;
     params.mem_size   = cache.buf.size;
@@ -160,8 +129,11 @@ static bool kv_cache_init(
 }
 
 // load the model's weights from a stream
-bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & model, gpt_vocab & vocab) {
+bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+    if(mem_req != nullptr) {
+        *mem_req = 0;
+    }
 
     // verify magic
     {
@@ -278,6 +250,19 @@ bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & m
         printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
     }
 
+    if (mem_req != nullptr) {
+        *mem_req += ctx_size;
+        const int n_embd  = model.hparams.n_embd;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = n_embd*n_mem;
+
+        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+        return false;
+    }
+
+
     // create the ggml context
     {
         struct ggml_init_params params = {
@@ -411,7 +396,7 @@ bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & m
             }
 
             if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%lu, %lu], expected [%d, %d]\n",
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
                         __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
                 return false;
             }
@@ -456,6 +441,9 @@ bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & m
         printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
     }
 
+    model.scr0_buf.resize(256u * 1024 * 1024);
+    model.scr1_buf.resize(256u * 1024 * 1024);
+
     return true;
 }
 
@@ -501,25 +489,25 @@ bool gptj_eval(
     const int n_vocab = hparams.n_vocab;
     const int n_rot   = hparams.n_rot;
 
-    const size_t init_buf_size = 1024u*MB;
-    if (!model.buf.addr || model.buf.size < init_buf_size)
-        model.buf.resize(init_buf_size);
+    const size_t init_buf_size = 1024_MiB;
+    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
+        model.eval_buf.resize(init_buf_size);
 
-    if (mem_per_token > 0 && mem_per_token*N > model.buf.size) {
+    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
         const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
-        printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
+        printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.eval_buf.size, buf_size_new);
 
         // reallocate
-        model.buf.resize(buf_size_new);
-        if (model.buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.buf.size);
+        model.eval_buf.resize(buf_size_new);
+        if (model.eval_buf.addr == nullptr) {
+            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
             return false;
         }
     }
 
     struct ggml_init_params params = {
-        .mem_size   = model.buf.size,
-        .mem_buffer = model.buf.addr,
+        .mem_size   = model.eval_buf.size,
+        .mem_buffer = model.eval_buf.addr,
         .no_alloc = false
     };
 
@@ -535,7 +523,7 @@ bool gptj_eval(
 
     for (int il = 0; il < n_layer; ++il) {
         struct ggml_tensor * cur;
-
+        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
         // norm
         {
             cur = ggml_norm(ctx0, inpL);
@@ -630,6 +618,7 @@ bool gptj_eval(
 
         struct ggml_tensor * inpFF = cur;
 
+        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
         // feed-forward network
         // this is independent of the self-attention result, so it could be done in parallel to the self-attention
         {
@@ -663,6 +652,8 @@ bool gptj_eval(
         inpL = ggml_add(ctx0, cur, inpL);
     }
 
+    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
     // norm
     {
         inpL = ggml_norm(ctx0, inpL);
@@ -675,6 +666,8 @@ bool gptj_eval(
                 ggml_repeat(ctx0, model.ln_f_b, inpL));
     }
 
+    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
+
     // lm_head
     {
         inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL);
@@ -835,9 +828,19 @@ struct GPTJPrivate {
 GPTJ::GPTJ()
     : d_ptr(new GPTJPrivate) {
     d_ptr->model = new gptj_model;
+    d_ptr->model->ctx = nullptr;
     d_ptr->modelLoaded = false;
 }
 
+size_t GPTJ::requiredMem(const std::string &modelPath) {
+    gptj_model dummy_model;
+    gpt_vocab dummy_vocab;
+    size_t mem_req;
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    gptj_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
+    return mem_req;
+}
+
 bool GPTJ::loadModel(const std::string &modelPath) {
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
@@ -907,7 +910,7 @@ LLModel::Token GPTJ::sampleToken(PromptContext &promptCtx) const
         d_ptr->rng);
 }
 
-std::string_view GPTJ::tokenToString(Token id) const
+std::string GPTJ::tokenToString(Token id) const
 {
     return d_ptr->vocab.id_to_token[id];
 }
@@ -958,6 +961,11 @@ DLL_EXPORT const char *get_build_variant() {
 DLL_EXPORT bool magic_match(std::istream& f) {
     uint32_t magic = 0;
     f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    gptj_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    if (!(hparams.n_vocab >= 50300 && hparams.n_vocab <= 50400)) {
+        return false; // not a gptj.
+    }
     return magic == 0x67676d6c;
 }
 

gpt4all-backend/gptj_impl.h

@@ -15,8 +15,11 @@ public:
     GPTJ();
     ~GPTJ();
 
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
@@ -29,7 +32,7 @@ private:
 protected:
     std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
     Token sampleToken(PromptContext &ctx) const override;
-    std::string_view tokenToString(Token) const override;
+    std::string tokenToString(Token) const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
     const std::vector<Token>& endTokens() const override;

gpt4all-backend/llama.cpp.cmake

@@ -1,3 +1,11 @@
+#
+# Copyright (c) 2023 Nomic, Inc. All rights reserved.
+#
+# This software is licensed under the terms of the Software for Open Models License (SOM),
+# version 1.0, as detailed in the LICENSE_SOM.txt file. A copy of this license should accompany
+# this software. Except as expressly granted in the SOM license, all rights are reserved by Nomic, Inc.
+#
+
 cmake_minimum_required(VERSION 3.12) # Don't bump this version for no reason
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@@ -34,6 +42,7 @@ endif()
 #
 # Option list
 #
+# some of the options here are commented out so they can be set "dynamically" before calling include_ggml()
 
 # general
 option(LLAMA_STATIC                 "llama: static link libraries"                          OFF)
@@ -65,8 +74,13 @@ option(LLAMA_SANITIZE_UNDEFINED     "llama: enable undefined sanitizer"
 # 3rd party libs
 option(LLAMA_ACCELERATE             "llama: enable Accelerate framework"                    ON)
 option(LLAMA_OPENBLAS               "llama: use OpenBLAS"                                   OFF)
-option(LLAMA_CUBLAS                 "llama: use cuBLAS"                                     OFF)
-option(LLAMA_CLBLAST                "llama: use CLBlast"                                    OFF)
+#option(LLAMA_CUBLAS                 "llama: use cuBLAS"                                     OFF)
+#option(LLAMA_CLBLAST                "llama: use CLBlast"                                    OFF)
+#option(LLAMA_METAL                  "llama: use Metal"                                      OFF)
+#option(LLAMA_K_QUANTS               "llama: use k-quants"                                   ON)
+set(LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor")
+set(LLAMA_CUDA_DMMV_X "32" CACHE STRING "llama: x stride for dmmv CUDA kernels")
+set(LLAMA_CUDA_DMMV_Y "1" CACHE STRING  "llama: y block size for dmmv CUDA kernels")
 
 #
 # Compile flags
@@ -139,6 +153,130 @@ if (LLAMA_OPENBLAS)
     endif()
 endif()
 
+if (LLAMA_KOMPUTE)
+    add_compile_definitions(VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
+    find_package(Vulkan COMPONENTS glslc REQUIRED)
+    find_program(glslc_executable NAMES glslc HINTS Vulkan::glslc)
+    if (NOT glslc_executable)
+        message(FATAL_ERROR "glslc not found")
+    endif()
+
+    set(LLAMA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp-mainline)
+
+    function(compile_shader)
+      set(options)
+      set(oneValueArgs)
+      set(multiValueArgs SOURCES)
+      cmake_parse_arguments(compile_shader "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+      foreach(source ${compile_shader_SOURCES})
+        get_filename_component(OP_FILE ${source} NAME)
+        set(spv_file ${CMAKE_CURRENT_BINARY_DIR}/${OP_FILE}.spv)
+        add_custom_command(
+            OUTPUT ${spv_file}
+            DEPENDS ${LLAMA_DIR}/${source}
+            COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
+            COMMENT "Compiling ${source} to ${source}.spv"
+        )
+
+        get_filename_component(RAW_FILE_NAME ${spv_file} NAME)
+        set(FILE_NAME "shader${RAW_FILE_NAME}")
+        string(REPLACE ".comp.spv" ".h" HEADER_FILE ${FILE_NAME})
+        string(TOUPPER ${HEADER_FILE} HEADER_FILE_DEFINE)
+        string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}")
+        set(OUTPUT_HEADER_FILE "${HEADER_FILE}")
+        message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}")
+        add_custom_command(
+          OUTPUT ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
+          COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+          DEPENDS ${spv_file} xxd
+          COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
+        )
+      endforeach()
+    endfunction()
+
+    if (EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
+        message(STATUS "Kompute found")
+        add_subdirectory(${LLAMA_DIR}/kompute)
+
+        # Compile our shaders
+        compile_shader(SOURCES
+          kompute/op_scale.comp
+          kompute/op_add.comp
+          kompute/op_addrow.comp
+          kompute/op_mul.comp
+          kompute/op_mulrow.comp
+          kompute/op_silu.comp
+          kompute/op_relu.comp
+          kompute/op_gelu.comp
+          kompute/op_softmax.comp
+          kompute/op_norm.comp
+          kompute/op_rmsnorm.comp
+          kompute/op_diagmask.comp
+          kompute/op_mul_mat_f16.comp
+          kompute/op_mul_mat_q4_0.comp
+          kompute/op_mul_mat_q4_1.comp
+          kompute/op_getrows_f16.comp
+          kompute/op_getrows_q4_0.comp
+          kompute/op_getrows_q4_1.comp
+          kompute/op_rope.comp
+          kompute/op_cpy_f16_f16.comp
+          kompute/op_cpy_f16_f32.comp
+          kompute/op_cpy_f32_f16.comp
+          kompute/op_cpy_f32_f32.comp
+        )
+
+        # Create a custom target for our generated shaders
+        add_custom_target(generated_shaders DEPENDS
+          shaderop_scale.h
+          shaderop_add.h
+          shaderop_addrow.h
+          shaderop_mul.h
+          shaderop_mulrow.h
+          shaderop_silu.h
+          shaderop_relu.h
+          shaderop_gelu.h
+          shaderop_softmax.h
+          shaderop_norm.h
+          shaderop_rmsnorm.h
+          shaderop_diagmask.h
+          shaderop_mul_mat_f16.h
+          shaderop_mul_mat_q4_0.h
+          shaderop_mul_mat_q4_1.h
+          shaderop_getrows_f16.h
+          shaderop_getrows_q4_0.h
+          shaderop_getrows_q4_1.h
+          shaderop_rope.h
+          shaderop_cpy_f16_f16.h
+          shaderop_cpy_f16_f32.h
+          shaderop_cpy_f32_f16.h
+          shaderop_cpy_f32_f32.h
+        )
+
+        # Create a custom command that depends on the generated_shaders
+        add_custom_command(
+            OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp
+            COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp
+            DEPENDS generated_shaders
+            COMMENT "Ensuring shaders are generated before compiling ggml-vulkan.cpp"
+        )
+
+        # Add the stamp to the main sources to ensure dependency tracking
+        set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml-vulkan.cpp ${LLAMA_DIR}/ggml-vulkan.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp)
+        add_compile_definitions(GGML_USE_KOMPUTE)
+        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute)
+        set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CMAKE_BINARY_DIR})
+    else()
+        message(WARNING "Kompute not found")
+    endif()
+endif()
+
 if (LLAMA_ALL_WARNINGS)
     if (NOT MSVC)
         set(c_flags
@@ -210,86 +348,22 @@ endif()
 function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
     message(STATUS "Configuring ggml implementation target llama${SUFFIX} in ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}")
 
-    if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
-        message(STATUS "ARM detected")
-        if (MSVC)
-            # TODO: arm msvc?
-        else()
-            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
-                add_compile_options(-mcpu=native)
-            endif()
-            # TODO: armv6,7,8 version specific flags
-        endif()
-    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$")
-        message(STATUS "x86 detected")
-        if (MSVC)
-            if (LLAMA_AVX512)
-                add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX512>)
-                add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
-                # MSVC has no compile-time flags enabling specific
-                # AVX512 extensions, neither it defines the
-                # macros corresponding to the extensions.
-                # Do it manually.
-                if (LLAMA_AVX512_VBMI)
-                    add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
-                    add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
-                endif()
-                if (LLAMA_AVX512_VNNI)
-                    add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
-                    add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
-                endif()
-            elseif (LLAMA_AVX2)
-                add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX2>)
-                add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
-            elseif (LLAMA_AVX)
-                add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX>)
-                add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
-            endif()
-        else()
-            if (LLAMA_F16C)
-                add_compile_options(-mf16c)
-            endif()
-            if (LLAMA_FMA)
-                add_compile_options(-mfma)
-            endif()
-            if (LLAMA_AVX)
-                add_compile_options(-mavx)
-            endif()
-            if (LLAMA_AVX2)
-                add_compile_options(-mavx2)
-            endif()
-            if (LLAMA_AVX512)
-                add_compile_options(-mavx512f)
-                add_compile_options(-mavx512bw)
-            endif()
-            if (LLAMA_AVX512_VBMI)
-                add_compile_options(-mavx512vbmi)
-            endif()
-            if (LLAMA_AVX512_VNNI)
-                add_compile_options(-mavx512vnni)
-            endif()
-        endif()
-    else()
-        # TODO: support PowerPC
-        message(STATUS "Unknown architecture")
-    endif()
-
     #
     # Build libraries
     #
 
-    if (LLAMA_CUBLAS AND EXISTS ${DIRECTORY}/ggml-cuda.h)
+    set(GGML_CUBLAS_USE NO)
+    if (LLAMA_CUBLAS)
         cmake_minimum_required(VERSION 3.17)
 
         find_package(CUDAToolkit)
         if (CUDAToolkit_FOUND)
+            set(GGML_CUBLAS_USE YES)
             message(STATUS "cuBLAS found")
 
             enable_language(CUDA)
 
-            set(GGML_CUDA_SOURCES ${DIRECTORY}/ggml-cuda.cu ${DIRECTORY}/ggml-cuda.h)
-
-            add_compile_definitions(GGML_USE_CUBLAS)
+            set(GGML_SOURCES_CUDA ${DIRECTORY}/ggml-cuda.cu ${DIRECTORY}/ggml-cuda.h)
 
             if (LLAMA_STATIC)
                 set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static)
@@ -302,14 +376,19 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
         endif()
     endif()
 
-    if (LLAMA_CLBLAST AND EXISTS ${DIRECTORY}/ggml-opencl.h)
+    set(GGML_CLBLAST_USE NO)
+    if (LLAMA_CLBLAST)
         find_package(CLBlast)
         if (CLBlast_FOUND)
+            set(GGML_CLBLAST_USE YES)
             message(STATUS "CLBlast found")
 
-            set(GGML_OPENCL_SOURCES ${DIRECTORY}/ggml-opencl.c ${DIRECTORY}/ggml-opencl.h)
+            set(GGML_OPENCL_SOURCE_FILE ggml-opencl.cpp)
+            if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/${GGML_OPENCL_SOURCE_FILE})
+                set(GGML_OPENCL_SOURCE_FILE ggml-opencl.c)
+            endif()
 
-            add_compile_definitions(GGML_USE_CLBLAST)
+            set(GGML_OPENCL_SOURCES ${DIRECTORY}/${GGML_OPENCL_SOURCE_FILE} ${DIRECTORY}/ggml-opencl.h)
 
             set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} clblast)
         else()
@@ -317,15 +396,55 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
         endif()
     endif()
 
+    set(GGML_SOURCES_QUANT_K )
+    set(GGML_METAL_SOURCES )
+    if (LLAMA_K_QUANTS)
+        set(GGML_SOURCES_QUANT_K
+            ${DIRECTORY}/k_quants.h
+            ${DIRECTORY}/k_quants.c)
+
+        if (LLAMA_METAL)
+            find_library(FOUNDATION_LIBRARY         Foundation              REQUIRED)
+            find_library(METAL_FRAMEWORK            Metal                   REQUIRED)
+            find_library(METALKIT_FRAMEWORK         MetalKit                REQUIRED)
+            find_library(METALPERFORMANCE_FRAMEWORK MetalPerformanceShaders REQUIRED)
+
+            set(GGML_METAL_SOURCES ${DIRECTORY}/ggml-metal.m ${DIRECTORY}/ggml-metal.h)
+            # get full path to the file
+            #add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
+
+            # copy ggml-metal.metal to bin directory
+            configure_file(${DIRECTORY}/ggml-metal.metal bin/ggml-metal.metal COPYONLY)
+
+            set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS}
+                ${FOUNDATION_LIBRARY}
+                ${METAL_FRAMEWORK}
+                ${METALKIT_FRAMEWORK}
+                ${METALPERFORMANCE_FRAMEWORK}
+            )
+        endif()
+    endif()
+
     add_library(ggml${SUFFIX} OBJECT
                 ${DIRECTORY}/ggml.c
                 ${DIRECTORY}/ggml.h
-                ${GGML_CUDA_SOURCES}
-                ${GGML_OPENCL_SOURCES})
+                ${DIRECTORY}/ggml-alloc.c
+                ${DIRECTORY}/ggml-alloc.h
+                ${GGML_SOURCES_QUANT_K}
+                ${GGML_SOURCES_CUDA}
+                ${GGML_METAL_SOURCES}
+                ${GGML_OPENCL_SOURCES}
+                ${GGML_SOURCES_KOMPUTE})
 
+    if (LLAMA_K_QUANTS)
+        target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_K_QUANTS)
+    endif()
+
+    if (LLAMA_METAL AND GGML_METAL_SOURCES)
+        target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)
+    endif()
     target_include_directories(ggml${SUFFIX} PUBLIC ${DIRECTORY})
     target_compile_features(ggml${SUFFIX} PUBLIC c_std_11) # don't bump
-    target_link_libraries(ggml${SUFFIX} PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
 
     if (BUILD_SHARED_LIBS)
         set_target_properties(ggml${SUFFIX} PROPERTIES POSITION_INDEPENDENT_CODE ON)
@@ -338,14 +457,16 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
             set(LLAMA_UTIL_SOURCE_FILE llama_util.h)
         endif()
 
-        add_library(llama${SUFFIX}
+        add_library(llama${SUFFIX} STATIC
                     ${DIRECTORY}/llama.cpp
                     ${DIRECTORY}/llama.h
                     ${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
 
+        if (LLAMA_METAL AND GGML_METAL_SOURCES)
+            target_compile_definitions(llama${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)
+        endif()
         target_include_directories(llama${SUFFIX} PUBLIC ${DIRECTORY})
         target_compile_features(llama${SUFFIX} PUBLIC cxx_std_11) # don't bump
-        target_link_libraries(llama${SUFFIX} PRIVATE ggml${SUFFIX} ${LLAMA_EXTRA_LIBS})
 
         if (BUILD_SHARED_LIBS)
             set_target_properties(llama${SUFFIX} PROPERTIES POSITION_INDEPENDENT_CODE ON)
@@ -353,7 +474,7 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
         endif()
     endif()
 
-    if (GGML_CUDA_SOURCES)
+    if (GGML_SOURCES_CUDA)
         message(STATUS "GGML CUDA sources found, configuring CUDA architecture")
         set_property(TARGET ggml${SUFFIX} PROPERTY CUDA_ARCHITECTURES OFF)
         set_property(TARGET ggml${SUFFIX} PROPERTY CUDA_SELECT_NVCC_ARCH_FLAGS "Auto")
@@ -361,4 +482,97 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
             set_property(TARGET llama${SUFFIX} PROPERTY CUDA_ARCHITECTURES OFF)
         endif()
     endif()
+
+    if (GGML_CUBLAS_USE)
+        target_compile_definitions(ggml${SUFFIX} PRIVATE
+            GGML_USE_CUBLAS
+            GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X}
+            GGML_CUDA_DMMV_Y=${LLAMA_CUDA_DMMV_Y})
+        if (WITH_LLAMA)
+            target_compile_definitions(llama${SUFFIX} PRIVATE
+                GGML_USE_CUBLAS
+                GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X}
+                GGML_CUDA_DMMV_Y=${LLAMA_CUDA_DMMV_Y})
+        endif()
+    endif()
+    if (GGML_CLBLAST_USE)
+        if (WITH_LLAMA)
+            target_compile_definitions(llama${SUFFIX} PRIVATE GGML_USE_CLBLAST)
+        endif()
+        target_compile_definitions(ggml${SUFFIX} PRIVATE GGML_USE_CLBLAST)
+    endif()
+
+    if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
+        message(STATUS "ARM detected")
+        if (MSVC)
+            # TODO: arm msvc?
+        else()
+            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
+                target_compile_options(ggml${SUFFIX} PRIVATE -mcpu=native)
+            endif()
+            # TODO: armv6,7,8 version specific flags
+        endif()
+    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$")
+        message(STATUS "x86 detected")
+        if (MSVC)
+            if (LLAMA_AVX512)
+                target_compile_options(ggml${SUFFIX} PRIVATE
+                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX512>
+                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
+                # MSVC has no compile-time flags enabling specific
+                # AVX512 extensions, neither it defines the
+                # macros corresponding to the extensions.
+                # Do it manually.
+                if (LLAMA_AVX512_VBMI)
+                    target_compile_definitions(ggml${SUFFIX} PRIVATE
+                        $<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>
+                        $<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
+                endif()
+                if (LLAMA_AVX512_VNNI)
+                    target_compile_definitions(ggml${SUFFIX} PRIVATE
+                        $<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>
+                        $<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
+                endif()
+            elseif (LLAMA_AVX2)
+                target_compile_options(ggml${SUFFIX} PRIVATE
+                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX2>
+                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
+            elseif (LLAMA_AVX)
+                target_compile_options(ggml${SUFFIX} PRIVATE
+                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX>
+                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
+            endif()
+        else()
+            if (LLAMA_F16C)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mf16c)
+            endif()
+            if (LLAMA_FMA)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mfma)
+            endif()
+            if (LLAMA_AVX)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx)
+            endif()
+            if (LLAMA_AVX2)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx2)
+            endif()
+            if (LLAMA_AVX512)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512f)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512bw)
+            endif()
+            if (LLAMA_AVX512_VBMI)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512vbmi)
+            endif()
+            if (LLAMA_AVX512_VNNI)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512vnni)
+            endif()
+        endif()
+    else()
+        # TODO: support PowerPC
+        message(STATUS "Unknown architecture")
+    endif()
+
+    target_link_libraries(ggml${SUFFIX} PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
+    if (WITH_LLAMA)
+        target_link_libraries(llama${SUFFIX} PRIVATE ggml${SUFFIX} ${LLAMA_EXTRA_LIBS})
+    endif()
 endfunction()

gpt4all-backend/llamamodel.cpp

@@ -28,6 +28,9 @@
 #include <llama.h>
 #include <ggml.h>
 
+#ifdef GGML_USE_KOMPUTE
+#include "ggml-vulkan.h"
+#endif
 
 namespace {
 const char *modelType_ = "LLaMA";
@@ -97,6 +100,40 @@ LLamaModel::LLamaModel()
     d_ptr->modelLoaded = false;
 }
 
+// default hparams (LLaMA 7B)
+struct llama_file_hparams {
+    uint32_t n_vocab = 32000;
+    uint32_t n_embd  = 4096;
+    uint32_t n_mult  = 256;
+    uint32_t n_head  = 32;
+    uint32_t n_layer = 32;
+    uint32_t n_rot   = 64;
+    enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16;
+};
+
+size_t LLamaModel::requiredMem(const std::string &modelPath) {
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    fin.seekg(0, std::ios_base::end);
+    size_t filesize = fin.tellg();
+    fin.seekg(0, std::ios_base::beg);
+    uint32_t magic = 0;
+    fin.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    if (magic != 0x67676a74) return 0;
+    uint32_t version = 0;
+    fin.read(reinterpret_cast<char*>(&version), sizeof(version));
+    llama_file_hparams hparams;
+    fin.read(reinterpret_cast<char*>(&hparams.n_vocab), sizeof(hparams.n_vocab));
+    fin.read(reinterpret_cast<char*>(&hparams.n_embd), sizeof(hparams.n_embd));
+    fin.read(reinterpret_cast<char*>(&hparams.n_head), sizeof(hparams.n_head));
+    fin.read(reinterpret_cast<char*>(&hparams.n_layer), sizeof(hparams.n_layer));
+    fin.read(reinterpret_cast<char*>(&hparams.n_rot), sizeof(hparams.n_rot));
+    fin.read(reinterpret_cast<char*>(&hparams.ftype), sizeof(hparams.ftype));
+    const size_t n_ctx = 2048;
+    const size_t kvcache_element_size = 2; // fp16
+    const size_t est_kvcache_size = hparams.n_embd * hparams.n_layer * 2u * n_ctx * kvcache_element_size;
+    return filesize + est_kvcache_size;
+}
+
 bool LLamaModel::loadModel(const std::string &modelPath)
 {
     // load the model
@@ -115,6 +152,19 @@ bool LLamaModel::loadModel(const std::string &modelPath)
 #if LLAMA_DATE <= 230511
     d_ptr->params.n_parts  = params.n_parts;
 #endif
+#ifdef GGML_USE_METAL
+    std::cerr << "llama.cpp: using Metal" << std::endl;
+    // metal always runs the whole model if n_gpu_layers is not 0, at least
+    // currently
+    d_ptr->params.n_gpu_layers = 1;
+#endif
+#ifdef GGML_USE_KOMPUTE
+    if (ggml_vk_has_device()) {
+        // vulkan always runs the whole model if n_gpu_layers is not 0, at least
+        // currently
+        d_ptr->params.n_gpu_layers = 1;
+    }
+#endif
 
     d_ptr->ctx = llama_init_from_file(modelPath.c_str(), d_ptr->params);
     if (!d_ptr->ctx) {
@@ -122,6 +172,12 @@ bool LLamaModel::loadModel(const std::string &modelPath)
         return false;
     }
 
+#ifdef GGML_USE_KOMPUTE
+    if (ggml_vk_has_device()) {
+        std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
+    }
+#endif
+
     d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
     d_ptr->modelLoaded = true;
     fflush(stderr);
@@ -138,7 +194,9 @@ int32_t LLamaModel::threadCount() const {
 
 LLamaModel::~LLamaModel()
 {
-    llama_free(d_ptr->ctx);
+    if(d_ptr->ctx) {
+        llama_free(d_ptr->ctx);
+    }
 }
 
 bool LLamaModel::isModelLoaded() const
@@ -171,7 +229,7 @@ std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::
     return fres;
 }
 
-std::string_view LLamaModel::tokenToString(Token id) const
+std::string LLamaModel::tokenToString(Token id) const
 {
     return llama_token_to_str(d_ptr->ctx, id);
 }
@@ -187,7 +245,16 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
 
 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
 {
-    return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+    // When we recalculate context we could have erased the original BOS token... we need to replace it
+    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
+    if (useBOS) {
+        std::vector<int32_t> myTokens;
+        myTokens.push_back(llama_token_bos());
+        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
+        ctx.n_past += 1;
+        return llama_eval(d_ptr->ctx, myTokens.data(), myTokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+    } else
+        return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
 }
 
 int32_t LLamaModel::contextLength() const
@@ -201,6 +268,75 @@ const std::vector<LLModel::Token> &LLamaModel::endTokens() const
     return fres;
 }
 
+#if defined(GGML_USE_KOMPUTE)
+#include "ggml-vulkan.h"
+#endif
+
+std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryRequired)
+{
+#if defined(GGML_USE_KOMPUTE)
+    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(memoryRequired);
+
+    std::vector<LLModel::GPUDevice> devices;
+    for(const auto& vkDevice : vkDevices) {
+        LLModel::GPUDevice device;
+        device.index = vkDevice.index;
+        device.type = vkDevice.type;
+        device.heapSize = vkDevice.heapSize;
+        device.name = vkDevice.name;
+        device.vendor = vkDevice.vendor;
+
+        devices.push_back(device);
+    }
+
+    return devices;
+#else
+    return std::vector<LLModel::GPUDevice>();
+#endif
+}
+
+bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string& device)
+{
+#if defined(GGML_USE_KOMPUTE)
+    return ggml_vk_init_device(memoryRequired, device);
+#else
+    return false;
+#endif
+}
+
+bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device)
+{
+#if defined(GGML_USE_KOMPUTE)
+    ggml_vk_device vkDevice;
+    vkDevice.index = device.index;
+    vkDevice.type = device.type;
+    vkDevice.heapSize = device.heapSize;
+    vkDevice.name = device.name;
+    vkDevice.vendor = device.vendor;
+    return ggml_vk_init_device(vkDevice);
+#else
+    return false;
+#endif
+}
+
+bool LLamaModel::initializeGPUDevice(int device)
+{
+#if defined(GGML_USE_KOMPUTE)
+    return ggml_vk_init_device(device);
+#else
+    return false;
+#endif
+}
+
+bool LLamaModel::hasGPUDevice()
+{
+#if defined(GGML_USE_KOMPUTE)
+    return ggml_vk_has_device();
+#else
+    return false;
+#endif
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -228,7 +364,31 @@ DLL_EXPORT bool magic_match(std::istream& f) {
     // Check version
     uint32_t version = 0;
     f.read(reinterpret_cast<char*>(&version), sizeof(version));
-    return version LLAMA_VERSIONS;
+    if (!(version LLAMA_VERSIONS)) {
+        return false;
+    }
+    llama_file_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    if (!(hparams.n_vocab >= 32000 && hparams.n_vocab <= 32100)) {
+        return false; // not a llama.
+    }
+#ifdef GGML_USE_METAL
+    // Check quant supported on metal
+    // skip fields
+    switch(hparams.ftype) {
+        // currently supported on Metal https://github.com/ggerganov/llama.cpp/blob/ae9663f1887513e152839e91f61c513075a19422/ggml-metal.m#L51-L55
+        case LLAMA_FTYPE_MOSTLY_F16:
+        case LLAMA_FTYPE_MOSTLY_Q2_K:
+        case LLAMA_FTYPE_MOSTLY_Q4_0:
+        case LLAMA_FTYPE_MOSTLY_Q6_K:
+        case LLAMA_FTYPE_MOSTLY_Q4_K_S:
+        case LLAMA_FTYPE_MOSTLY_Q4_K_M:
+            return true;
+        default: // unsupported quant-type for Metal
+            return false;
+    }
+#endif
+    return true;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llamamodel_impl.h

@@ -15,20 +15,28 @@ public:
     LLamaModel();
     ~LLamaModel();
 
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
+    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) override;
+    bool initializeGPUDevice(size_t memoryRequired, const std::string& device) override;
+    bool initializeGPUDevice(const GPUDevice &device) override;
+    bool initializeGPUDevice(int device) override;
+    bool hasGPUDevice() override;
 
 private:
     LLamaPrivate *d_ptr;
 
 protected:
     std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string_view tokenToString(Token) const override;
+    std::string tokenToString(Token) const override;
     Token sampleToken(PromptContext& ctx) const override;
     bool evalTokens(PromptContext& ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;

gpt4all-backend/llmodel.cpp

@@ -1,5 +1,6 @@
 #include "llmodel.h"
 #include "dlhandle.h"
+#include "sysinfo.h"
 
 #include <iostream>
 #include <string>
@@ -9,11 +10,14 @@
 #include <cassert>
 #include <cstdlib>
 #include <sstream>
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
 
-std::string LLModel::m_implementations_search_path = ".";
+std::string s_implementations_search_path = ".";
 
 static bool has_at_least_minimal_hardware() {
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(_M_X64)
     #ifndef _MSC_VER
         return __builtin_cpu_supports("avx");
     #else
@@ -27,7 +31,7 @@ static bool has_at_least_minimal_hardware() {
 }
 
 static bool requires_avxonly() {
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(_M_X64)
     #ifndef _MSC_VER
         return !__builtin_cpu_supports("avx2");
     #else
@@ -40,41 +44,42 @@ static bool requires_avxonly() {
 #endif
 }
 
-LLModel::Implementation::Implementation(Dlhandle &&dlhandle_) : dlhandle(new Dlhandle(std::move(dlhandle_))) {
-    auto get_model_type = dlhandle->get<const char *()>("get_model_type");
+LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
+    : m_dlhandle(new Dlhandle(std::move(dlhandle_))) {
+    auto get_model_type = m_dlhandle->get<const char *()>("get_model_type");
     assert(get_model_type);
-    modelType = get_model_type();
-    auto get_build_variant = dlhandle->get<const char *()>("get_build_variant");
+    m_modelType = get_model_type();
+    auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
-    buildVariant = get_build_variant();
-    magicMatch = dlhandle->get<bool(std::ifstream&)>("magic_match");
-    assert(magicMatch);
-    construct_ = dlhandle->get<LLModel *()>("construct");
-    assert(construct_);
+    m_buildVariant = get_build_variant();
+    m_magicMatch = m_dlhandle->get<bool(std::ifstream&)>("magic_match");
+    assert(m_magicMatch);
+    m_construct = m_dlhandle->get<LLModel *()>("construct");
+    assert(m_construct);
 }
 
 LLModel::Implementation::Implementation(Implementation &&o)
-    : construct_(o.construct_)
-    , modelType(o.modelType)
-    , buildVariant(o.buildVariant)
-    , magicMatch(o.magicMatch)
-    , dlhandle(o.dlhandle) {
-    o.dlhandle = nullptr;
+    : m_magicMatch(o.m_magicMatch)
+    , m_construct(o.m_construct)
+    , m_modelType(o.m_modelType)
+    , m_buildVariant(o.m_buildVariant)
+    , m_dlhandle(o.m_dlhandle) {
+    o.m_dlhandle = nullptr;
 }
 
 LLModel::Implementation::~Implementation() {
-    if (dlhandle) delete dlhandle;
+    if (m_dlhandle) delete m_dlhandle;
 }
 
 bool LLModel::Implementation::isImplementation(const Dlhandle &dl) {
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
-const std::vector<LLModel::Implementation> &LLModel::implementationList() {
+const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
     // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
     // individual models without the cleanup of the static list interfering
-    static auto* libs = new std::vector<LLModel::Implementation>([] () {
-        std::vector<LLModel::Implementation> fres;
+    static auto* libs = new std::vector<Implementation>([] () {
+        std::vector<Implementation> fres;
 
         auto search_in_directory = [&](const std::string& paths) {
             std::stringstream ss(paths);
@@ -98,7 +103,7 @@ const std::vector<LLModel::Implementation> &LLModel::implementationList() {
             }
         };
 
-        search_in_directory(m_implementations_search_path);
+        search_in_directory(s_implementations_search_path);
 
         return fres;
     }());
@@ -106,36 +111,71 @@ const std::vector<LLModel::Implementation> &LLModel::implementationList() {
     return *libs;
 }
 
-const LLModel::Implementation* LLModel::implementation(std::ifstream& f, const std::string& buildVariant) {
+const LLModel::Implementation* LLModel::Implementation::implementation(std::ifstream& f, const std::string& buildVariant) {
     for (const auto& i : implementationList()) {
         f.seekg(0);
-        if (!i.magicMatch(f)) continue;
-        if (buildVariant != i.buildVariant) continue;
+        if (!i.m_magicMatch(f)) continue;
+        if (buildVariant != i.m_buildVariant) continue;
         return &i;
     }
     return nullptr;
 }
 
-LLModel *LLModel::construct(const std::string &modelPath, std::string buildVariant) {
+LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant) {
 
     if (!has_at_least_minimal_hardware())
         return nullptr;
 
-    //TODO: Auto-detect CUDA/OpenCL
-    if (buildVariant == "auto") {
-        if (requires_avxonly()) {
-            buildVariant = "avxonly";
-        } else {
-            buildVariant = "default";
-        }
-    }
     // Read magic
     std::ifstream f(modelPath, std::ios::binary);
     if (!f) return nullptr;
     // Get correct implementation
-    auto impl = implementation(f, buildVariant);
-    if (!impl) return nullptr;
+    const Implementation* impl = nullptr;
+
+    #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
+        if (buildVariant == "auto") {
+            size_t total_mem = getSystemTotalRAMInBytes();
+            impl = implementation(f, "metal");
+            if(impl) {
+                LLModel* metalimpl = impl->m_construct();
+                metalimpl->m_implementation = impl;
+                size_t req_mem = metalimpl->requiredMem(modelPath);
+                float req_to_total = (float) req_mem / (float) total_mem;
+                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
+                if (req_to_total >= 0.53) {
+                    delete metalimpl;
+                    impl = nullptr;
+                } else {
+                    return metalimpl;
+                }
+            }
+        }
+    #endif
+
+    if (!impl) {
+        //TODO: Auto-detect CUDA/OpenCL
+        if (buildVariant == "auto") {
+            if (requires_avxonly()) {
+                buildVariant = "avxonly";
+            } else {
+                buildVariant = "default";
+            }
+        }
+        impl = implementation(f, buildVariant);
+        if (!impl) return nullptr;
+    }
     f.close();
+
     // Construct and return llmodel implementation
-    return impl->construct();
+    auto fres = impl->m_construct();
+    fres->m_implementation = impl;
+    return fres;
+}
+
+void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
+    s_implementations_search_path = path;
+}
+
+const std::string& LLModel::Implementation::implementationsSearchPath() {
+    return s_implementations_search_path;
 }

gpt4all-backend/llmodel.h

@@ -9,33 +9,37 @@
 #include <cstdint>
 #include <limits>
 
-class Dlhandle;
+#define LLMODEL_MAX_PROMPT_BATCH 128
 
+class Dlhandle;
 class LLModel {
 public:
     using Token = int32_t;
-
     class Implementation {
-        LLModel *(*construct_)();
-
     public:
         Implementation(Dlhandle&&);
         Implementation(const Implementation&) = delete;
         Implementation(Implementation&&);
         ~Implementation();
 
+        std::string_view modelType() const { return m_modelType; }
+        std::string_view buildVariant() const { return m_buildVariant; }
+
         static bool isImplementation(const Dlhandle&);
+        static const std::vector<Implementation>& implementationList();
+        static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
+        static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
+        static void setImplementationsSearchPath(const std::string& path);
+        static const std::string& implementationsSearchPath();
 
-        std::string_view modelType, buildVariant;
-        bool (*magicMatch)(std::ifstream& f);
-        Dlhandle *dlhandle;
+    private:
+        bool (*m_magicMatch)(std::ifstream& f);
+        LLModel *(*m_construct)();
 
-        // The only way an implementation should be constructed
-        LLModel *construct() const {
-            auto fres = construct_();
-            fres->m_implementation = this;
-            return fres;
-        }
+    private:
+        std::string_view m_modelType;
+        std::string_view m_buildVariant;
+        Dlhandle *m_dlhandle;
     };
 
     struct PromptContext {
@@ -54,20 +58,36 @@ public:
             // window
     };
 
+    struct GPUDevice {
+        int index = 0;
+        int type = 0;
+        size_t heapSize = 0;
+        std::string name;
+        std::string vendor;
+    };
+
     explicit LLModel() {}
     virtual ~LLModel() {}
 
+    virtual bool supportsEmbedding() const = 0;
+    virtual bool supportsCompletion() const = 0;
     virtual bool loadModel(const std::string &modelPath) = 0;
     virtual bool isModelLoaded() const = 0;
+    virtual size_t requiredMem(const std::string &modelPath) = 0;
     virtual size_t stateSize() const { return 0; }
     virtual size_t saveState(uint8_t */*dest*/) const { return 0; }
     virtual size_t restoreState(const uint8_t */*src*/) { return 0; }
+
+    // This method requires the model to return true from supportsCompletion otherwise it will throw
+    // an error
     virtual void prompt(const std::string &prompt,
                         std::function<bool(int32_t)> promptCallback,
                         std::function<bool(int32_t, const std::string&)> responseCallback,
                         std::function<bool(bool)> recalculateCallback,
                         PromptContext &ctx);
 
+    virtual std::vector<float> embedding(const std::string &text);
+
     virtual void setThreadCount(int32_t /*n_threads*/) {}
     virtual int32_t threadCount() const { return 1; }
 
@@ -75,22 +95,17 @@ public:
         return *m_implementation;
     }
 
-    static const std::vector<Implementation>& implementationList();
-    static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
-    static LLModel *construct(const std::string &modelPath, std::string buildVariant = "default");
-
-    static inline void setImplementationsSearchPath(const std::string& path) {
-        m_implementations_search_path = path;
-    }
-    static inline const std::string& implementationsSearchPath() {
-        return m_implementations_search_path;
-    }
+    virtual std::vector<GPUDevice> availableGPUDevices(size_t /*memoryRequired*/) { return std::vector<GPUDevice>(); }
+    virtual bool initializeGPUDevice(size_t /*memoryRequired*/, const std::string& /*device*/) { return false; }
+    virtual bool initializeGPUDevice(const GPUDevice &/*device*/) { return false; }
+    virtual bool initializeGPUDevice(int /*device*/) { return false; }
+    virtual bool hasGPUDevice() { return false; }
 
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
     virtual std::vector<Token> tokenize(PromptContext &, const std::string&) const = 0;
-    virtual std::string_view tokenToString(Token) const = 0;
+    virtual std::string tokenToString(Token) const = 0;
     virtual Token sampleToken(PromptContext &ctx) const = 0;
     virtual bool evalTokens(PromptContext &/*ctx*/, const std::vector<int32_t>& /*tokens*/) const = 0;
     virtual int32_t contextLength() const = 0;
@@ -101,6 +116,9 @@ protected:
     void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate);
 
     const Implementation *m_implementation = nullptr;
-    static std::string m_implementations_search_path;
+
+private:
+    friend class LLMImplementation;
 };
+
 #endif // LLMODEL_H

gpt4all-backend/llmodel_c.cpp

@@ -5,10 +5,10 @@
 #include <cerrno>
 #include <utility>
 
-
 struct LLModelWrapper {
     LLModel *llModel = nullptr;
     LLModel::PromptContext promptContext;
+    ~LLModelWrapper() { delete llModel; }
 };
 
 
@@ -25,33 +25,44 @@ llmodel_model llmodel_model_create(const char *model_path) {
 
 llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error) {
     auto wrapper = new LLModelWrapper;
-    llmodel_error new_error{};
+    int error_code = 0;
 
     try {
-        wrapper->llModel = LLModel::construct(model_path, build_variant);
+        wrapper->llModel = LLModel::Implementation::construct(model_path, build_variant);
     } catch (const std::exception& e) {
-        new_error.code = EINVAL;
+        error_code = EINVAL;
         last_error_message = e.what();
     }
 
     if (!wrapper->llModel) {
         delete std::exchange(wrapper, nullptr);
         // Get errno and error message if none
-        if (new_error.code == 0) {
-            new_error.code = errno;
-            last_error_message = strerror(errno);
+        if (error_code == 0) {
+            if (errno != 0) {
+                error_code = errno;
+                last_error_message = std::strerror(error_code);
+            } else {
+                error_code = ENOTSUP;
+                last_error_message = "Model format not supported (no matching implementation found)";
+            }
         }
-        // Set message pointer
-        new_error.message = last_error_message.c_str();
         // Set error argument
-        if (error) *error = new_error;
+        if (error) {
+            error->message = last_error_message.c_str();
+            error->code = error_code;
+        }
     }
     return reinterpret_cast<llmodel_model*>(wrapper);
 }
 
 void llmodel_model_destroy(llmodel_model model) {
+    delete reinterpret_cast<LLModelWrapper*>(model);
+}
+
+size_t llmodel_required_mem(llmodel_model model, const char *model_path)
+{
     LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    delete wrapper->llModel;
+    return wrapper->llModel->requiredMem(model_path);
 }
 
 bool llmodel_loadModel(llmodel_model model, const char *model_path)
@@ -116,6 +127,9 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     std::function<bool(bool)> recalc_func =
         std::bind(&recalculate_wrapper, std::placeholders::_1, reinterpret_cast<void*>(recalculate_callback));
 
+    if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
+        wrapper->promptContext.tokens.resize(ctx->n_past);
+
     // Copy the C prompt context
     wrapper->promptContext.n_past = ctx->n_past;
     wrapper->promptContext.n_ctx = ctx->n_ctx;
@@ -151,6 +165,29 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->context_erase = wrapper->promptContext.contextErase;
 }
 
+float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size)
+{
+    if (model == nullptr || text == nullptr || !strlen(text)) {
+        *embedding_size = 0;
+        return nullptr;
+    }
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    std::vector<float> embeddingVector = wrapper->llModel->embedding(text);
+    float *embedding = (float *)malloc(embeddingVector.size() * sizeof(float));
+    if (embedding == nullptr) {
+        *embedding_size = 0;
+        return nullptr;
+    }
+    std::copy(embeddingVector.begin(), embeddingVector.end(), embedding);
+    *embedding_size = embeddingVector.size();
+    return embedding;
+}
+
+void llmodel_free_embedding(float *ptr)
+{
+    free(ptr);
+}
+
 void llmodel_setThreadCount(llmodel_model model, int32_t n_threads)
 {
     LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
@@ -165,10 +202,64 @@ int32_t llmodel_threadCount(llmodel_model model)
 
 void llmodel_set_implementation_search_path(const char *path)
 {
-    LLModel::setImplementationsSearchPath(path);
+    LLModel::Implementation::setImplementationsSearchPath(path);
 }
 
 const char *llmodel_get_implementation_search_path()
 {
-    return LLModel::implementationsSearchPath().c_str();
+    return LLModel::Implementation::implementationsSearchPath().c_str();
+}
+
+struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    std::vector<LLModel::GPUDevice> devices = wrapper->llModel->availableGPUDevices(memoryRequired);
+
+    // Set the num_devices
+    *num_devices = devices.size();
+
+    if (*num_devices == 0) return nullptr;  // Return nullptr if no devices are found
+
+    // Allocate memory for the output array
+    struct llmodel_gpu_device* output = (struct llmodel_gpu_device*) malloc(*num_devices * sizeof(struct llmodel_gpu_device));
+
+    for (int i = 0; i < *num_devices; i++) {
+        output[i].index = devices[i].index;
+        output[i].type = devices[i].type;
+        output[i].heapSize = devices[i].heapSize;
+        output[i].name = strdup(devices[i].name.c_str());  // Convert std::string to char* and allocate memory
+        output[i].vendor = strdup(devices[i].vendor.c_str());  // Convert std::string to char* and allocate memory
+    }
+
+    return output;
+}
+
+bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(memoryRequired, std::string(device));
+}
+
+bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device)
+{
+    LLModel::GPUDevice d;
+    d.index = device->index;
+    d.type = device->type;
+    d.heapSize = device->heapSize;
+    d.name = device->name;
+    d.vendor = device->vendor;
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(d);
+}
+
+bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(device);
+}
+
+bool llmodel_has_gpu_device(llmodel_model model)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->hasGPUDevice();
 }

gpt4all-backend/llmodel_c.h

@@ -56,8 +56,18 @@ struct llmodel_prompt_context {
     int32_t repeat_last_n;  // last n tokens to penalize
     float context_erase;    // percent of context to erase if we exceed the context window
 };
+
+struct llmodel_gpu_device {
+    int index = 0;
+    int type = 0;           // same as VkPhysicalDeviceType
+    size_t heapSize = 0;
+    const char * name;
+    const char * vendor;
+};
+
 #ifndef __cplusplus
 typedef struct llmodel_prompt_context llmodel_prompt_context;
+typedef struct llmodel_gpu_device llmodel_gpu_device;
 #endif
 
 /**
@@ -107,6 +117,14 @@ llmodel_model llmodel_model_create2(const char *model_path, const char *build_va
  */
 void llmodel_model_destroy(llmodel_model model);
 
+/**
+ * Estimate RAM requirement for a model file
+ * @param model A pointer to the llmodel_model instance.
+ * @param model_path A string representing the path to the model file.
+ * @return size greater than 0 if the model was parsed successfully, 0 if file could not be parsed.
+ */
+size_t llmodel_required_mem(llmodel_model model, const char *model_path);
+
 /**
  * Load a model from a file.
  * @param model A pointer to the llmodel_model instance.
@@ -163,6 +181,25 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
                     llmodel_recalculate_callback recalculate_callback,
                     llmodel_prompt_context *ctx);
 
+/**
+ * Generate an embedding using the model.
+ * NOTE: If given NULL pointers for the model or text, or an empty text, a NULL pointer will be
+ * returned. Bindings should signal an error when NULL is the return value.
+ * @param model A pointer to the llmodel_model instance.
+ * @param text A string representing the text to generate an embedding for.
+ * @param embedding_size A pointer to a size_t type that will be set by the call indicating the length
+ * of the returned floating point array.
+ * @return A pointer to an array of floating point values passed to the calling method which then will
+ * be responsible for lifetime of this memory.
+ */
+float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size);
+
+/**
+ * Frees the memory allocated by the llmodel_embedding function.
+ * @param ptr A pointer to the embedding as returned from llmodel_embedding.
+ */
+void llmodel_free_embedding(float *ptr);
+
 /**
  * Set the number of threads to be used by the model.
  * @param model A pointer to the llmodel_model instance.
@@ -191,6 +228,50 @@ void llmodel_set_implementation_search_path(const char *path);
  */
 const char *llmodel_get_implementation_search_path();
 
+/**
+ * Get a list of available GPU devices given the memory required.
+ * @return A pointer to an array of llmodel_gpu_device's whose number is given by num_devices.
+ */
+struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices);
+
+/**
+ * Initializes a GPU device based on a specified string criterion.
+ *
+ * This function initializes a GPU device based on a string identifier provided. The function
+ * allows initialization based on general device type ("gpu"), vendor name ("amd", "nvidia", "intel"),
+ * or any specific device name.
+ *
+ * @param memoryRequired The amount of memory (in bytes) required by the application or task
+ *                       that will utilize the GPU device.
+ * @param device A string specifying the desired criterion for GPU device selection. It can be:
+ *               - "gpu": To initialize the best available GPU.
+ *               - "amd", "nvidia", or "intel": To initialize the best available GPU from that vendor.
+ *               - A specific GPU device name: To initialize a GPU with that exact name.
+ *
+ * @return True if the GPU device is successfully initialized based on the provided string
+ *         criterion. Returns false if the desired GPU device could not be initialized.
+ */
+bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device);
+
+/**
+ * Initializes a GPU device by specifying a valid gpu device pointer.
+ * @param device A gpu device pointer.
+ * @return True if the GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device);
+
+/**
+ * Initializes a GPU device by its index.
+ * @param device An integer representing the index of the GPU device to be initialized.
+ * @return True if the GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
+
+/**
+ * @return True if a GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_has_gpu_device(llmodel_model model);
+
 #ifdef __cplusplus
 }
 #endif

gpt4all-backend/llmodel_shared.cpp

@@ -33,7 +33,14 @@ void LLModel::prompt(const std::string &prompt,
                      PromptContext &promptCtx)
 {
     if (!isModelLoaded()) {
-        std::cerr << implementation().modelType << " ERROR: prompt won't work with an unloaded model!\n";
+        std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
+        return;
+    }
+
+    if (!supportsCompletion()) {
+        std::string errorMessage = "ERROR: this model does not support text completion or chat!\n";
+        responseCallback(-1, errorMessage);
+        std::cerr << implementation().modelType() << errorMessage;
         return;
     }
 
@@ -45,13 +52,14 @@ void LLModel::prompt(const std::string &prompt,
 
     if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
         responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed.");
-        std::cerr << implementation().modelType << " ERROR: The prompt is" << embd_inp.size() <<
-            "tokens and the context window is" << promptCtx.n_ctx << "!\n";
+        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
+            " tokens and the context window is " << promptCtx.n_ctx << "!\n";
         return;
     }
 
     promptCtx.n_predict = std::min(promptCtx.n_predict, promptCtx.n_ctx - (int) embd_inp.size());
     promptCtx.n_past = std::min(promptCtx.n_past, promptCtx.n_ctx);
+    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
 
     // process the prompt in batches
     size_t i = 0;
@@ -63,7 +71,7 @@ void LLModel::prompt(const std::string &prompt,
         if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
             const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
             // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n";
+            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
             promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
             promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
@@ -71,7 +79,7 @@ void LLModel::prompt(const std::string &prompt,
         }
 
         if (!evalTokens(promptCtx, batch)) {
-            std::cerr << implementation().modelType << " ERROR: Failed to process prompt\n";
+            std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
             return;
         }
 
@@ -102,7 +110,7 @@ void LLModel::prompt(const std::string &prompt,
         if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
             const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
             // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n";
+            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
             promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
             promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
@@ -110,7 +118,7 @@ void LLModel::prompt(const std::string &prompt,
         }
 
         if (!evalTokens(promptCtx, { id })) {
-            std::cerr << implementation().modelType << " ERROR: Failed to predict next token\n";
+            std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
             return;
         }
 
@@ -121,7 +129,7 @@ void LLModel::prompt(const std::string &prompt,
             if (id == token) return;
         }
 
-        const std::string_view str = tokenToString(id);
+        const std::string str = tokenToString(id);
 
         // Check if the provided str is part of our reverse prompts
         bool foundPartialReversePrompt = false;
@@ -157,3 +165,12 @@ void LLModel::prompt(const std::string &prompt,
         cachedTokens.clear();
     }
 }
+
+std::vector<float> LLModel::embedding(const std::string &/*text*/)
+{
+    if (!supportsCompletion()) {
+        std::string errorMessage = "ERROR: this model does not support generating embeddings!\n";
+        std::cerr << implementation().modelType() << errorMessage;
+    }
+    return std::vector<float>();
+}

gpt4all-backend/llmodel_shared.h

@@ -0,0 +1,92 @@
+#pragma once
+#include <cstdint>
+#include <cstddef>
+#include <vector>
+#include <ggml.h>
+
+#if defined(GGML_USE_KOMPUTE)
+#include "ggml-vulkan.h"
+struct llm_buffer {
+    uint8_t * addr = NULL;
+    size_t size = 0;
+    ggml_vk_memory memory;
+
+    llm_buffer() = default;
+
+    void resize(size_t size) {
+        free();
+
+        if (!ggml_vk_has_device()) {
+            this->addr = new uint8_t[size];
+            this->size = size;
+        } else {
+            this->memory = ggml_vk_allocate(size);
+            this->addr = (uint8_t*)memory.data;
+            this->size = size;
+        }
+    }
+
+    void free() {
+        if (!memory.primaryMemory) {
+            delete[] addr;
+        } else if (memory.data) {
+            ggml_vk_free_memory(memory);
+        }
+        this->addr = NULL;
+        this->size = 0;
+    }
+
+    ~llm_buffer() {
+        free();
+    }
+
+    // disable copy and move
+    llm_buffer(const llm_buffer&) = delete;
+    llm_buffer(llm_buffer&&) = delete;
+    llm_buffer& operator=(const llm_buffer&) = delete;
+    llm_buffer& operator=(llm_buffer&&) = delete;
+};
+#else
+struct llm_buffer {
+    uint8_t * addr = NULL;
+    size_t size = 0;
+
+    void resize(size_t size) {
+        delete[] addr;
+        addr = new uint8_t[size];
+        this->size = size;
+    }
+
+    ~llm_buffer() {
+        delete[] addr;
+    }
+};
+#endif
+
+struct llm_kv_cache {
+    struct ggml_tensor * k;
+    struct ggml_tensor * v;
+
+    struct ggml_context * ctx = NULL;
+
+    llm_buffer buf;
+
+    int n; // number of tokens currently in the cache
+
+    ~llm_kv_cache() {
+        if (ctx) {
+            ggml_free(ctx);
+        }
+    }
+};
+
+#if LLAMA_DATE >= 230519
+inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads) {
+    struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
+    if (plan.work_size > 0) {
+        buf.resize(plan.work_size);
+        plan.work_data = buf.addr;
+    }
+    ggml_graph_compute(graph, &plan);
+}
+#endif

gpt4all-backend/mpt.cpp

@@ -2,8 +2,10 @@
 #include "mpt_impl.h"
 
 #include "utils.h"
+#include "llmodel_shared.h"
 
 #include <cassert>
+#include <cinttypes>
 #include <cmath>
 #include <cstdio>
 #include <cstring>
@@ -33,8 +35,6 @@
 
 namespace {
 const char *modelType_ = "MPT";
-
-static const size_t MB = 1024*1024;
 }
 
 // default hparams (MPT 7B)
@@ -64,39 +64,6 @@ struct mpt_layer {
     struct ggml_tensor * ffn_down_proj_w;
 };
 
-struct mpt_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-
-    void resize(size_t size) {
-        delete[] addr;
-        addr = new uint8_t[size];
-        this->size = size;
-    }
-
-    ~mpt_buffer() {
-        fflush(stdout);
-        delete[] addr;
-    }
-};
-
-struct mpt_kv_cache {
-    struct ggml_tensor * k;
-    struct ggml_tensor * v;
-
-    struct ggml_context * ctx = NULL;
-
-    mpt_buffer buf;
-
-    int n; // number of tokens currently in the cache
-
-    ~mpt_kv_cache() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
 struct mpt_model {
     mpt_hparams hparams;
 
@@ -109,12 +76,14 @@ struct mpt_model {
 
     std::vector<mpt_layer> layers;
 
-    struct mpt_kv_cache kv_self;
+    struct llm_kv_cache kv_self;
     struct ggml_context * ctx;
     std::map<std::string, struct ggml_tensor *> tensors;
 
 
-    mpt_buffer buf;
+    llm_buffer eval_buf;
+    llm_buffer scr0_buf;
+    llm_buffer scr1_buf;
 
     ~mpt_model() {
         if (ctx) {
@@ -125,7 +94,7 @@ struct mpt_model {
 
 static bool kv_cache_init(
         const struct mpt_hparams & hparams,
-             struct mpt_kv_cache & cache,
+             struct llm_kv_cache & cache,
                          ggml_type   wtype,
                                int   n_ctx) {
     const int n_embd  = hparams.n_embd;
@@ -134,7 +103,7 @@ static bool kv_cache_init(
     const int64_t n_mem      = (int64_t)n_layer*n_ctx;
     const int64_t n_elements = n_embd*n_mem;
 
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB);
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
 
     struct ggml_init_params params;
     params.mem_size   = cache.buf.size;
@@ -154,9 +123,13 @@ static bool kv_cache_init(
     return true;
 }
 
-// load the model's weights from a stream
-bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & model, gpt_vocab & vocab) {
+// load the model's weights from a stream. if mem_req ptr is passed the model is
+// only partially parsed to estimate required memory
+bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & model, gpt_vocab & vocab, size_t * mem_req) {
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+    if (mem_req != nullptr) {
+        *mem_req = 0;
+    }
 
     // verify magic
     {
@@ -278,6 +251,18 @@ bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & mod
         printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
     }
 
+    if (mem_req != nullptr) {
+        *mem_req += ctx_size;
+        const int n_embd  = model.hparams.n_embd;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = n_embd*n_mem;
+
+        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+        return false;
+    }
+
     // create the ggml context
     {
         struct ggml_init_params params = {
@@ -388,8 +373,8 @@ bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & mod
             }
 
             if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
-                        __func__, name.data(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
+                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
                 return false;
             }
 
@@ -421,6 +406,9 @@ bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & mod
         printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
     }
 
+    model.scr0_buf.resize(256u * 1024 * 1024);
+    model.scr1_buf.resize(256u * 1024 * 1024);
+
     return true;
 }
 
@@ -433,7 +421,7 @@ bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vo
         return false;
     }
 
-    bool loaded = mpt_model_load(fname, fin, model, vocab);
+    bool loaded = mpt_model_load(fname, fin, model, vocab, nullptr);
     fin.close();
     return loaded;
 }
@@ -455,25 +443,25 @@ bool mpt_eval(
     const int n_head  = hparams.n_head;
     const int n_vocab = hparams.n_vocab;
 
-    const size_t init_buf_size = 1024u*MB;
-    if (!model.buf.addr || model.buf.size < init_buf_size)
-        model.buf.resize(init_buf_size);
+    const size_t init_buf_size = 1024_MiB;
+    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
+        model.eval_buf.resize(init_buf_size);
 
-    if (mem_per_token > 0 && mem_per_token*N > model.buf.size) {
+    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
         const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
         // printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
 
         // reallocate
-        model.buf.resize(buf_size_new);
-        if (model.buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.buf.size);
+        model.eval_buf.resize(buf_size_new);
+        if (model.eval_buf.addr == nullptr) {
+            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
             return false;
         }
     }
 
     struct ggml_init_params params = {
-        .mem_size   = model.buf.size,
-        .mem_buffer = model.buf.addr,
+        .mem_size   = model.eval_buf.size,
+        .mem_buffer = model.eval_buf.addr,
         .no_alloc = false
     };
 
@@ -488,6 +476,7 @@ bool mpt_eval(
     struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
 
     for (int il = 0; il < n_layer; ++il) {
+        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
 
         struct ggml_tensor * inpSA = inpL;
         struct ggml_tensor * cur = inpSA;
@@ -579,7 +568,7 @@ bool mpt_eval(
                     cur);
         }
 
-
+        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
         // residual
         struct ggml_tensor * resSA = ggml_add(ctx0, cur, inpSA);
         // feed-forward network
@@ -604,6 +593,7 @@ bool mpt_eval(
         // self-attention + FF
         inpL = ggml_add(ctx0, cur, resSA);
     }
+    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
 
     struct ggml_tensor * out = inpL;
     // -> logits
@@ -612,6 +602,7 @@ bool mpt_eval(
         out = ggml_mul(ctx0,
                     ggml_repeat(ctx0, model.norm_f_w, out),
                     out);
+        ggml_set_scratch(ctx0, { 0, 0, nullptr, });
         out = ggml_mul_mat(ctx0, model.wte, out);
     }
 
@@ -759,9 +750,19 @@ struct MPTPrivate {
 MPT::MPT()
     : d_ptr(new MPTPrivate) {
     d_ptr->model = new mpt_model;
+    d_ptr->model->ctx = nullptr;
     d_ptr->modelLoaded = false;
 }
 
+size_t MPT::requiredMem(const std::string &modelPath) {
+    mpt_model dummy_model;
+    gpt_vocab dummy_vocab;
+    size_t mem_req;
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    mpt_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
+    return mem_req;
+}
+
 bool MPT::loadModel(const std::string &modelPath) {
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
@@ -769,8 +770,8 @@ bool MPT::loadModel(const std::string &modelPath) {
     auto fin = std::ifstream(modelPath, std::ios::binary);
 
     // load the model
-    if (!mpt_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab)) {
-        std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
+    if (!mpt_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab, nullptr)) {
+        std::cerr << "MPT ERROR: failed to load model from " <<  modelPath;
         return false;
     }
 
@@ -820,7 +821,7 @@ std::vector<LLModel::Token> MPT::tokenize(PromptContext &, const std::string &st
     return ::gpt_tokenize(d_ptr->vocab, str);
 }
 
-std::string_view MPT::tokenToString(Token id) const
+std::string MPT::tokenToString(Token id) const
 {
     return d_ptr->vocab.id_to_token[id];
 }

gpt4all-backend/mpt_impl.h

@@ -15,8 +15,11 @@ public:
     MPT();
     ~MPT();
 
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
@@ -28,7 +31,7 @@ private:
 
 protected:
     std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string_view tokenToString(Token) const override;
+    std::string tokenToString(Token) const override;
     Token sampleToken(PromptContext &ctx) const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;

gpt4all-backend/replit_impl.h

@@ -0,0 +1,43 @@
+#ifndef REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of replit.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef REPLIT_H
+#define REPLIT_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include "llmodel.h"
+
+#define GGML_QNT_VERSION_FACTOR 1000 // do not change this
+
+struct ReplitPrivate;
+class Replit : public LLModel {
+public:
+    Replit();
+    ~Replit();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string & modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    ReplitPrivate *d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::string tokenToString(Token) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // REPLIT_H

gpt4all-backend/scripts/convert_bert_hf_to_ggml.py

@@ -0,0 +1,102 @@
+import sys
+import struct
+import json
+import torch
+import numpy as np
+
+from transformers import AutoModel, AutoTokenizer
+
+if len(sys.argv) < 3:
+    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = sys.argv[1]
+fname_out = sys.argv[1] + "/ggml-model.bin"
+
+with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f:
+    encoder = json.load(f)
+
+with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
+    hparams = json.load(f)
+
+with open(dir_model + "/vocab.txt", "r", encoding="utf-8") as f:
+    vocab = f.readlines()
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+    fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
+
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+model = AutoModel.from_pretrained(dir_model, low_cpu_mem_usage=True)
+print (model)
+
+print(tokenizer.encode('I believe the meaning of life is'))
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+fout = open(fname_out, "wb")
+
+print(hparams)
+
+fout.write(struct.pack("i", 0x62657274)) # magic: ggml in hex
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["max_position_embeddings"]))
+fout.write(struct.pack("i", hparams["hidden_size"]))
+fout.write(struct.pack("i", hparams["intermediate_size"]))
+fout.write(struct.pack("i", hparams["num_attention_heads"]))
+fout.write(struct.pack("i", hparams["num_hidden_layers"]))
+fout.write(struct.pack("i", ftype))
+
+for i in range(hparams["vocab_size"]):
+    text = vocab[i][:-1] # strips newline at the end
+    #print(f"{i}:{text}")
+    data = bytes(text, 'utf-8')
+    fout.write(struct.pack("i", len(data)))
+    fout.write(data)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
+        continue
+    print("Processing variable: " + name + " with shape: ", data.shape)
+
+    n_dims = len(data.shape);
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+            print("  Converting to float16")
+            data = data.astype(np.float16)
+            l_type = 1
+    else:
+        l_type = 0
+
+    # header
+    str = name.encode('utf-8')
+    fout.write(struct.pack("iii", n_dims, len(str), l_type))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str);
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/scripts/convert_falcon_hf_to_ggml.py

@@ -0,0 +1,143 @@
+# Based on: https://github.com/KerfuffleV2/ggml-falcon/blob/feat-improve-falcon-convert-hf/examples/falcon/convert-hf-to-ggml.py
+# Convert Hugging Face fine-tuned bloom-like models to ggml format
+#
+# Usage:
+#
+#   python3 convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]
+#
+# This script is similar to "convert-pt-to-ggml.py"
+#
+
+import io
+import os
+import sys
+import struct
+import json
+import code
+import torch
+import numpy as np
+import gc
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
+
+# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a significant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+if len(sys.argv) < 3:
+    print("INFO: GGML V1 files produced are meant to be finalized through examples/falcon_quantize which will bring them to latest version and precision of choice");
+    print("Usage: python convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]")
+    print("  model_directory: name of the directory and model you convert (it should be a subdirectory)")
+    print("  output-directory: directory where the output file will be written")
+    print("  use-f32:    if present, use float32 instead of float16 (f32 is recommended)")
+    sys.exit(1)
+
+# num_parts = int(sys.argv[1])
+dir_model = sys.argv[1] # name and dir of model
+dir_out = sys.argv[2]   # output directory
+
+# make sure the output directory exists
+os.makedirs(dir_out, exist_ok=True)
+
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+ftype = 1
+if len(sys.argv) > 3:
+    ftype = 0
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+# print(tokenizer)
+config = AutoConfig.from_pretrained(dir_model, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(dir_model, trust_remote_code=True, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)
+hparams = config.to_dict()
+
+n_head = hparams["n_head"]
+n_head_kv = hparams["n_head_kv"] if "n_head_kv" in hparams else 1
+head_dim = hparams["hidden_size"] // n_head
+print("* Loading model from: ", dir_model)
+
+fname_out = dir_out + f"/ggml-model-{dir_model.split('/')[-1]}-{ftype_str[ftype]}.bin"
+fout = open(fname_out, "wb")
+fout.write(struct.pack("i", 0x67676a74)) # magic: ggmf in hex (version 1) - possibly change to ggfc ?
+fout.write(struct.pack("i", 1)) # version
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["hidden_size"]))
+fout.write(struct.pack("i", n_head))
+fout.write(struct.pack("i", n_head_kv))
+fout.write(struct.pack("i", hparams["n_layer"]))
+fout.write(struct.pack("i", 40 if "n_head_kv" in hparams else 7)) # obsolete field that breaks ggml compatibility - todo again remove one day
+fout.write(struct.pack("i", ftype))
+
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+byte_encoder = bytes_to_unicode()
+byte_decoder = {v:k for k, v in byte_encoder.items()}
+
+for i in range(hparams["vocab_size"]):
+    text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+    fout.write(struct.pack("i", len(text)))
+    fout.write(text)
+    fout.write(struct.pack("f", 0.0)) # falcon uses bpe on RefinedWeb - no probability scores used
+
+model = model.state_dict()
+for name in model.keys():
+    src = name
+    # The original query_key_value tensor contains n_head_kv "kv groups",
+    # each consisting of n_head/n_head_kv query weights followed by one key
+    # and one value weight (shared by all query heads in the kv group).
+    # This layout makes it a big pain to work with in GGML.
+    # So we rearrange them here,, so that we have n_head query weights
+    # followed by n_head_kv key weights followed by n_head_kv value weights,
+    # in contiguous fashion.
+
+    if "query_key_value" in src:
+        qkv = model[src].view(
+            n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)
+
+        q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head)
+        k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)
+        v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)
+
+        model[src] = torch.cat((q,k,v)).reshape_as(model[src])
+    data = model[src].squeeze()
+    n_dims = len(data.shape)
+    # default type is fp32
+    ftype_cur = 1 if ftype == 1 and n_dims > 1 else 0
+    data = data.to(dtype = torch.float16 if ftype_cur == 1 else torch.float32).numpy()
+    print(f'  |', name, data.shape, '->', data.dtype)
+    # header
+    str = name.encode('utf-8')
+    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str)
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/scripts/convert_replit_hf_to_ggml.py

@@ -0,0 +1,113 @@
+from pathlib import Path
+import sys
+import struct
+import json
+import numpy as np
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import sentencepiece.sentencepiece_model_pb2 as model
+
+if len(sys.argv) < 3:
+    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+
+# output in the same directory as the model
+dir_model = sys.argv[1]
+fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b.bin"
+
+
+with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
+    hparams = json.load(f)
+
+sp_proto = model.ModelProto()
+sp_proto.ParseFromString(open(Path(sys.argv[1]) / "spiece.model", "rb").read())
+
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+    fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".bin"
+
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    dir_model, low_cpu_mem_usage=True, trust_remote_code=True
+)
+# print (model)
+
+# print(tokenizer.encode('I believe the meaning of life is'))
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+fout = open(fname_out, "wb")
+
+print(hparams)
+
+fout.write(struct.pack("i", 0x7265706c))  # magic: repl in hex
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["max_seq_len"]))
+fout.write(struct.pack("i", hparams["d_model"]))
+fout.write(struct.pack("i", hparams["n_heads"]))
+fout.write(struct.pack("i", hparams["n_layers"]))
+fout.write(struct.pack("i", ftype))
+
+
+# TODO: temporary hack to not deal with implementing the tokenizer
+for piece in sp_proto.pieces:
+    encoded_piece = piece.piece.encode("utf-8")
+    fout.write(struct.pack("i", len(encoded_piece)))
+    fout.write(encoded_piece)
+    fout.write(struct.pack("f", piece.score))
+
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable: " + name + " with shape: ", data.shape)
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0
+    if ftype != 0:
+        if name[-7:] == ".weight" and n_dims == 2:
+            print("  Converting to float16")
+            data = data.astype(np.float16)
+            ftype_cur = 1
+        else:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+    else:
+        if data.dtype != np.float32:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+
+    # header
+    str = name.encode("utf-8")
+    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str)
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/starcoder_impl.h

@@ -0,0 +1,42 @@
+#ifndef STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of starcoder.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef STARCODER_H
+#define STARCODER_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <memory>
+#include "llmodel.h"
+
+struct StarcoderPrivate;
+class Starcoder : public LLModel {
+public:
+    Starcoder();
+    ~Starcoder();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    std::unique_ptr<StarcoderPrivate> d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // STARCODER_H

gpt4all-backend/sysinfo.h

@@ -0,0 +1,61 @@
+#ifndef SYSINFO_H
+#define SYSINFO_H
+
+#include <fstream>
+#include <string>
+#include <sstream>
+#include <iomanip>
+
+#if defined(__linux__)
+#include <unistd.h>
+#elif defined(__APPLE__)
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#elif defined(_WIN32)
+#include <windows.h>
+#endif
+
+static long long getSystemTotalRAMInBytes()
+{
+    long long totalRAM = 0;
+
+#if defined(__linux__)
+    std::ifstream file("/proc/meminfo");
+    std::string line;
+    while (std::getline(file, line)) {
+        if (line.find("MemTotal") != std::string::npos) {
+            std::string memTotalStr = line.substr(line.find(":") + 1);
+            memTotalStr.erase(0, memTotalStr.find_first_not_of(" "));
+            memTotalStr = memTotalStr.substr(0, memTotalStr.find(" "));
+            totalRAM = std::stoll(memTotalStr) * 1024;  // Convert from KB to bytes
+            break;
+        }
+    }
+    file.close();
+#elif defined(__APPLE__)
+    int mib[2] = {CTL_HW, HW_MEMSIZE};
+    size_t length = sizeof(totalRAM);
+    sysctl(mib, 2, &totalRAM, &length, NULL, 0);
+#elif defined(_WIN32)
+    MEMORYSTATUSEX memoryStatus;
+    memoryStatus.dwLength = sizeof(memoryStatus);
+    GlobalMemoryStatusEx(&memoryStatus);
+    totalRAM = memoryStatus.ullTotalPhys;
+#endif
+
+    return totalRAM;
+}
+
+static double getSystemTotalRAMInGB()
+{
+    return static_cast<double>(getSystemTotalRAMInBytes()) / (1024 * 1024 * 1024);
+}
+
+static std::string getSystemTotalRAMInGBString()
+{
+    std::stringstream ss;
+    ss << std::fixed << std::setprecision(2) << getSystemTotalRAMInGB() << " GB";
+    return ss.str();
+}
+
+#endif // SYSINFO_H

gpt4all-backend/utils.cpp

@@ -230,8 +230,21 @@ gpt_vocab::id gpt_sample_top_k_top_p(
     int n_logits = actualVocabSize;
 
     const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_size);
-    const auto * plogits = logits.data() + logits.size() - n_logits;
+    const auto * plogits = logits.data();
 
+    if (temp <= 0) {
+        // select the token with the highest logit directly
+        float max_logit = plogits[0];
+        gpt_vocab::id max_id = 0;
+
+        for (int i = 1; i < n_logits; ++i) {
+            if (plogits[i] > max_logit) {
+                max_logit = plogits[i];
+                max_id = i;
+            }
+        }
+        return max_id;
+    }
     std::vector<std::pair<double, gpt_vocab::id>> logits_id;
     logits_id.reserve(n_logits);
 

gpt4all-backend/utils.h

@@ -8,6 +8,13 @@
 #include <random>
 #include <thread>
 
+//
+// General purpose inline functions
+//
+constexpr inline unsigned long long operator ""_MiB(unsigned long long bytes) {
+    return bytes*1024*1024;
+}
+
 //
 // CLI argument parsing
 //

gpt4all-bindings/cli/README.md

@@ -0,0 +1,44 @@
+# GPT4All Command-Line Interface (CLI)
+
+GPT4All on the command-line.
+
+## Documentation
+<https://docs.gpt4all.io/gpt4all_cli.html>
+
+## Quickstart
+
+The CLI is based on the `gpt4all` Python bindings and the `typer` package.
+
+The following shows one way to get started with the CLI, the documentation has more information.
+Typically, you will want to replace `python` with `python3` on _Unix-like_ systems and `py -3` on
+_Windows_. Also, it's assumed you have all the necessary Python components already installed.
+
+The CLI is a self-contained Python script named [app.py] ([download][app.py-download]). As long as
+its package dependencies are present, you can download and run it from wherever you like.
+
+[app.py]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-bindings/cli/app.py
+[app.py-download]: https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-bindings/cli/app.py
+
+```shell
+# optional but recommended: create and use a virtual environment
+python -m venv gpt4all-cli
+```
+_Windows_ and _Unix-like_ systems differ slightly in how you activate a _virtual environment_:
+- _Unix-like_, typically: `. gpt4all-cli/bin/activate`
+- _Windows_: `gpt4all-cli\Scripts\activate`
+
+Then:
+```shell
+# pip-install the necessary packages; omit '--user' if using a virtual environment
+python -m pip install --user --upgrade gpt4all typer
+# run the CLI
+python app.py repl
+```
+By default, it will automatically download the `groovy` model to `.cache/gpt4all/` in your user
+directory, if necessary.
+
+If you have already saved a model beforehand, specify its path with the `-m`/`--model` argument,
+for example:
+```shell
+python app.py repl --model /home/user/my-gpt4all-models/GPT4All-13B-snoozy.ggmlv3.q4_0.bin
+```

gpt4all-bindings/cli/app.py

@@ -1,9 +1,19 @@
+"""GPT4All CLI
+
+The GPT4All CLI is a self-contained script based on the `gpt4all` and `typer` packages. It offers a
+REPL to communicate with a language model similar to the chat GUI application, but more basic.
+"""
+
+import io
+import pkg_resources  # should be present as a dependency of gpt4all
 import sys
 import typer
 
+from collections import namedtuple
 from typing_extensions import Annotated
 from gpt4all import GPT4All
 
+
 MESSAGES = [
     {"role": "system", "content": "You are a helpful assistant."},
     {"role": "user", "content": "Hello there."},
@@ -17,7 +27,9 @@ SPECIAL_COMMANDS = {
     "/help": lambda _: print("Special commands: /reset, /exit, /help and /clear"),
 }
 
-VERSION = "0.1.0"
+VersionInfo = namedtuple('VersionInfo', ['major', 'minor', 'micro'])
+VERSION_INFO = VersionInfo(1, 0, 2)
+VERSION = '.'.join(map(str, VERSION_INFO))  # convert to string form, like: '1.2.3'
 
 CLI_START_MESSAGE = f"""
     
@@ -33,12 +45,6 @@ Type /help for special commands.
                                                     
 """
 
-def _cli_override_response_callback(token_id, response):
-    resp = response.decode("utf-8")
-    print(resp, end="", flush=True)
-    return True
-
-
 # create typer app
 app = typer.Typer()
 
@@ -53,6 +59,7 @@ def repl(
         typer.Option("--n-threads", "-t", help="Number of threads to use for chatbot"),
     ] = None,
 ):
+    """The CLI read-eval-print loop."""
     gpt4all_instance = GPT4All(model)
 
     # if threads are passed, set them
@@ -68,11 +75,23 @@ def repl(
     else:
         print(f"\nUsing {gpt4all_instance.model.thread_count()} threads", end="")
 
-    # overwrite _response_callback on model
-    gpt4all_instance.model._response_callback = _cli_override_response_callback
-
     print(CLI_START_MESSAGE)
 
+    use_new_loop = False
+    try:
+        version = pkg_resources.Environment()['gpt4all'][0].version
+        version_major = int(version.split('.')[0])
+        if version_major >= 1:
+            use_new_loop = True
+    except:
+        pass  # fall back to old loop
+    if use_new_loop:
+        _new_loop(gpt4all_instance)
+    else:
+        _old_loop(gpt4all_instance)
+
+
+def _old_loop(gpt4all_instance):
     while True:
         message = input(" ⇢  ")
 
@@ -103,16 +122,58 @@ def repl(
             context_erase=0.0,
             # required kwargs for cli ux (incremental response)
             verbose=False,
-            std_passthrough=True,
+            streaming=True,
         )
         # record assistant's response to messages
         MESSAGES.append(full_response.get("choices")[0].get("message"))
         print() # newline before next prompt
 
 
+def _new_loop(gpt4all_instance):
+    with gpt4all_instance.chat_session():
+        while True:
+            message = input(" ⇢  ")
+
+            # Check if special command and take action
+            if message in SPECIAL_COMMANDS:
+                SPECIAL_COMMANDS[message](MESSAGES)
+                continue
+
+            # if regular message, append to messages
+            MESSAGES.append({"role": "user", "content": message})
+
+            # execute chat completion and ignore the full response since 
+            # we are outputting it incrementally
+            response_generator = gpt4all_instance.generate(
+                message,
+                # preferential kwargs for chat ux
+                max_tokens=200,
+                temp=0.9,
+                top_k=40,
+                top_p=0.9,
+                repeat_penalty=1.1,
+                repeat_last_n=64,
+                n_batch=9,
+                # required kwargs for cli ux (incremental response)
+                streaming=True,
+            )
+            response = io.StringIO()
+            for token in response_generator:
+                print(token, end='', flush=True)
+                response.write(token)
+
+            # record assistant's response to messages
+            response_message = {'role': 'assistant', 'content': response.getvalue()}
+            response.close()
+            gpt4all_instance.current_chat_session.append(response_message)
+            MESSAGES.append(response_message)
+            print() # newline before next prompt
+
+
 @app.command()
 def version():
-    print("gpt4all-cli v0.1.0")
+    """The CLI version command."""
+    print(f"gpt4all-cli v{VERSION}")
 
 
 if __name__ == "__main__":

gpt4all-bindings/cli/developer_notes.md

@@ -0,0 +1,25 @@
+# Developing the CLI
+## Documentation
+Documentation can be found in three places:
+- `app.py` docstrings & comments
+- a Readme: `gpt4all-bindings/cli/README.md`
+- the actual CLI documentation: `gpt4all-bindings/python/docs/gpt4all_cli.md`
+
+The _docstrings_ are meant for programmatic use. Since the CLI is primarily geared towards users and
+not to build on top, they're kept terse.
+
+The _Readme_ is mostly meant for users and includes:
+- a link to the _CLI documentation_ (on the [website])
+- a Quickstart section with some guidance on how to get started with a sane setup
+
+The _CLI documentation_ and other documentation are located in the above mentioned `docs/` folder.
+They're in Markdown format and built for the [website]. Of the three, they should be the most
+detailed.
+
+[website]: https://docs.gpt4all.io/gpt4all_cli.html
+
+
+## Versioning
+The version number should now follow the `gpt4all` PyPI package, so compatibility is more clear.
+
+The one place to change it is the `namedtuple` called `VERSION_INFO`.

gpt4all-bindings/csharp/Directory.Build.props

@@ -5,7 +5,7 @@
         <Company></Company>
         <Copyright></Copyright>
         <NeutralLanguage>en-US</NeutralLanguage>
-        <Version>0.6.1-alpha</Version>
+        <Version>0.6.3-alpha</Version>
         <VersionSuffix>$(VersionSuffix)</VersionSuffix>
         <Version Condition=" '$(VersionSuffix)' != '' ">$(Version)$(VersionSuffix)</Version>
         <TreatWarningsAsErrors>true</TreatWarningsAsErrors>

gpt4all-bindings/csharp/Gpt4All.Samples/Gpt4All.Samples.csproj

@@ -1,18 +1,32 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net7.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
-  </ItemGroup>
+    <ItemGroup>
+        <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
+    </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Properties\" />
-  </ItemGroup>
+    <ItemGroup>
+        <!-- Windows -->
+        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
+        <!-- Linux -->
+        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
+        <!-- MacOS -->
+        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
+    </ItemGroup>
 
+    <ItemGroup>
+        <!-- Windows -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+        <!-- Linux -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+         <!-- MacOS -->
+        <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+        <Content Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.metal" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+    </ItemGroup>
 </Project>

gpt4all-bindings/csharp/Gpt4All.Tests/Constants.cs

@@ -1,10 +1,9 @@
-namespace Gpt4All.Tests
+namespace Gpt4All.Tests;
+
+public static class Constants
 {
-    public static class Constants
-    {
-        public const string MODELS_BASE_DIR = "../../../models";
-        public const string LLAMA_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-l13b-snoozy.bin";
-        public const string GPTJ_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-j-v1.3-groovy.bin";
-        public const string MPT_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-mpt-7b-chat.bin";
-    }
+    public const string MODELS_BASE_DIR = "../../../models";
+    public const string LLAMA_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-l13b-snoozy.bin";
+    public const string GPTJ_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-j-v1.3-groovy.bin";
+    public const string MPT_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-mpt-7b-chat.bin";
 }

gpt4all-bindings/csharp/Gpt4All.Tests/Gpt4All.Tests.csproj

@@ -1,27 +1,59 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net6.0</TargetFramework>
+        <TargetFramework>net7.0</TargetFramework>
         <Nullable>enable</Nullable>
 
         <IsPackable>false</IsPackable>
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
-        <PackageReference Include="xunit" Version="2.4.1" />
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2" />
+        <PackageReference Include="xunit" Version="2.4.2" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             <PrivateAssets>all</PrivateAssets>
         </PackageReference>
-        <PackageReference Include="coverlet.collector" Version="3.1.0">
+        <PackageReference Include="coverlet.collector" Version="6.0.0">
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             <PrivateAssets>all</PrivateAssets>
         </PackageReference>
     </ItemGroup>
 
     <ItemGroup>
-      <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
+        <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+        <!-- Windows -->
+        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
+        <!-- Linux -->
+        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
+        <!-- MacOS -->
+        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <!-- Windows -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+        <!-- Linux -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+         <!-- MacOS -->
+        <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Update="Roslynator.Analyzers" Version="4.3.0">
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+      </PackageReference>
+      <PackageReference Update="Roslynator.CodeAnalysis.Analyzers" Version="4.3.0">
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+      </PackageReference>
+      <PackageReference Update="Roslynator.Formatting.Analyzers" Version="4.3.0">
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+      </PackageReference>
+    </ItemGroup>
 </Project>

gpt4all-bindings/csharp/Gpt4All.Tests/ModelFactoryTests.cs

@@ -1,4 +1,4 @@
-using Xunit;
+using Xunit;
 
 namespace Gpt4All.Tests;
 
@@ -12,20 +12,23 @@ public class ModelFactoryTests
     }
 
     [Fact]
+    [Trait(Traits.SkipOnCI, "True")]
     public void CanLoadLlamaModel()
     {
-        using var model = _modelFactory.LoadLlamaModel(Constants.LLAMA_MODEL_PATH);
+        using var model = _modelFactory.LoadModel(Constants.LLAMA_MODEL_PATH);
     }
 
     [Fact]
+    [Trait(Traits.SkipOnCI, "True")]
     public void CanLoadGptjModel()
     {
-        using var model = _modelFactory.LoadGptjModel(Constants.GPTJ_MODEL_PATH);
+        using var model = _modelFactory.LoadModel(Constants.GPTJ_MODEL_PATH);
     }
 
     [Fact]
+    [Trait(Traits.SkipOnCI, "True")]
     public void CanLoadMptModel()
     {
-        using var model = _modelFactory.LoadMptModel(Constants.MPT_MODEL_PATH);
+        using var model = _modelFactory.LoadModel(Constants.MPT_MODEL_PATH);
     }
 }

gpt4all-bindings/csharp/Gpt4All.Tests/NativeLibraryLoaderTests.cs

@@ -0,0 +1,56 @@
+using System.IO;
+using Gpt4All.LibraryLoader;
+using Xunit;
+
+namespace Gpt4All.Tests;
+
+public class NativeLibraryLoaderTests
+{
+    [Fact]
+    public void NativeLibraryShouldLoad()
+    {
+        var result = NativeLibraryLoader.LoadNativeLibrary(bypassLoading: false);
+        Assert.True(result.IsSuccess);
+    }
+
+    private const string LLModelLib = "libllmodel.{0}";
+
+    [PlatformSpecificFact(Platforms.Windows)]
+    public void NativeLibraryShouldLoad_Windows()
+    {
+        var libraryLoader = new WindowsLibraryLoader();
+
+        var libraryPath = Path.Combine(
+            Environment.CurrentDirectory,
+            string.Format(LLModelLib, "dll"));
+
+        var result = libraryLoader.OpenLibrary(libraryPath);
+        Assert.True(result.IsSuccess);
+    }
+
+    [PlatformSpecificFact(Platforms.Linux)]
+    public void NativeLibraryShouldLoad_Linux()
+    {
+        var libraryLoader = new LinuxLibraryLoader();
+
+        var libraryPath = Path.Combine(
+            Environment.CurrentDirectory,
+            string.Format(LLModelLib, "so"));
+
+        var result = libraryLoader.OpenLibrary(libraryPath);
+        Assert.True(result.IsSuccess);
+    }
+
+    [PlatformSpecificFact(Platforms.MacOS)]
+    public void NativeLibraryShouldLoad_MacOS()
+    {
+        var libraryLoader = new MacOsLibraryLoader();
+
+        var libraryPath = Path.Combine(
+            Environment.CurrentDirectory,
+            string.Format(LLModelLib, "dylib"));
+
+        var result = libraryLoader.OpenLibrary(libraryPath);
+        Assert.True(result.IsSuccess);
+    }
+}

gpt4all-bindings/csharp/Gpt4All.Tests/PlatformSpecificFactAttribute.cs

@@ -0,0 +1,27 @@
+using Xunit;
+
+namespace Gpt4All.Tests;
+
+public static class Platforms
+{
+    public const string Windows = "windows";
+    public const string Linux = "linux";
+    public const string MacOS = "macOS";
+}
+
+/// <summary>
+/// This attribute ensures the Fact is only run on the specified platform.
+/// </summary>
+/// <remarks>
+/// <see cref="OperatingSystem.IsOSPlatform(string)"/> for info about the platform string.
+/// </remarks>
+public class PlatformSpecificFactAttribute : FactAttribute
+{
+    public PlatformSpecificFactAttribute(string platform)
+    {
+        if (!OperatingSystem.IsOSPlatform(platform))
+        {
+            Skip = $"Test only runs on {platform}.";
+        }
+    }
+}

gpt4all-bindings/csharp/Gpt4All.Tests/Traits.cs

@@ -0,0 +1,6 @@
+namespace Gpt4All.Tests;
+
+public static class Traits
+{
+    public const string SkipOnCI = "SKIP_ON_CI";
+}

gpt4all-bindings/csharp/Gpt4All/Bindings/LLModel.cs

@@ -1,247 +1,222 @@
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
-
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Arguments for the response processing callback
-/// </summary>
-/// <param name="TokenId">The token id of the response</param>
-/// <param name="Response"> The response string. NOTE: a token_id of -1 indicates the string is an error string</param>
-/// <return>
-/// A bool indicating whether the model should keep generating
-/// </return>
-public record ModelResponseEventArgs(int TokenId, string Response)
-{
-    public bool IsError => TokenId == -1;
-}
-
-/// <summary>
-/// Arguments for the prompt processing callback
-/// </summary>
-/// <param name="TokenId">The token id of the prompt</param>
-/// <return>
-/// A bool indicating whether the model should keep processing
-/// </return>
-public record ModelPromptEventArgs(int TokenId)
-{
-}
-
-/// <summary>
-/// Arguments for the recalculating callback
-/// </summary>
-/// <param name="IsRecalculating"> whether the model is recalculating the context.</param>
-/// <return>
-/// A bool indicating whether the model should keep generating
-/// </return>
-public record ModelRecalculatingEventArgs(bool IsRecalculating);
-
-/// <summary>
-/// Base class and universal wrapper for GPT4All language models built around llmodel C-API.
-/// </summary>
-public class LLModel : ILLModel
-{
-    protected readonly IntPtr _handle;
-    private readonly ModelType _modelType;
-    private readonly ILogger _logger;
-    private bool _disposed;
-
-    public ModelType ModelType => _modelType;
-
-    internal LLModel(IntPtr handle, ModelType modelType, ILogger? logger = null)
-    {
-        _handle = handle;
-        _modelType = modelType;
-        _logger = logger ?? NullLogger.Instance;
-    }
-
-    /// <summary>
-    /// Create a new model from a pointer
-    /// </summary>
-    /// <param name="handle">Pointer to underlying model</param>
-    /// <param name="modelType">The model type</param>
-    public static LLModel Create(IntPtr handle, ModelType modelType, ILogger? logger = null)
-    {
-        return new LLModel(handle, modelType, logger: logger);
-    }
-
-    /// <summary>
-    /// Generate a response using the model
-    /// </summary>
-    /// <param name="text">The input promp</param>
-    /// <param name="context">The context</param>
-    /// <param name="promptCallback">A callback function for handling the processing of prompt</param>
-    /// <param name="responseCallback">A callback function for handling the generated response</param>
-    /// <param name="recalculateCallback">A callback function for handling recalculation requests</param>
-    /// <param name="cancellationToken"></param>
-    public void Prompt(
-        string text,
-        LLModelPromptContext context,
-        Func<ModelPromptEventArgs, bool>? promptCallback = null,
-        Func<ModelResponseEventArgs, bool>? responseCallback = null,
-        Func<ModelRecalculatingEventArgs, bool>? recalculateCallback = null,
-        CancellationToken cancellationToken = default)
-    {
-        GC.KeepAlive(promptCallback);
-        GC.KeepAlive(responseCallback);
-        GC.KeepAlive(recalculateCallback);
-        GC.KeepAlive(cancellationToken);
-
-        _logger.LogInformation("Prompt input='{Prompt}' ctx={Context}", text, context.Dump());
-
-        NativeMethods.llmodel_prompt(
-            _handle,
-            text,
-            (tokenId) =>
-            {
-                if (cancellationToken.IsCancellationRequested) return false;
-                if (promptCallback == null) return true;
-                var args = new ModelPromptEventArgs(tokenId);
-                return promptCallback(args);
-            },
-            (tokenId, response) =>
-            {
-                if (cancellationToken.IsCancellationRequested)
-                {
-                    _logger.LogDebug("ResponseCallback evt=CancellationRequested");
-                    return false;
-                }
-
-                if (responseCallback == null) return true;
-                var args = new ModelResponseEventArgs(tokenId, response);
-                return responseCallback(args);
-            },
-            (isRecalculating) =>
-            {
-                if (cancellationToken.IsCancellationRequested) return false;
-                if (recalculateCallback == null) return true;
-                var args = new ModelRecalculatingEventArgs(isRecalculating);
-                return recalculateCallback(args);
-            },
-            ref context.UnderlyingContext
-        );
-    }
-
-    /// <summary>
-    ///  Set the number of threads to be used by the model.
-    /// </summary>
-    /// <param name="threadCount">The new thread count</param>
-    public void SetThreadCount(int threadCount)
-    {
-        NativeMethods.llmodel_setThreadCount(_handle, threadCount);
-    }
-
-    /// <summary>
-    /// Get  the number of threads used by the model.
-    /// </summary>
-    /// <returns>the number of threads used by the model</returns>
-    public int GetThreadCount()
-    {
-        return NativeMethods.llmodel_threadCount(_handle);
-    }
-
-    /// <summary>
-    /// Get the size of the internal state of the model.
-    /// </summary>
-    /// <remarks>
-    /// This state data is specific to the type of model you have created.
-    /// </remarks>
-    /// <returns>the size in bytes of the internal state of the model</returns>
-    public ulong GetStateSizeBytes()
-    {
-        return NativeMethods.llmodel_get_state_size(_handle);
-    }
-
-    /// <summary>
-    /// Saves the internal state of the model to the specified destination address.
-    /// </summary>
-    /// <param name="source">A pointer to the src</param>
-    /// <returns>The number of bytes copied</returns>
-    public unsafe ulong SaveStateData(byte* source)
-    {
-        return NativeMethods.llmodel_save_state_data(_handle, source);
-    }
-
-    /// <summary>
-    /// Restores the internal state of the model using data from the specified address.
-    /// </summary>
-    /// <param name="destination">A pointer to destination</param>
-    /// <returns>the number of bytes read</returns>
-    public unsafe ulong RestoreStateData(byte* destination)
-    {
-        return NativeMethods.llmodel_restore_state_data(_handle, destination);
-    }
-
-    /// <summary>
-    /// Check if the model is loaded.
-    /// </summary>
-    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
-    public bool IsLoaded()
-    {
-        return NativeMethods.llmodel_isModelLoaded(_handle);
-    }
-
-    /// <summary>
-    /// Load the model from a file.
-    /// </summary>
-    /// <param name="modelPath">The path to the model file.</param>
-    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
-    public bool Load(string modelPath)
-    {
-        return NativeMethods.llmodel_loadModel(_handle, modelPath);
-    }
-
-    protected void Destroy()
-    {
-        NativeMethods.llmodel_model_destroy(_handle);
-    }
-
-    protected void DestroyLLama()
-    {
-        NativeMethods.llmodel_llama_destroy(_handle);
-    }
-
-    protected void DestroyGptj()
-    {
-        NativeMethods.llmodel_gptj_destroy(_handle);
-    }
-
-    protected void DestroyMtp()
-    {
-        NativeMethods.llmodel_mpt_destroy(_handle);
-    }
-
-    protected virtual void Dispose(bool disposing)
-    {
-        if (_disposed) return;
-
-        if (disposing)
-        {
-            // dispose managed state
-        }
-
-        switch (_modelType)
-        {
-            case ModelType.LLAMA:
-                DestroyLLama();
-                break;
-            case ModelType.GPTJ:
-                DestroyGptj();
-                break;
-            case ModelType.MPT:
-                DestroyMtp();
-                break;
-            default:
-                Destroy();
-                break;
-        }
-
-        _disposed = true;
-    }
-
-    public void Dispose()
-    {
-        Dispose(disposing: true);
-        GC.SuppressFinalize(this);
-    }
-}
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace Gpt4All.Bindings;
+
+/// <summary>
+/// Arguments for the response processing callback
+/// </summary>
+/// <param name="TokenId">The token id of the response</param>
+/// <param name="Response"> The response string. NOTE: a token_id of -1 indicates the string is an error string</param>
+/// <return>
+/// A bool indicating whether the model should keep generating
+/// </return>
+public record ModelResponseEventArgs(int TokenId, string Response)
+{
+    public bool IsError => TokenId == -1;
+}
+
+/// <summary>
+/// Arguments for the prompt processing callback
+/// </summary>
+/// <param name="TokenId">The token id of the prompt</param>
+/// <return>
+/// A bool indicating whether the model should keep processing
+/// </return>
+public record ModelPromptEventArgs(int TokenId)
+{
+}
+
+/// <summary>
+/// Arguments for the recalculating callback
+/// </summary>
+/// <param name="IsRecalculating"> whether the model is recalculating the context.</param>
+/// <return>
+/// A bool indicating whether the model should keep generating
+/// </return>
+public record ModelRecalculatingEventArgs(bool IsRecalculating);
+
+/// <summary>
+/// Base class and universal wrapper for GPT4All language models built around llmodel C-API.
+/// </summary>
+public class LLModel : ILLModel
+{
+    protected readonly IntPtr _handle;
+    private readonly ModelType _modelType;
+    private readonly ILogger _logger;
+    private bool _disposed;
+
+    public ModelType ModelType => _modelType;
+
+    internal LLModel(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    {
+        _handle = handle;
+        _modelType = modelType;
+        _logger = logger ?? NullLogger.Instance;
+    }
+
+    /// <summary>
+    /// Create a new model from a pointer
+    /// </summary>
+    /// <param name="handle">Pointer to underlying model</param>
+    /// <param name="modelType">The model type</param>
+    public static LLModel Create(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    {
+        return new LLModel(handle, modelType, logger: logger);
+    }
+
+    /// <summary>
+    /// Generate a response using the model
+    /// </summary>
+    /// <param name="text">The input promp</param>
+    /// <param name="context">The context</param>
+    /// <param name="promptCallback">A callback function for handling the processing of prompt</param>
+    /// <param name="responseCallback">A callback function for handling the generated response</param>
+    /// <param name="recalculateCallback">A callback function for handling recalculation requests</param>
+    /// <param name="cancellationToken"></param>
+    public void Prompt(
+        string text,
+        LLModelPromptContext context,
+        Func<ModelPromptEventArgs, bool>? promptCallback = null,
+        Func<ModelResponseEventArgs, bool>? responseCallback = null,
+        Func<ModelRecalculatingEventArgs, bool>? recalculateCallback = null,
+        CancellationToken cancellationToken = default)
+    {
+        GC.KeepAlive(promptCallback);
+        GC.KeepAlive(responseCallback);
+        GC.KeepAlive(recalculateCallback);
+        GC.KeepAlive(cancellationToken);
+
+        _logger.LogInformation("Prompt input='{Prompt}' ctx={Context}", text, context.Dump());
+
+        NativeMethods.llmodel_prompt(
+            _handle,
+            text,
+            (tokenId) =>
+            {
+                if (cancellationToken.IsCancellationRequested) return false;
+                if (promptCallback == null) return true;
+                var args = new ModelPromptEventArgs(tokenId);
+                return promptCallback(args);
+            },
+            (tokenId, response) =>
+            {
+                if (cancellationToken.IsCancellationRequested)
+                {
+                    _logger.LogDebug("ResponseCallback evt=CancellationRequested");
+                    return false;
+                }
+
+                if (responseCallback == null) return true;
+                var args = new ModelResponseEventArgs(tokenId, response);
+                return responseCallback(args);
+            },
+            (isRecalculating) =>
+            {
+                if (cancellationToken.IsCancellationRequested) return false;
+                if (recalculateCallback == null) return true;
+                var args = new ModelRecalculatingEventArgs(isRecalculating);
+                return recalculateCallback(args);
+            },
+            ref context.UnderlyingContext
+        );
+    }
+
+    /// <summary>
+    ///  Set the number of threads to be used by the model.
+    /// </summary>
+    /// <param name="threadCount">The new thread count</param>
+    public void SetThreadCount(int threadCount)
+    {
+        NativeMethods.llmodel_setThreadCount(_handle, threadCount);
+    }
+
+    /// <summary>
+    /// Get  the number of threads used by the model.
+    /// </summary>
+    /// <returns>the number of threads used by the model</returns>
+    public int GetThreadCount()
+    {
+        return NativeMethods.llmodel_threadCount(_handle);
+    }
+
+    /// <summary>
+    /// Get the size of the internal state of the model.
+    /// </summary>
+    /// <remarks>
+    /// This state data is specific to the type of model you have created.
+    /// </remarks>
+    /// <returns>the size in bytes of the internal state of the model</returns>
+    public ulong GetStateSizeBytes()
+    {
+        return NativeMethods.llmodel_get_state_size(_handle);
+    }
+
+    /// <summary>
+    /// Saves the internal state of the model to the specified destination address.
+    /// </summary>
+    /// <param name="source">A pointer to the src</param>
+    /// <returns>The number of bytes copied</returns>
+    public unsafe ulong SaveStateData(byte* source)
+    {
+        return NativeMethods.llmodel_save_state_data(_handle, source);
+    }
+
+    /// <summary>
+    /// Restores the internal state of the model using data from the specified address.
+    /// </summary>
+    /// <param name="destination">A pointer to destination</param>
+    /// <returns>the number of bytes read</returns>
+    public unsafe ulong RestoreStateData(byte* destination)
+    {
+        return NativeMethods.llmodel_restore_state_data(_handle, destination);
+    }
+
+    /// <summary>
+    /// Check if the model is loaded.
+    /// </summary>
+    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
+    public bool IsLoaded()
+    {
+        return NativeMethods.llmodel_isModelLoaded(_handle);
+    }
+
+    /// <summary>
+    /// Load the model from a file.
+    /// </summary>
+    /// <param name="modelPath">The path to the model file.</param>
+    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
+    public bool Load(string modelPath)
+    {
+        return NativeMethods.llmodel_loadModel(_handle, modelPath);
+    }
+
+    protected void Destroy()
+    {
+        NativeMethods.llmodel_model_destroy(_handle);
+    }
+    protected virtual void Dispose(bool disposing)
+    {
+        if (_disposed) return;
+
+        if (disposing)
+        {
+            // dispose managed state
+        }
+
+        switch (_modelType)
+        {
+            default:
+                Destroy();
+                break;
+        }
+
+        _disposed = true;
+    }
+
+    public void Dispose()
+    {
+        Dispose(disposing: true);
+        GC.SuppressFinalize(this);
+    }
+}

gpt4all-bindings/csharp/Gpt4All/Bindings/LLPromptContext.cs

@@ -1,138 +1,138 @@
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Wrapper around the llmodel_prompt_context structure for holding the prompt context.
-/// </summary>
-/// <remarks>
-/// The implementation takes care of all the memory handling of the raw logits pointer and the
-/// raw tokens pointer.Attempting to resize them or modify them in any way can lead to undefined behavior
-/// </remarks>
-public unsafe class LLModelPromptContext
-{
-    private llmodel_prompt_context _ctx;
-
-    internal ref llmodel_prompt_context UnderlyingContext => ref _ctx;
-
-    public LLModelPromptContext()
-    {
-        _ctx = new();
-    }
-
-    /// <summary>
-    /// logits of current context
-    /// </summary>
-    public Span<float> Logits => new(_ctx.logits, (int)_ctx.logits_size);
-
-    /// <summary>
-    /// the size of the raw logits vector
-    /// </summary>
-    public nuint LogitsSize
-    {
-        get => _ctx.logits_size;
-        set => _ctx.logits_size = value;
-    }
-
-    /// <summary>
-    /// current tokens in the context window
-    /// </summary>
-    public Span<int> Tokens => new(_ctx.tokens, (int)_ctx.tokens_size);
-
-    /// <summary>
-    /// the size of the raw tokens vector
-    /// </summary>
-    public nuint TokensSize
-    {
-        get => _ctx.tokens_size;
-        set => _ctx.tokens_size = value;
-    }
-
-    /// <summary>
-    /// top k logits to sample from
-    /// </summary>
-    public int TopK
-    {
-        get => _ctx.top_k;
-        set => _ctx.top_k = value;
-    }
-
-    /// <summary>
-    /// nucleus sampling probability threshold
-    /// </summary>
-    public float TopP
-    {
-        get => _ctx.top_p;
-        set => _ctx.top_p = value;
-    }
-
-    /// <summary>
-    /// temperature to adjust model's output distribution
-    /// </summary>
-    public float Temperature
-    {
-        get => _ctx.temp;
-        set => _ctx.temp = value;
-    }
-
-    /// <summary>
-    /// number of tokens in past conversation
-    /// </summary>
-    public int PastNum
-    {
-        get => _ctx.n_past;
-        set => _ctx.n_past = value;
-    }
-
-    /// <summary>
-    /// number of predictions to generate in parallel
-    /// </summary>
-    public int Batches
-    {
-        get => _ctx.n_batch;
-        set => _ctx.n_batch = value;
-    }
-
-    /// <summary>
-    /// number of tokens to predict
-    /// </summary>
-    public int TokensToPredict
-    {
-        get => _ctx.n_predict;
-        set => _ctx.n_predict = value;
-    }
-
-    /// <summary>
-    /// penalty factor for repeated tokens
-    /// </summary>
-    public float RepeatPenalty
-    {
-        get => _ctx.repeat_penalty;
-        set => _ctx.repeat_penalty = value;
-    }
-
-    /// <summary>
-    /// last n tokens to penalize
-    /// </summary>
-    public int RepeatLastN
-    {
-        get => _ctx.repeat_last_n;
-        set => _ctx.repeat_last_n = value;
-    }
-
-    /// <summary>
-    /// number of tokens possible in context window
-    /// </summary>
-    public int ContextSize
-    {
-        get => _ctx.n_ctx;
-        set => _ctx.n_ctx = value;
-    }
-
-    /// <summary>
-    /// percent of context to erase if we exceed the context window
-    /// </summary>
-    public float ContextErase
-    {
-        get => _ctx.context_erase;
-        set => _ctx.context_erase = value;
-    }
-}
+namespace Gpt4All.Bindings;
+
+/// <summary>
+/// Wrapper around the llmodel_prompt_context structure for holding the prompt context.
+/// </summary>
+/// <remarks>
+/// The implementation takes care of all the memory handling of the raw logits pointer and the
+/// raw tokens pointer.Attempting to resize them or modify them in any way can lead to undefined behavior
+/// </remarks>
+public unsafe class LLModelPromptContext
+{
+    private llmodel_prompt_context _ctx;
+
+    internal ref llmodel_prompt_context UnderlyingContext => ref _ctx;
+
+    public LLModelPromptContext()
+    {
+        _ctx = new();
+    }
+
+    /// <summary>
+    /// logits of current context
+    /// </summary>
+    public Span<float> Logits => new(_ctx.logits, (int)_ctx.logits_size);
+
+    /// <summary>
+    /// the size of the raw logits vector
+    /// </summary>
+    public nuint LogitsSize
+    {
+        get => _ctx.logits_size;
+        set => _ctx.logits_size = value;
+    }
+
+    /// <summary>
+    /// current tokens in the context window
+    /// </summary>
+    public Span<int> Tokens => new(_ctx.tokens, (int)_ctx.tokens_size);
+
+    /// <summary>
+    /// the size of the raw tokens vector
+    /// </summary>
+    public nuint TokensSize
+    {
+        get => _ctx.tokens_size;
+        set => _ctx.tokens_size = value;
+    }
+
+    /// <summary>
+    /// top k logits to sample from
+    /// </summary>
+    public int TopK
+    {
+        get => _ctx.top_k;
+        set => _ctx.top_k = value;
+    }
+
+    /// <summary>
+    /// nucleus sampling probability threshold
+    /// </summary>
+    public float TopP
+    {
+        get => _ctx.top_p;
+        set => _ctx.top_p = value;
+    }
+
+    /// <summary>
+    /// temperature to adjust model's output distribution
+    /// </summary>
+    public float Temperature
+    {
+        get => _ctx.temp;
+        set => _ctx.temp = value;
+    }
+
+    /// <summary>
+    /// number of tokens in past conversation
+    /// </summary>
+    public int PastNum
+    {
+        get => _ctx.n_past;
+        set => _ctx.n_past = value;
+    }
+
+    /// <summary>
+    /// number of predictions to generate in parallel
+    /// </summary>
+    public int Batches
+    {
+        get => _ctx.n_batch;
+        set => _ctx.n_batch = value;
+    }
+
+    /// <summary>
+    /// number of tokens to predict
+    /// </summary>
+    public int TokensToPredict
+    {
+        get => _ctx.n_predict;
+        set => _ctx.n_predict = value;
+    }
+
+    /// <summary>
+    /// penalty factor for repeated tokens
+    /// </summary>
+    public float RepeatPenalty
+    {
+        get => _ctx.repeat_penalty;
+        set => _ctx.repeat_penalty = value;
+    }
+
+    /// <summary>
+    /// last n tokens to penalize
+    /// </summary>
+    public int RepeatLastN
+    {
+        get => _ctx.repeat_last_n;
+        set => _ctx.repeat_last_n = value;
+    }
+
+    /// <summary>
+    /// number of tokens possible in context window
+    /// </summary>
+    public int ContextSize
+    {
+        get => _ctx.n_ctx;
+        set => _ctx.n_ctx = value;
+    }
+
+    /// <summary>
+    /// percent of context to erase if we exceed the context window
+    /// </summary>
+    public float ContextErase
+    {
+        get => _ctx.context_erase;
+        set => _ctx.context_erase = value;
+    }
+}

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeMethods.cs

@@ -1,126 +1,108 @@
-using System.Runtime.InteropServices;
-
-namespace Gpt4All.Bindings;
-
-public unsafe partial struct llmodel_prompt_context
-{
-    public float* logits;
-
-    [NativeTypeName("size_t")]
-    public nuint logits_size;
-
-    [NativeTypeName("int32_t *")]
-    public int* tokens;
-
-    [NativeTypeName("size_t")]
-    public nuint tokens_size;
-
-    [NativeTypeName("int32_t")]
-    public int n_past;
-
-    [NativeTypeName("int32_t")]
-    public int n_ctx;
-
-    [NativeTypeName("int32_t")]
-    public int n_predict;
-
-    [NativeTypeName("int32_t")]
-    public int top_k;
-
-    public float top_p;
-
-    public float temp;
-
-    [NativeTypeName("int32_t")]
-    public int n_batch;
-
-    public float repeat_penalty;
-
-    [NativeTypeName("int32_t")]
-    public int repeat_last_n;
-
-    public float context_erase;
-}
-
-internal static unsafe partial class NativeMethods
-{
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelResponseCallback(int token_id, [MarshalAs(UnmanagedType.LPUTF8Str)] string response);
-
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelPromptCallback(int token_id);
-
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelRecalculateCallback(bool isRecalculating);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_gptj_create();
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_gptj_destroy([NativeTypeName("llmodel_model")] IntPtr gptj);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_mpt_create();
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_mpt_destroy([NativeTypeName("llmodel_model")] IntPtr mpt);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_llama_create();
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_llama_destroy([NativeTypeName("llmodel_model")] IntPtr llama);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_model_create(
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_model_destroy([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public static extern bool llmodel_loadModel(
-        [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-
-    [return: MarshalAs(UnmanagedType.I1)]
-    public static extern bool llmodel_isModelLoaded([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_get_state_size([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_save_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("uint8_t *")] byte* dest);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_restore_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("const uint8_t *")] byte* src);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    public static extern void llmodel_prompt(
-        [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string prompt,
-        LlmodelPromptCallback prompt_callback,
-        LlmodelResponseCallback response_callback,
-        LlmodelRecalculateCallback recalculate_callback,
-        ref llmodel_prompt_context ctx);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_setThreadCount([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("int32_t")] int n_threads);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("int32_t")]
-    public static extern int llmodel_threadCount([NativeTypeName("llmodel_model")] IntPtr model);
-}
+using System.Runtime.InteropServices;
+
+namespace Gpt4All.Bindings;
+
+public unsafe partial struct llmodel_prompt_context
+{
+    public float* logits;
+
+    [NativeTypeName("size_t")]
+    public nuint logits_size;
+
+    [NativeTypeName("int32_t *")]
+    public int* tokens;
+
+    [NativeTypeName("size_t")]
+    public nuint tokens_size;
+
+    [NativeTypeName("int32_t")]
+    public int n_past;
+
+    [NativeTypeName("int32_t")]
+    public int n_ctx;
+
+    [NativeTypeName("int32_t")]
+    public int n_predict;
+
+    [NativeTypeName("int32_t")]
+    public int top_k;
+
+    public float top_p;
+
+    public float temp;
+
+    [NativeTypeName("int32_t")]
+    public int n_batch;
+
+    public float repeat_penalty;
+
+    [NativeTypeName("int32_t")]
+    public int repeat_last_n;
+
+    public float context_erase;
+}
+#pragma warning disable CA2101
+internal static unsafe partial class NativeMethods
+{
+    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public delegate bool LlmodelResponseCallback(int token_id, [MarshalAs(UnmanagedType.LPUTF8Str)] string response);
+
+    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public delegate bool LlmodelPromptCallback(int token_id);
+
+    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public delegate bool LlmodelRecalculateCallback(bool isRecalculating);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
+    [return: NativeTypeName("llmodel_model")]
+    public static extern IntPtr llmodel_model_create2(
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path,
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string build_variant,
+        out IntPtr error);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    public static extern void llmodel_model_destroy([NativeTypeName("llmodel_model")] IntPtr model);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public static extern bool llmodel_loadModel(
+        [NativeTypeName("llmodel_model")] IntPtr model,
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+
+    [return: MarshalAs(UnmanagedType.I1)]
+    public static extern bool llmodel_isModelLoaded([NativeTypeName("llmodel_model")] IntPtr model);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("uint64_t")]
+    public static extern ulong llmodel_get_state_size([NativeTypeName("llmodel_model")] IntPtr model);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("uint64_t")]
+    public static extern ulong llmodel_save_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("uint8_t *")] byte* dest);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("uint64_t")]
+    public static extern ulong llmodel_restore_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("const uint8_t *")] byte* src);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
+    public static extern void llmodel_prompt(
+        [NativeTypeName("llmodel_model")] IntPtr model,
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string prompt,
+        LlmodelPromptCallback prompt_callback,
+        LlmodelResponseCallback response_callback,
+        LlmodelRecalculateCallback recalculate_callback,
+        ref llmodel_prompt_context ctx);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    public static extern void llmodel_setThreadCount([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("int32_t")] int n_threads);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("int32_t")]
+    public static extern int llmodel_threadCount([NativeTypeName("llmodel_model")] IntPtr model);
+}
+#pragma warning restore CA2101

gpt4all-bindings/csharp/Gpt4All/Gpt4All.cs

@@ -1,8 +1,11 @@
 using System.Diagnostics;
+using System.Runtime.CompilerServices;
 using Gpt4All.Bindings;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 
+[assembly: InternalsVisibleTo("Gpt4All.Tests")]
+
 namespace Gpt4All;
 
 public class Gpt4All : IGpt4AllModel

gpt4all-bindings/csharp/Gpt4All/Gpt4All.csproj

@@ -1,27 +1,22 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-	<TargetFrameworks>net6.0</TargetFrameworks>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-  </PropertyGroup>
-
-	<ItemGroup>
-		<!-- Windows -->
-		<None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
-		<!-- Linux -->
-		<None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
-	</ItemGroup>
-
-	<ItemGroup>
-		<!-- Windows -->
-		<None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-		<!-- Linux -->
-		<None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-	</ItemGroup>
-
-	<ItemGroup>
-	  <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
-	</ItemGroup>
+    <PropertyGroup>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    </PropertyGroup>
+    <ItemGroup>
+        <!-- Windows -->
+        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
+        <!-- Linux -->
+        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
+        <!-- MacOS -->
+        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
+        <Content Include="..\runtimes\osx\native\*.metal" Pack="true" PackagePath="contentFiles\any\any;content">
+            <PackageCopyToOutput>true</PackageCopyToOutput>
+        </Content>
+    </ItemGroup>
+    <ItemGroup>
+        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
+    </ItemGroup>
 </Project>

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/ILibraryLoader.cs

@@ -0,0 +1,6 @@
+namespace Gpt4All.LibraryLoader;
+
+public interface ILibraryLoader
+{
+    LoadResult OpenLibrary(string? fileName);
+}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LinuxLibraryLoader.cs

@@ -0,0 +1,53 @@
+using System.Runtime.InteropServices;
+
+namespace Gpt4All.LibraryLoader;
+
+internal class LinuxLibraryLoader : ILibraryLoader
+{
+#pragma warning disable CA2101
+    [DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
+#pragma warning restore CA2101
+    public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags);
+
+#pragma warning disable CA2101
+    [DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
+#pragma warning restore CA2101
+    public static extern IntPtr NativeOpenLibraryLibdl2(string? filename, int flags);
+
+    [DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
+    public static extern IntPtr GetLoadError();
+
+    [DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
+    public static extern IntPtr GetLoadError2();
+
+    public LoadResult OpenLibrary(string? fileName)
+    {
+        IntPtr loadedLib;
+        try
+        {
+            // open with rtls lazy flag
+            loadedLib = NativeOpenLibraryLibdl2(fileName, 0x00001);
+        }
+        catch (DllNotFoundException)
+        {
+            loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001);
+        }
+
+        if (loadedLib == IntPtr.Zero)
+        {
+            string errorMessage;
+            try
+            {
+                errorMessage = Marshal.PtrToStringAnsi(GetLoadError2()) ?? "Unknown error";
+            }
+            catch (DllNotFoundException)
+            {
+                errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error";
+            }
+
+            return LoadResult.Failure(errorMessage);
+        }
+
+        return LoadResult.Success;
+    }
+}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LoadResult.cs

@@ -0,0 +1,20 @@
+namespace Gpt4All.LibraryLoader;
+
+public class LoadResult
+{
+    private LoadResult(bool isSuccess, string? errorMessage)
+    {
+        IsSuccess = isSuccess;
+        ErrorMessage = errorMessage;
+    }
+
+    public static LoadResult Success { get; } = new(true, null);
+
+    public static LoadResult Failure(string errorMessage)
+    {
+        return new(false, errorMessage);
+    }
+
+    public bool IsSuccess { get; }
+    public string? ErrorMessage { get; }
+}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/MacOsLibraryLoader.cs

@@ -0,0 +1,28 @@
+using System.Runtime.InteropServices;
+
+namespace Gpt4All.LibraryLoader;
+
+internal class MacOsLibraryLoader : ILibraryLoader
+{
+#pragma warning disable CA2101
+    [DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
+#pragma warning restore CA2101
+    public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags);
+
+    [DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
+    public static extern IntPtr GetLoadError();
+
+    public LoadResult OpenLibrary(string? fileName)
+    {
+        var loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001);
+
+        if (loadedLib == IntPtr.Zero)
+        {
+            var errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error";
+
+            return LoadResult.Failure(errorMessage);
+        }
+
+        return LoadResult.Success;
+    }
+}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/NativeLibraryLoader.cs

@@ -0,0 +1,81 @@
+#if !IOS && !MACCATALYST && !TVOS && !ANDROID
+using System.Runtime.InteropServices;
+#endif
+
+namespace Gpt4All.LibraryLoader;
+
+public static class NativeLibraryLoader
+{
+    private static ILibraryLoader? defaultLibraryLoader;
+
+    /// <summary>
+    /// Sets the library loader used to load the native libraries. Overwrite this only if you want some custom loading.
+    /// </summary>
+    /// <param name="libraryLoader">The library loader to be used.</param>
+    public static void SetLibraryLoader(ILibraryLoader libraryLoader)
+    {
+        defaultLibraryLoader = libraryLoader;
+    }
+
+    internal static LoadResult LoadNativeLibrary(string? path = default, bool bypassLoading = true)
+    {
+        // If the user has handled loading the library themselves, we don't need to do anything.
+        if (bypassLoading)
+        {
+            return LoadResult.Success;
+        }
+
+        var architecture = RuntimeInformation.OSArchitecture switch
+        {
+            Architecture.X64 => "x64",
+            Architecture.X86 => "x86",
+            Architecture.Arm => "arm",
+            Architecture.Arm64 => "arm64",
+            _ => throw new PlatformNotSupportedException(
+                $"Unsupported OS platform, architecture: {RuntimeInformation.OSArchitecture}")
+        };
+
+        var (platform, extension) = Environment.OSVersion.Platform switch
+        {
+            _ when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => ("win", "dll"),
+            _ when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => ("linux", "so"),
+            _ when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => ("osx", "dylib"),
+            _ => throw new PlatformNotSupportedException(
+                $"Unsupported OS platform, architecture: {RuntimeInformation.OSArchitecture}")
+        };
+
+        // If the user hasn't set the path, we'll try to find it ourselves.
+        if (string.IsNullOrEmpty(path))
+        {
+            var libraryName = "libllmodel";
+            var assemblySearchPath = new[]
+            {
+                AppDomain.CurrentDomain.RelativeSearchPath,
+                Path.GetDirectoryName(typeof(NativeLibraryLoader).Assembly.Location),
+                Path.GetDirectoryName(Environment.GetCommandLineArgs()[0])
+            }.FirstOrDefault(it => !string.IsNullOrEmpty(it));
+            // Search for the library dll within the assembly search path. If it doesn't exist, for whatever reason, use the default path.
+            path = Directory.EnumerateFiles(assemblySearchPath ?? string.Empty, $"{libraryName}.{extension}", SearchOption.AllDirectories).FirstOrDefault() ?? Path.Combine("runtimes", $"{platform}-{architecture}", $"{libraryName}.{extension}");
+        }
+
+        if (defaultLibraryLoader != null)
+        {
+            return defaultLibraryLoader.OpenLibrary(path);
+        }
+
+        if (!File.Exists(path))
+        {
+            throw new FileNotFoundException($"Native Library not found in path {path}. " +
+                                            $"Verify you have have included the native Gpt4All library in your application.");
+        }
+
+        ILibraryLoader libraryLoader = platform switch
+        {
+            "win" => new WindowsLibraryLoader(),
+            "osx" => new MacOsLibraryLoader(),
+            "linux" => new LinuxLibraryLoader(),
+            _ => throw new PlatformNotSupportedException($"Currently {platform} platform is not supported")
+        };
+        return libraryLoader.OpenLibrary(path);
+    }
+}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/WindowsLibraryLoader.cs

@@ -0,0 +1,24 @@
+using System.ComponentModel;
+using System.Runtime.InteropServices;
+
+namespace Gpt4All.LibraryLoader;
+
+internal class WindowsLibraryLoader : ILibraryLoader
+{
+    public LoadResult OpenLibrary(string? fileName)
+    {
+        var loadedLib = LoadLibrary(fileName);
+
+        if (loadedLib == IntPtr.Zero)
+        {
+            var errorCode = Marshal.GetLastWin32Error();
+            var errorMessage = new Win32Exception(errorCode).Message;
+            return LoadResult.Failure(errorMessage);
+        }
+
+        return LoadResult.Success;
+    }
+
+    [DllImport("kernel32", SetLastError = true, CharSet = CharSet.Auto)]
+    private static extern IntPtr LoadLibrary([MarshalAs(UnmanagedType.LPWStr)] string? lpFileName);
+}

gpt4all-bindings/csharp/Gpt4All/Model/Gpt4AllModelFactory.cs

@@ -1,61 +1,58 @@
-using System.Diagnostics;
-using Microsoft.Extensions.Logging;
-using Gpt4All.Bindings;
-using Microsoft.Extensions.Logging.Abstractions;
-
-namespace Gpt4All;
-
-public class Gpt4AllModelFactory : IGpt4AllModelFactory
-{
-    private readonly ILoggerFactory _loggerFactory;
-    private readonly ILogger _logger;
-
-    public Gpt4AllModelFactory(ILoggerFactory? loggerFactory = null)
-    {
-        _loggerFactory = loggerFactory ?? NullLoggerFactory.Instance;
-        _logger = _loggerFactory.CreateLogger<Gpt4AllModelFactory>();
-    }
-
-    private IGpt4AllModel CreateModel(string modelPath, ModelType? modelType = null)
-    {
-        var modelType_ = modelType ?? ModelFileUtils.GetModelTypeFromModelFileHeader(modelPath);
-
-        _logger.LogInformation("Creating model path={ModelPath} type={ModelType}", modelPath, modelType_);
-
-        var handle = modelType_ switch
-        {
-            ModelType.LLAMA => NativeMethods.llmodel_llama_create(),
-            ModelType.GPTJ => NativeMethods.llmodel_gptj_create(),
-            ModelType.MPT => NativeMethods.llmodel_mpt_create(),
-            _ => NativeMethods.llmodel_model_create(modelPath),
-        };
-
-        _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
-        _logger.LogInformation("Model loading started");
-
-        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath);
-
-        _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
-
-        if (loadedSuccessfully == false)
-        {
-            throw new Exception($"Failed to load model: '{modelPath}'");
-        }
-
-        var logger = _loggerFactory.CreateLogger<LLModel>();
-
-        var underlyingModel = LLModel.Create(handle, modelType_, logger: logger);
-
-        Debug.Assert(underlyingModel.IsLoaded());
-
-        return new Gpt4All(underlyingModel, logger: logger);
-    }
-
-    public IGpt4AllModel LoadModel(string modelPath) => CreateModel(modelPath, modelType: null);
-
-    public IGpt4AllModel LoadMptModel(string modelPath) => CreateModel(modelPath, ModelType.MPT);
-
-    public IGpt4AllModel LoadGptjModel(string modelPath) => CreateModel(modelPath, ModelType.GPTJ);
-
-    public IGpt4AllModel LoadLlamaModel(string modelPath) => CreateModel(modelPath, ModelType.LLAMA);
-}
+using System.Diagnostics;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Logging;
+using Gpt4All.Bindings;
+using Gpt4All.LibraryLoader;
+
+namespace Gpt4All;
+
+public class Gpt4AllModelFactory : IGpt4AllModelFactory
+{
+    private readonly ILoggerFactory _loggerFactory;
+    private readonly ILogger _logger;
+    private static bool bypassLoading;
+    private static string? libraryPath;
+
+    private static readonly Lazy<LoadResult> libraryLoaded = new(() =>
+    {
+        return NativeLibraryLoader.LoadNativeLibrary(Gpt4AllModelFactory.libraryPath, Gpt4AllModelFactory.bypassLoading);
+    }, true);
+
+    public Gpt4AllModelFactory(string? libraryPath = default, bool bypassLoading = true, ILoggerFactory? loggerFactory = null)
+    {
+        _loggerFactory = loggerFactory ?? NullLoggerFactory.Instance;
+        _logger = _loggerFactory.CreateLogger<Gpt4AllModelFactory>();
+        Gpt4AllModelFactory.libraryPath = libraryPath;
+        Gpt4AllModelFactory.bypassLoading = bypassLoading;
+
+        if (!libraryLoaded.Value.IsSuccess)
+        {
+            throw new Exception($"Failed to load native gpt4all library. Error: {libraryLoaded.Value.ErrorMessage}");
+        }
+    }
+
+    private IGpt4AllModel CreateModel(string modelPath)
+    {
+        var modelType_ = ModelFileUtils.GetModelTypeFromModelFileHeader(modelPath);
+        _logger.LogInformation("Creating model path={ModelPath} type={ModelType}", modelPath, modelType_);
+        IntPtr error;
+        var handle = NativeMethods.llmodel_model_create2(modelPath, "auto", out error);
+        _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
+        _logger.LogInformation("Model loading started");
+        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath);
+        _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
+        if (!loadedSuccessfully)
+        {
+            throw new Exception($"Failed to load model: '{modelPath}'");
+        }
+
+        var logger = _loggerFactory.CreateLogger<LLModel>();
+        var underlyingModel = LLModel.Create(handle, modelType_, logger: logger);
+
+        Debug.Assert(underlyingModel.IsLoaded());
+
+        return new Gpt4All(underlyingModel, logger: logger);
+    }
+
+    public IGpt4AllModel LoadModel(string modelPath) => CreateModel(modelPath);
+}

gpt4all-bindings/csharp/Gpt4All/Model/IGpt4AllModelFactory.cs

@@ -1,12 +1,6 @@
-namespace Gpt4All;
-
-public interface IGpt4AllModelFactory
-{
-    IGpt4AllModel LoadGptjModel(string modelPath);
-
-    IGpt4AllModel LoadLlamaModel(string modelPath);
-
-    IGpt4AllModel LoadModel(string modelPath);
-
-    IGpt4AllModel LoadMptModel(string modelPath);
-}
+namespace Gpt4All;
+
+public interface IGpt4AllModelFactory
+{
+    IGpt4AllModel LoadModel(string modelPath);
+}

gpt4all-bindings/csharp/Gpt4All/Model/ModelType.cs

@@ -1,11 +1,11 @@
-namespace Gpt4All;
-
-/// <summary>
-/// The supported model types
-/// </summary>
-public enum ModelType
-{
-    LLAMA = 0,
-    GPTJ,
-    MPT
-}
+namespace Gpt4All;
+
+/// <summary>
+/// The supported model types
+/// </summary>
+public enum ModelType
+{
+    LLAMA = 0,
+    GPTJ,
+    MPT
+}

gpt4all-bindings/csharp/Gpt4All/Prediction/ITextPrediction.cs

@@ -1,31 +1,31 @@
-namespace Gpt4All;
-
-/// <summary>
-/// Interface for text prediction services
-/// </summary>
-public interface ITextPrediction
-{
-    /// <summary>
-    /// Get prediction results for the prompt and provided options.
-    /// </summary>
-    /// <param name="text">The text to complete</param>
-    /// <param name="opts">The prediction settings</param>
-    /// <param name="cancellationToken">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
-    /// <returns>The prediction result generated by the model</returns>
-    Task<ITextPredictionResult> GetPredictionAsync(
-        string text,
-        PredictRequestOptions opts,
-        CancellationToken cancellation = default);
-
-    /// <summary>
-    /// Get streaming prediction results for the prompt and provided options.
-    /// </summary>
-    /// <param name="text">The text to complete</param>
-    /// <param name="opts">The prediction settings</param>
-    /// <param name="cancellationToken">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
-    /// <returns>The prediction result generated by the model</returns>
-    Task<ITextPredictionStreamingResult> GetStreamingPredictionAsync(
-        string text,
-        PredictRequestOptions opts,
-        CancellationToken cancellationToken = default);
-}
+namespace Gpt4All;
+
+/// <summary>
+/// Interface for text prediction services
+/// </summary>
+public interface ITextPrediction
+{
+    /// <summary>
+    /// Get prediction results for the prompt and provided options.
+    /// </summary>
+    /// <param name="text">The text to complete</param>
+    /// <param name="opts">The prediction settings</param>
+    /// <param name="cancellation">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns>The prediction result generated by the model</returns>
+    Task<ITextPredictionResult> GetPredictionAsync(
+        string text,
+        PredictRequestOptions opts,
+        CancellationToken cancellation = default);
+
+    /// <summary>
+    /// Get streaming prediction results for the prompt and provided options.
+    /// </summary>
+    /// <param name="text">The text to complete</param>
+    /// <param name="opts">The prediction settings</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns>The prediction result generated by the model</returns>
+    Task<ITextPredictionStreamingResult> GetStreamingPredictionAsync(
+        string text,
+        PredictRequestOptions opts,
+        CancellationToken cancellationToken = default);
+}

gpt4all-bindings/csharp/build_linux.sh

@@ -5,4 +5,6 @@ mkdir runtimes/linux-x64/build
 cmake -S ../../gpt4all-backend -B runtimes/linux-x64/build
 cmake --build runtimes/linux-x64/build --parallel --config Release
 cp runtimes/linux-x64/build/libllmodel.so  runtimes/linux-x64/native/libllmodel.so
-cp runtimes/linux-x64/build/llama.cpp/libllama.so runtimes/linux-x64/native/libllama.so
+cp runtimes/linux-x64/build/libgptj*.so  runtimes/linux-x64/native/
+cp runtimes/linux-x64/build/libllama*.so  runtimes/linux-x64/native/
+cp runtimes/linux-x64/build/libmpt*.so  runtimes/linux-x64/native/

gpt4all-bindings/csharp/build_win-mingw.ps1

@@ -13,4 +13,4 @@ cmake --build $BUILD_DIR --parallel --config Release
 
 # copy native dlls
 cp "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll" $LIBS_DIR
-cp "$BUILD_DIR\*.dll" $LIBS_DIR
+cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR

gpt4all-bindings/csharp/build_win-msvc.ps1

@@ -2,4 +2,5 @@ Remove-Item -Force -Recurse .\runtimes\win-x64\msvc -ErrorAction SilentlyContinu
 mkdir .\runtimes\win-x64\msvc\build | Out-Null
 cmake -G "Visual Studio 17 2022" -A X64 -S ..\..\gpt4all-backend -B .\runtimes\win-x64\msvc\build
 cmake --build .\runtimes\win-x64\msvc\build --parallel --config Release
-cp .\runtimes\win-x64\msvc\build\bin\Release\*.dll .\runtimes\win-x64
+cp .\runtimes\win-x64\msvc\build\bin\Release\*.dll .\runtimes\win-x64
+mv .\runtimes\win-x64\llmodel.dll .\runtimes\win-x64\libllmodel.dll

gpt4all-bindings/golang/README.md

@@ -45,7 +45,7 @@ To use the bindings in your own software:
 
 - Import `github.com/nomic-ai/gpt4all/gpt4all-bindings/golang`;
 - Compile `libgpt4all.a` (you can use `make libgpt4all.a` in the bindings/go directory);
-- Link your go binary against whisper by setting the environment variables `C_INCLUDE_PATH` and `LIBRARY_PATH` to point to the `binding.h` file directory and `libgpt4all.a` file directory respectively.
+- Link your go binary by setting the environment variables `C_INCLUDE_PATH` and `LIBRARY_PATH` to point to the `binding.h` file directory and `libgpt4all.a` file directory respectively.
 - Note: you need to have *.so/*.dynlib/*.dll files of the implementation nearby the binary produced by the binding in order to make this to work
 
 ## Testing

gpt4all-bindings/golang/binding.cpp

@@ -24,11 +24,12 @@ void* load_model(const char *fname, int n_threads) {
                 __func__, new_error.message);
         return nullptr;
     }
-    llmodel_setThreadCount(model,  n_threads);
     if (!llmodel_loadModel(model, fname)) {
+        llmodel_model_destroy(model);
         return nullptr;
     }
 
+    llmodel_setThreadCount(model,  n_threads);
     return model;
 }
 

gpt4all-bindings/golang/gpt4all.go

@@ -10,6 +10,7 @@ package gpt4all
 //                            float top_p, float temp, int n_batch,float ctx_erase);
 // void free_model(void *state_ptr);
 // extern unsigned char getTokenCallback(void *, char *);
+// void llmodel_set_implementation_search_path(const char *path);
 import "C"
 import (
 	"fmt"
@@ -27,6 +28,10 @@ type Model struct {
 func New(model string, opts ...ModelOption) (*Model, error) {
 	ops := NewModelOptions(opts...)
 
+	if ops.LibrarySearchPath != "" {
+		C.llmodel_set_implementation_search_path(C.CString(ops.LibrarySearchPath))
+	}
+
 	state := C.load_model(C.CString(model), C.int(ops.Threads))
 
 	if state == nil {

gpt4all-bindings/golang/options.go

@@ -24,7 +24,8 @@ var DefaultModelOptions ModelOptions = ModelOptions{
 }
 
 type ModelOptions struct {
-	Threads int
+	Threads           int
+	LibrarySearchPath string
 }
 type ModelOption func(p *ModelOptions)
 
@@ -100,6 +101,13 @@ func SetThreads(c int) ModelOption {
 	}
 }
 
+// SetLibrarySearchPath sets the dynamic libraries used by gpt4all for the various ggml implementations.
+func SetLibrarySearchPath(t string) ModelOption {
+	return func(p *ModelOptions) {
+		p.LibrarySearchPath = t
+	}
+}
+
 // Create a new PredictOptions object with the given options.
 func NewModelOptions(opts ...ModelOption) ModelOptions {
 	p := DefaultModelOptions

gpt4all-bindings/java/.gitignore

@@ -0,0 +1,5 @@
+# Make sure native directory never gets commited to git for the project.
+/src/main/resources/native
+
+# IntelliJ project file
+*.iml

gpt4all-bindings/java/Developer_docs.md

@@ -0,0 +1,80 @@
+# Java Bindings Developer documents.
+
+This document is meant to anyone looking to build the Java bindings from source, test a build locally and perform a release.
+
+## Building locally
+
+Maven is the build tool used by the project. Maven version of 3.8 or higher is recommended. Make sure the **mvn** 
+is available on the command path.
+
+The project builds to Java version 11 target so make sure that a JDK at version 11 or newer is installed.
+
+### Setting up location of native shared libraries
+The property **native.libs.location** in pom.xml may need to be set:
+```
+    <properties>
+        ...
+        <native.libs.location>C:\Users\felix\dev\gpt4all_java_bins\release_1_1_3_Jun22_2023</native.libs.location>
+    </properties>
+```
+All the native shared libraries bundled with the Java binding jar will be copied from this location.
+The directory structure is **native/linux**, **native/macos**, **native/windows**. These directories are copied
+into the **src/main/resources** folder during the build process.
+
+For the purposes of local testing, none of these directories have to be present or just one OS type may be present.
+
+If none of the native libraries are present in **native.libs.location** the shared libraries will be searched for
+in location path set by **LLModel.LIBRARY_SEARCH_PATH** static variable in Java source code that is using the bindings.
+
+Alternately you can copy the shared libraries into the **src/resources/native/linux** before 
+you build, but note **src/main/resources/native** is on the .gitignore, so it will not be committed to sources.
+
+### Building
+
+To package the bindings jar run:
+```
+mvn package
+```
+This will build two jars. One has only the Java bindings and the other is a fat jar that will have required dependencies included as well.
+
+To package and install the Java bindings to your local maven repository run:
+```
+mvn install
+```
+
+### Using in a sample application
+
+You can check out a sample project that uses the java bindings here:
+https://github.com/felix-zaslavskiy/gpt4all-java-bindings-sample.git
+
+1. First, update the dependency of java bindings to whatever you have installed in local repository such as **1.1.4-SNAPSHOT**
+2. Second, update **Main.java** and set **baseModelPath** to the correct location of model weight files.
+
+3. To make a runnable jar run:
+```
+mvn package
+```
+
+A fat jar is also created which is easy to run from command line:
+```
+java -jar target/gpt4all-java-bindings-sample-1.0-SNAPSHOT-jar-with-dependencies.jar
+```
+
+### Publish a public release.
+
+For publishing a new version to maven central repository requires password and signing keys which F.Z. currently maintains, so
+he is responsible for making a public release.
+
+The procedure is as follows:
+
+For a snapshot release
+Run:
+```
+mvn deploy -P signing-profile
+```
+
+For a non-snapshot release
+Run:
+```
+mvn clean deploy -P signing-profile,release
+```

gpt4all-bindings/java/README.md

@@ -0,0 +1,126 @@
+# Java bindings
+
+Java bindings let you load a gpt4all library into your Java application and execute text 
+generation using an intuitive and easy to use API. No GPU is required because gpt4all executes on the CPU.
+The gpt4all models are quantized to easily fit into system RAM and use about 4 to 7GB of system RAM.
+
+## Getting Started
+You can add Java bindings into your Java project by adding the following dependency to your project:
+
+**Maven**
+```
+<dependency>
+    <groupId>com.hexadevlabs</groupId>
+    <artifactId>gpt4all-java-binding</artifactId>
+    <version>1.1.5</version>
+</dependency>
+```
+**Gradle**
+```
+implementation 'com.hexadevlabs:gpt4all-java-binding:1.1.5'
+```
+
+To add the library dependency for another build system see [Maven Central Java bindings](https://central.sonatype.com/artifact/com.hexadevlabs/gpt4all-java-binding/).
+
+To download model binary weights file use a URL such as [`https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin`](https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin).
+
+For information about other models available see the [model file list](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-chat#manual-download-of-models).
+
+### Sample code
+```java
+public class Example {
+    public static void main(String[] args) {
+
+        String prompt = "### Human:\nWhat is the meaning of life\n### Assistant:";
+
+        // Replace the hardcoded path with the actual path where your model file resides
+        String modelFilePath = "C:\\Users\\felix\\AppData\\Local\\nomic.ai\\GPT4All\\ggml-gpt4all-j-v1.3-groovy.bin";
+        
+        try (LLModel model = new LLModel(Path.of(modelFilePath))) {
+
+            // May generate up to 4096 tokens but generally stops early
+            LLModel.GenerationConfig config = LLModel.config()
+                    .withNPredict(4096).build();
+
+            // Will also stream to standard output
+            String fullGeneration = model.generate(prompt, config, true);
+
+        } catch (Exception e) {
+            // Exceptions generally may happen if the model file fails to load 
+            // for a number of reasons such as a file not found. 
+            // It is possible that Java may not be able to dynamically load the native shared library or 
+            // the llmodel shared library may not be able to dynamically load the backend 
+            // implementation for the model file you provided.
+            // 
+            // Once the LLModel class is successfully loaded into memory the text generation calls 
+            // generally should not throw exceptions.
+            e.printStackTrace(); // Printing here but in a production system you may want to take some action.
+        }
+    }
+
+}
+```
+
+For a Maven-based sample project that uses this library see this [sample project](https://github.com/felix-zaslavskiy/gpt4all-java-bindings-sample)
+
+### Additional considerations
+#### Logger warnings
+The Java bindings library may produce a warning if you don't have a SLF4J binding included in your project:
+```
+SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
+SLF4J: Defaulting to no-operation (NOP) logger implementation
+SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
+```
+The Java bindings only use logging for informational 
+purposes, so a logger is not essential to correctly use the library. You can ignore this warning if you don't have SLF4J bindings
+in your project.
+
+To add a simple logger using a Maven dependency you may use:
+```
+<dependency>
+    <groupId>org.slf4j</groupId>
+    <artifactId>slf4j-simple</artifactId>
+    <version>1.7.36</version>
+</dependency>
+```
+
+#### Loading your native libraries
+1. the Java bindings package JAR comes bundled with a native library files for Windows, macOS and Linux. These library files are 
+copied to a temporary directory and loaded at runtime. For advanced users who may want to package shared libraries into Docker containers 
+or want to use a custom build of the shared libraries and ignore the once bundled with the Java package they have option 
+to load libraries from your local directory by setting a static property to the location of library files.
+There are no guarantees of compatibility if used in such a way so be careful if you really want to do it.
+
+For example:
+```java
+class Example {
+    public static void main(String[] args) {
+        // gpt4all native shared libraries location
+        LLModel.LIBRARY_SEARCH_PATH = "C:\\Users\\felix\\gpt4all\\lib\\"; 
+        // ... use the library normally
+    }
+}
+```
+2. Not every AVX-only shared library is bundled with the JAR right now to reduce size. Only libgptj-avx is included.
+If you are running into issues please let us know using the [gpt4all project issue tracker](https://github.com/nomic-ai/gpt4all/issues).
+
+3. For Windows the native library included in jar depends on specific Microsoft C and C++ (MSVC) runtime libraries which may not be installed on your system.
+If this is the case you can easily download and install the latest x64 Microsoft Visual C++ Redistributable package from https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170
+
+4. When running Java in a Docker container it is advised to use eclipse-temurin:17-jre parent image. Alpine based parent images don't work due to the native library dependencies.
+
+## Version history
+1. Version **1.1.2**:
+   - Java bindings is compatible with gpt4ll version 2.4.6
+   - Initial stable release with the initial feature set
+2. Version **1.1.3**:
+   - Java bindings is compatible with gpt4all version 2.4.8
+   - Add static GPT4ALL_VERSION to signify gpt4all version of the bindings
+   - Add PromptIsTooLongException for prompts that are longer than context size.
+   - Replit model support to include Metal Mac hardware support.
+3. Version **1.1.4**:
+   - Java bindings is compatible with gpt4all version 2.4.11
+   - Falcon model support included.
+4. Version **1.1.5**:
+   - Add a check for model file readability before loading model.
+   

gpt4all-bindings/java/TODO.md

@@ -0,0 +1,6 @@
+## Needed
+1. Integrate with circleci build pipeline like the C# binding.
+
+## These are just ideas
+1. Better Chat completions function.
+2. Chat completion that returns result in OpenAI compatible format.

gpt4all-bindings/java/pom.xml

@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.hexadevlabs</groupId>
+    <artifactId>gpt4all-java-binding</artifactId>
+    <version>1.1.5</version>
+    <packaging>jar</packaging>
+
+    <properties>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <native.libs.location>C:\Users\felix\dev\gpt4all_java_bins\release_1_1_4_July8_2023</native.libs.location>
+    </properties>
+
+    <name>${project.groupId}:${project.artifactId}</name>
+    <description>Java bindings for GPT4ALL LLM</description>
+    <url>https://github.com/nomic-ai/gpt4all</url>
+    <licenses>
+        <license>
+            <name>The Apache License, Version 2.0</name>
+            <url>https://github.com/nomic-ai/gpt4all/blob/main/LICENSE.txt</url>
+        </license>
+    </licenses>
+    <developers>
+        <developer>
+            <name>Felix Zaslavskiy</name>
+            <email>felixz@hexadevlabs.com</email>
+            <organizationUrl>https://github.com/felix-zaslavskiy/</organizationUrl>
+        </developer>
+    </developers>
+    <scm>
+        <connection>scm:git:git://github.com/nomic-ai/gpt4all.git</connection>
+        <developerConnection>scm:git:ssh://github.com/nomic-ai/gpt4all.git</developerConnection>
+        <url>https://github.com/nomic-ai/gpt4all/tree/main</url>
+    </scm>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.github.jnr</groupId>
+            <artifactId>jnr-ffi</artifactId>
+            <version>2.2.13</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.36</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>5.9.2</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-junit-jupiter</artifactId>
+            <version>5.4.0</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>5.4.0</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <distributionManagement>
+        <snapshotRepository>
+            <id>ossrh</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/snapshots</url>
+        </snapshotRepository>
+        <repository>
+            <id>ossrh</id>
+            <url>https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/</url>
+        </repository>
+    </distributionManagement>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+            </resource>
+            <resource>
+                <directory>${project.build.directory}/generated-resources</directory>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>3.0.0</version>
+                <configuration>
+                    <forkCount>0</forkCount>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-resources-plugin</artifactId>
+                <version>3.3.1</version>
+                <executions>
+                    <execution>
+                        <id>copy-resources</id>
+                        <!-- Here the phase you need -->
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/generated-resources</outputDirectory>
+                            <resources>
+                                <resource>
+                                    <directory>${native.libs.location}</directory>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <version>1.6.13</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <serverId>ossrh</serverId>
+                    <nexusUrl>https://s01.oss.sonatype.org/</nexusUrl>
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>3.5.0</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.6.0</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+         </plugins>
+
+    </build>
+
+    <profiles>
+        <profile>
+        <id>signing-profile</id>
+        <!-- activation conditions here, if any -->
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-gpg-plugin</artifactId>
+                        <version>3.1.0</version>
+                        <executions>
+                            <execution>
+                                <id>sign-artifacts</id>
+                                <phase>verify</phase>
+                                <goals>
+                                    <goal>sign</goal>
+                                </goals>
+
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+</project>