Update translate_eval.py

Signed-off-by: Zach Nussbaum <zanussbaum@gmail.com>

.circleci/config.yml

@@ -1,194 +1,18 @@
 version: 2.1
+setup: true
 orbs:
-  win: circleci/windows@5.0
-  python: circleci/python@1.2
-
-jobs:
-  build-py-docs:
-    docker:
-      - image: circleci/python:3.8
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          # TODO: eventually this will be cleaned up so we aren't building
-          # new dependencies each time unnecessarily.
-          # This will be introduced once we setup branch and path filtering
-          command: |
-            sudo apt-get update
-            sudo apt-get -y install python3 python3-pip
-            sudo pip3 install awscli --upgrade
-            sudo pip3 install mkdocs mkdocs-material mkautodoc 'mkdocstrings[python]'
-      - run:
-          name: Make Documentation
-          command: |
-            cd gpt4all-bindings/python/
-            mkdocs build
-      - run:
-          name: Deploy Documentation
-          command: |
-            cd gpt4all-bindings/python/
-            aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat
-      - run:
-          name: Invalidate docs.gpt4all.io cloudfront
-          command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"
-
-            
-
-  build-py-linux:
-    docker:
-      - image: circleci/python:3.8
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y cmake build-essential
-            pip install setuptools wheel cmake
-      - run:
-          name: Build C library
-          command: |
-            git submodule init
-            git submodule update
-            cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake ..
-            cmake --build . --parallel
-      - run:
-          name: Build wheel
-          command: |
-            cd gpt4all-bindings/python/
-            python setup.py bdist_wheel --plat-name=manylinux1_x86_64
-      - persist_to_workspace:
-          root: gpt4all-bindings/python/dist
-          paths:
-            - "*.whl"
-
-  build-py-macos:
-    macos:
-      xcode: "14.2.0"
-    resource_class: macos.m1.large.gen1
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            brew install cmake
-            pip install setuptools wheel cmake
-      - run:
-          name: Build C library
-          command: |
-            git submodule init
-            git submodule update
-            cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake .. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
-            cmake --build . --parallel
-      - run:
-          name: Build wheel
-          command: |
-            cd gpt4all-bindings/python
-            python setup.py bdist_wheel --plat-name=macosx_10_9_universal2
-      - persist_to_workspace:
-          root: gpt4all-bindings/python/dist
-          paths:
-            - "*.whl"
-
-  build-py-windows:
-    executor:
-      name: win/default
-    steps:
-      - checkout
-      - run:
-          name: Install MinGW64
-          command: choco install -y mingw --force --no-progress
-      - run:
-          name: Add MinGW64 to PATH
-          command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
-      - run:
-          name: Install dependencies
-          command: choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
-      - run:
-          name: Install Python dependencies
-          command: pip install setuptools wheel cmake
-      - run:
-          name: Build C library
-          command: |
-            git submodule init
-            git submodule update
-            cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake -G "MinGW Makefiles" .. 
-            cmake --build . --parallel
-      - run:
-          name: Build wheel
-          # TODO: As part of this task, we need to move mingw64 binaries into package.
-          # This is terrible and needs a more robust solution eventually.
-          command: |
-            cd gpt4all-bindings/python
-            cd gpt4all
-            mkdir llmodel_DO_NOT_MODIFY
-            mkdir llmodel_DO_NOT_MODIFY/build/
-            cp 'C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
-            cd ..
-            python setup.py bdist_wheel --plat-name=win_amd64
-      - persist_to_workspace:
-          root: gpt4all-bindings/python/dist
-          paths:
-            - "*.whl"
-
-  store-and-upload-wheels:
-    docker:
-      - image: circleci/python:3.8
-    steps:
-      - setup_remote_docker
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Install dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y cmake build-essential
-            pip install setuptools wheel twine
-      - run:
-          name: Upload Python package
-          command: |
-            twine upload /tmp/workspace/*.whl --username __token__ --password $PYPI_CRED
-      - store_artifacts:
-          path: /tmp/workspace
+  path-filtering: circleci/path-filtering@0.0.1
 
 workflows:
-  version: 2
-  deploy-docs:
+  version: 2.1
+  generate-config:
     jobs:
-      - build-py-docs:
-          filters:
-            branches:
-              only:
-                - main
-  # build-py-deploy:
-  #   jobs:
-  #     - build-py-linux:
-  #         filters:
-  #           branches:
-  #             only:
-  #     - build-py-macos:
-  #         filters:
-  #           branches:
-  #             only:
-  #     - build-py-windows:
-  #         filters:
-  #           branches:
-  #             only:
-  #     - store-and-upload-wheels:
-  #         filters:
-  #           branches:
-  #             only:
-  #         requires:
-  #           - build-py-windows
-  #           - build-py-linux
-  #           - build-py-macos
+      - path-filtering/filter:
+          base-revision: main
+          config-path: .circleci/continue_config.yml
+          mapping: |
+            gpt4all-bindings/python/.* run-python-workflow true
+            gpt4all-bindings/csharp/.* run-csharp-workflow true
+            gpt4all-backend/.* run-chat-workflow true
+            gpt4all-chat/.* run-chat-workflow true
+            .* run-default-workflow true

.circleci/continue_config.yml

@@ -0,0 +1,748 @@
+version: 2.1
+orbs:
+  win: circleci/windows@5.0
+  python: circleci/python@1.2
+
+parameters:
+  run-default-workflow:
+    type: boolean
+    default: false
+  run-python-workflow:
+    type: boolean
+    default: false
+  run-chat-workflow:
+    type: boolean
+    default: false
+  run-csharp-workflow:
+    type: boolean
+    default: false
+
+jobs:
+  default-job:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - run: echo "CircleCI pipeline triggered"
+
+  build-gpt4all-chat-linux:
+    machine:
+      image: ubuntu-2204:2023.04.2
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - linux-qt-cache
+      - run: 
+          name: Setup Linux and Dependencies
+          command: |
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
+              chmod +x qt-unified-linux-x64-4.6.0-online.run
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: linux-qt-cache
+          paths:
+            - ~/Qt
+      - run:
+          name: Build
+          command: |
+            export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
+            mkdir build
+            cd build
+            ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S ../gpt4all-chat -B .
+            ~/Qt/Tools/CMake/bin/cmake --build . --target all
+
+  build-gpt4all-chat-windows:
+    machine:
+      image: 'windows-server-2019-vs2019:2022.08.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - windows-qt-cache
+      - run:
+          name: Installing Qt
+          command: |
+            if (-not (Test-Path C:\Qt)) {
+              Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+            }
+      - save_cache:  # this is the new step to save cache
+          key: windows-qt-cache
+          paths:
+            - C:\Qt
+      - run:
+          name: Build
+          command: |
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
+            mkdir build
+            cd build
+            & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
+              "-DCMAKE_GENERATOR:STRING=Ninja" `
+              "-DCMAKE_BUILD_TYPE=Release" `
+              "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
+              "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
+              "-S ..\gpt4all-chat" `
+              "-B ."
+            & "C:\Qt\Tools\Ninja\ninja.exe"
+
+  build-gpt4all-chat-macos:
+    macos:
+      xcode: 14.0.0
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - macos-qt-cache_v2
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
+              hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: macos-qt-cache_v2
+          paths:
+            - ~/Qt
+      - run:
+          name: Build
+          command: |
+            mkdir build
+            cd build
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
+              -DCMAKE_GENERATOR:STRING=Ninja \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
+              -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
+              -S ../gpt4all-chat \
+              -B .
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
+
+  build-py-docs:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get -y install python3 python3-pip
+            sudo pip3 install awscli --upgrade
+            sudo pip3 install mkdocs mkdocs-material mkautodoc 'mkdocstrings[python]'
+      - run:
+          name: Make Documentation
+          command: |
+            cd gpt4all-bindings/python/
+            mkdocs build
+      - run:
+          name: Deploy Documentation
+          command: |
+            cd gpt4all-bindings/python/
+            aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat
+      - run:
+          name: Invalidate docs.gpt4all.io cloudfront
+          command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"
+
+  build-py-linux:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y cmake build-essential
+            pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            cmake ..
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          command: |
+            cd gpt4all-bindings/python/
+            python setup.py bdist_wheel --plat-name=manylinux1_x86_64
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  build-py-macos:
+    macos:
+      xcode: "14.2.0"
+    resource_class: macos.m1.large.gen1
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            brew install cmake
+            pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            cmake .. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          command: |
+            cd gpt4all-bindings/python
+            python setup.py bdist_wheel --plat-name=macosx_10_9_universal2
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  build-py-windows:
+    executor:
+      name: win/default
+    steps:
+      - checkout
+      - run:
+          name: Install MinGW64
+          command: choco install -y mingw --force --no-progress
+      - run:
+          name: Add MinGW64 to PATH
+          command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
+      - run:
+          name: Install dependencies
+          command: choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+      - run:
+          name: Install Python dependencies
+          command: pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            cmake -G "MinGW Makefiles" .. 
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          # TODO: As part of this task, we need to move mingw64 binaries into package.
+          # This is terrible and needs a more robust solution eventually.
+          command: |
+            cd gpt4all-bindings/python
+            cd gpt4all
+            mkdir llmodel_DO_NOT_MODIFY
+            mkdir llmodel_DO_NOT_MODIFY/build/
+            cp 'C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
+            cd ..
+            python setup.py bdist_wheel --plat-name=win_amd64
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  store-and-upload-wheels:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - setup_remote_docker
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y cmake build-essential
+            pip install setuptools wheel twine
+      - run:
+          name: Upload Python package
+          command: |
+            twine upload /tmp/workspace/*.whl --username __token__ --password $PYPI_CRED
+      - store_artifacts:
+          path: /tmp/workspace
+
+  build-bindings-backend-linux:
+    machine:
+      image: ubuntu-2204:2023.04.2
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y cmake build-essential
+      - run:
+          name: Build Libraries
+          command: |
+            cd gpt4all-backend
+            mkdir -p runtimes/build
+            cd runtimes/build
+            cmake ../..
+            cmake --build . --parallel --config Release
+            mkdir ../linux-x64
+            cp -L *.so ../linux-x64 # otherwise persist_to_workspace seems to mess symlinks
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/linux-x64/*.so
+
+  build-bindings-backend-macos:
+    macos:
+      xcode: "14.0.0"
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install dependencies
+          command: |
+            brew install cmake
+      - run:
+          name: Build Libraries
+          command: |
+            cd gpt4all-backend
+            mkdir -p runtimes/build
+            cd runtimes/build
+            cmake ../.. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake --build . --parallel --config Release
+            mkdir ../osx-x64
+            cp -L *.dylib ../osx-x64
+            cp ../../llama.cpp-mainline/*.metal ../osx-x64
+            ls ../osx-x64
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/osx-x64/*.dylib
+            - runtimes/osx-x64/*.metal
+
+  build-bindings-backend-windows:
+    executor:
+      name: win/default
+      size: large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install MinGW64
+          command: choco install -y mingw --force --no-progress
+      - run:
+          name: Install dependencies
+          command: |
+            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+      - run:
+          name: Build Libraries
+          command: |
+            $MinGWBin = "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
+            $Env:Path += ";$MinGwBin"
+            $Env:Path += ";C:\Program Files\CMake\bin"
+            cd gpt4all-backend
+            mkdir runtimes/win-x64
+            cd runtimes/win-x64
+            cmake -G "MinGW Makefiles" ../..
+            cmake --build . --parallel --config Release
+            cp "$MinGWBin\libgcc*.dll" .
+            cp "$MinGWBin\libstdc++*.dll" .
+            cp "$MinGWBin\libwinpthread*.dll" .
+            cp bin/*.dll .
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/win-x64/*.dll
+
+  build-bindings-backend-windows-msvc:
+    machine:
+      image: 'windows-server-2022-gui:2023.03.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install dependencies
+          command: |
+            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'  
+      - run:
+          name: Build Libraries
+          command: |
+            $Env:Path += ";C:\Program Files\CMake\bin"
+            cd gpt4all-backend
+            mkdir runtimes/win-x64_msvc
+            cd runtimes/win-x64_msvc
+            cmake -G "Visual Studio 17 2022" -A X64 ../..
+            cmake --build . --parallel --config Release
+            cp bin/Release/*.dll .
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/win-x64_msvc/*.dll
+
+  build-csharp-linux:
+    docker:
+      - image: mcr.microsoft.com/dotnet/sdk:7.0-jammy # Ubuntu 22.04
+    steps:
+      - when:
+          condition: << pipeline.parameters.run-csharp-workflow >>
+          steps:
+            - checkout
+            - attach_workspace:
+                at: /tmp/workspace
+            - run:
+                name: "Prepare Native Libs"
+                command: |
+                  cd gpt4all-bindings/csharp
+                  mkdir -p runtimes/linux-x64/native
+                  cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
+                  ls -R runtimes
+            - restore_cache:
+                keys:
+                  - gpt4all-csharp-nuget-packages-nix
+            - run:
+                name: "Install project dependencies"
+                command: |
+                  cd gpt4all-bindings/csharp
+                  dotnet restore Gpt4All
+            - save_cache:
+                paths:
+                  - ~/.nuget/packages
+                key: gpt4all-csharp-nuget-packages-nix
+            - run:
+                name: Build C# Project
+                command: |
+                  cd gpt4all-bindings/csharp
+                  dotnet build Gpt4All --configuration Release --nologo
+            - run:
+                name: "Run C# Tests"
+                command: |
+                  cd gpt4all-bindings/csharp
+                  dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+            - run:
+                name: Test results
+                command: |
+                    cd gpt4all-bindings/csharp/Gpt4All.Tests
+                    dotnet tool install -g trx2junit
+                    export PATH="$PATH:$HOME/.dotnet/tools"
+                    trx2junit TestResults/*.trx
+            - store_test_results:
+                path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+
+  build-csharp-windows:
+    executor:
+      name: win/default
+      size: large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - when:
+          condition: << pipeline.parameters.run-csharp-workflow >>
+          steps:
+            - checkout
+            - restore_cache:
+                keys:
+                  - gpt4all-csharp-nuget-packages-win
+            - attach_workspace:
+                at: C:\Users\circleci\workspace
+            - run:
+                name: "Prepare Native Libs"
+                command: |
+                  cd gpt4all-bindings/csharp
+                  mkdir -p runtimes\win-x64\native
+                  cp C:\Users\circleci\workspace\runtimes\win-x64\*.dll runtimes\win-x64\native\
+                  ls -R runtimes
+            - run:
+                name: "Install project dependencies"
+                command: |
+                  cd gpt4all-bindings/csharp
+                  dotnet.exe restore Gpt4All
+            - save_cache:
+                paths:
+                  - C:\Users\circleci\.nuget\packages
+                key: gpt4all-csharp-nuget-packages-win
+            - run:
+                name: Build C# Project
+                command: |
+                  cd gpt4all-bindings/csharp
+                  dotnet.exe build Gpt4All --configuration Release --nologo
+            - run:
+                name: "Run C# Tests"
+                command: |
+                  cd gpt4all-bindings/csharp
+                  dotnet.exe test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+            - run:
+                name: Test results
+                command: |
+                    cd gpt4all-bindings/csharp/Gpt4All.Tests
+                    dotnet tool install -g trx2junit
+                    $Env:Path += ";$Env:USERPROFILE\.dotnet\tools"
+                    trx2junit TestResults/*.trx
+            - store_test_results:
+                path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+
+  build-csharp-macos:
+    macos:
+      xcode: "14.0.0"
+    steps:
+      - when:
+          condition: << pipeline.parameters.run-csharp-workflow >>
+          steps:
+            - checkout
+            - restore_cache:
+                keys:
+                  - gpt4all-csharp-nuget-packages-nix
+            - run:
+                name: Install dependencies
+                command: |
+                  brew install --cask dotnet-sdk
+            - attach_workspace:
+                at: /tmp/workspace
+            - run:
+                name: "Prepare Native Libs"
+                command: |
+                  cd gpt4all-bindings/csharp
+                  mkdir -p runtimes/osx/native
+                  cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
+                  cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
+                  ls -R runtimes
+            - run:
+                name: "Install project dependencies"
+                command: |
+                  cd gpt4all-bindings/csharp
+                  dotnet restore Gpt4All
+            - save_cache:
+                paths:
+                  - ~/.nuget/packages
+                key: gpt4all-csharp-nuget-packages-nix
+            - run:
+                name: Build C# Project
+                command: |
+                  cd gpt4all-bindings/csharp
+                  dotnet build Gpt4All --configuration Release --nologo
+            - run:
+                name: "Run C# Tests"
+                command: |
+                  cd gpt4all-bindings/csharp
+                  dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+            - run:
+                name: Test results
+                command: |
+                    cd gpt4all-bindings/csharp/Gpt4All.Tests
+                    dotnet tool install -g trx2junit
+                    export PATH="$PATH:$HOME/.dotnet/tools"
+                    trx2junit TestResults/*.trx
+            - store_test_results:
+                path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+
+  store-and-upload-nupkgs:
+    docker:
+      - image: mcr.microsoft.com/dotnet/sdk:6.0-jammy # Ubuntu 22.04
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - checkout
+      - restore_cache:
+           keys:
+             - gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: NuGet Pack
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes/linux-x64/native
+            cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
+            mkdir -p runtimes/win-x64/native
+            cp /tmp/workspace/runtimes/win-x64/*.dll runtimes/win-x64/native/
+            mkdir -p runtimes/osx/native
+            cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
+            cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
+            dotnet pack ./Gpt4All/Gpt4All.csproj -p:IncludeSymbols=true -p:SymbolPackageFormat=snupkg -c Release
+            dotnet nuget push ./Gpt4All/bin/Release/Gpt4All.*.nupkg -s $NUGET_URL -k $NUGET_TOKEN --skip-duplicate
+      - store_artifacts:
+          path: gpt4all-bindings/csharp/Gpt4All/bin/Release
+
+workflows:
+  version: 2
+  default:
+    when: << pipeline.parameters.run-default-workflow >>
+    jobs:
+      - default-job
+  build-and-test-gpt4all-chat:
+    when: << pipeline.parameters.run-chat-workflow >>
+    jobs:
+      - hold:
+          type: approval
+      - build-gpt4all-chat-linux:
+          requires:
+            - hold
+      - build-gpt4all-chat-windows:
+          requires:
+            - hold
+      - build-gpt4all-chat-macos:
+          requires:
+            - hold
+  deploy-docs:
+    when: << pipeline.parameters.run-python-workflow >>
+    jobs:
+      - build-py-docs:
+          filters:
+            branches:
+              only:
+                - main
+  build-py-deploy:
+    when: << pipeline.parameters.run-python-workflow >>
+    jobs:
+      - pypi-hold:
+          type: approval
+      - hold:
+          type: approval
+      - build-py-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-py-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-py-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - store-and-upload-wheels:
+          filters:
+            branches:
+              only:
+          requires:
+            - pypi-hold
+            - build-py-windows
+            - build-py-linux
+            - build-py-macos
+  build-bindings:
+    when: 
+        or:
+         - << pipeline.parameters.run-python-workflow >>
+         - << pipeline.parameters.run-csharp-workflow >>
+    jobs:
+      - hold:
+          type: approval
+      - nuget-hold:
+          type: approval
+      - build-bindings-backend-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-bindings-backend-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-bindings-backend-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-bindings-backend-windows-msvc:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      # CSharp Jobs
+      - build-csharp-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - build-bindings-backend-linux
+      - build-csharp-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - build-bindings-backend-windows
+      - build-csharp-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - build-bindings-backend-macos
+      - store-and-upload-nupkgs:
+          filters:
+            branches:
+              only:
+          requires:
+            - nuget-hold
+            - build-csharp-windows
+            - build-csharp-linux
+            - build-csharp-macos
+        

.codespellrc

@@ -1,4 +1,3 @@
 [codespell]
-skip = .git,*.pdf,*.svg
-#
-# ignore-words-list =
+ignore-words-list = blong, belong, afterall
+skip = .git,*.pdf,*.svg,*.lock

.gitignore

@@ -1,3 +1,6 @@
+*.arrow
+squad_*
+*sbert_embedded*
 *.pkl
 ckpts*
 .deepspeed_env
@@ -178,3 +181,6 @@ CMakeLists.txt.user
 gpt4all-chat/models/*
 build_*
 build-*
+
+# IntelliJ
+.idea/

.gitmodules

@@ -3,7 +3,7 @@
 	url = https://github.com/ggerganov/llama.cpp.git
 [submodule "llama.cpp-230511"]
 	path = gpt4all-backend/llama.cpp-230511
-	url = https://github.com/manyoso/llama.cpp.git
+	url = https://github.com/nomic-ai/llama.cpp
 [submodule "llama.cpp-mainline"]
 	path = gpt4all-backend/llama.cpp-mainline
-	url = https://github.com/ggerganov/llama.cpp.git
+	url = https://github.com/nomic-ai/llama.cpp.git

README.md

@@ -25,7 +25,7 @@ GPT4All is made possible by our compute partner <a href="https://www.paperspace.
   <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
 </p>
 <p align="center">
-Run on an M1 Mac (not sped up!)
+Run on an M1 macOS Device (not sped up!)
 </p>
 
 ## GPT4All: An ecosystem of open-source on-edge large language models.
@@ -43,20 +43,12 @@ Run any GPT4All model natively on your home desktop with the auto-updating deskt
 
 Direct Installer Links:
 
-* [Mac/OSX](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
+* [macOS](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
 
 * [Windows](https://gpt4all.io/installers/gpt4all-installer-win64.exe)
 
 * [Ubuntu](https://gpt4all.io/installers/gpt4all-installer-linux.run)
 
-If you have older hardware that only supports avx and not avx2 you can use these.
-
-* [Mac/OSX - avx-only](https://gpt4all.io/installers/gpt4all-installer-darwin-avx-only.dmg)
-
-* [Windows - avx-only](https://gpt4all.io/installers/gpt4all-installer-win64-avx-only.exe)
-
-* [Ubuntu - avx-only](https://gpt4all.io/installers/gpt4all-installer-linux-avx-only.run)
-
 Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io/)
 
 ### Chat Client building and running
@@ -69,6 +61,7 @@ Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/java">:computer: Official Java Bindings</a>
 
 
 ## Contributing

gpt4all-api/.gitignore

@@ -0,0 +1,112 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+app/__pycache__/
+gpt4all_api/__pycache__/
+gpt4all_api/app/api_v1/__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# VS Code
+.vscode/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+*.lock
+*.cache

gpt4all-api/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2023 Nomic, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

gpt4all-api/README.md

@@ -1,2 +1,71 @@
-# GPT4All API
-This directory will contain code to build out a RESTful API for GPT4All models. Exact details TBD, but as an MVP, user should be able to send requests to list, download, and generate text with different models.
+# GPT4All REST API
+This directory contains the source code to run and build docker images that run a FastAPI app
+for serving inference from GPT4All models. The API matches the OpenAI API spec.
+
+## Tutorial
+
+### Starting the app
+
+First build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
+```bash
+DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
+```
+
+Then, start the backend with:
+
+```bash
+docker compose up --build
+```
+
+#### Spinning up your app
+Run `docker compose up` to spin up the backend. Monitor the logs for errors in-case you forgot to set an environment variable above.
+
+
+#### Development
+Run
+
+```bash
+docker compose up --build
+```
+and edit files in the `api` directory. The api will hot-reload on changes.
+
+You can run the unit tests with
+
+```bash
+make test
+```
+
+#### Viewing API documentation
+
+Once the FastAPI ap is started you can access its documentation and test the search endpoint by going to:
+```
+localhost:80/docs
+```
+
+This documentation should match the OpenAI OpenAPI spec located at https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+#### Running inference
+```python
+import openai
+openai.api_base = "http://localhost:4891/v1"
+
+openai.api_key = "not needed for a local LLM"
+
+
+def test_completion():
+    model = "gpt4all-j-v1.3-groovy"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Completion.create(
+        model=model,
+        prompt=prompt,
+        max_tokens=50,
+        temperature=0.28,
+        top_p=0.95,
+        n=1,
+        echo=True,
+        stream=False
+    )
+    assert len(response['choices'][0]['text']) > len(prompt)
+    print(response)
+```

gpt4all-api/docker-compose.yaml

@@ -0,0 +1,18 @@
+version: "3.5"
+
+services:
+  gpt4all_api:
+    image: gpt4all_api
+    container_name: gpt4all_api
+    restart: always #restart on error (usually code compilation from save during bad state)
+    ports:
+      - "4891:4891"
+    environment:
+      - APP_ENVIRONMENT=dev
+      - WEB_CONCURRENCY=2
+      - LOGLEVEL=debug
+      - PORT=4891
+      - model=ggml-mpt-7b-chat.bin
+    volumes:
+      - './gpt4all_api/app:/app'
+    command: ["/start-reload.sh"]

gpt4all-api/gpt4all_api/Dockerfile.buildkit

@@ -0,0 +1,23 @@
+# syntax=docker/dockerfile:1.0.0-experimental
+FROM tiangolo/uvicorn-gunicorn:python3.11
+
+ARG MODEL_BIN=ggml-mpt-7b-chat.bin
+
+# Put first so anytime this file changes other cached layers are invalidated.
+COPY gpt4all_api/requirements.txt /requirements.txt
+
+RUN pip install --upgrade pip
+
+# Run various pip install commands with ssh keys from host machine.
+RUN --mount=type=ssh pip install -r /requirements.txt && \
+  rm -Rf /root/.cache && rm -Rf /tmp/pip-install*
+
+# Finally, copy app and client.
+COPY gpt4all_api/app /app
+
+RUN mkdir -p /models
+
+# Include the following line to bake a model into the image and not have to download it on API start.
+RUN wget -q --show-progress=off https://gpt4all.io/models/${MODEL_BIN} -P /models \
+  && md5sum /models/${MODEL_BIN}
+

gpt4all-api/gpt4all_api/README.md

@@ -0,0 +1 @@
+# FastAPI app for serving GPT4All models

gpt4all-api/gpt4all_api/app/api_v1/api.py

@@ -0,0 +1,8 @@
+from api_v1.routes import chat, completions, engines
+from fastapi import APIRouter
+
+router = APIRouter()
+
+router.include_router(chat.router)
+router.include_router(completions.router)
+router.include_router(engines.router)

gpt4all-api/gpt4all_api/app/api_v1/events.py

@@ -0,0 +1,26 @@
+import logging
+from fastapi import HTTPException
+from fastapi.responses import JSONResponse
+from starlette.requests import Request
+from api_v1.settings import settings
+log = logging.getLogger(__name__)
+
+
+startup_msg_fmt = """
+ Starting up GPT4All API
+"""
+
+
+async def on_http_error(request: Request, exc: HTTPException):
+    return JSONResponse({'detail': exc.detail}, status_code=exc.status_code)
+
+
+async def on_startup(app):
+    startup_msg = startup_msg_fmt.format(settings=settings)
+    log.info(startup_msg)
+
+def startup_event_handler(app):
+    async def start_app() -> None:
+        await on_startup(app)
+
+    return start_app

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -0,0 +1,63 @@
+from fastapi import APIRouter, Depends, Response, Security, status
+from pydantic import BaseModel, Field
+from typing import List, Dict
+import logging
+import time
+from api_v1.settings import settings
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+
+class ChatCompletionMessage(BaseModel):
+    role: str
+    content: str
+
+class ChatCompletionRequest(BaseModel):
+    model: str = Field(..., description='The model to generate a completion from.')
+    messages: List[ChatCompletionMessage] = Field(..., description='The model to generate a completion from.')
+
+
+class ChatCompletionChoice(BaseModel):
+    message: ChatCompletionMessage
+    index: int
+    finish_reason: str
+
+class ChatCompletionUsage(BaseModel):
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+
+class ChatCompletionResponse(BaseModel):
+    id: str
+    object: str = 'text_completion'
+    created: int
+    model: str
+    choices: List[ChatCompletionChoice]
+    usage: ChatCompletionUsage
+
+
+router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
+
+@router.post("/completions", response_model=ChatCompletionResponse)
+async def chat_completion(request: ChatCompletionRequest):
+    '''
+    Completes a GPT4All model response.
+    '''
+
+    return ChatCompletionResponse(
+        id='asdf',
+        created=time.time(),
+        model=request.model,
+        choices=[{}],
+        usage={
+            'prompt_tokens': 0,
+            'completion_tokens': 0,
+            'total_tokens': 0
+        }
+    )
+
+

gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py

@@ -0,0 +1,128 @@
+import json
+
+from fastapi import APIRouter, Depends, Response, Security, status
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel, Field
+from typing import List, Dict, Iterable, AsyncIterable
+import logging
+from uuid import uuid4
+from api_v1.settings import settings
+from gpt4all import GPT4All
+import time
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+class CompletionRequest(BaseModel):
+    model: str = Field(..., description='The model to generate a completion from.')
+    prompt: str = Field(..., description='The prompt to begin completing from.')
+    max_tokens: int = Field(7, description='Max tokens to generate')
+    temperature: float = Field(0, description='Model temperature')
+    top_p: float = Field(1.0, description='top_p')
+    n: int = Field(1, description='')
+    stream: bool = Field(False, description='Stream responses')
+
+
+class CompletionChoice(BaseModel):
+    text: str
+    index: int
+    logprobs: float
+    finish_reason: str
+
+
+class CompletionUsage(BaseModel):
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+
+
+class CompletionResponse(BaseModel):
+    id: str
+    object: str = 'text_completion'
+    created: int
+    model: str
+    choices: List[CompletionChoice]
+    usage: CompletionUsage
+
+
+class CompletionStreamResponse(BaseModel):
+    id: str
+    object: str = 'text_completion'
+    created: int
+    model: str
+    choices: List[CompletionChoice]
+
+
+router = APIRouter(prefix="/completions", tags=["Completion Endpoints"])
+
+
+def stream_completion(output: Iterable, base_response: CompletionStreamResponse):
+    """
+    Streams a GPT4All output to the client.
+
+    Args:
+        output: The output of GPT4All.generate(), which is an iterable of tokens.
+        base_response: The base response object, which is cloned and modified for each token.
+
+    Returns:
+        A Generator of CompletionStreamResponse objects, which are serialized to JSON Event Stream format.
+    """
+    for token in output:
+        chunk = base_response.copy()
+        chunk.choices = [dict(CompletionChoice(
+            text=token,
+            index=0,
+            logprobs=-1,
+            finish_reason=''
+        ))]
+        yield f"data: {json.dumps(dict(chunk))}\n\n"
+
+
+@router.post("/", response_model=CompletionResponse)
+async def completions(request: CompletionRequest):
+    '''
+    Completes a GPT4All model response.
+    '''
+
+    model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+    output = model.generate(prompt=request.prompt,
+                            n_predict=request.max_tokens,
+                            streaming=request.stream,
+                            top_k=20,
+                            top_p=request.top_p,
+                            temp=request.temperature,
+                            n_batch=1024,
+                            repeat_penalty=1.2,
+                            repeat_last_n=10)
+
+    # If streaming, we need to return a StreamingResponse
+    if request.stream:
+        base_chunk = CompletionStreamResponse(
+            id=str(uuid4()),
+            created=time.time(),
+            model=request.model,
+            choices=[]
+        )
+        return StreamingResponse((response for response in stream_completion(output, base_chunk)),
+                                 media_type="text/event-stream")
+    else:
+        return CompletionResponse(
+            id=str(uuid4()),
+            created=time.time(),
+            model=request.model,
+            choices=[dict(CompletionChoice(
+                text=output,
+                index=0,
+                logprobs=-1,
+                finish_reason='stop'
+            ))],
+            usage={
+                'prompt_tokens': 0, #TODO how to compute this?
+                'completion_tokens': 0,
+                'total_tokens': 0
+            }
+        )

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -0,0 +1,38 @@
+from fastapi import APIRouter, Depends, Response, Security, status
+from pydantic import BaseModel, Field
+from typing import List, Dict
+import logging
+from api_v1.settings import settings
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+class ListEnginesResponse(BaseModel):
+    data: List[Dict] = Field(..., description="All available models.")
+
+class EngineResponse(BaseModel):
+    data: List[Dict] = Field(..., description="All available models.")
+
+router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
+
+@router.get("/", response_model=ListEnginesResponse)
+async def list_engines():
+    '''
+    List all available GPT4All models from
+    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json
+    '''
+    raise NotImplementedError()
+    return ListEnginesResponse(data=[])
+
+
+@router.get("/{engine_id}", response_model=EngineResponse)
+async def retrieve_engine(engine_id: str):
+    '''
+
+    '''
+
+    raise NotImplementedError()
+    return EngineResponse()
+

gpt4all-api/gpt4all_api/app/api_v1/routes/health.py

@@ -0,0 +1,12 @@
+import logging
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+log = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/health", tags=["Health"])
+
+
+@router.get('/', response_class=JSONResponse)
+async def health_check():
+    """Runs a health check on this instance of the API."""
+    return JSONResponse({'status': 'ok'}, headers={'Access-Control-Allow-Origin': '*'})

gpt4all-api/gpt4all_api/app/api_v1/settings.py

@@ -0,0 +1,10 @@
+from pydantic import BaseSettings
+
+
+class Settings(BaseSettings):
+    app_environment = 'dev'
+    model: str = 'ggml-mpt-7b-chat.bin'
+    gpt4all_path: str = '/models'
+
+
+settings = Settings()

gpt4all-api/gpt4all_api/app/docs.py

@@ -0,0 +1,3 @@
+desc = 'GPT4All API'
+
+endpoint_paths = {'health': '/health'}

gpt4all-api/gpt4all_api/app/main.py

@@ -0,0 +1,61 @@
+import os
+import docs
+import logging
+from fastapi import FastAPI, HTTPException, Request
+from starlette.middleware.cors import CORSMiddleware
+from fastapi.logger import logger as fastapi_logger
+from api_v1.settings import settings
+from api_v1.api import router as v1_router
+from api_v1 import events
+import os
+
+logger = logging.getLogger(__name__)
+
+app = FastAPI(title='GPT4All API', description=docs.desc)
+
+#CORS Configuration (in-case you want to deploy)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["GET", "POST", "OPTIONS"],
+    allow_headers=["*"],
+)
+
+logger.info('Adding v1 endpoints..')
+
+# add v1
+app.include_router(v1_router, prefix='/v1')
+app.add_event_handler('startup', events.startup_event_handler(app))
+app.add_exception_handler(HTTPException, events.on_http_error)
+
+@app.on_event("startup")
+async def startup():
+    global model
+    logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
+    from gpt4all import GPT4All
+    model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+    logger.info("GPT4All API is ready.")
+
+@app.on_event("shutdown")
+async def shutdown():
+    logger.info("Shutting down API")
+
+
+# This is needed to get logs to show up in the app
+if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
+    gunicorn_error_logger = logging.getLogger("gunicorn.error")
+    gunicorn_logger = logging.getLogger("gunicorn")
+
+    root_logger = logging.getLogger()
+    fastapi_logger.setLevel(gunicorn_logger.level)
+    fastapi_logger.handlers = gunicorn_error_logger.handlers
+    root_logger.setLevel(gunicorn_logger.level)
+
+    uvicorn_logger = logging.getLogger("uvicorn.access")
+    uvicorn_logger.handlers = gunicorn_error_logger.handlers
+else:
+    # https://github.com/tiangolo/fastapi/issues/2019
+    LOG_FORMAT2 = "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
+    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -0,0 +1,51 @@
+"""
+Use the OpenAI python API to test gpt4all models.
+"""
+import openai
+openai.api_base = "http://localhost:4891/v1"
+
+openai.api_key = "not needed for a local LLM"
+
+
+def test_completion():
+    model = "gpt4all-j-v1.3-groovy"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Completion.create(
+        model=model,
+        prompt=prompt,
+        max_tokens=50,
+        temperature=0.28,
+        top_p=0.95,
+        n=1,
+        echo=True,
+        stream=False
+    )
+    assert len(response['choices'][0]['text']) > len(prompt)
+    print(response)
+
+
+def test_streaming_completion():
+    model = "gpt4all-j-v1.3-groovy"
+    prompt = "Who is Michael Jordan?"
+    tokens = []
+    for resp in openai.Completion.create(
+            model=model,
+            prompt=prompt,
+            max_tokens=50,
+            temperature=0.28,
+            top_p=0.95,
+            n=1,
+            echo=True,
+            stream=True):
+        tokens.append(resp.choices[0].text)
+
+    assert (len(tokens) > 0)
+    assert (len("".join(tokens)) > len(prompt))
+
+# def test_chat_completions():
+#     model = "gpt4all-j-v1.3-groovy"
+#     prompt = "Who is Michael Jordan?"
+#     response = openai.ChatCompletion.create(
+#         model=model,
+#         messages=[]
+#     )

gpt4all-api/gpt4all_api/requirements.txt

@@ -0,0 +1,10 @@
+aiohttp>=3.6.2
+aiofiles
+pydantic>=1.4.0
+requests>=2.24.0
+ujson>=2.0.2
+fastapi>=0.95.0
+Jinja2>=3.0
+gpt4all==1.0.1
+pytest
+openai

gpt4all-api/makefile

@@ -0,0 +1,37 @@
+ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+APP_NAME:=gpt4all_api
+PYTHON:=python3.8
+
+all: dependencies
+
+fresh: clean dependencies
+
+testenv: clean_testenv test_build
+	docker compose up --build
+
+testenv_d: clean_testenv test_build
+	docker compose up --build -d
+
+test:
+	docker compose exec gpt4all_api pytest -svv --disable-warnings -p no:cacheprovider /app/tests
+
+test_build:
+    DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
+
+clean_testenv:
+	docker compose down -v
+
+fresh_testenv: clean_testenv testenv
+
+venv:
+	if [ ! -d $(ROOT_DIR)/env ]; then $(PYTHON) -m venv $(ROOT_DIR)/env; fi
+
+dependencies: venv
+	source $(ROOT_DIR)/env/bin/activate; yes w | python -m pip install -r $(ROOT_DIR)/gpt4all_api/requirements.txt
+
+clean: clean_testenv
+	# Remove existing environment
+	rm -rf $(ROOT_DIR)/env;
+	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
+
+

gpt4all-backend/CMakeLists.txt

@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.16)
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if(APPLE)
   option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
@@ -9,7 +10,9 @@ if(APPLE)
     set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
   else()
     # Build for the host architecture on macOS
-    set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
+    if(NOT CMAKE_OSX_ARCHITECTURES)
+      set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
+    endif()
   endif()
 endif()
 
@@ -39,6 +42,9 @@ endif()
 include(llama.cpp.cmake)
 
 set(BUILD_VARIANTS default avxonly)
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    set(BUILD_VARIANTS ${BUILD_VARIANTS} metal)
+endif()
 
 set(CMAKE_VERBOSE_MAKEFILE ON)
 
@@ -54,10 +60,20 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     set(LLAMA_F16C ${GPT4ALL_ALLOW_NON_AVX})
     set(LLAMA_FMA  ${GPT4ALL_ALLOW_NON_AVX})
 
+    if (BUILD_VARIANT STREQUAL metal)
+        set(LLAMA_METAL YES)
+    else()
+        set(LLAMA_METAL NO)
+    endif()
+
     # Include GGML
+    set(LLAMA_K_QUANTS YES)
     include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
-    include_ggml(llama.cpp-230511 -230511-${BUILD_VARIANT} ON)
-    include_ggml(llama.cpp-230519 -230519-${BUILD_VARIANT} ON)
+    if (NOT LLAMA_METAL)
+        set(LLAMA_K_QUANTS NO)
+        include_ggml(llama.cpp-230511 -230511-${BUILD_VARIANT} ON)
+        include_ggml(llama.cpp-230519 -230519-${BUILD_VARIANT} ON)
+    endif()
 
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
@@ -65,13 +81,14 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         message(STATUS "Configuring model implementation target ${TARGET_NAME}")
         # Link to ggml/llama
         target_link_libraries(${TARGET_NAME}
-            PUBLIC ${BASE_LIB}-${BUILD_VARIANT})
+            PRIVATE ${BASE_LIB}-${BUILD_VARIANT})
         # Let it know about its build variant
         target_compile_definitions(${TARGET_NAME}
             PRIVATE GGML_BUILD_VARIANT="${BUILD_VARIANT}")
         # Enable IPO if possible
-        set_property(TARGET ${TARGET_NAME}
-                     PROPERTY INTERPROCEDURAL_OPTIMIZATION ${IPO_SUPPORTED})
+# FIXME: Doesn't work with msvc reliably. See https://github.com/nomic-ai/gpt4all/issues/841
+#        set_property(TARGET ${TARGET_NAME}
+#                     PROPERTY INTERPROCEDURAL_OPTIMIZATION ${IPO_SUPPORTED})
     endfunction()
 
     # Add each individual implementations
@@ -81,25 +98,34 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    add_library(llamamodel-230519-${BUILD_VARIANT} SHARED
-        llamamodel.cpp llmodel_shared.cpp)
-    target_compile_definitions(llamamodel-230519-${BUILD_VARIANT} PRIVATE
-        LLAMA_VERSIONS===2 LLAMA_DATE=230519)
-    prepare_target(llamamodel-230519 llama-230519)
+    add_library(replit-mainline-${BUILD_VARIANT} SHARED
+    replit.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+    prepare_target(replit-mainline llama-mainline)
 
-    add_library(llamamodel-230511-${BUILD_VARIANT} SHARED
-        llamamodel.cpp llmodel_shared.cpp)
-    target_compile_definitions(llamamodel-230511-${BUILD_VARIANT} PRIVATE
-        LLAMA_VERSIONS=<=1 LLAMA_DATE=230511)
-    prepare_target(llamamodel-230511 llama-230511)
+    if (NOT LLAMA_METAL)
+        add_library(llamamodel-230519-${BUILD_VARIANT} SHARED
+            llamamodel.cpp llmodel_shared.cpp)
+        target_compile_definitions(llamamodel-230519-${BUILD_VARIANT} PRIVATE
+            LLAMA_VERSIONS===2 LLAMA_DATE=230519)
+        prepare_target(llamamodel-230519 llama-230519)
+        add_library(llamamodel-230511-${BUILD_VARIANT} SHARED
+            llamamodel.cpp llmodel_shared.cpp)
+        target_compile_definitions(llamamodel-230511-${BUILD_VARIANT} PRIVATE
+            LLAMA_VERSIONS=<=1 LLAMA_DATE=230511)
+        prepare_target(llamamodel-230511 llama-230511)
 
-    add_library(gptj-${BUILD_VARIANT} SHARED
-        gptj.cpp utils.h utils.cpp llmodel_shared.cpp)
-    prepare_target(gptj ggml-230511)
+        add_library(gptj-${BUILD_VARIANT} SHARED
+            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        prepare_target(gptj ggml-230511)
 
-    add_library(mpt-${BUILD_VARIANT} SHARED
-        mpt.cpp utils.h utils.cpp llmodel_shared.cpp)
-    prepare_target(mpt ggml-230511)
+        add_library(falcon-${BUILD_VARIANT} SHARED
+            falcon.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        prepare_target(falcon llama-mainline)
+
+        add_library(mpt-${BUILD_VARIANT} SHARED
+            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        prepare_target(mpt ggml-230511)
+    endif()
 endforeach()
 
 add_library(llmodel

gpt4all-backend/dlhandle.h

@@ -18,7 +18,7 @@ public:
     };
 
     Dlhandle() : chandle(nullptr) {}
-    Dlhandle(const std::string& fpath, int flags = RTLD_LAZY) {
+    Dlhandle(const std::string& fpath, int flags = RTLD_LAZY | RTLD_LOCAL) {
         chandle = dlopen(fpath.c_str(), flags);
         if (!chandle) {
             throw Exception("dlopen(\""+fpath+"\"): "+dlerror());

gpt4all-backend/falcon.cpp

@@ -0,0 +1,983 @@
+#define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#include "falcon_impl.h"
+#include "llama.h"
+#include "llama-util.h"
+#include "utils.h"
+#include "llmodel_shared.h"
+
+#include <cassert>
+#include <cinttypes>
+#include <iostream>
+#include <sstream>
+
+namespace {
+const char *modelType_ = "Falcon";
+}
+
+// commented out 40B support as it presently would require forking ggml/llama.cpp
+// can re-add once mainline ggml supports it
+
+#define FALCON_MAGIC 0x67676a74
+
+// default hparams (Falcon 7B)
+struct falcon_hparams {
+    int32_t n_vocab = 65024;
+    int32_t n_embd  = 4544;
+    int32_t n_head  = 71;
+    int32_t n_head_kv = 1;
+    int32_t n_layer = 32;
+    int32_t falcon_version = 7; // 7 for Falcon-7B, 40 for Falcon-40B
+    int32_t ftype   = 1;
+    int32_t n_ctx   = 2048;
+};
+
+struct falcon_layer {
+    // normalization
+    struct ggml_tensor* input_layernorm;
+    struct ggml_tensor* input_layernorm_b;
+    //struct ggml_tensor* attention_norm;    // Falcon-40B only
+    //struct ggml_tensor* attention_norm_b;  // Falcon-40B only
+
+    // attention
+    struct ggml_tensor* query_key_value;
+    struct ggml_tensor* wo;
+
+    // ff
+    struct ggml_tensor* ffn_up;
+    struct ggml_tensor* ffn_down;
+};
+
+struct falcon_model {
+    falcon_hparams hparams;
+
+    struct ggml_tensor* tok_embeddings;
+    struct ggml_tensor* output_norm;
+    struct ggml_tensor* output_norm_b;
+    struct ggml_tensor* lm_head;
+
+    std::vector<falcon_layer> layers;
+
+    // key + value memory
+    llm_kv_cache kv_self;
+
+    struct ggml_context* ctx;
+    std::map<std::string, struct ggml_tensor*> tensors;
+
+    llm_buffer eval_buf;
+    llm_buffer scr0_buf;
+    llm_buffer scr1_buf;
+};
+
+static bool kv_cache_init(
+        const struct falcon_hparams & hparams,
+              struct llm_kv_cache & cache,
+                         ggml_type   wtype,
+                               int   n_ctx) {
+    const int n_embd  = hparams.n_embd;
+    const int dim_head  = n_embd / hparams.n_head;
+    const int dim_kv = dim_head * hparams.n_head_kv;
+    const int n_layer = hparams.n_layer;
+
+    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
+    const int64_t n_elements = dim_kv * n_mem;
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+    struct ggml_init_params params;
+    params.mem_size   = cache.buf.size;
+    params.mem_buffer = cache.buf.addr;
+    params.no_alloc   = false;
+
+    cache.ctx = ggml_init(params);
+    if (!cache.ctx) {
+        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
+        return false;
+    }
+
+    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+    return true;
+}
+
+// load the model's weights from a file
+bool falcon_model_load(const std::string & fname, falcon_model & model, gpt_vocab & vocab, size_t *mem_req) {
+    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+    if (mem_req) {
+        *mem_req = 0;
+    }
+
+    auto fin = std::ifstream(fname, std::ios::binary);
+    if (!fin) {
+        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
+        return false;
+    }
+
+    // verify magic
+    {
+        uint32_t magic;
+        fin.read((char *) &magic, sizeof(magic));
+        if (magic != FALCON_MAGIC) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
+            return false;
+        }
+    }
+
+    uint32_t format_version;
+    fin.read((char *) &format_version, sizeof(format_version));
+
+    // load hparams
+    {
+        auto & hparams = model.hparams;
+
+        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
+        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
+        fin.read((char *) &hparams.n_head_kv, sizeof(hparams.n_head_kv));
+        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
+        fin.read((char *) &hparams.falcon_version, sizeof(hparams.falcon_version));
+        fin.read((char *) &hparams.ftype,   sizeof(hparams.ftype));
+
+        if (hparams.falcon_version != 7) { // && hparams.falcon_version != 40) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad Falcon version: %d)\n", __func__, fname.c_str(), hparams.falcon_version);
+            return false;
+        }
+
+        const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
+
+        printf("%s: n_vocab   = %d\n", __func__, hparams.n_vocab);
+        printf("%s: n_embd    = %d\n", __func__, hparams.n_embd);
+        printf("%s: n_head    = %d\n", __func__, hparams.n_head);
+        printf("%s: n_head_kv = %d\n", __func__, hparams.n_head_kv);
+        printf("%s: n_layer   = %d\n", __func__, hparams.n_layer);
+        printf("%s: ftype     = %d\n", __func__, hparams.ftype);
+        printf("%s: qntvr     = %d\n", __func__, qntvr);
+
+        hparams.ftype %= GGML_QNT_VERSION_FACTOR;
+    }
+
+    // load vocab
+    {
+        const int32_t n_vocab = model.hparams.n_vocab;
+
+        std::string word;
+        std::vector<char> buf(128);
+
+        for (int i = 0; i < n_vocab; i++) {
+            uint32_t len;
+            fin.read((char *) &len, sizeof(len));
+
+            buf.resize(len);
+            fin.read((char *) buf.data(), len);
+            word.assign(buf.data(), len);
+
+            uint32_t dummy;
+            fin.read((char *) &dummy, sizeof(dummy));
+
+            vocab.token_to_id[word] = i;
+            vocab.id_to_token[i] = word;
+        }
+    }
+
+    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
+    // in order to save memory and also to speed up the computation
+    ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
+    if (wtype == GGML_TYPE_COUNT) {
+        fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
+                __func__, fname.c_str(), model.hparams.ftype);
+        return false;
+    }
+
+    auto & ctx = model.ctx;
+
+    size_t ctx_size = 0;
+
+    {
+        const auto& hparams = model.hparams;
+
+        const int n_embd = hparams.n_embd;
+        const int n_head = hparams.n_head;
+        const int n_head_kv = hparams.n_head_kv;
+        const int n_layer = hparams.n_layer;
+        const int n_ctx = hparams.n_ctx;
+        const int n_ff = 4 * model.hparams.n_embd;
+        const int n_vocab = hparams.n_vocab;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // tok_embeddings
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm_b
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // lm_head
+
+        // if (hparams.version == 40) { // Falcon-40B
+        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm
+        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm_b
+        // }
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm_b
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * (n_head_kv * 2 + n_head) * head_dim);  // query_key_value
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_embd);  // wo
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_ff);  // ffn_up
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_ff * n_embd);  // ffn_down
+ 
+        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
+    }
+
+    if (mem_req) {
+        const int n_embd  = model.hparams.n_embd;
+        const int dim_head  = n_embd / model.hparams.n_head;
+        const int dim_kv = dim_head * model.hparams.n_head_kv;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = dim_kv * n_mem;
+        size_t kv_cache_size = 2u*n_elements*ggml_type_size(wtype) + 2_MiB;
+        *mem_req = ctx_size + kv_cache_size;
+        return false;
+    }
+
+    // create the ggml context
+    {
+        struct ggml_init_params params = {
+            .mem_size   = ctx_size,
+            .mem_buffer = NULL,
+            .no_alloc   = false,
+        };
+
+        model.ctx = ggml_init(params);
+        if (!model.ctx) {
+            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
+            return false;
+        }
+    }
+
+    // prepare memory for the weights
+    {
+        const auto& hparams = model.hparams;
+
+        const int n_embd = hparams.n_embd;
+        const int n_head = hparams.n_head;
+        const int n_head_kv = hparams.n_head_kv;
+        const int n_layer = hparams.n_layer;
+        const int n_ff = 4 * model.hparams.n_embd;
+        const int n_vocab = hparams.n_vocab;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        model.layers.resize(n_layer);
+
+        model.tok_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
+
+        model.output_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.output_norm_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
+
+        // map by name
+        model.tensors["transformer.word_embeddings.weight"] =
+            model.tok_embeddings;
+
+        model.tensors["transformer.ln_f.weight"] = model.output_norm;
+        model.tensors["transformer.ln_f.bias"] = model.output_norm_b;
+        model.tensors["lm_head.weight"] = model.lm_head;
+
+        for (int i = 0; i < n_layer; ++i) {
+            auto& layer = model.layers[i];
+
+            layer.input_layernorm =
+                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.input_layernorm_b =
+                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+            // if (hparams.version == 40) { // for Falcon-40B only
+            //     layer.attention_norm =
+            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            //     layer.attention_norm_b =
+            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            // }
+
+            // query_key_value shape for config.multi_query == True:
+            layer.query_key_value = ggml_new_tensor_2d(
+                ctx, wtype, n_embd, (n_head_kv * 2 + n_head) * head_dim);
+            layer.wo = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
+
+            layer.ffn_up = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);
+            layer.ffn_down = ggml_new_tensor_2d(ctx, wtype, n_ff, n_embd);
+
+            // map by name
+            // if (hparams.version == 40) {
+            //     // Falcon-40B:
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_mlp.weight"] = layer.input_layernorm;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_mlp.bias"] = layer.input_layernorm_b;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_attn.weight"] = layer.attention_norm;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_attn.bias"] = layer.attention_norm_b;
+            // } else {
+            // Falcon-7B:
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".input_layernorm.weight"] = layer.input_layernorm;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".input_layernorm.bias"] = layer.input_layernorm_b;
+            //}
+
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".self_attention.query_key_value.weight"] =
+                layer.query_key_value;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".self_attention.dense.weight"] = layer.wo;
+
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".mlp.dense_h_to_4h.weight"] = layer.ffn_up;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".mlp.dense_4h_to_h.weight"] = layer.ffn_down;
+        }
+    }
+
+    // key + value memory
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_layer = hparams.n_layer;
+        const int n_ctx   = hparams.n_ctx;
+        const int n_head_kv = hparams.n_head_kv;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        const int64_t n_mem      = n_layer*n_ctx;
+        const int64_t n_elements = head_dim*n_mem;
+
+        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F32, model.hparams.n_ctx)) {
+            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
+            ggml_free(ctx);
+            return false;
+        }
+        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
+
+        printf("%s: memory_size = %8.2f MB, n_mem = %" PRId64 "\n", __func__, memory_size/1024.0/1024.0, n_mem);
+    }
+
+    // load weights
+    {
+        int n_tensors = 0;
+        size_t total_size = 0;
+
+        printf("%s: ", __func__);
+
+        while (true) {
+            int32_t n_dims;
+            int32_t length;
+            int32_t ttype;
+
+            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
+            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
+
+            if (fin.eof()) {
+                break;
+            }
+
+            int32_t nelements = 1;
+            int32_t ne[2] = { 1, 1 };
+            for (int i = 0; i < n_dims; ++i) {
+                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+                nelements *= ne[i];
+            }
+
+            std::string name(length, 0);
+            fin.read(&name[0], length);
+            fin.seekg(-static_cast<ptrdiff_t>(fin.tellg()) & 31, std::ios_base::cur);
+
+            if (model.tensors.find(name.data()) == model.tensors.end()) {
+                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
+                return false;
+            }
+
+            auto tensor = model.tensors[name.data()];
+            if (ggml_nelements(tensor) != nelements) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
+                return false;
+            }
+
+            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%5d, %5d], expected [%5d, %5d]\n",
+                        __func__, name.data(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
+                return false;
+            }
+
+            // for debugging
+            if (0) {
+                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
+            }
+
+            const size_t bpe = ggml_type_size(ggml_type(ttype));
+
+            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
+                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
+                return false;
+            }
+
+            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
+
+            total_size += ggml_nbytes(tensor);
+            if (++n_tensors % 8 == 0) {
+                printf(".");
+                fflush(stdout);
+            }
+        }
+
+        printf(" done\n");
+
+        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
+    }
+
+    fin.close();
+
+    model.eval_buf.resize(1280u * 1024 * 1024);
+    model.scr0_buf.resize(256u * 1024 * 1024);
+    model.scr1_buf.resize(256u * 1024 * 1024);
+    return true;
+}
+
+
+// evaluate the transformer
+//
+//   - model:     the model
+//   - n_threads: number of threads to use
+//   - n_past:    the context size so far
+//   - embd_inp:  the embeddings of the tokens in the context
+//   - embd_w:    the predicted logits for the next token
+//
+bool falcon_eval(
+        const falcon_model & model,
+        const int n_threads,
+        const int n_past,
+        const std::vector<gpt_vocab::id> & embd_inp,
+              std::vector<float>         & embd_w,
+              size_t                     & mem_per_token) {
+    const int N = embd_inp.size();
+
+    const auto & hparams = model.hparams;
+
+    const int n_embd  = hparams.n_embd;
+    const int n_layer = hparams.n_layer;
+    const int n_ctx   = hparams.n_ctx;
+    const int n_head  = hparams.n_head;
+    const int n_head_kv = hparams.n_head_kv;
+    const int n_vocab = hparams.n_vocab;
+    const int version = hparams.falcon_version;
+    const size_t head_dim = n_embd / n_head;
+
+   struct ggml_init_params eval_ctx_params = {
+        .mem_size = model.eval_buf.size,
+        .mem_buffer = model.eval_buf.addr,
+        .no_alloc = false,
+    };
+
+    struct ggml_context * ctx0 = ggml_init(eval_ctx_params);
+    struct ggml_cgraph gf = {};
+    gf.n_threads = n_threads;
+
+    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
+
+    // wte
+    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.tok_embeddings, embd);
+    struct ggml_tensor* repeat_dummy = ggml_new_tensor_3d(ctx0, inpL->type, head_dim, N + n_past, n_head);
+
+    ggml_type wtype = GGML_TYPE_F32;
+    const int sizeof_wtype = ggml_type_sizef(wtype);
+
+    for (int il = 0; il < n_layer; ++il) {
+        struct ggml_tensor * cur;
+        struct ggml_tensor * layernorm_output;
+
+        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
+        // self-attention
+        {
+            layernorm_output = ggml_norm(ctx0, inpL);
+
+            layernorm_output = ggml_add(ctx0,
+                    ggml_mul(ctx0,
+                        ggml_repeat(ctx0, model.layers[il].input_layernorm, layernorm_output),
+                        layernorm_output),
+                    ggml_repeat(ctx0, model.layers[il].input_layernorm_b, layernorm_output));
+
+            // if (version == 40) { // Falcon-40B only
+            //     cur = ggml_norm(ctx0, inpL);
+
+            //     cur = ggml_add(ctx0,
+            //             ggml_mul(ctx0,
+            //                 ggml_repeat(ctx0, model.layers[il].attention_norm, cur),
+            //                 cur),
+            //             ggml_repeat(ctx0, model.layers[il].attention_norm_b, cur));
+            // }
+            // else {
+            cur = layernorm_output;
+            // }
+
+            // compute QKV
+
+            cur = ggml_mul_mat(ctx0, model.layers[il].query_key_value, cur);
+
+            // Note that the strides for Kcur, Vcur are set up so that the
+            // resulting views are misaligned with the tensor's storage
+            // (by applying the K/V offset we shift the tensor's original
+            // view to stick out behind the viewed QKV tensor's allocated
+            // memory, so to say). This is ok because no actual accesses
+            // happen to that out-of-range memory, but it can require some
+            // trickery when trying to accurately dump these views for
+            // debugging.
+
+            struct ggml_tensor * Qcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                0);
+
+            struct ggml_tensor * Kcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head_kv, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                head_dim * n_head * sizeof_wtype);
+
+            struct ggml_tensor * Vcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head_kv, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                head_dim * (n_head + n_head_kv) * sizeof_wtype);
+
+            // using mode = 2 for neox mode
+            Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, head_dim, 2);
+            Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, head_dim, 2);
+
+            // store key and value to memory
+            {
+                struct ggml_tensor* k = ggml_view_1d(
+                    ctx0, model.kv_self.k, N * n_head_kv * head_dim,
+                    (ggml_element_size(model.kv_self.k) * n_head_kv * head_dim) *
+                        (il * n_ctx + n_past));
+                struct ggml_tensor* v = ggml_view_1d(
+                    ctx0, model.kv_self.v, N * n_head_kv * head_dim,
+                    (ggml_element_size(model.kv_self.v) * n_head_kv * head_dim) *
+                        (il * n_ctx + n_past));
+
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
+            }
+
+            struct ggml_tensor * K = ggml_permute(
+                ctx0,
+                ggml_view_3d(
+                    ctx0,
+                    model.kv_self.k,
+                    head_dim, n_head_kv, n_past + N,
+                    head_dim * sizeof_wtype,
+                    head_dim * n_head_kv * sizeof_wtype,
+                    il * n_ctx * ggml_element_size(model.kv_self.k) * n_head_kv * head_dim),
+                0, 2, 1, 3);
+
+            // K * Q
+
+            // changed from repeat2 back to repeat, will not support 40B!
+            K = ggml_cont(ctx0, ggml_repeat(ctx0, K, repeat_dummy));
+
+            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
+
+            // KQ_scaled = KQ / sqrt(n_embd/n_head)
+            struct ggml_tensor * KQ_scaled =
+                ggml_scale_inplace(ctx0,
+                        KQ,
+                        ggml_new_f32(ctx0, 1.0f/sqrt(float(head_dim)))
+                        );
+
+            // KQ_masked = mask_past(KQ_scaled)
+            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
+
+            // KQ = soft_max(KQ_masked)
+            struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
+
+            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
+            struct ggml_tensor* V = ggml_permute(
+                ctx0,
+                ggml_view_3d(
+                    ctx0,
+                    model.kv_self.v,
+                    head_dim, n_head_kv, n_past + N,
+                    head_dim * sizeof_wtype,
+                    head_dim * n_head_kv * sizeof_wtype,
+                    il * n_ctx * ggml_element_size(model.kv_self.v) * n_head_kv * head_dim),
+                0, 2, 1, 3);
+
+            // changed from repeat2 back to repeat, will not support 40B!
+            V = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_repeat(ctx0, V, repeat_dummy)));
+
+            // KQV = transpose(V) * KQ_soft_max
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
+
+            // KQV_merged = KQV.permute(0, 2, 1, 3)
+            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
+
+            // cur = KQV_merged.contiguous().view(n_embd, N)
+            cur = ggml_cpy(ctx0,
+                    KQV_merged,
+                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+
+            // projection
+            {
+                cur = ggml_mul_mat(ctx0,
+                        model.layers[il].wo,
+                        cur);
+            }
+        }
+
+        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
+
+        struct ggml_tensor* inpFF = layernorm_output;
+        struct ggml_tensor* attn_out = ggml_cpy(
+            ctx0, cur, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+
+        {
+            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_up, inpFF);
+            cur = ggml_gelu(ctx0, cur);
+            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_down, cur);
+        }
+
+        cur = ggml_add(ctx0, cur, attn_out);
+        cur = ggml_add(ctx0, cur, inpL);
+        // input for next layer
+        inpL = cur;
+    }
+
+    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
+    // norm
+    {
+        inpL = ggml_norm(ctx0, inpL);
+
+        // inpL = ln_f_g*inpL + ln_f_b
+        inpL = ggml_add(ctx0,
+                ggml_mul(ctx0,
+                    ggml_repeat(ctx0, model.output_norm, inpL),
+                    inpL),
+                ggml_repeat(ctx0, model.output_norm_b, inpL));
+    }
+
+    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
+
+    // lm_head
+    {
+        inpL = ggml_mul_mat(ctx0, model.lm_head, inpL);
+
+        //inpL = ggml_add(ctx0,
+        //        ggml_repeat(ctx0, model.lmh_b, inpL),
+        //        inpL);
+    }
+
+    // logits -> probs
+    //inpL = ggml_soft_max_inplace(ctx0, inpL);
+
+    // run the computation
+    ggml_build_forward_expand(&gf, inpL);
+    ggml_graph_compute       (ctx0, &gf);
+
+    //if (n_past%100 == 0) {
+    //    ggml_graph_print   (&gf);
+    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
+    //}
+
+    //embd_w.resize(n_vocab*N);
+    //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
+
+    // return result for just the last token
+    embd_w.resize(n_vocab);
+    memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
+
+    if (mem_per_token == 0) {
+        mem_per_token = ggml_used_mem(ctx0)/N;
+    }
+    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
+
+    ggml_free(ctx0);
+
+    return true;
+}
+
+
+#define MAX_RNG_STATE 64*1024
+size_t falcon_get_state_size(const falcon_model &model) {
+    const size_t s_rng_size        = sizeof(size_t);
+    const size_t s_rng             = MAX_RNG_STATE;
+    const size_t s_kv_size         = sizeof(size_t);
+    const size_t s_kv_ntok         = sizeof(int);
+    const size_t s_kv              = model.kv_self.buf.size;
+    const size_t s_total = (
+        + s_rng_size
+        + s_rng
+        + s_kv_size
+        + s_kv_ntok
+        + s_kv
+    );
+    return s_total;
+}
+
+size_t falcon_copy_state_data(const falcon_model &model, const std::mt19937 &rng, uint8_t *dest)
+{
+    uint8_t * out = dest;
+    // copy rng
+    {
+        std::stringstream rng_ss;
+        rng_ss << rng;
+
+        const size_t rng_size = rng_ss.str().size();
+        char rng_buf[MAX_RNG_STATE];
+
+        memset(&rng_buf[0], 0, MAX_RNG_STATE);
+        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
+
+        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
+        memcpy(out, &rng_buf[0], MAX_RNG_STATE); out += MAX_RNG_STATE;
+    }
+
+    // copy kv cache
+    {
+        const size_t kv_size = model.kv_self.buf.size;
+        const int    kv_ntok = model.kv_self.n;
+
+        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
+        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
+
+        if (kv_size) {
+            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
+        }
+    }
+
+    const size_t written  = out - dest;
+    assert(written == falcon_get_state_size(model));
+    fflush(stdout);
+    return written;
+}
+
+size_t falcon_set_state_data(falcon_model *model, std::mt19937 *rng, const uint8_t *src)
+{
+    const uint8_t * in = src;
+
+    // set rng
+    {
+        size_t rng_size;
+        char   rng_buf[MAX_RNG_STATE];
+
+        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
+        memcpy(&rng_buf[0], in, MAX_RNG_STATE); in += MAX_RNG_STATE;
+
+        std::stringstream rng_ss;
+        rng_ss.str(std::string(&rng_buf[0], rng_size));
+        rng_ss >> *rng;
+
+        assert(rng_ss.fail() == false);
+    }
+
+    // set kv cache
+    {
+        size_t kv_size;
+        int kv_ntok;
+
+        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
+        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
+
+        if (kv_size) {
+            assert(model->kv_self.buf.size == kv_size);
+
+            void * k_data = model->kv_self.k->data; // remember data pointers
+            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
+
+            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
+
+            model->kv_self.k->data = k_data; // restore correct data pointers
+            model->kv_self.v->data = v_data;
+
+        }
+
+        model->kv_self.n = kv_ntok;
+    }
+
+    const size_t nread    = in - src;
+    assert(nread == falcon_get_state_size(*model));
+    fflush(stdout);
+    return nread;
+}
+
+struct FalconPrivate {
+    const std::string modelPath;
+    bool modelLoaded;
+    gpt_vocab vocab;
+    falcon_model *model = nullptr;
+    int64_t n_threads = 0;
+    size_t mem_per_token = 0;
+    std::mt19937 rng;
+};
+
+Falcon::Falcon() : d_ptr(new FalconPrivate) {
+    d_ptr->model = new falcon_model;
+    d_ptr->model->ctx = nullptr;
+    d_ptr->modelLoaded = false;
+}
+
+Falcon::~Falcon() {
+    if(d_ptr->model->ctx) {
+        ggml_free(d_ptr->model->ctx);
+        d_ptr->model->ctx = nullptr;
+    }
+    delete d_ptr->model;
+}
+
+bool Falcon::loadModel(const std::string &modelPath)
+{
+    std::mt19937 rng(time(NULL));
+    d_ptr->rng = rng;
+
+    // load the model
+    if (!falcon_model_load(modelPath, *d_ptr->model, d_ptr->vocab, nullptr)) {
+        std::cerr << "FALCON ERROR: failed to load model from " <<  modelPath;
+        return false;
+    }
+
+    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    d_ptr->modelLoaded = true;
+    fflush(stdout);
+    return true;
+}
+
+bool Falcon::isModelLoaded() const
+{
+    return d_ptr -> modelLoaded;
+}
+
+size_t Falcon::requiredMem(const std::string &modelPath)
+{
+    falcon_model dummy_model;
+    gpt_vocab dummy_vocab;
+    size_t mem_req;
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    falcon_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
+    return mem_req;
+}
+
+size_t Falcon::stateSize() const
+{
+    return falcon_get_state_size(*d_ptr->model);
+}
+
+size_t Falcon::saveState(uint8_t *dest) const
+{
+    return falcon_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
+}
+
+size_t Falcon::restoreState(const uint8_t *src)
+{
+    return falcon_set_state_data(d_ptr->model, &d_ptr->rng, src);
+}
+
+void Falcon::setThreadCount(int32_t n_threads)
+{
+    d_ptr->n_threads = n_threads;
+}
+
+int32_t Falcon::threadCount() const
+{
+    return d_ptr->n_threads;
+}
+
+std::vector<LLModel::Token> Falcon::tokenize(PromptContext &, const std::string &str) const
+{
+    return ::gpt_tokenize(d_ptr->vocab, str);
+}
+
+LLModel::Token Falcon::sampleToken(PromptContext &promptCtx) const
+{
+    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
+    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
+        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
+        n_prev_toks,
+        promptCtx.logits,
+        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
+        promptCtx.repeat_penalty,
+        d_ptr->rng);
+}
+
+std::string Falcon::tokenToString(Token id) const
+{
+    return d_ptr->vocab.id_to_token[id];
+}
+
+bool Falcon::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
+{
+    // determine the required inference memory per token:
+    static bool initialized = false;
+    if (!initialized) {
+        falcon_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
+            d_ptr->mem_per_token);
+        initialized = true;
+    }
+
+    return falcon_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
+}
+
+int32_t Falcon::contextLength() const
+{
+    return d_ptr->model->hparams.n_ctx;
+}
+
+const std::vector<LLModel::Token> &Falcon::endTokens() const
+{
+    static const std::vector<LLModel::Token> out = { 11 };
+    return out;
+}
+
+#if defined(_WIN32)
+#define DLL_EXPORT __declspec(dllexport)
+#else
+#define DLL_EXPORT __attribute__ ((visibility ("default")))
+#endif
+
+extern "C" {
+DLL_EXPORT bool is_g4a_backend_model_implementation() {
+    return true;
+}
+
+DLL_EXPORT const char *get_model_type() {
+    return modelType_;
+}
+
+DLL_EXPORT const char *get_build_variant() {
+    return GGML_BUILD_VARIANT;
+}
+
+DLL_EXPORT bool magic_match(std::istream& f) {
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    uint32_t version = 0;
+    f.read(reinterpret_cast<char*>(&version), sizeof(version));
+    if (magic != FALCON_MAGIC) {
+         return false; 
+    }
+    falcon_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    // we're matching the file format of existing pre-converted models
+    // compatible with ctransformers llama.cpp based format, which also
+    // unfortunately shares its magic number what llama uses, so we now
+    // differentiate by n_vocab
+    // give some wiggle room over the max to allow for finetunes that expand the
+    // vocabulary
+    if (!(hparams.n_vocab >= 65024 && hparams.n_vocab <= 65100)) {
+        return false;
+    }
+    if (hparams.falcon_version != 7) {
+        return false;
+    }
+    return true;
+}
+
+DLL_EXPORT LLModel *construct() {
+    return new Falcon;
+}
+}

gpt4all-backend/falcon_impl.h

@@ -0,0 +1,40 @@
+#ifndef FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of falcon.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef FALCON_H
+#define FALCON_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <memory>
+#include "llmodel.h"
+
+struct FalconPrivate;
+class Falcon : public LLModel {
+public:
+    Falcon();
+    ~Falcon();
+
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    std::unique_ptr<FalconPrivate> d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // Falcon_H

gpt4all-backend/gptj.cpp

@@ -2,8 +2,10 @@
 #include "gptj_impl.h"
 
 #include "utils.h"
+#include "llmodel_shared.h"
 
 #include <cassert>
+#include <cinttypes>
 #include <cmath>
 #include <cstdio>
 #include <cstring>
@@ -30,8 +32,6 @@
 
 namespace {
 const char *modelType_ = "GPT-J";
-
-static const size_t MB = 1024*1024;
 }
 
 // default hparams (GPT-J 6B)
@@ -65,39 +65,6 @@ struct gptj_layer {
     struct ggml_tensor * c_mlp_proj_b;
 };
 
-struct gptj_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-
-    void resize(size_t size) {
-        delete[] addr;
-        addr = new uint8_t[size];
-        this->size = size;
-    }
-
-    ~gptj_buffer() {
-        fflush(stdout);
-        delete[] addr;
-    }
-};
-
-struct gptj_kv_cache {
-    struct ggml_tensor * k;
-    struct ggml_tensor * v;
-
-    struct ggml_context * ctx = NULL;
-
-    gptj_buffer buf;
-
-    int n; // number of tokens currently in the cache
-
-    ~gptj_kv_cache() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
 struct gptj_model {
     gptj_hparams hparams;
 
@@ -113,13 +80,15 @@ struct gptj_model {
     std::vector<gptj_layer> layers;
 
     // key + value memory
-    struct gptj_kv_cache kv_self;
+    struct llm_kv_cache kv_self;
 
     //
     struct ggml_context * ctx;
     std::map<std::string, struct ggml_tensor *> tensors;
 
-    gptj_buffer buf;
+    llm_buffer eval_buf;
+    llm_buffer scr0_buf;
+    llm_buffer scr1_buf;
 
     ~gptj_model() {
         if (ctx) {
@@ -130,7 +99,7 @@ struct gptj_model {
 
 static bool kv_cache_init(
         const struct gptj_hparams & hparams,
-             struct gptj_kv_cache & cache,
+              struct llm_kv_cache & cache,
                          ggml_type   wtype,
                                int   n_ctx) {
     const int n_embd  = hparams.n_embd;
@@ -139,7 +108,7 @@ static bool kv_cache_init(
     const int64_t n_mem      = (int64_t)n_layer*n_ctx;
     const int64_t n_elements = n_embd*n_mem;
 
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB);
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
 
     struct ggml_init_params params;
     params.mem_size   = cache.buf.size;
@@ -160,8 +129,11 @@ static bool kv_cache_init(
 }
 
 // load the model's weights from a stream
-bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & model, gpt_vocab & vocab) {
+bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+    if(mem_req != nullptr) {
+        *mem_req = 0;
+    }
 
     // verify magic
     {
@@ -278,6 +250,19 @@ bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & m
         printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
     }
 
+    if (mem_req != nullptr) {
+        *mem_req += ctx_size;
+        const int n_embd  = model.hparams.n_embd;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = n_embd*n_mem;
+
+        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+        return false;
+    }
+
+
     // create the ggml context
     {
         struct ggml_init_params params = {
@@ -411,7 +396,7 @@ bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & m
             }
 
             if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%lu, %lu], expected [%d, %d]\n",
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
                         __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
                 return false;
             }
@@ -456,6 +441,9 @@ bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & m
         printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
     }
 
+    model.scr0_buf.resize(256u * 1024 * 1024);
+    model.scr1_buf.resize(256u * 1024 * 1024);
+
     return true;
 }
 
@@ -501,25 +489,25 @@ bool gptj_eval(
     const int n_vocab = hparams.n_vocab;
     const int n_rot   = hparams.n_rot;
 
-    const size_t init_buf_size = 1024u*MB;
-    if (!model.buf.addr || model.buf.size < init_buf_size)
-        model.buf.resize(init_buf_size);
+    const size_t init_buf_size = 1024_MiB;
+    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
+        model.eval_buf.resize(init_buf_size);
 
-    if (mem_per_token > 0 && mem_per_token*N > model.buf.size) {
+    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
         const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
-        printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
+        printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.eval_buf.size, buf_size_new);
 
         // reallocate
-        model.buf.resize(buf_size_new);
-        if (model.buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.buf.size);
+        model.eval_buf.resize(buf_size_new);
+        if (model.eval_buf.addr == nullptr) {
+            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
             return false;
         }
     }
 
     struct ggml_init_params params = {
-        .mem_size   = model.buf.size,
-        .mem_buffer = model.buf.addr,
+        .mem_size   = model.eval_buf.size,
+        .mem_buffer = model.eval_buf.addr,
         .no_alloc = false
     };
 
@@ -535,7 +523,7 @@ bool gptj_eval(
 
     for (int il = 0; il < n_layer; ++il) {
         struct ggml_tensor * cur;
-
+        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
         // norm
         {
             cur = ggml_norm(ctx0, inpL);
@@ -630,6 +618,7 @@ bool gptj_eval(
 
         struct ggml_tensor * inpFF = cur;
 
+        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
         // feed-forward network
         // this is independent of the self-attention result, so it could be done in parallel to the self-attention
         {
@@ -663,6 +652,8 @@ bool gptj_eval(
         inpL = ggml_add(ctx0, cur, inpL);
     }
 
+    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
     // norm
     {
         inpL = ggml_norm(ctx0, inpL);
@@ -675,6 +666,8 @@ bool gptj_eval(
                 ggml_repeat(ctx0, model.ln_f_b, inpL));
     }
 
+    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
+
     // lm_head
     {
         inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL);
@@ -835,9 +828,19 @@ struct GPTJPrivate {
 GPTJ::GPTJ()
     : d_ptr(new GPTJPrivate) {
     d_ptr->model = new gptj_model;
+    d_ptr->model->ctx = nullptr;
     d_ptr->modelLoaded = false;
 }
 
+size_t GPTJ::requiredMem(const std::string &modelPath) {
+    gptj_model dummy_model;
+    gpt_vocab dummy_vocab;
+    size_t mem_req;
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    gptj_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
+    return mem_req;
+}
+
 bool GPTJ::loadModel(const std::string &modelPath) {
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
@@ -907,7 +910,7 @@ LLModel::Token GPTJ::sampleToken(PromptContext &promptCtx) const
         d_ptr->rng);
 }
 
-std::string_view GPTJ::tokenToString(Token id) const
+std::string GPTJ::tokenToString(Token id) const
 {
     return d_ptr->vocab.id_to_token[id];
 }

gpt4all-backend/gptj_impl.h

@@ -17,6 +17,7 @@ public:
 
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
@@ -29,7 +30,7 @@ private:
 protected:
     std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
     Token sampleToken(PromptContext &ctx) const override;
-    std::string_view tokenToString(Token) const override;
+    std::string tokenToString(Token) const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
     const std::vector<Token>& endTokens() const override;

gpt4all-backend/llama.cpp.cmake

@@ -34,6 +34,7 @@ endif()
 #
 # Option list
 #
+# some of the options here are commented out so they can be set "dynamically" before calling include_ggml()
 
 # general
 option(LLAMA_STATIC                 "llama: static link libraries"                          OFF)
@@ -65,8 +66,13 @@ option(LLAMA_SANITIZE_UNDEFINED     "llama: enable undefined sanitizer"
 # 3rd party libs
 option(LLAMA_ACCELERATE             "llama: enable Accelerate framework"                    ON)
 option(LLAMA_OPENBLAS               "llama: use OpenBLAS"                                   OFF)
-option(LLAMA_CUBLAS                 "llama: use cuBLAS"                                     OFF)
-option(LLAMA_CLBLAST                "llama: use CLBlast"                                    OFF)
+#option(LLAMA_CUBLAS                 "llama: use cuBLAS"                                     OFF)
+#option(LLAMA_CLBLAST                "llama: use CLBlast"                                    OFF)
+#option(LLAMA_METAL                  "llama: use Metal"                                      OFF)
+#option(LLAMA_K_QUANTS               "llama: use k-quants"                                   ON)
+set(LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor")
+set(LLAMA_CUDA_DMMV_X "32" CACHE STRING "llama: x stride for dmmv CUDA kernels")
+set(LLAMA_CUDA_DMMV_Y "1" CACHE STRING  "llama: y block size for dmmv CUDA kernels")
 
 #
 # Compile flags
@@ -210,86 +216,22 @@ endif()
 function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
     message(STATUS "Configuring ggml implementation target llama${SUFFIX} in ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}")
 
-    if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
-        message(STATUS "ARM detected")
-        if (MSVC)
-            # TODO: arm msvc?
-        else()
-            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
-                add_compile_options(-mcpu=native)
-            endif()
-            # TODO: armv6,7,8 version specific flags
-        endif()
-    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$")
-        message(STATUS "x86 detected")
-        if (MSVC)
-            if (LLAMA_AVX512)
-                add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX512>)
-                add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
-                # MSVC has no compile-time flags enabling specific
-                # AVX512 extensions, neither it defines the
-                # macros corresponding to the extensions.
-                # Do it manually.
-                if (LLAMA_AVX512_VBMI)
-                    add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
-                    add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
-                endif()
-                if (LLAMA_AVX512_VNNI)
-                    add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
-                    add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
-                endif()
-            elseif (LLAMA_AVX2)
-                add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX2>)
-                add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
-            elseif (LLAMA_AVX)
-                add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX>)
-                add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
-            endif()
-        else()
-            if (LLAMA_F16C)
-                add_compile_options(-mf16c)
-            endif()
-            if (LLAMA_FMA)
-                add_compile_options(-mfma)
-            endif()
-            if (LLAMA_AVX)
-                add_compile_options(-mavx)
-            endif()
-            if (LLAMA_AVX2)
-                add_compile_options(-mavx2)
-            endif()
-            if (LLAMA_AVX512)
-                add_compile_options(-mavx512f)
-                add_compile_options(-mavx512bw)
-            endif()
-            if (LLAMA_AVX512_VBMI)
-                add_compile_options(-mavx512vbmi)
-            endif()
-            if (LLAMA_AVX512_VNNI)
-                add_compile_options(-mavx512vnni)
-            endif()
-        endif()
-    else()
-        # TODO: support PowerPC
-        message(STATUS "Unknown architecture")
-    endif()
-
     #
     # Build libraries
     #
 
-    if (LLAMA_CUBLAS AND EXISTS ${DIRECTORY}/ggml-cuda.h)
+    set(GGML_CUBLAS_USE NO)
+    if (LLAMA_CUBLAS)
         cmake_minimum_required(VERSION 3.17)
 
         find_package(CUDAToolkit)
         if (CUDAToolkit_FOUND)
+            set(GGML_CUBLAS_USE YES)
             message(STATUS "cuBLAS found")
 
             enable_language(CUDA)
 
-            set(GGML_CUDA_SOURCES ${DIRECTORY}/ggml-cuda.cu ${DIRECTORY}/ggml-cuda.h)
-
-            add_compile_definitions(GGML_USE_CUBLAS)
+            set(GGML_SOURCES_CUDA ${DIRECTORY}/ggml-cuda.cu ${DIRECTORY}/ggml-cuda.h)
 
             if (LLAMA_STATIC)
                 set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static)
@@ -302,14 +244,19 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
         endif()
     endif()
 
-    if (LLAMA_CLBLAST AND EXISTS ${DIRECTORY}/ggml-opencl.h)
+    set(GGML_CLBLAST_USE NO)
+    if (LLAMA_CLBLAST)
         find_package(CLBlast)
         if (CLBlast_FOUND)
+            set(GGML_CLBLAST_USE YES)
             message(STATUS "CLBlast found")
 
-            set(GGML_OPENCL_SOURCES ${DIRECTORY}/ggml-opencl.c ${DIRECTORY}/ggml-opencl.h)
+            set(GGML_OPENCL_SOURCE_FILE ggml-opencl.cpp)
+            if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/${GGML_OPENCL_SOURCE_FILE})
+                set(GGML_OPENCL_SOURCE_FILE ggml-opencl.c)
+            endif()
 
-            add_compile_definitions(GGML_USE_CLBLAST)
+            set(GGML_OPENCL_SOURCES ${DIRECTORY}/${GGML_OPENCL_SOURCE_FILE} ${DIRECTORY}/ggml-opencl.h)
 
             set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} clblast)
         else()
@@ -317,15 +264,52 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
         endif()
     endif()
 
+    set(GGML_SOURCES_QUANT_K )
+    set(GGML_METAL_SOURCES )
+    if (LLAMA_K_QUANTS)
+        set(GGML_SOURCES_QUANT_K
+            ${DIRECTORY}/k_quants.h
+            ${DIRECTORY}/k_quants.c)
+
+        if (LLAMA_METAL)
+            find_library(FOUNDATION_LIBRARY         Foundation              REQUIRED)
+            find_library(METAL_FRAMEWORK            Metal                   REQUIRED)
+            find_library(METALKIT_FRAMEWORK         MetalKit                REQUIRED)
+            find_library(METALPERFORMANCE_FRAMEWORK MetalPerformanceShaders REQUIRED)
+
+            set(GGML_METAL_SOURCES ${DIRECTORY}/ggml-metal.m ${DIRECTORY}/ggml-metal.h)
+            # get full path to the file
+            #add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
+
+            # copy ggml-metal.metal to bin directory
+            configure_file(${DIRECTORY}/ggml-metal.metal bin/ggml-metal.metal COPYONLY)
+
+            set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS}
+                ${FOUNDATION_LIBRARY}
+                ${METAL_FRAMEWORK}
+                ${METALKIT_FRAMEWORK}
+                ${METALPERFORMANCE_FRAMEWORK}
+            )
+        endif()
+    endif()
+
     add_library(ggml${SUFFIX} OBJECT
                 ${DIRECTORY}/ggml.c
                 ${DIRECTORY}/ggml.h
-                ${GGML_CUDA_SOURCES}
+                ${GGML_SOURCES_QUANT_K}
+                ${GGML_SOURCES_CUDA}
+                ${GGML_METAL_SOURCES}
                 ${GGML_OPENCL_SOURCES})
 
+    if (LLAMA_K_QUANTS)
+        target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_K_QUANTS)
+    endif()
+
+    if (LLAMA_METAL AND GGML_METAL_SOURCES)
+        target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)
+    endif()
     target_include_directories(ggml${SUFFIX} PUBLIC ${DIRECTORY})
     target_compile_features(ggml${SUFFIX} PUBLIC c_std_11) # don't bump
-    target_link_libraries(ggml${SUFFIX} PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
 
     if (BUILD_SHARED_LIBS)
         set_target_properties(ggml${SUFFIX} PROPERTIES POSITION_INDEPENDENT_CODE ON)
@@ -338,14 +322,16 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
             set(LLAMA_UTIL_SOURCE_FILE llama_util.h)
         endif()
 
-        add_library(llama${SUFFIX}
+        add_library(llama${SUFFIX} STATIC
                     ${DIRECTORY}/llama.cpp
                     ${DIRECTORY}/llama.h
                     ${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
 
+        if (LLAMA_METAL AND GGML_METAL_SOURCES)
+            target_compile_definitions(llama${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)
+        endif()
         target_include_directories(llama${SUFFIX} PUBLIC ${DIRECTORY})
         target_compile_features(llama${SUFFIX} PUBLIC cxx_std_11) # don't bump
-        target_link_libraries(llama${SUFFIX} PRIVATE ggml${SUFFIX} ${LLAMA_EXTRA_LIBS})
 
         if (BUILD_SHARED_LIBS)
             set_target_properties(llama${SUFFIX} PROPERTIES POSITION_INDEPENDENT_CODE ON)
@@ -353,7 +339,7 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
         endif()
     endif()
 
-    if (GGML_CUDA_SOURCES)
+    if (GGML_SOURCES_CUDA)
         message(STATUS "GGML CUDA sources found, configuring CUDA architecture")
         set_property(TARGET ggml${SUFFIX} PROPERTY CUDA_ARCHITECTURES OFF)
         set_property(TARGET ggml${SUFFIX} PROPERTY CUDA_SELECT_NVCC_ARCH_FLAGS "Auto")
@@ -361,4 +347,97 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
             set_property(TARGET llama${SUFFIX} PROPERTY CUDA_ARCHITECTURES OFF)
         endif()
     endif()
+
+    if (GGML_CUBLAS_USE)
+        target_compile_definitions(ggml${SUFFIX} PRIVATE
+            GGML_USE_CUBLAS
+            GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X}
+            GGML_CUDA_DMMV_Y=${LLAMA_CUDA_DMMV_Y})
+        if (WITH_LLAMA)
+            target_compile_definitions(llama${SUFFIX} PRIVATE
+                GGML_USE_CUBLAS
+                GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X}
+                GGML_CUDA_DMMV_Y=${LLAMA_CUDA_DMMV_Y})
+        endif()
+    endif()
+    if (GGML_CLBLAST_USE)
+        if (WITH_LLAMA)
+            target_compile_definitions(llama${SUFFIX} PRIVATE GGML_USE_CLBLAST)
+        endif()
+        target_compile_definitions(ggml${SUFFIX} PRIVATE GGML_USE_CLBLAST)
+    endif()
+
+    if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
+        message(STATUS "ARM detected")
+        if (MSVC)
+            # TODO: arm msvc?
+        else()
+            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
+                target_compile_options(ggml${SUFFIX} PRIVATE -mcpu=native)
+            endif()
+            # TODO: armv6,7,8 version specific flags
+        endif()
+    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$")
+        message(STATUS "x86 detected")
+        if (MSVC)
+            if (LLAMA_AVX512)
+                target_compile_options(ggml${SUFFIX} PRIVATE
+                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX512>
+                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
+                # MSVC has no compile-time flags enabling specific
+                # AVX512 extensions, neither it defines the
+                # macros corresponding to the extensions.
+                # Do it manually.
+                if (LLAMA_AVX512_VBMI)
+                    target_compile_definitions(ggml${SUFFIX} PRIVATE
+                        $<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>
+                        $<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
+                endif()
+                if (LLAMA_AVX512_VNNI)
+                    target_compile_definitions(ggml${SUFFIX} PRIVATE
+                        $<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>
+                        $<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
+                endif()
+            elseif (LLAMA_AVX2)
+                target_compile_options(ggml${SUFFIX} PRIVATE
+                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX2>
+                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
+            elseif (LLAMA_AVX)
+                target_compile_options(ggml${SUFFIX} PRIVATE
+                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX>
+                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
+            endif()
+        else()
+            if (LLAMA_F16C)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mf16c)
+            endif()
+            if (LLAMA_FMA)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mfma)
+            endif()
+            if (LLAMA_AVX)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx)
+            endif()
+            if (LLAMA_AVX2)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx2)
+            endif()
+            if (LLAMA_AVX512)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512f)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512bw)
+            endif()
+            if (LLAMA_AVX512_VBMI)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512vbmi)
+            endif()
+            if (LLAMA_AVX512_VNNI)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512vnni)
+            endif()
+        endif()
+    else()
+        # TODO: support PowerPC
+        message(STATUS "Unknown architecture")
+    endif()
+
+    target_link_libraries(ggml${SUFFIX} PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
+    if (WITH_LLAMA)
+        target_link_libraries(llama${SUFFIX} PRIVATE ggml${SUFFIX} ${LLAMA_EXTRA_LIBS})
+    endif()
 endfunction()

gpt4all-backend/llamamodel.cpp

@@ -97,6 +97,40 @@ LLamaModel::LLamaModel()
     d_ptr->modelLoaded = false;
 }
 
+// default hparams (LLaMA 7B)
+struct llama_file_hparams {
+    uint32_t n_vocab = 32000;
+    uint32_t n_embd  = 4096;
+    uint32_t n_mult  = 256;
+    uint32_t n_head  = 32;
+    uint32_t n_layer = 32;
+    uint32_t n_rot   = 64;
+    enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16;
+};
+
+size_t LLamaModel::requiredMem(const std::string &modelPath) {
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    fin.seekg(0, std::ios_base::end);
+    size_t filesize = fin.tellg();
+    fin.seekg(0, std::ios_base::beg);
+    uint32_t magic = 0;
+    fin.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    if (magic != 0x67676a74) return 0;
+    uint32_t version = 0;
+    fin.read(reinterpret_cast<char*>(&version), sizeof(version));
+    llama_file_hparams hparams;
+    fin.read(reinterpret_cast<char*>(&hparams.n_vocab), sizeof(hparams.n_vocab));
+    fin.read(reinterpret_cast<char*>(&hparams.n_embd), sizeof(hparams.n_embd));
+    fin.read(reinterpret_cast<char*>(&hparams.n_head), sizeof(hparams.n_head));
+    fin.read(reinterpret_cast<char*>(&hparams.n_layer), sizeof(hparams.n_layer));
+    fin.read(reinterpret_cast<char*>(&hparams.n_rot), sizeof(hparams.n_rot));
+    fin.read(reinterpret_cast<char*>(&hparams.ftype), sizeof(hparams.ftype));
+    const size_t n_ctx = 2048;
+    const size_t kvcache_element_size = 2; // fp16
+    const size_t est_kvcache_size = hparams.n_embd * hparams.n_layer * 2u * n_ctx * kvcache_element_size;
+    return filesize + est_kvcache_size;
+}
+
 bool LLamaModel::loadModel(const std::string &modelPath)
 {
     // load the model
@@ -115,6 +149,12 @@ bool LLamaModel::loadModel(const std::string &modelPath)
 #if LLAMA_DATE <= 230511
     d_ptr->params.n_parts  = params.n_parts;
 #endif
+#ifdef GGML_USE_METAL
+    std::cerr << "llama.cpp: using Metal" << std::endl;
+    // metal always runs the whole model if n_gpu_layers is not 0, at least
+    // currently
+    d_ptr->params.n_gpu_layers = 1;
+#endif
 
     d_ptr->ctx = llama_init_from_file(modelPath.c_str(), d_ptr->params);
     if (!d_ptr->ctx) {
@@ -138,7 +178,9 @@ int32_t LLamaModel::threadCount() const {
 
 LLamaModel::~LLamaModel()
 {
-    llama_free(d_ptr->ctx);
+    if(d_ptr->ctx) {
+        llama_free(d_ptr->ctx);
+    }
 }
 
 bool LLamaModel::isModelLoaded() const
@@ -171,7 +213,7 @@ std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::
     return fres;
 }
 
-std::string_view LLamaModel::tokenToString(Token id) const
+std::string LLamaModel::tokenToString(Token id) const
 {
     return llama_token_to_str(d_ptr->ctx, id);
 }
@@ -187,7 +229,16 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
 
 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
 {
-    return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+    // When we recalculate context we could have erased the original BOS token... we need to replace it
+    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
+    if (useBOS) {
+        std::vector<int32_t> myTokens;
+        myTokens.push_back(llama_token_bos());
+        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
+        ctx.n_past += 1;
+        return llama_eval(d_ptr->ctx, myTokens.data(), myTokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+    } else
+        return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
 }
 
 int32_t LLamaModel::contextLength() const
@@ -228,7 +279,31 @@ DLL_EXPORT bool magic_match(std::istream& f) {
     // Check version
     uint32_t version = 0;
     f.read(reinterpret_cast<char*>(&version), sizeof(version));
-    return version LLAMA_VERSIONS;
+    if (!(version LLAMA_VERSIONS)) {
+        return false;
+    }
+    llama_file_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    if (!(hparams.n_vocab >= 32000 && hparams.n_vocab <= 32100)) {
+        return false; // not a llama.
+    }
+#ifdef GGML_USE_METAL
+    // Check quant supported on metal
+    // skip fields
+    switch(hparams.ftype) {
+        // currently supported on Metal https://github.com/ggerganov/llama.cpp/blob/ae9663f1887513e152839e91f61c513075a19422/ggml-metal.m#L51-L55
+        case LLAMA_FTYPE_MOSTLY_F16:
+        case LLAMA_FTYPE_MOSTLY_Q2_K:
+        case LLAMA_FTYPE_MOSTLY_Q4_0:
+        case LLAMA_FTYPE_MOSTLY_Q6_K:
+        case LLAMA_FTYPE_MOSTLY_Q4_K_S:
+        case LLAMA_FTYPE_MOSTLY_Q4_K_M:
+            return true;
+        default: // unsupported quant-type for Metal
+            return false;
+    }
+#endif
+    return true;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llamamodel_impl.h

@@ -17,6 +17,7 @@ public:
 
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
@@ -28,7 +29,7 @@ private:
 
 protected:
     std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string_view tokenToString(Token) const override;
+    std::string tokenToString(Token) const override;
     Token sampleToken(PromptContext& ctx) const override;
     bool evalTokens(PromptContext& ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;

gpt4all-backend/llmodel.cpp

@@ -1,5 +1,6 @@
 #include "llmodel.h"
 #include "dlhandle.h"
+#include "sysinfo.h"
 
 #include <iostream>
 #include <string>
@@ -9,17 +10,19 @@
 #include <cassert>
 #include <cstdlib>
 #include <sstream>
+#ifdef _MSC_VER
+#include <windows.h>
+#include <processthreadsapi.h>
+#endif
 
-std::string LLModel::m_implementations_search_path = ".";
+std::string s_implementations_search_path = ".";
 
 static bool has_at_least_minimal_hardware() {
 #ifdef __x86_64__
     #ifndef _MSC_VER
         return __builtin_cpu_supports("avx");
     #else
-        int cpuInfo[4];
-        __cpuid(cpuInfo, 1);
-        return cpuInfo[2] & (1 << 28);
+        return IsProcessorFeaturePresent(PF_AVX_INSTRUCTIONS_AVAILABLE);
     #endif
 #else
     return true; // Don't know how to handle non-x86_64
@@ -31,9 +34,7 @@ static bool requires_avxonly() {
     #ifndef _MSC_VER
         return !__builtin_cpu_supports("avx2");
     #else
-        int cpuInfo[4];
-        __cpuidex(cpuInfo, 7, 0);
-        return !(cpuInfo[1] & (1 << 5));
+        return !IsProcessorFeaturePresent(PF_AVX2_INSTRUCTIONS_AVAILABLE);
     #endif
 #else
     return false; // Don't know how to handle non-x86_64
@@ -98,7 +99,7 @@ const std::vector<LLModel::Implementation> &LLModel::implementationList() {
             }
         };
 
-        search_in_directory(m_implementations_search_path);
+        search_in_directory(s_implementations_search_path);
 
         return fres;
     }());
@@ -121,21 +122,52 @@ LLModel *LLModel::construct(const std::string &modelPath, std::string buildVaria
     if (!has_at_least_minimal_hardware())
         return nullptr;
 
-    //TODO: Auto-detect CUDA/OpenCL
-    if (buildVariant == "auto") {
-        if (requires_avxonly()) {
-            buildVariant = "avxonly";
-        } else {
-            buildVariant = "default";
-        }
-    }
     // Read magic
     std::ifstream f(modelPath, std::ios::binary);
     if (!f) return nullptr;
     // Get correct implementation
-    auto impl = implementation(f, buildVariant);
-    if (!impl) return nullptr;
+    const LLModel::Implementation* impl = nullptr;
+
+    #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
+        if (buildVariant == "auto") {
+            size_t total_mem = getSystemTotalRAMInBytes();
+            impl = implementation(f, "metal");
+            if(impl) {
+                LLModel* metalimpl = impl->construct();
+                size_t req_mem = metalimpl->requiredMem(modelPath);
+                float req_to_total = (float) req_mem / (float) total_mem;
+                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
+                if (req_to_total >= 0.53) {
+                    delete metalimpl;
+                    impl = nullptr;
+                } else {
+                    return metalimpl;
+                }
+            }
+        }
+    #endif
+
+    if (!impl) {
+        //TODO: Auto-detect CUDA/OpenCL
+        if (buildVariant == "auto") {
+            if (requires_avxonly()) {
+                buildVariant = "avxonly";
+            } else {
+                buildVariant = "default";
+            }
+        }
+        impl = implementation(f, buildVariant);
+        if (!impl) return nullptr;
+    }
     f.close();
     // Construct and return llmodel implementation
     return impl->construct();
 }
+
+void LLModel::setImplementationsSearchPath(const std::string& path) {
+    s_implementations_search_path = path;
+}
+
+const std::string& LLModel::implementationsSearchPath() {
+    return s_implementations_search_path;
+}

gpt4all-backend/llmodel.h

@@ -9,6 +9,8 @@
 #include <cstdint>
 #include <limits>
 
+#define LLMODEL_MAX_PROMPT_BATCH 128
+
 class Dlhandle;
 
 class LLModel {
@@ -59,6 +61,7 @@ public:
 
     virtual bool loadModel(const std::string &modelPath) = 0;
     virtual bool isModelLoaded() const = 0;
+    virtual size_t requiredMem(const std::string &modelPath) = 0;
     virtual size_t stateSize() const { return 0; }
     virtual size_t saveState(uint8_t */*dest*/) const { return 0; }
     virtual size_t restoreState(const uint8_t */*src*/) { return 0; }
@@ -77,20 +80,16 @@ public:
 
     static const std::vector<Implementation>& implementationList();
     static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
-    static LLModel *construct(const std::string &modelPath, std::string buildVariant = "default");
+    static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
 
-    static inline void setImplementationsSearchPath(const std::string& path) {
-        m_implementations_search_path = path;
-    }
-    static inline const std::string& implementationsSearchPath() {
-        return m_implementations_search_path;
-    }
+    static void setImplementationsSearchPath(const std::string& path);
+    static const std::string& implementationsSearchPath();
 
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
     virtual std::vector<Token> tokenize(PromptContext &, const std::string&) const = 0;
-    virtual std::string_view tokenToString(Token) const = 0;
+    virtual std::string tokenToString(Token) const = 0;
     virtual Token sampleToken(PromptContext &ctx) const = 0;
     virtual bool evalTokens(PromptContext &/*ctx*/, const std::vector<int32_t>& /*tokens*/) const = 0;
     virtual int32_t contextLength() const = 0;
@@ -101,6 +100,5 @@ protected:
     void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate);
 
     const Implementation *m_implementation = nullptr;
-    static std::string m_implementations_search_path;
 };
 #endif // LLMODEL_H

gpt4all-backend/llmodel_c.cpp

@@ -9,6 +9,7 @@
 struct LLModelWrapper {
     LLModel *llModel = nullptr;
     LLModel::PromptContext promptContext;
+    ~LLModelWrapper() { delete llModel; }
 };
 
 
@@ -25,33 +26,44 @@ llmodel_model llmodel_model_create(const char *model_path) {
 
 llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error) {
     auto wrapper = new LLModelWrapper;
-    llmodel_error new_error{};
+    int error_code = 0;
 
     try {
         wrapper->llModel = LLModel::construct(model_path, build_variant);
     } catch (const std::exception& e) {
-        new_error.code = EINVAL;
+        error_code = EINVAL;
         last_error_message = e.what();
     }
 
     if (!wrapper->llModel) {
         delete std::exchange(wrapper, nullptr);
         // Get errno and error message if none
-        if (new_error.code == 0) {
-            new_error.code = errno;
-            last_error_message = strerror(errno);
+        if (error_code == 0) {
+            if (errno != 0) {
+                error_code = errno;
+                last_error_message = std::strerror(error_code);
+            } else {
+                error_code = ENOTSUP;
+                last_error_message = "Model format not supported (no matching implementation found)";
+            }
         }
-        // Set message pointer
-        new_error.message = last_error_message.c_str();
         // Set error argument
-        if (error) *error = new_error;
+        if (error) {
+            error->message = last_error_message.c_str();
+            error->code = error_code;
+        }
     }
     return reinterpret_cast<llmodel_model*>(wrapper);
 }
 
 void llmodel_model_destroy(llmodel_model model) {
+    delete reinterpret_cast<LLModelWrapper*>(model);
+}
+
+size_t llmodel_required_mem(llmodel_model model, const char *model_path)
+{
     LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    delete wrapper->llModel;
+    return wrapper->llModel->requiredMem(model_path);
 }
 
 bool llmodel_loadModel(llmodel_model model, const char *model_path)
@@ -116,6 +128,9 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     std::function<bool(bool)> recalc_func =
         std::bind(&recalculate_wrapper, std::placeholders::_1, reinterpret_cast<void*>(recalculate_callback));
 
+    if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
+        wrapper->promptContext.tokens.resize(ctx->n_past);
+
     // Copy the C prompt context
     wrapper->promptContext.n_past = ctx->n_past;
     wrapper->promptContext.n_ctx = ctx->n_ctx;

gpt4all-backend/llmodel_c.h

@@ -107,6 +107,14 @@ llmodel_model llmodel_model_create2(const char *model_path, const char *build_va
  */
 void llmodel_model_destroy(llmodel_model model);
 
+/**
+ * Estimate RAM requirement for a model file
+ * @param model A pointer to the llmodel_model instance.
+ * @param model_path A string representing the path to the model file.
+ * @return size greater than 0 if the model was parsed successfully, 0 if file could not be parsed.
+ */
+size_t llmodel_required_mem(llmodel_model model, const char *model_path);
+
 /**
  * Load a model from a file.
  * @param model A pointer to the llmodel_model instance.

gpt4all-backend/llmodel_shared.cpp

@@ -52,6 +52,7 @@ void LLModel::prompt(const std::string &prompt,
 
     promptCtx.n_predict = std::min(promptCtx.n_predict, promptCtx.n_ctx - (int) embd_inp.size());
     promptCtx.n_past = std::min(promptCtx.n_past, promptCtx.n_ctx);
+    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
 
     // process the prompt in batches
     size_t i = 0;
@@ -121,7 +122,7 @@ void LLModel::prompt(const std::string &prompt,
             if (id == token) return;
         }
 
-        const std::string_view str = tokenToString(id);
+        const std::string str = tokenToString(id);
 
         // Check if the provided str is part of our reverse prompts
         bool foundPartialReversePrompt = false;

gpt4all-backend/llmodel_shared.h

@@ -0,0 +1,36 @@
+#pragma once
+#include <cstdint>
+#include <cstddef>
+#include <ggml.h>
+
+struct llm_buffer {
+    uint8_t * addr = NULL;
+    size_t size = 0;
+
+    void resize(size_t size) {
+        delete[] addr;
+        addr = new uint8_t[size];
+        this->size = size;
+    }
+
+    ~llm_buffer() {
+        delete[] addr;
+    }
+};
+
+struct llm_kv_cache {
+    struct ggml_tensor * k;
+    struct ggml_tensor * v;
+
+    struct ggml_context * ctx = NULL;
+
+    llm_buffer buf;
+
+    int n; // number of tokens currently in the cache
+
+    ~llm_kv_cache() {
+        if (ctx) {
+            ggml_free(ctx);
+        }
+    }
+};

gpt4all-backend/mpt.cpp

@@ -2,8 +2,10 @@
 #include "mpt_impl.h"
 
 #include "utils.h"
+#include "llmodel_shared.h"
 
 #include <cassert>
+#include <cinttypes>
 #include <cmath>
 #include <cstdio>
 #include <cstring>
@@ -33,8 +35,6 @@
 
 namespace {
 const char *modelType_ = "MPT";
-
-static const size_t MB = 1024*1024;
 }
 
 // default hparams (MPT 7B)
@@ -64,39 +64,6 @@ struct mpt_layer {
     struct ggml_tensor * ffn_down_proj_w;
 };
 
-struct mpt_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-
-    void resize(size_t size) {
-        delete[] addr;
-        addr = new uint8_t[size];
-        this->size = size;
-    }
-
-    ~mpt_buffer() {
-        fflush(stdout);
-        delete[] addr;
-    }
-};
-
-struct mpt_kv_cache {
-    struct ggml_tensor * k;
-    struct ggml_tensor * v;
-
-    struct ggml_context * ctx = NULL;
-
-    mpt_buffer buf;
-
-    int n; // number of tokens currently in the cache
-
-    ~mpt_kv_cache() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
 struct mpt_model {
     mpt_hparams hparams;
 
@@ -109,12 +76,14 @@ struct mpt_model {
 
     std::vector<mpt_layer> layers;
 
-    struct mpt_kv_cache kv_self;
+    struct llm_kv_cache kv_self;
     struct ggml_context * ctx;
     std::map<std::string, struct ggml_tensor *> tensors;
 
 
-    mpt_buffer buf;
+    llm_buffer eval_buf;
+    llm_buffer scr0_buf;
+    llm_buffer scr1_buf;
 
     ~mpt_model() {
         if (ctx) {
@@ -125,7 +94,7 @@ struct mpt_model {
 
 static bool kv_cache_init(
         const struct mpt_hparams & hparams,
-             struct mpt_kv_cache & cache,
+             struct llm_kv_cache & cache,
                          ggml_type   wtype,
                                int   n_ctx) {
     const int n_embd  = hparams.n_embd;
@@ -134,7 +103,7 @@ static bool kv_cache_init(
     const int64_t n_mem      = (int64_t)n_layer*n_ctx;
     const int64_t n_elements = n_embd*n_mem;
 
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB);
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
 
     struct ggml_init_params params;
     params.mem_size   = cache.buf.size;
@@ -154,9 +123,13 @@ static bool kv_cache_init(
     return true;
 }
 
-// load the model's weights from a stream
-bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & model, gpt_vocab & vocab) {
+// load the model's weights from a stream. if mem_req ptr is passed the model is
+// only partially parsed to estimate required memory
+bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & model, gpt_vocab & vocab, size_t * mem_req) {
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+    if (mem_req != nullptr) {
+        *mem_req = 0;
+    }
 
     // verify magic
     {
@@ -278,6 +251,18 @@ bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & mod
         printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
     }
 
+    if (mem_req != nullptr) {
+        *mem_req += ctx_size;
+        const int n_embd  = model.hparams.n_embd;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = n_embd*n_mem;
+
+        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+        return false;
+    }
+
     // create the ggml context
     {
         struct ggml_init_params params = {
@@ -388,8 +373,8 @@ bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & mod
             }
 
             if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
-                        __func__, name.data(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
+                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
                 return false;
             }
 
@@ -421,6 +406,9 @@ bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & mod
         printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
     }
 
+    model.scr0_buf.resize(256u * 1024 * 1024);
+    model.scr1_buf.resize(256u * 1024 * 1024);
+
     return true;
 }
 
@@ -433,7 +421,7 @@ bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vo
         return false;
     }
 
-    bool loaded = mpt_model_load(fname, fin, model, vocab);
+    bool loaded = mpt_model_load(fname, fin, model, vocab, nullptr);
     fin.close();
     return loaded;
 }
@@ -455,25 +443,25 @@ bool mpt_eval(
     const int n_head  = hparams.n_head;
     const int n_vocab = hparams.n_vocab;
 
-    const size_t init_buf_size = 1024u*MB;
-    if (!model.buf.addr || model.buf.size < init_buf_size)
-        model.buf.resize(init_buf_size);
+    const size_t init_buf_size = 1024_MiB;
+    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
+        model.eval_buf.resize(init_buf_size);
 
-    if (mem_per_token > 0 && mem_per_token*N > model.buf.size) {
+    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
         const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
         // printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
 
         // reallocate
-        model.buf.resize(buf_size_new);
-        if (model.buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.buf.size);
+        model.eval_buf.resize(buf_size_new);
+        if (model.eval_buf.addr == nullptr) {
+            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
             return false;
         }
     }
 
     struct ggml_init_params params = {
-        .mem_size   = model.buf.size,
-        .mem_buffer = model.buf.addr,
+        .mem_size   = model.eval_buf.size,
+        .mem_buffer = model.eval_buf.addr,
         .no_alloc = false
     };
 
@@ -488,6 +476,7 @@ bool mpt_eval(
     struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
 
     for (int il = 0; il < n_layer; ++il) {
+        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
 
         struct ggml_tensor * inpSA = inpL;
         struct ggml_tensor * cur = inpSA;
@@ -579,7 +568,7 @@ bool mpt_eval(
                     cur);
         }
 
-
+        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
         // residual
         struct ggml_tensor * resSA = ggml_add(ctx0, cur, inpSA);
         // feed-forward network
@@ -604,6 +593,7 @@ bool mpt_eval(
         // self-attention + FF
         inpL = ggml_add(ctx0, cur, resSA);
     }
+    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
 
     struct ggml_tensor * out = inpL;
     // -> logits
@@ -612,6 +602,7 @@ bool mpt_eval(
         out = ggml_mul(ctx0,
                     ggml_repeat(ctx0, model.norm_f_w, out),
                     out);
+        ggml_set_scratch(ctx0, { 0, 0, nullptr, });
         out = ggml_mul_mat(ctx0, model.wte, out);
     }
 
@@ -759,9 +750,19 @@ struct MPTPrivate {
 MPT::MPT()
     : d_ptr(new MPTPrivate) {
     d_ptr->model = new mpt_model;
+    d_ptr->model->ctx = nullptr;
     d_ptr->modelLoaded = false;
 }
 
+size_t MPT::requiredMem(const std::string &modelPath) {
+    mpt_model dummy_model;
+    gpt_vocab dummy_vocab;
+    size_t mem_req;
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    mpt_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
+    return mem_req;
+}
+
 bool MPT::loadModel(const std::string &modelPath) {
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
@@ -769,8 +770,8 @@ bool MPT::loadModel(const std::string &modelPath) {
     auto fin = std::ifstream(modelPath, std::ios::binary);
 
     // load the model
-    if (!mpt_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab)) {
-        std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
+    if (!mpt_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab, nullptr)) {
+        std::cerr << "MPT ERROR: failed to load model from " <<  modelPath;
         return false;
     }
 
@@ -820,7 +821,7 @@ std::vector<LLModel::Token> MPT::tokenize(PromptContext &, const std::string &st
     return ::gpt_tokenize(d_ptr->vocab, str);
 }
 
-std::string_view MPT::tokenToString(Token id) const
+std::string MPT::tokenToString(Token id) const
 {
     return d_ptr->vocab.id_to_token[id];
 }

gpt4all-backend/mpt_impl.h

@@ -17,6 +17,7 @@ public:
 
     bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
@@ -28,7 +29,7 @@ private:
 
 protected:
     std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string_view tokenToString(Token) const override;
+    std::string tokenToString(Token) const override;
     Token sampleToken(PromptContext &ctx) const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;

gpt4all-backend/replit_impl.h

@@ -0,0 +1,41 @@
+#ifndef REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of replit.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef REPLIT_H
+#define REPLIT_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include "llmodel.h"
+
+#define GGML_QNT_VERSION_FACTOR 1000 // do not change this
+
+struct ReplitPrivate;
+class Replit : public LLModel {
+public:
+    Replit();
+    ~Replit();
+
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string & modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    ReplitPrivate *d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::string tokenToString(Token) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // REPLIT_H

gpt4all-backend/scripts/convert_falcon_hf_to_ggml.py

@@ -0,0 +1,143 @@
+# Based on: https://github.com/KerfuffleV2/ggml-falcon/blob/feat-improve-falcon-convert-hf/examples/falcon/convert-hf-to-ggml.py
+# Convert Hugging Face fine-tuned bloom-like models to ggml format
+#
+# Usage:
+#
+#   python3 convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]
+#
+# This script is similar to "convert-pt-to-ggml.py"
+#
+
+import io
+import os
+import sys
+import struct
+import json
+import code
+import torch
+import numpy as np
+import gc
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
+
+# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a significant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+if len(sys.argv) < 3:
+    print("INFO: GGML V1 files produced are meant to be finalized through examples/falcon_quantize which will bring them to latest version and precision of choice");
+    print("Usage: python convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]")
+    print("  model_directory: name of the directory and model you convert (it should be a subdirectory)")
+    print("  output-directory: directory where the output file will be written")
+    print("  use-f32:    if present, use float32 instead of float16 (f32 is recommended)")
+    sys.exit(1)
+
+# num_parts = int(sys.argv[1])
+dir_model = sys.argv[1] # name and dir of model
+dir_out = sys.argv[2]   # output directory
+
+# make sure the output directory exists
+os.makedirs(dir_out, exist_ok=True)
+
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+ftype = 1
+if len(sys.argv) > 3:
+    ftype = 0
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+# print(tokenizer)
+config = AutoConfig.from_pretrained(dir_model, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(dir_model, trust_remote_code=True, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)
+hparams = config.to_dict()
+
+n_head = hparams["n_head"]
+n_head_kv = hparams["n_head_kv"] if "n_head_kv" in hparams else 1
+head_dim = hparams["hidden_size"] // n_head
+print("* Loading model from: ", dir_model)
+
+fname_out = dir_out + f"/ggml-model-{dir_model.split('/')[-1]}-{ftype_str[ftype]}.bin"
+fout = open(fname_out, "wb")
+fout.write(struct.pack("i", 0x67676a74)) # magic: ggmf in hex (version 1) - possibly change to ggfc ?
+fout.write(struct.pack("i", 1)) # version
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["hidden_size"]))
+fout.write(struct.pack("i", n_head))
+fout.write(struct.pack("i", n_head_kv))
+fout.write(struct.pack("i", hparams["n_layer"]))
+fout.write(struct.pack("i", 40 if "n_head_kv" in hparams else 7)) # obsolete field that breaks ggml compatibility - todo again remove one day
+fout.write(struct.pack("i", ftype))
+
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+byte_encoder = bytes_to_unicode()
+byte_decoder = {v:k for k, v in byte_encoder.items()}
+
+for i in range(hparams["vocab_size"]):
+    text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+    fout.write(struct.pack("i", len(text)))
+    fout.write(text)
+    fout.write(struct.pack("f", 0.0)) # falcon uses bpe on RefinedWeb - no probability scores used
+
+model = model.state_dict()
+for name in model.keys():
+    src = name
+    # The original query_key_value tensor contains n_head_kv "kv groups",
+    # each consisting of n_head/n_head_kv query weights followed by one key
+    # and one value weight (shared by all query heads in the kv group).
+    # This layout makes it a big pain to work with in GGML.
+    # So we rearrange them here,, so that we have n_head query weights
+    # followed by n_head_kv key weights followed by n_head_kv value weights,
+    # in contiguous fashion.
+
+    if "query_key_value" in src:
+        qkv = model[src].view(
+            n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)
+
+        q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head)
+        k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)
+        v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)
+
+        model[src] = torch.cat((q,k,v)).reshape_as(model[src])
+    data = model[src].squeeze()
+    n_dims = len(data.shape)
+    # default type is fp32
+    ftype_cur = 1 if ftype == 1 and n_dims > 1 else 0
+    data = data.to(dtype = torch.float16 if ftype_cur == 1 else torch.float32).numpy()
+    print(f'  |', name, data.shape, '->', data.dtype)
+    # header
+    str = name.encode('utf-8')
+    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str)
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/scripts/convert_replit_hf_to_ggml.py

@@ -0,0 +1,113 @@
+from pathlib import Path
+import sys
+import struct
+import json
+import numpy as np
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import sentencepiece.sentencepiece_model_pb2 as model
+
+if len(sys.argv) < 3:
+    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+
+# output in the same directory as the model
+dir_model = sys.argv[1]
+fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b.bin"
+
+
+with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
+    hparams = json.load(f)
+
+sp_proto = model.ModelProto()
+sp_proto.ParseFromString(open(Path(sys.argv[1]) / "spiece.model", "rb").read())
+
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+    fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".bin"
+
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    dir_model, low_cpu_mem_usage=True, trust_remote_code=True
+)
+# print (model)
+
+# print(tokenizer.encode('I believe the meaning of life is'))
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+fout = open(fname_out, "wb")
+
+print(hparams)
+
+fout.write(struct.pack("i", 0x7265706c))  # magic: repl in hex
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["max_seq_len"]))
+fout.write(struct.pack("i", hparams["d_model"]))
+fout.write(struct.pack("i", hparams["n_heads"]))
+fout.write(struct.pack("i", hparams["n_layers"]))
+fout.write(struct.pack("i", ftype))
+
+
+# TODO: temporary hack to not deal with implementing the tokenizer
+for piece in sp_proto.pieces:
+    encoded_piece = piece.piece.encode("utf-8")
+    fout.write(struct.pack("i", len(encoded_piece)))
+    fout.write(encoded_piece)
+    fout.write(struct.pack("f", piece.score))
+
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable: " + name + " with shape: ", data.shape)
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0
+    if ftype != 0:
+        if name[-7:] == ".weight" and n_dims == 2:
+            print("  Converting to float16")
+            data = data.astype(np.float16)
+            ftype_cur = 1
+        else:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+    else:
+        if data.dtype != np.float32:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+
+    # header
+    str = name.encode("utf-8")
+    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str)
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/sysinfo.h

@@ -0,0 +1,61 @@
+#ifndef SYSINFO_H
+#define SYSINFO_H
+
+#include <fstream>
+#include <string>
+#include <sstream>
+#include <iomanip>
+
+#if defined(__linux__)
+#include <unistd.h>
+#elif defined(__APPLE__)
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#elif defined(_WIN32)
+#include <windows.h>
+#endif
+
+static long long getSystemTotalRAMInBytes()
+{
+    long long totalRAM = 0;
+
+#if defined(__linux__)
+    std::ifstream file("/proc/meminfo");
+    std::string line;
+    while (std::getline(file, line)) {
+        if (line.find("MemTotal") != std::string::npos) {
+            std::string memTotalStr = line.substr(line.find(":") + 1);
+            memTotalStr.erase(0, memTotalStr.find_first_not_of(" "));
+            memTotalStr = memTotalStr.substr(0, memTotalStr.find(" "));
+            totalRAM = std::stoll(memTotalStr) * 1024;  // Convert from KB to bytes
+            break;
+        }
+    }
+    file.close();
+#elif defined(__APPLE__)
+    int mib[2] = {CTL_HW, HW_MEMSIZE};
+    size_t length = sizeof(totalRAM);
+    sysctl(mib, 2, &totalRAM, &length, NULL, 0);
+#elif defined(_WIN32)
+    MEMORYSTATUSEX memoryStatus;
+    memoryStatus.dwLength = sizeof(memoryStatus);
+    GlobalMemoryStatusEx(&memoryStatus);
+    totalRAM = memoryStatus.ullTotalPhys;
+#endif
+
+    return totalRAM;
+}
+
+static double getSystemTotalRAMInGB()
+{
+    return static_cast<double>(getSystemTotalRAMInBytes()) / (1024 * 1024 * 1024);
+}
+
+static std::string getSystemTotalRAMInGBString()
+{
+    std::stringstream ss;
+    ss << std::fixed << std::setprecision(2) << getSystemTotalRAMInGB() << " GB";
+    return ss.str();
+}
+
+#endif // SYSINFO_H

gpt4all-backend/utils.cpp

@@ -230,8 +230,21 @@ gpt_vocab::id gpt_sample_top_k_top_p(
     int n_logits = actualVocabSize;
 
     const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_size);
-    const auto * plogits = logits.data() + logits.size() - n_logits;
+    const auto * plogits = logits.data();
 
+    if (temp <= 0) {
+        // select the token with the highest logit directly
+        float max_logit = plogits[0];
+        gpt_vocab::id max_id = 0;
+
+        for (int i = 1; i < n_logits; ++i) {
+            if (plogits[i] > max_logit) {
+                max_logit = plogits[i];
+                max_id = i;
+            }
+        }
+        return max_id;
+    }
     std::vector<std::pair<double, gpt_vocab::id>> logits_id;
     logits_id.reserve(n_logits);
 

gpt4all-backend/utils.h

@@ -8,6 +8,13 @@
 #include <random>
 #include <thread>
 
+//
+// General purpose inline functions
+//
+constexpr inline unsigned long long operator ""_MiB(unsigned long long bytes) {
+    return bytes*1024*1024;
+}
+
 //
 // CLI argument parsing
 //

gpt4all-bindings/cli/README.md

@@ -0,0 +1,44 @@
+# GPT4All Command-Line Interface (CLI)
+
+GPT4All on the command-line.
+
+## Documentation
+<https://docs.gpt4all.io/gpt4all_cli.html>
+
+## Quickstart
+
+The CLI is based on the `gpt4all` Python bindings and the `typer` package.
+
+The following shows one way to get started with the CLI, the documentation has more information.
+Typically, you will want to replace `python` with `python3` on _Unix-like_ systems and `py -3` on
+_Windows_. Also, it's assumed you have all the necessary Python components already installed.
+
+The CLI is a self-contained Python script named [app.py] ([download][app.py-download]). As long as
+its package dependencies are present, you can download and run it from wherever you like.
+
+[app.py]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-bindings/cli/app.py
+[app.py-download]: https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-bindings/cli/app.py
+
+```shell
+# optional but recommended: create and use a virtual environment
+python -m venv gpt4all-cli
+```
+_Windows_ and _Unix-like_ systems differ slightly in how you activate a _virtual environment_:
+- _Unix-like_, typically: `. gpt4all-cli/bin/activate`
+- _Windows_: `gpt4all-cli\Scripts\activate`
+
+Then:
+```shell
+# pip-install the necessary packages; omit '--user' if using a virtual environment
+python -m pip install --user --upgrade gpt4all typer
+# run the CLI
+python app.py repl
+```
+By default, it will automatically download the `groovy` model to `.cache/gpt4all/` in your user
+directory, if necessary.
+
+If you have already saved a model beforehand, specify its path with the `-m`/`--model` argument,
+for example:
+```shell
+python app.py repl --model /home/user/my-gpt4all-models/GPT4All-13B-snoozy.ggmlv3.q4_0.bin
+```

gpt4all-bindings/cli/app.py

@@ -1,9 +1,19 @@
+"""GPT4All CLI
+
+The GPT4All CLI is a self-contained script based on the `gpt4all` and `typer` packages. It offers a
+REPL to communicate with a language model similar to the chat GUI application, but more basic.
+"""
+
+import io
+import pkg_resources  # should be present as a dependency of gpt4all
 import sys
 import typer
 
+from collections import namedtuple
 from typing_extensions import Annotated
 from gpt4all import GPT4All
 
+
 MESSAGES = [
     {"role": "system", "content": "You are a helpful assistant."},
     {"role": "user", "content": "Hello there."},
@@ -17,7 +27,9 @@ SPECIAL_COMMANDS = {
     "/help": lambda _: print("Special commands: /reset, /exit, /help and /clear"),
 }
 
-VERSION = "0.1.0"
+VersionInfo = namedtuple('VersionInfo', ['major', 'minor', 'micro'])
+VERSION_INFO = VersionInfo(1, 0, 2)
+VERSION = '.'.join(map(str, VERSION_INFO))  # convert to string form, like: '1.2.3'
 
 CLI_START_MESSAGE = f"""
     
@@ -33,12 +45,6 @@ Type /help for special commands.
                                                     
 """
 
-def _cli_override_response_callback(token_id, response):
-    resp = response.decode("utf-8")
-    print(resp, end="", flush=True)
-    return True
-
-
 # create typer app
 app = typer.Typer()
 
@@ -53,6 +59,7 @@ def repl(
         typer.Option("--n-threads", "-t", help="Number of threads to use for chatbot"),
     ] = None,
 ):
+    """The CLI read-eval-print loop."""
     gpt4all_instance = GPT4All(model)
 
     # if threads are passed, set them
@@ -68,11 +75,23 @@ def repl(
     else:
         print(f"\nUsing {gpt4all_instance.model.thread_count()} threads", end="")
 
-    # overwrite _response_callback on model
-    gpt4all_instance.model._response_callback = _cli_override_response_callback
-
     print(CLI_START_MESSAGE)
 
+    use_new_loop = False
+    try:
+        version = pkg_resources.Environment()['gpt4all'][0].version
+        version_major = int(version.split('.')[0])
+        if version_major >= 1:
+            use_new_loop = True
+    except:
+        pass  # fall back to old loop
+    if use_new_loop:
+        _new_loop(gpt4all_instance)
+    else:
+        _old_loop(gpt4all_instance)
+
+
+def _old_loop(gpt4all_instance):
     while True:
         message = input(" ⇢  ")
 
@@ -103,16 +122,58 @@ def repl(
             context_erase=0.0,
             # required kwargs for cli ux (incremental response)
             verbose=False,
-            std_passthrough=True,
+            streaming=True,
         )
         # record assistant's response to messages
         MESSAGES.append(full_response.get("choices")[0].get("message"))
         print() # newline before next prompt
 
 
+def _new_loop(gpt4all_instance):
+    with gpt4all_instance.chat_session():
+        while True:
+            message = input(" ⇢  ")
+
+            # Check if special command and take action
+            if message in SPECIAL_COMMANDS:
+                SPECIAL_COMMANDS[message](MESSAGES)
+                continue
+
+            # if regular message, append to messages
+            MESSAGES.append({"role": "user", "content": message})
+
+            # execute chat completion and ignore the full response since 
+            # we are outputting it incrementally
+            response_generator = gpt4all_instance.generate(
+                message,
+                # preferential kwargs for chat ux
+                max_tokens=200,
+                temp=0.9,
+                top_k=40,
+                top_p=0.9,
+                repeat_penalty=1.1,
+                repeat_last_n=64,
+                n_batch=9,
+                # required kwargs for cli ux (incremental response)
+                streaming=True,
+            )
+            response = io.StringIO()
+            for token in response_generator:
+                print(token, end='', flush=True)
+                response.write(token)
+
+            # record assistant's response to messages
+            response_message = {'role': 'assistant', 'content': response.getvalue()}
+            response.close()
+            gpt4all_instance.current_chat_session.append(response_message)
+            MESSAGES.append(response_message)
+            print() # newline before next prompt
+
+
 @app.command()
 def version():
-    print("gpt4all-cli v0.1.0")
+    """The CLI version command."""
+    print(f"gpt4all-cli v{VERSION}")
 
 
 if __name__ == "__main__":

gpt4all-bindings/cli/developer_notes.md

@@ -0,0 +1,25 @@
+# Developing the CLI
+## Documentation
+Documentation can be found in three places:
+- `app.py` docstrings & comments
+- a Readme: `gpt4all-bindings/cli/README.md`
+- the actual CLI documentation: `gpt4all-bindings/python/docs/gpt4all_cli.md`
+
+The _docstrings_ are meant for programmatic use. Since the CLI is primarily geared towards users and
+not to build on top, they're kept terse.
+
+The _Readme_ is mostly meant for users and includes:
+- a link to the _CLI documentation_ (on the [website])
+- a Quickstart section with some guidance on how to get started with a sane setup
+
+The _CLI documentation_ and other documentation are located in the above mentioned `docs/` folder.
+They're in Markdown format and built for the [website]. Of the three, they should be the most
+detailed.
+
+[website]: https://docs.gpt4all.io/gpt4all_cli.html
+
+
+## Versioning
+The version number should now follow the `gpt4all` PyPI package, so compatibility is more clear.
+
+The one place to change it is the `namedtuple` called `VERSION_INFO`.

gpt4all-bindings/csharp/Directory.Build.props

@@ -5,7 +5,7 @@
         <Company></Company>
         <Copyright></Copyright>
         <NeutralLanguage>en-US</NeutralLanguage>
-        <Version>0.6.1-alpha</Version>
+        <Version>0.6.3-alpha</Version>
         <VersionSuffix>$(VersionSuffix)</VersionSuffix>
         <Version Condition=" '$(VersionSuffix)' != '' ">$(Version)$(VersionSuffix)</Version>
         <TreatWarningsAsErrors>true</TreatWarningsAsErrors>

gpt4all-bindings/csharp/Gpt4All.Samples/Gpt4All.Samples.csproj

@@ -1,18 +1,32 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net7.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
-  </ItemGroup>
+    <ItemGroup>
+        <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
+    </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Properties\" />
-  </ItemGroup>
+    <ItemGroup>
+        <!-- Windows -->
+        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
+        <!-- Linux -->
+        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
+        <!-- MacOS -->
+        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
+    </ItemGroup>
 
+    <ItemGroup>
+        <!-- Windows -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+        <!-- Linux -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+         <!-- MacOS -->
+        <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+        <Content Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.metal" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+    </ItemGroup>
 </Project>

gpt4all-bindings/csharp/Gpt4All.Tests/Constants.cs

@@ -1,10 +1,9 @@
-namespace Gpt4All.Tests
+namespace Gpt4All.Tests;
+
+public static class Constants
 {
-    public static class Constants
-    {
-        public const string MODELS_BASE_DIR = "../../../models";
-        public const string LLAMA_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-l13b-snoozy.bin";
-        public const string GPTJ_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-j-v1.3-groovy.bin";
-        public const string MPT_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-mpt-7b-chat.bin";
-    }
+    public const string MODELS_BASE_DIR = "../../../models";
+    public const string LLAMA_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-l13b-snoozy.bin";
+    public const string GPTJ_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-j-v1.3-groovy.bin";
+    public const string MPT_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-mpt-7b-chat.bin";
 }

gpt4all-bindings/csharp/Gpt4All.Tests/Gpt4All.Tests.csproj

@@ -1,27 +1,59 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net6.0</TargetFramework>
+        <TargetFramework>net7.0</TargetFramework>
         <Nullable>enable</Nullable>
 
         <IsPackable>false</IsPackable>
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
-        <PackageReference Include="xunit" Version="2.4.1" />
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2" />
+        <PackageReference Include="xunit" Version="2.4.2" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             <PrivateAssets>all</PrivateAssets>
         </PackageReference>
-        <PackageReference Include="coverlet.collector" Version="3.1.0">
+        <PackageReference Include="coverlet.collector" Version="6.0.0">
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             <PrivateAssets>all</PrivateAssets>
         </PackageReference>
     </ItemGroup>
 
     <ItemGroup>
-      <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
+        <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+        <!-- Windows -->
+        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
+        <!-- Linux -->
+        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
+        <!-- MacOS -->
+        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <!-- Windows -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+        <!-- Linux -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+         <!-- MacOS -->
+        <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Update="Roslynator.Analyzers" Version="4.3.0">
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+      </PackageReference>
+      <PackageReference Update="Roslynator.CodeAnalysis.Analyzers" Version="4.3.0">
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+      </PackageReference>
+      <PackageReference Update="Roslynator.Formatting.Analyzers" Version="4.3.0">
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+      </PackageReference>
+    </ItemGroup>
 </Project>

gpt4all-bindings/csharp/Gpt4All.Tests/ModelFactoryTests.cs

@@ -1,4 +1,4 @@
-using Xunit;
+using Xunit;
 
 namespace Gpt4All.Tests;
 
@@ -12,20 +12,23 @@ public class ModelFactoryTests
     }
 
     [Fact]
+    [Trait(Traits.SkipOnCI, "True")]
     public void CanLoadLlamaModel()
     {
-        using var model = _modelFactory.LoadLlamaModel(Constants.LLAMA_MODEL_PATH);
+        using var model = _modelFactory.LoadModel(Constants.LLAMA_MODEL_PATH);
     }
 
     [Fact]
+    [Trait(Traits.SkipOnCI, "True")]
     public void CanLoadGptjModel()
     {
-        using var model = _modelFactory.LoadGptjModel(Constants.GPTJ_MODEL_PATH);
+        using var model = _modelFactory.LoadModel(Constants.GPTJ_MODEL_PATH);
     }
 
     [Fact]
+    [Trait(Traits.SkipOnCI, "True")]
     public void CanLoadMptModel()
     {
-        using var model = _modelFactory.LoadMptModel(Constants.MPT_MODEL_PATH);
+        using var model = _modelFactory.LoadModel(Constants.MPT_MODEL_PATH);
     }
 }

gpt4all-bindings/csharp/Gpt4All.Tests/NativeLibraryLoaderTests.cs

@@ -0,0 +1,56 @@
+using System.IO;
+using Gpt4All.LibraryLoader;
+using Xunit;
+
+namespace Gpt4All.Tests;
+
+public class NativeLibraryLoaderTests
+{
+    [Fact]
+    public void NativeLibraryShouldLoad()
+    {
+        var result = NativeLibraryLoader.LoadNativeLibrary(bypassLoading: false);
+        Assert.True(result.IsSuccess);
+    }
+
+    private const string LLModelLib = "libllmodel.{0}";
+
+    [PlatformSpecificFact(Platforms.Windows)]
+    public void NativeLibraryShouldLoad_Windows()
+    {
+        var libraryLoader = new WindowsLibraryLoader();
+
+        var libraryPath = Path.Combine(
+            Environment.CurrentDirectory,
+            string.Format(LLModelLib, "dll"));
+
+        var result = libraryLoader.OpenLibrary(libraryPath);
+        Assert.True(result.IsSuccess);
+    }
+
+    [PlatformSpecificFact(Platforms.Linux)]
+    public void NativeLibraryShouldLoad_Linux()
+    {
+        var libraryLoader = new LinuxLibraryLoader();
+
+        var libraryPath = Path.Combine(
+            Environment.CurrentDirectory,
+            string.Format(LLModelLib, "so"));
+
+        var result = libraryLoader.OpenLibrary(libraryPath);
+        Assert.True(result.IsSuccess);
+    }
+
+    [PlatformSpecificFact(Platforms.MacOS)]
+    public void NativeLibraryShouldLoad_MacOS()
+    {
+        var libraryLoader = new MacOsLibraryLoader();
+
+        var libraryPath = Path.Combine(
+            Environment.CurrentDirectory,
+            string.Format(LLModelLib, "dylib"));
+
+        var result = libraryLoader.OpenLibrary(libraryPath);
+        Assert.True(result.IsSuccess);
+    }
+}

gpt4all-bindings/csharp/Gpt4All.Tests/PlatformSpecificFactAttribute.cs

@@ -0,0 +1,27 @@
+using Xunit;
+
+namespace Gpt4All.Tests;
+
+public static class Platforms
+{
+    public const string Windows = "windows";
+    public const string Linux = "linux";
+    public const string MacOS = "macOS";
+}
+
+/// <summary>
+/// This attribute ensures the Fact is only run on the specified platform.
+/// </summary>
+/// <remarks>
+/// <see cref="OperatingSystem.IsOSPlatform(string)"/> for info about the platform string.
+/// </remarks>
+public class PlatformSpecificFactAttribute : FactAttribute
+{
+    public PlatformSpecificFactAttribute(string platform)
+    {
+        if (!OperatingSystem.IsOSPlatform(platform))
+        {
+            Skip = $"Test only runs on {platform}.";
+        }
+    }
+}

gpt4all-bindings/csharp/Gpt4All.Tests/Traits.cs

@@ -0,0 +1,6 @@
+namespace Gpt4All.Tests;
+
+public static class Traits
+{
+    public const string SkipOnCI = "SKIP_ON_CI";
+}

gpt4all-bindings/csharp/Gpt4All/Bindings/LLModel.cs

@@ -1,247 +1,222 @@
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
-
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Arguments for the response processing callback
-/// </summary>
-/// <param name="TokenId">The token id of the response</param>
-/// <param name="Response"> The response string. NOTE: a token_id of -1 indicates the string is an error string</param>
-/// <return>
-/// A bool indicating whether the model should keep generating
-/// </return>
-public record ModelResponseEventArgs(int TokenId, string Response)
-{
-    public bool IsError => TokenId == -1;
-}
-
-/// <summary>
-/// Arguments for the prompt processing callback
-/// </summary>
-/// <param name="TokenId">The token id of the prompt</param>
-/// <return>
-/// A bool indicating whether the model should keep processing
-/// </return>
-public record ModelPromptEventArgs(int TokenId)
-{
-}
-
-/// <summary>
-/// Arguments for the recalculating callback
-/// </summary>
-/// <param name="IsRecalculating"> whether the model is recalculating the context.</param>
-/// <return>
-/// A bool indicating whether the model should keep generating
-/// </return>
-public record ModelRecalculatingEventArgs(bool IsRecalculating);
-
-/// <summary>
-/// Base class and universal wrapper for GPT4All language models built around llmodel C-API.
-/// </summary>
-public class LLModel : ILLModel
-{
-    protected readonly IntPtr _handle;
-    private readonly ModelType _modelType;
-    private readonly ILogger _logger;
-    private bool _disposed;
-
-    public ModelType ModelType => _modelType;
-
-    internal LLModel(IntPtr handle, ModelType modelType, ILogger? logger = null)
-    {
-        _handle = handle;
-        _modelType = modelType;
-        _logger = logger ?? NullLogger.Instance;
-    }
-
-    /// <summary>
-    /// Create a new model from a pointer
-    /// </summary>
-    /// <param name="handle">Pointer to underlying model</param>
-    /// <param name="modelType">The model type</param>
-    public static LLModel Create(IntPtr handle, ModelType modelType, ILogger? logger = null)
-    {
-        return new LLModel(handle, modelType, logger: logger);
-    }
-
-    /// <summary>
-    /// Generate a response using the model
-    /// </summary>
-    /// <param name="text">The input promp</param>
-    /// <param name="context">The context</param>
-    /// <param name="promptCallback">A callback function for handling the processing of prompt</param>
-    /// <param name="responseCallback">A callback function for handling the generated response</param>
-    /// <param name="recalculateCallback">A callback function for handling recalculation requests</param>
-    /// <param name="cancellationToken"></param>
-    public void Prompt(
-        string text,
-        LLModelPromptContext context,
-        Func<ModelPromptEventArgs, bool>? promptCallback = null,
-        Func<ModelResponseEventArgs, bool>? responseCallback = null,
-        Func<ModelRecalculatingEventArgs, bool>? recalculateCallback = null,
-        CancellationToken cancellationToken = default)
-    {
-        GC.KeepAlive(promptCallback);
-        GC.KeepAlive(responseCallback);
-        GC.KeepAlive(recalculateCallback);
-        GC.KeepAlive(cancellationToken);
-
-        _logger.LogInformation("Prompt input='{Prompt}' ctx={Context}", text, context.Dump());
-
-        NativeMethods.llmodel_prompt(
-            _handle,
-            text,
-            (tokenId) =>
-            {
-                if (cancellationToken.IsCancellationRequested) return false;
-                if (promptCallback == null) return true;
-                var args = new ModelPromptEventArgs(tokenId);
-                return promptCallback(args);
-            },
-            (tokenId, response) =>
-            {
-                if (cancellationToken.IsCancellationRequested)
-                {
-                    _logger.LogDebug("ResponseCallback evt=CancellationRequested");
-                    return false;
-                }
-
-                if (responseCallback == null) return true;
-                var args = new ModelResponseEventArgs(tokenId, response);
-                return responseCallback(args);
-            },
-            (isRecalculating) =>
-            {
-                if (cancellationToken.IsCancellationRequested) return false;
-                if (recalculateCallback == null) return true;
-                var args = new ModelRecalculatingEventArgs(isRecalculating);
-                return recalculateCallback(args);
-            },
-            ref context.UnderlyingContext
-        );
-    }
-
-    /// <summary>
-    ///  Set the number of threads to be used by the model.
-    /// </summary>
-    /// <param name="threadCount">The new thread count</param>
-    public void SetThreadCount(int threadCount)
-    {
-        NativeMethods.llmodel_setThreadCount(_handle, threadCount);
-    }
-
-    /// <summary>
-    /// Get  the number of threads used by the model.
-    /// </summary>
-    /// <returns>the number of threads used by the model</returns>
-    public int GetThreadCount()
-    {
-        return NativeMethods.llmodel_threadCount(_handle);
-    }
-
-    /// <summary>
-    /// Get the size of the internal state of the model.
-    /// </summary>
-    /// <remarks>
-    /// This state data is specific to the type of model you have created.
-    /// </remarks>
-    /// <returns>the size in bytes of the internal state of the model</returns>
-    public ulong GetStateSizeBytes()
-    {
-        return NativeMethods.llmodel_get_state_size(_handle);
-    }
-
-    /// <summary>
-    /// Saves the internal state of the model to the specified destination address.
-    /// </summary>
-    /// <param name="source">A pointer to the src</param>
-    /// <returns>The number of bytes copied</returns>
-    public unsafe ulong SaveStateData(byte* source)
-    {
-        return NativeMethods.llmodel_save_state_data(_handle, source);
-    }
-
-    /// <summary>
-    /// Restores the internal state of the model using data from the specified address.
-    /// </summary>
-    /// <param name="destination">A pointer to destination</param>
-    /// <returns>the number of bytes read</returns>
-    public unsafe ulong RestoreStateData(byte* destination)
-    {
-        return NativeMethods.llmodel_restore_state_data(_handle, destination);
-    }
-
-    /// <summary>
-    /// Check if the model is loaded.
-    /// </summary>
-    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
-    public bool IsLoaded()
-    {
-        return NativeMethods.llmodel_isModelLoaded(_handle);
-    }
-
-    /// <summary>
-    /// Load the model from a file.
-    /// </summary>
-    /// <param name="modelPath">The path to the model file.</param>
-    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
-    public bool Load(string modelPath)
-    {
-        return NativeMethods.llmodel_loadModel(_handle, modelPath);
-    }
-
-    protected void Destroy()
-    {
-        NativeMethods.llmodel_model_destroy(_handle);
-    }
-
-    protected void DestroyLLama()
-    {
-        NativeMethods.llmodel_llama_destroy(_handle);
-    }
-
-    protected void DestroyGptj()
-    {
-        NativeMethods.llmodel_gptj_destroy(_handle);
-    }
-
-    protected void DestroyMtp()
-    {
-        NativeMethods.llmodel_mpt_destroy(_handle);
-    }
-
-    protected virtual void Dispose(bool disposing)
-    {
-        if (_disposed) return;
-
-        if (disposing)
-        {
-            // dispose managed state
-        }
-
-        switch (_modelType)
-        {
-            case ModelType.LLAMA:
-                DestroyLLama();
-                break;
-            case ModelType.GPTJ:
-                DestroyGptj();
-                break;
-            case ModelType.MPT:
-                DestroyMtp();
-                break;
-            default:
-                Destroy();
-                break;
-        }
-
-        _disposed = true;
-    }
-
-    public void Dispose()
-    {
-        Dispose(disposing: true);
-        GC.SuppressFinalize(this);
-    }
-}
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace Gpt4All.Bindings;
+
+/// <summary>
+/// Arguments for the response processing callback
+/// </summary>
+/// <param name="TokenId">The token id of the response</param>
+/// <param name="Response"> The response string. NOTE: a token_id of -1 indicates the string is an error string</param>
+/// <return>
+/// A bool indicating whether the model should keep generating
+/// </return>
+public record ModelResponseEventArgs(int TokenId, string Response)
+{
+    public bool IsError => TokenId == -1;
+}
+
+/// <summary>
+/// Arguments for the prompt processing callback
+/// </summary>
+/// <param name="TokenId">The token id of the prompt</param>
+/// <return>
+/// A bool indicating whether the model should keep processing
+/// </return>
+public record ModelPromptEventArgs(int TokenId)
+{
+}
+
+/// <summary>
+/// Arguments for the recalculating callback
+/// </summary>
+/// <param name="IsRecalculating"> whether the model is recalculating the context.</param>
+/// <return>
+/// A bool indicating whether the model should keep generating
+/// </return>
+public record ModelRecalculatingEventArgs(bool IsRecalculating);
+
+/// <summary>
+/// Base class and universal wrapper for GPT4All language models built around llmodel C-API.
+/// </summary>
+public class LLModel : ILLModel
+{
+    protected readonly IntPtr _handle;
+    private readonly ModelType _modelType;
+    private readonly ILogger _logger;
+    private bool _disposed;
+
+    public ModelType ModelType => _modelType;
+
+    internal LLModel(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    {
+        _handle = handle;
+        _modelType = modelType;
+        _logger = logger ?? NullLogger.Instance;
+    }
+
+    /// <summary>
+    /// Create a new model from a pointer
+    /// </summary>
+    /// <param name="handle">Pointer to underlying model</param>
+    /// <param name="modelType">The model type</param>
+    public static LLModel Create(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    {
+        return new LLModel(handle, modelType, logger: logger);
+    }
+
+    /// <summary>
+    /// Generate a response using the model
+    /// </summary>
+    /// <param name="text">The input promp</param>
+    /// <param name="context">The context</param>
+    /// <param name="promptCallback">A callback function for handling the processing of prompt</param>
+    /// <param name="responseCallback">A callback function for handling the generated response</param>
+    /// <param name="recalculateCallback">A callback function for handling recalculation requests</param>
+    /// <param name="cancellationToken"></param>
+    public void Prompt(
+        string text,
+        LLModelPromptContext context,
+        Func<ModelPromptEventArgs, bool>? promptCallback = null,
+        Func<ModelResponseEventArgs, bool>? responseCallback = null,
+        Func<ModelRecalculatingEventArgs, bool>? recalculateCallback = null,
+        CancellationToken cancellationToken = default)
+    {
+        GC.KeepAlive(promptCallback);
+        GC.KeepAlive(responseCallback);
+        GC.KeepAlive(recalculateCallback);
+        GC.KeepAlive(cancellationToken);
+
+        _logger.LogInformation("Prompt input='{Prompt}' ctx={Context}", text, context.Dump());
+
+        NativeMethods.llmodel_prompt(
+            _handle,
+            text,
+            (tokenId) =>
+            {
+                if (cancellationToken.IsCancellationRequested) return false;
+                if (promptCallback == null) return true;
+                var args = new ModelPromptEventArgs(tokenId);
+                return promptCallback(args);
+            },
+            (tokenId, response) =>
+            {
+                if (cancellationToken.IsCancellationRequested)
+                {
+                    _logger.LogDebug("ResponseCallback evt=CancellationRequested");
+                    return false;
+                }
+
+                if (responseCallback == null) return true;
+                var args = new ModelResponseEventArgs(tokenId, response);
+                return responseCallback(args);
+            },
+            (isRecalculating) =>
+            {
+                if (cancellationToken.IsCancellationRequested) return false;
+                if (recalculateCallback == null) return true;
+                var args = new ModelRecalculatingEventArgs(isRecalculating);
+                return recalculateCallback(args);
+            },
+            ref context.UnderlyingContext
+        );
+    }
+
+    /// <summary>
+    ///  Set the number of threads to be used by the model.
+    /// </summary>
+    /// <param name="threadCount">The new thread count</param>
+    public void SetThreadCount(int threadCount)
+    {
+        NativeMethods.llmodel_setThreadCount(_handle, threadCount);
+    }
+
+    /// <summary>
+    /// Get  the number of threads used by the model.
+    /// </summary>
+    /// <returns>the number of threads used by the model</returns>
+    public int GetThreadCount()
+    {
+        return NativeMethods.llmodel_threadCount(_handle);
+    }
+
+    /// <summary>
+    /// Get the size of the internal state of the model.
+    /// </summary>
+    /// <remarks>
+    /// This state data is specific to the type of model you have created.
+    /// </remarks>
+    /// <returns>the size in bytes of the internal state of the model</returns>
+    public ulong GetStateSizeBytes()
+    {
+        return NativeMethods.llmodel_get_state_size(_handle);
+    }
+
+    /// <summary>
+    /// Saves the internal state of the model to the specified destination address.
+    /// </summary>
+    /// <param name="source">A pointer to the src</param>
+    /// <returns>The number of bytes copied</returns>
+    public unsafe ulong SaveStateData(byte* source)
+    {
+        return NativeMethods.llmodel_save_state_data(_handle, source);
+    }
+
+    /// <summary>
+    /// Restores the internal state of the model using data from the specified address.
+    /// </summary>
+    /// <param name="destination">A pointer to destination</param>
+    /// <returns>the number of bytes read</returns>
+    public unsafe ulong RestoreStateData(byte* destination)
+    {
+        return NativeMethods.llmodel_restore_state_data(_handle, destination);
+    }
+
+    /// <summary>
+    /// Check if the model is loaded.
+    /// </summary>
+    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
+    public bool IsLoaded()
+    {
+        return NativeMethods.llmodel_isModelLoaded(_handle);
+    }
+
+    /// <summary>
+    /// Load the model from a file.
+    /// </summary>
+    /// <param name="modelPath">The path to the model file.</param>
+    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
+    public bool Load(string modelPath)
+    {
+        return NativeMethods.llmodel_loadModel(_handle, modelPath);
+    }
+
+    protected void Destroy()
+    {
+        NativeMethods.llmodel_model_destroy(_handle);
+    }
+    protected virtual void Dispose(bool disposing)
+    {
+        if (_disposed) return;
+
+        if (disposing)
+        {
+            // dispose managed state
+        }
+
+        switch (_modelType)
+        {
+            default:
+                Destroy();
+                break;
+        }
+
+        _disposed = true;
+    }
+
+    public void Dispose()
+    {
+        Dispose(disposing: true);
+        GC.SuppressFinalize(this);
+    }
+}

gpt4all-bindings/csharp/Gpt4All/Bindings/LLPromptContext.cs

@@ -1,138 +1,138 @@
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Wrapper around the llmodel_prompt_context structure for holding the prompt context.
-/// </summary>
-/// <remarks>
-/// The implementation takes care of all the memory handling of the raw logits pointer and the
-/// raw tokens pointer.Attempting to resize them or modify them in any way can lead to undefined behavior
-/// </remarks>
-public unsafe class LLModelPromptContext
-{
-    private llmodel_prompt_context _ctx;
-
-    internal ref llmodel_prompt_context UnderlyingContext => ref _ctx;
-
-    public LLModelPromptContext()
-    {
-        _ctx = new();
-    }
-
-    /// <summary>
-    /// logits of current context
-    /// </summary>
-    public Span<float> Logits => new(_ctx.logits, (int)_ctx.logits_size);
-
-    /// <summary>
-    /// the size of the raw logits vector
-    /// </summary>
-    public nuint LogitsSize
-    {
-        get => _ctx.logits_size;
-        set => _ctx.logits_size = value;
-    }
-
-    /// <summary>
-    /// current tokens in the context window
-    /// </summary>
-    public Span<int> Tokens => new(_ctx.tokens, (int)_ctx.tokens_size);
-
-    /// <summary>
-    /// the size of the raw tokens vector
-    /// </summary>
-    public nuint TokensSize
-    {
-        get => _ctx.tokens_size;
-        set => _ctx.tokens_size = value;
-    }
-
-    /// <summary>
-    /// top k logits to sample from
-    /// </summary>
-    public int TopK
-    {
-        get => _ctx.top_k;
-        set => _ctx.top_k = value;
-    }
-
-    /// <summary>
-    /// nucleus sampling probability threshold
-    /// </summary>
-    public float TopP
-    {
-        get => _ctx.top_p;
-        set => _ctx.top_p = value;
-    }
-
-    /// <summary>
-    /// temperature to adjust model's output distribution
-    /// </summary>
-    public float Temperature
-    {
-        get => _ctx.temp;
-        set => _ctx.temp = value;
-    }
-
-    /// <summary>
-    /// number of tokens in past conversation
-    /// </summary>
-    public int PastNum
-    {
-        get => _ctx.n_past;
-        set => _ctx.n_past = value;
-    }
-
-    /// <summary>
-    /// number of predictions to generate in parallel
-    /// </summary>
-    public int Batches
-    {
-        get => _ctx.n_batch;
-        set => _ctx.n_batch = value;
-    }
-
-    /// <summary>
-    /// number of tokens to predict
-    /// </summary>
-    public int TokensToPredict
-    {
-        get => _ctx.n_predict;
-        set => _ctx.n_predict = value;
-    }
-
-    /// <summary>
-    /// penalty factor for repeated tokens
-    /// </summary>
-    public float RepeatPenalty
-    {
-        get => _ctx.repeat_penalty;
-        set => _ctx.repeat_penalty = value;
-    }
-
-    /// <summary>
-    /// last n tokens to penalize
-    /// </summary>
-    public int RepeatLastN
-    {
-        get => _ctx.repeat_last_n;
-        set => _ctx.repeat_last_n = value;
-    }
-
-    /// <summary>
-    /// number of tokens possible in context window
-    /// </summary>
-    public int ContextSize
-    {
-        get => _ctx.n_ctx;
-        set => _ctx.n_ctx = value;
-    }
-
-    /// <summary>
-    /// percent of context to erase if we exceed the context window
-    /// </summary>
-    public float ContextErase
-    {
-        get => _ctx.context_erase;
-        set => _ctx.context_erase = value;
-    }
-}
+namespace Gpt4All.Bindings;
+
+/// <summary>
+/// Wrapper around the llmodel_prompt_context structure for holding the prompt context.
+/// </summary>
+/// <remarks>
+/// The implementation takes care of all the memory handling of the raw logits pointer and the
+/// raw tokens pointer.Attempting to resize them or modify them in any way can lead to undefined behavior
+/// </remarks>
+public unsafe class LLModelPromptContext
+{
+    private llmodel_prompt_context _ctx;
+
+    internal ref llmodel_prompt_context UnderlyingContext => ref _ctx;
+
+    public LLModelPromptContext()
+    {
+        _ctx = new();
+    }
+
+    /// <summary>
+    /// logits of current context
+    /// </summary>
+    public Span<float> Logits => new(_ctx.logits, (int)_ctx.logits_size);
+
+    /// <summary>
+    /// the size of the raw logits vector
+    /// </summary>
+    public nuint LogitsSize
+    {
+        get => _ctx.logits_size;
+        set => _ctx.logits_size = value;
+    }
+
+    /// <summary>
+    /// current tokens in the context window
+    /// </summary>
+    public Span<int> Tokens => new(_ctx.tokens, (int)_ctx.tokens_size);
+
+    /// <summary>
+    /// the size of the raw tokens vector
+    /// </summary>
+    public nuint TokensSize
+    {
+        get => _ctx.tokens_size;
+        set => _ctx.tokens_size = value;
+    }
+
+    /// <summary>
+    /// top k logits to sample from
+    /// </summary>
+    public int TopK
+    {
+        get => _ctx.top_k;
+        set => _ctx.top_k = value;
+    }
+
+    /// <summary>
+    /// nucleus sampling probability threshold
+    /// </summary>
+    public float TopP
+    {
+        get => _ctx.top_p;
+        set => _ctx.top_p = value;
+    }
+
+    /// <summary>
+    /// temperature to adjust model's output distribution
+    /// </summary>
+    public float Temperature
+    {
+        get => _ctx.temp;
+        set => _ctx.temp = value;
+    }
+
+    /// <summary>
+    /// number of tokens in past conversation
+    /// </summary>
+    public int PastNum
+    {
+        get => _ctx.n_past;
+        set => _ctx.n_past = value;
+    }
+
+    /// <summary>
+    /// number of predictions to generate in parallel
+    /// </summary>
+    public int Batches
+    {
+        get => _ctx.n_batch;
+        set => _ctx.n_batch = value;
+    }
+
+    /// <summary>
+    /// number of tokens to predict
+    /// </summary>
+    public int TokensToPredict
+    {
+        get => _ctx.n_predict;
+        set => _ctx.n_predict = value;
+    }
+
+    /// <summary>
+    /// penalty factor for repeated tokens
+    /// </summary>
+    public float RepeatPenalty
+    {
+        get => _ctx.repeat_penalty;
+        set => _ctx.repeat_penalty = value;
+    }
+
+    /// <summary>
+    /// last n tokens to penalize
+    /// </summary>
+    public int RepeatLastN
+    {
+        get => _ctx.repeat_last_n;
+        set => _ctx.repeat_last_n = value;
+    }
+
+    /// <summary>
+    /// number of tokens possible in context window
+    /// </summary>
+    public int ContextSize
+    {
+        get => _ctx.n_ctx;
+        set => _ctx.n_ctx = value;
+    }
+
+    /// <summary>
+    /// percent of context to erase if we exceed the context window
+    /// </summary>
+    public float ContextErase
+    {
+        get => _ctx.context_erase;
+        set => _ctx.context_erase = value;
+    }
+}

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeMethods.cs

@@ -1,126 +1,108 @@
-using System.Runtime.InteropServices;
-
-namespace Gpt4All.Bindings;
-
-public unsafe partial struct llmodel_prompt_context
-{
-    public float* logits;
-
-    [NativeTypeName("size_t")]
-    public nuint logits_size;
-
-    [NativeTypeName("int32_t *")]
-    public int* tokens;
-
-    [NativeTypeName("size_t")]
-    public nuint tokens_size;
-
-    [NativeTypeName("int32_t")]
-    public int n_past;
-
-    [NativeTypeName("int32_t")]
-    public int n_ctx;
-
-    [NativeTypeName("int32_t")]
-    public int n_predict;
-
-    [NativeTypeName("int32_t")]
-    public int top_k;
-
-    public float top_p;
-
-    public float temp;
-
-    [NativeTypeName("int32_t")]
-    public int n_batch;
-
-    public float repeat_penalty;
-
-    [NativeTypeName("int32_t")]
-    public int repeat_last_n;
-
-    public float context_erase;
-}
-
-internal static unsafe partial class NativeMethods
-{
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelResponseCallback(int token_id, [MarshalAs(UnmanagedType.LPUTF8Str)] string response);
-
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelPromptCallback(int token_id);
-
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelRecalculateCallback(bool isRecalculating);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_gptj_create();
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_gptj_destroy([NativeTypeName("llmodel_model")] IntPtr gptj);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_mpt_create();
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_mpt_destroy([NativeTypeName("llmodel_model")] IntPtr mpt);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_llama_create();
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_llama_destroy([NativeTypeName("llmodel_model")] IntPtr llama);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_model_create(
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_model_destroy([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public static extern bool llmodel_loadModel(
-        [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-
-    [return: MarshalAs(UnmanagedType.I1)]
-    public static extern bool llmodel_isModelLoaded([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_get_state_size([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_save_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("uint8_t *")] byte* dest);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_restore_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("const uint8_t *")] byte* src);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    public static extern void llmodel_prompt(
-        [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string prompt,
-        LlmodelPromptCallback prompt_callback,
-        LlmodelResponseCallback response_callback,
-        LlmodelRecalculateCallback recalculate_callback,
-        ref llmodel_prompt_context ctx);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_setThreadCount([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("int32_t")] int n_threads);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("int32_t")]
-    public static extern int llmodel_threadCount([NativeTypeName("llmodel_model")] IntPtr model);
-}
+using System.Runtime.InteropServices;
+
+namespace Gpt4All.Bindings;
+
+public unsafe partial struct llmodel_prompt_context
+{
+    public float* logits;
+
+    [NativeTypeName("size_t")]
+    public nuint logits_size;
+
+    [NativeTypeName("int32_t *")]
+    public int* tokens;
+
+    [NativeTypeName("size_t")]
+    public nuint tokens_size;
+
+    [NativeTypeName("int32_t")]
+    public int n_past;
+
+    [NativeTypeName("int32_t")]
+    public int n_ctx;
+
+    [NativeTypeName("int32_t")]
+    public int n_predict;
+
+    [NativeTypeName("int32_t")]
+    public int top_k;
+
+    public float top_p;
+
+    public float temp;
+
+    [NativeTypeName("int32_t")]
+    public int n_batch;
+
+    public float repeat_penalty;
+
+    [NativeTypeName("int32_t")]
+    public int repeat_last_n;
+
+    public float context_erase;
+}
+#pragma warning disable CA2101
+internal static unsafe partial class NativeMethods
+{
+    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public delegate bool LlmodelResponseCallback(int token_id, [MarshalAs(UnmanagedType.LPUTF8Str)] string response);
+
+    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public delegate bool LlmodelPromptCallback(int token_id);
+
+    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public delegate bool LlmodelRecalculateCallback(bool isRecalculating);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
+    [return: NativeTypeName("llmodel_model")]
+    public static extern IntPtr llmodel_model_create2(
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path,
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string build_variant,
+        out IntPtr error);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    public static extern void llmodel_model_destroy([NativeTypeName("llmodel_model")] IntPtr model);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public static extern bool llmodel_loadModel(
+        [NativeTypeName("llmodel_model")] IntPtr model,
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+
+    [return: MarshalAs(UnmanagedType.I1)]
+    public static extern bool llmodel_isModelLoaded([NativeTypeName("llmodel_model")] IntPtr model);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("uint64_t")]
+    public static extern ulong llmodel_get_state_size([NativeTypeName("llmodel_model")] IntPtr model);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("uint64_t")]
+    public static extern ulong llmodel_save_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("uint8_t *")] byte* dest);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("uint64_t")]
+    public static extern ulong llmodel_restore_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("const uint8_t *")] byte* src);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
+    public static extern void llmodel_prompt(
+        [NativeTypeName("llmodel_model")] IntPtr model,
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string prompt,
+        LlmodelPromptCallback prompt_callback,
+        LlmodelResponseCallback response_callback,
+        LlmodelRecalculateCallback recalculate_callback,
+        ref llmodel_prompt_context ctx);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    public static extern void llmodel_setThreadCount([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("int32_t")] int n_threads);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("int32_t")]
+    public static extern int llmodel_threadCount([NativeTypeName("llmodel_model")] IntPtr model);
+}
+#pragma warning restore CA2101

gpt4all-bindings/csharp/Gpt4All/Gpt4All.cs

@@ -1,8 +1,11 @@
 using System.Diagnostics;
+using System.Runtime.CompilerServices;
 using Gpt4All.Bindings;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 
+[assembly: InternalsVisibleTo("Gpt4All.Tests")]
+
 namespace Gpt4All;
 
 public class Gpt4All : IGpt4AllModel

gpt4all-bindings/csharp/Gpt4All/Gpt4All.csproj

@@ -1,27 +1,22 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-	<TargetFrameworks>net6.0</TargetFrameworks>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-  </PropertyGroup>
-
-	<ItemGroup>
-		<!-- Windows -->
-		<None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
-		<!-- Linux -->
-		<None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
-	</ItemGroup>
-
-	<ItemGroup>
-		<!-- Windows -->
-		<None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-		<!-- Linux -->
-		<None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-	</ItemGroup>
-
-	<ItemGroup>
-	  <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
-	</ItemGroup>
+    <PropertyGroup>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    </PropertyGroup>
+    <ItemGroup>
+        <!-- Windows -->
+        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
+        <!-- Linux -->
+        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
+        <!-- MacOS -->
+        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
+        <Content Include="..\runtimes\osx\native\*.metal" Pack="true" PackagePath="contentFiles\any\any;content">
+            <PackageCopyToOutput>true</PackageCopyToOutput>
+        </Content>
+    </ItemGroup>
+    <ItemGroup>
+        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
+    </ItemGroup>
 </Project>

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/ILibraryLoader.cs

@@ -0,0 +1,6 @@
+namespace Gpt4All.LibraryLoader;
+
+public interface ILibraryLoader
+{
+    LoadResult OpenLibrary(string? fileName);
+}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LinuxLibraryLoader.cs

@@ -0,0 +1,53 @@
+using System.Runtime.InteropServices;
+
+namespace Gpt4All.LibraryLoader;
+
+internal class LinuxLibraryLoader : ILibraryLoader
+{
+#pragma warning disable CA2101
+    [DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
+#pragma warning restore CA2101
+    public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags);
+
+#pragma warning disable CA2101
+    [DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
+#pragma warning restore CA2101
+    public static extern IntPtr NativeOpenLibraryLibdl2(string? filename, int flags);
+
+    [DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
+    public static extern IntPtr GetLoadError();
+
+    [DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
+    public static extern IntPtr GetLoadError2();
+
+    public LoadResult OpenLibrary(string? fileName)
+    {
+        IntPtr loadedLib;
+        try
+        {
+            // open with rtls lazy flag
+            loadedLib = NativeOpenLibraryLibdl2(fileName, 0x00001);
+        }
+        catch (DllNotFoundException)
+        {
+            loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001);
+        }
+
+        if (loadedLib == IntPtr.Zero)
+        {
+            string errorMessage;
+            try
+            {
+                errorMessage = Marshal.PtrToStringAnsi(GetLoadError2()) ?? "Unknown error";
+            }
+            catch (DllNotFoundException)
+            {
+                errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error";
+            }
+
+            return LoadResult.Failure(errorMessage);
+        }
+
+        return LoadResult.Success;
+    }
+}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LoadResult.cs

@@ -0,0 +1,20 @@
+namespace Gpt4All.LibraryLoader;
+
+public class LoadResult
+{
+    private LoadResult(bool isSuccess, string? errorMessage)
+    {
+        IsSuccess = isSuccess;
+        ErrorMessage = errorMessage;
+    }
+
+    public static LoadResult Success { get; } = new(true, null);
+
+    public static LoadResult Failure(string errorMessage)
+    {
+        return new(false, errorMessage);
+    }
+
+    public bool IsSuccess { get; }
+    public string? ErrorMessage { get; }
+}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/MacOsLibraryLoader.cs

@@ -0,0 +1,28 @@
+using System.Runtime.InteropServices;
+
+namespace Gpt4All.LibraryLoader;
+
+internal class MacOsLibraryLoader : ILibraryLoader
+{
+#pragma warning disable CA2101
+    [DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
+#pragma warning restore CA2101
+    public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags);
+
+    [DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
+    public static extern IntPtr GetLoadError();
+
+    public LoadResult OpenLibrary(string? fileName)
+    {
+        var loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001);
+
+        if (loadedLib == IntPtr.Zero)
+        {
+            var errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error";
+
+            return LoadResult.Failure(errorMessage);
+        }
+
+        return LoadResult.Success;
+    }
+}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/NativeLibraryLoader.cs

@@ -0,0 +1,81 @@
+#if !IOS && !MACCATALYST && !TVOS && !ANDROID
+using System.Runtime.InteropServices;
+#endif
+
+namespace Gpt4All.LibraryLoader;
+
+public static class NativeLibraryLoader
+{
+    private static ILibraryLoader? defaultLibraryLoader;
+
+    /// <summary>
+    /// Sets the library loader used to load the native libraries. Overwrite this only if you want some custom loading.
+    /// </summary>
+    /// <param name="libraryLoader">The library loader to be used.</param>
+    public static void SetLibraryLoader(ILibraryLoader libraryLoader)
+    {
+        defaultLibraryLoader = libraryLoader;
+    }
+
+    internal static LoadResult LoadNativeLibrary(string? path = default, bool bypassLoading = true)
+    {
+        // If the user has handled loading the library themselves, we don't need to do anything.
+        if (bypassLoading)
+        {
+            return LoadResult.Success;
+        }
+
+        var architecture = RuntimeInformation.OSArchitecture switch
+        {
+            Architecture.X64 => "x64",
+            Architecture.X86 => "x86",
+            Architecture.Arm => "arm",
+            Architecture.Arm64 => "arm64",
+            _ => throw new PlatformNotSupportedException(
+                $"Unsupported OS platform, architecture: {RuntimeInformation.OSArchitecture}")
+        };
+
+        var (platform, extension) = Environment.OSVersion.Platform switch
+        {
+            _ when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => ("win", "dll"),
+            _ when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => ("linux", "so"),
+            _ when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => ("osx", "dylib"),
+            _ => throw new PlatformNotSupportedException(
+                $"Unsupported OS platform, architecture: {RuntimeInformation.OSArchitecture}")
+        };
+
+        // If the user hasn't set the path, we'll try to find it ourselves.
+        if (string.IsNullOrEmpty(path))
+        {
+            var libraryName = "libllmodel";
+            var assemblySearchPath = new[]
+            {
+                AppDomain.CurrentDomain.RelativeSearchPath,
+                Path.GetDirectoryName(typeof(NativeLibraryLoader).Assembly.Location),
+                Path.GetDirectoryName(Environment.GetCommandLineArgs()[0])
+            }.FirstOrDefault(it => !string.IsNullOrEmpty(it));
+            // Search for the library dll within the assembly search path. If it doesn't exist, for whatever reason, use the default path.
+            path = Directory.EnumerateFiles(assemblySearchPath ?? string.Empty, $"{libraryName}.{extension}", SearchOption.AllDirectories).FirstOrDefault() ?? Path.Combine("runtimes", $"{platform}-{architecture}", $"{libraryName}.{extension}");
+        }
+
+        if (defaultLibraryLoader != null)
+        {
+            return defaultLibraryLoader.OpenLibrary(path);
+        }
+
+        if (!File.Exists(path))
+        {
+            throw new FileNotFoundException($"Native Library not found in path {path}. " +
+                                            $"Verify you have have included the native Gpt4All library in your application.");
+        }
+
+        ILibraryLoader libraryLoader = platform switch
+        {
+            "win" => new WindowsLibraryLoader(),
+            "osx" => new MacOsLibraryLoader(),
+            "linux" => new LinuxLibraryLoader(),
+            _ => throw new PlatformNotSupportedException($"Currently {platform} platform is not supported")
+        };
+        return libraryLoader.OpenLibrary(path);
+    }
+}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/WindowsLibraryLoader.cs

@@ -0,0 +1,24 @@
+using System.ComponentModel;
+using System.Runtime.InteropServices;
+
+namespace Gpt4All.LibraryLoader;
+
+internal class WindowsLibraryLoader : ILibraryLoader
+{
+    public LoadResult OpenLibrary(string? fileName)
+    {
+        var loadedLib = LoadLibrary(fileName);
+
+        if (loadedLib == IntPtr.Zero)
+        {
+            var errorCode = Marshal.GetLastWin32Error();
+            var errorMessage = new Win32Exception(errorCode).Message;
+            return LoadResult.Failure(errorMessage);
+        }
+
+        return LoadResult.Success;
+    }
+
+    [DllImport("kernel32", SetLastError = true, CharSet = CharSet.Auto)]
+    private static extern IntPtr LoadLibrary([MarshalAs(UnmanagedType.LPWStr)] string? lpFileName);
+}

gpt4all-bindings/csharp/Gpt4All/Model/Gpt4AllModelFactory.cs

@@ -1,61 +1,58 @@
-using System.Diagnostics;
-using Microsoft.Extensions.Logging;
-using Gpt4All.Bindings;
-using Microsoft.Extensions.Logging.Abstractions;
-
-namespace Gpt4All;
-
-public class Gpt4AllModelFactory : IGpt4AllModelFactory
-{
-    private readonly ILoggerFactory _loggerFactory;
-    private readonly ILogger _logger;
-
-    public Gpt4AllModelFactory(ILoggerFactory? loggerFactory = null)
-    {
-        _loggerFactory = loggerFactory ?? NullLoggerFactory.Instance;
-        _logger = _loggerFactory.CreateLogger<Gpt4AllModelFactory>();
-    }
-
-    private IGpt4AllModel CreateModel(string modelPath, ModelType? modelType = null)
-    {
-        var modelType_ = modelType ?? ModelFileUtils.GetModelTypeFromModelFileHeader(modelPath);
-
-        _logger.LogInformation("Creating model path={ModelPath} type={ModelType}", modelPath, modelType_);
-
-        var handle = modelType_ switch
-        {
-            ModelType.LLAMA => NativeMethods.llmodel_llama_create(),
-            ModelType.GPTJ => NativeMethods.llmodel_gptj_create(),
-            ModelType.MPT => NativeMethods.llmodel_mpt_create(),
-            _ => NativeMethods.llmodel_model_create(modelPath),
-        };
-
-        _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
-        _logger.LogInformation("Model loading started");
-
-        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath);
-
-        _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
-
-        if (loadedSuccessfully == false)
-        {
-            throw new Exception($"Failed to load model: '{modelPath}'");
-        }
-
-        var logger = _loggerFactory.CreateLogger<LLModel>();
-
-        var underlyingModel = LLModel.Create(handle, modelType_, logger: logger);
-
-        Debug.Assert(underlyingModel.IsLoaded());
-
-        return new Gpt4All(underlyingModel, logger: logger);
-    }
-
-    public IGpt4AllModel LoadModel(string modelPath) => CreateModel(modelPath, modelType: null);
-
-    public IGpt4AllModel LoadMptModel(string modelPath) => CreateModel(modelPath, ModelType.MPT);
-
-    public IGpt4AllModel LoadGptjModel(string modelPath) => CreateModel(modelPath, ModelType.GPTJ);
-
-    public IGpt4AllModel LoadLlamaModel(string modelPath) => CreateModel(modelPath, ModelType.LLAMA);
-}
+using System.Diagnostics;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Logging;
+using Gpt4All.Bindings;
+using Gpt4All.LibraryLoader;
+
+namespace Gpt4All;
+
+public class Gpt4AllModelFactory : IGpt4AllModelFactory
+{
+    private readonly ILoggerFactory _loggerFactory;
+    private readonly ILogger _logger;
+    private static bool bypassLoading;
+    private static string? libraryPath;
+
+    private static readonly Lazy<LoadResult> libraryLoaded = new(() =>
+    {
+        return NativeLibraryLoader.LoadNativeLibrary(Gpt4AllModelFactory.libraryPath, Gpt4AllModelFactory.bypassLoading);
+    }, true);
+
+    public Gpt4AllModelFactory(string? libraryPath = default, bool bypassLoading = true, ILoggerFactory? loggerFactory = null)
+    {
+        _loggerFactory = loggerFactory ?? NullLoggerFactory.Instance;
+        _logger = _loggerFactory.CreateLogger<Gpt4AllModelFactory>();
+        Gpt4AllModelFactory.libraryPath = libraryPath;
+        Gpt4AllModelFactory.bypassLoading = bypassLoading;
+
+        if (!libraryLoaded.Value.IsSuccess)
+        {
+            throw new Exception($"Failed to load native gpt4all library. Error: {libraryLoaded.Value.ErrorMessage}");
+        }
+    }
+
+    private IGpt4AllModel CreateModel(string modelPath)
+    {
+        var modelType_ = ModelFileUtils.GetModelTypeFromModelFileHeader(modelPath);
+        _logger.LogInformation("Creating model path={ModelPath} type={ModelType}", modelPath, modelType_);
+        IntPtr error;
+        var handle = NativeMethods.llmodel_model_create2(modelPath, "auto", out error);
+        _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
+        _logger.LogInformation("Model loading started");
+        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath);
+        _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
+        if (!loadedSuccessfully)
+        {
+            throw new Exception($"Failed to load model: '{modelPath}'");
+        }
+
+        var logger = _loggerFactory.CreateLogger<LLModel>();
+        var underlyingModel = LLModel.Create(handle, modelType_, logger: logger);
+
+        Debug.Assert(underlyingModel.IsLoaded());
+
+        return new Gpt4All(underlyingModel, logger: logger);
+    }
+
+    public IGpt4AllModel LoadModel(string modelPath) => CreateModel(modelPath);
+}

gpt4all-bindings/csharp/Gpt4All/Model/IGpt4AllModelFactory.cs

@@ -1,12 +1,6 @@
-namespace Gpt4All;
-
-public interface IGpt4AllModelFactory
-{
-    IGpt4AllModel LoadGptjModel(string modelPath);
-
-    IGpt4AllModel LoadLlamaModel(string modelPath);
-
-    IGpt4AllModel LoadModel(string modelPath);
-
-    IGpt4AllModel LoadMptModel(string modelPath);
-}
+namespace Gpt4All;
+
+public interface IGpt4AllModelFactory
+{
+    IGpt4AllModel LoadModel(string modelPath);
+}

gpt4all-bindings/csharp/Gpt4All/Model/ModelType.cs

@@ -1,11 +1,11 @@
-namespace Gpt4All;
-
-/// <summary>
-/// The supported model types
-/// </summary>
-public enum ModelType
-{
-    LLAMA = 0,
-    GPTJ,
-    MPT
-}
+namespace Gpt4All;
+
+/// <summary>
+/// The supported model types
+/// </summary>
+public enum ModelType
+{
+    LLAMA = 0,
+    GPTJ,
+    MPT
+}

gpt4all-bindings/csharp/Gpt4All/Prediction/ITextPrediction.cs

@@ -1,31 +1,31 @@
-namespace Gpt4All;
-
-/// <summary>
-/// Interface for text prediction services
-/// </summary>
-public interface ITextPrediction
-{
-    /// <summary>
-    /// Get prediction results for the prompt and provided options.
-    /// </summary>
-    /// <param name="text">The text to complete</param>
-    /// <param name="opts">The prediction settings</param>
-    /// <param name="cancellationToken">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
-    /// <returns>The prediction result generated by the model</returns>
-    Task<ITextPredictionResult> GetPredictionAsync(
-        string text,
-        PredictRequestOptions opts,
-        CancellationToken cancellation = default);
-
-    /// <summary>
-    /// Get streaming prediction results for the prompt and provided options.
-    /// </summary>
-    /// <param name="text">The text to complete</param>
-    /// <param name="opts">The prediction settings</param>
-    /// <param name="cancellationToken">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
-    /// <returns>The prediction result generated by the model</returns>
-    Task<ITextPredictionStreamingResult> GetStreamingPredictionAsync(
-        string text,
-        PredictRequestOptions opts,
-        CancellationToken cancellationToken = default);
-}
+namespace Gpt4All;
+
+/// <summary>
+/// Interface for text prediction services
+/// </summary>
+public interface ITextPrediction
+{
+    /// <summary>
+    /// Get prediction results for the prompt and provided options.
+    /// </summary>
+    /// <param name="text">The text to complete</param>
+    /// <param name="opts">The prediction settings</param>
+    /// <param name="cancellation">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns>The prediction result generated by the model</returns>
+    Task<ITextPredictionResult> GetPredictionAsync(
+        string text,
+        PredictRequestOptions opts,
+        CancellationToken cancellation = default);
+
+    /// <summary>
+    /// Get streaming prediction results for the prompt and provided options.
+    /// </summary>
+    /// <param name="text">The text to complete</param>
+    /// <param name="opts">The prediction settings</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns>The prediction result generated by the model</returns>
+    Task<ITextPredictionStreamingResult> GetStreamingPredictionAsync(
+        string text,
+        PredictRequestOptions opts,
+        CancellationToken cancellationToken = default);
+}

gpt4all-bindings/csharp/build_linux.sh

@@ -5,4 +5,6 @@ mkdir runtimes/linux-x64/build
 cmake -S ../../gpt4all-backend -B runtimes/linux-x64/build
 cmake --build runtimes/linux-x64/build --parallel --config Release
 cp runtimes/linux-x64/build/libllmodel.so  runtimes/linux-x64/native/libllmodel.so
-cp runtimes/linux-x64/build/llama.cpp/libllama.so runtimes/linux-x64/native/libllama.so
+cp runtimes/linux-x64/build/libgptj*.so  runtimes/linux-x64/native/
+cp runtimes/linux-x64/build/libllama*.so  runtimes/linux-x64/native/
+cp runtimes/linux-x64/build/libmpt*.so  runtimes/linux-x64/native/

gpt4all-bindings/csharp/build_win-mingw.ps1

@@ -13,4 +13,4 @@ cmake --build $BUILD_DIR --parallel --config Release
 
 # copy native dlls
 cp "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll" $LIBS_DIR
-cp "$BUILD_DIR\*.dll" $LIBS_DIR
+cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR

gpt4all-bindings/csharp/build_win-msvc.ps1

@@ -2,4 +2,5 @@ Remove-Item -Force -Recurse .\runtimes\win-x64\msvc -ErrorAction SilentlyContinu
 mkdir .\runtimes\win-x64\msvc\build | Out-Null
 cmake -G "Visual Studio 17 2022" -A X64 -S ..\..\gpt4all-backend -B .\runtimes\win-x64\msvc\build
 cmake --build .\runtimes\win-x64\msvc\build --parallel --config Release
-cp .\runtimes\win-x64\msvc\build\bin\Release\*.dll .\runtimes\win-x64
+cp .\runtimes\win-x64\msvc\build\bin\Release\*.dll .\runtimes\win-x64
+mv .\runtimes\win-x64\llmodel.dll .\runtimes\win-x64\libllmodel.dll

gpt4all-bindings/golang/README.md

@@ -45,7 +45,7 @@ To use the bindings in your own software:
 
 - Import `github.com/nomic-ai/gpt4all/gpt4all-bindings/golang`;
 - Compile `libgpt4all.a` (you can use `make libgpt4all.a` in the bindings/go directory);
-- Link your go binary against whisper by setting the environment variables `C_INCLUDE_PATH` and `LIBRARY_PATH` to point to the `binding.h` file directory and `libgpt4all.a` file directory respectively.
+- Link your go binary by setting the environment variables `C_INCLUDE_PATH` and `LIBRARY_PATH` to point to the `binding.h` file directory and `libgpt4all.a` file directory respectively.
 - Note: you need to have *.so/*.dynlib/*.dll files of the implementation nearby the binary produced by the binding in order to make this to work
 
 ## Testing

gpt4all-bindings/golang/binding.cpp

@@ -24,11 +24,12 @@ void* load_model(const char *fname, int n_threads) {
                 __func__, new_error.message);
         return nullptr;
     }
-    llmodel_setThreadCount(model,  n_threads);
     if (!llmodel_loadModel(model, fname)) {
+        llmodel_model_destroy(model);
         return nullptr;
     }
 
+    llmodel_setThreadCount(model,  n_threads);
     return model;
 }
 

gpt4all-bindings/golang/gpt4all.go

@@ -10,6 +10,7 @@ package gpt4all
 //                            float top_p, float temp, int n_batch,float ctx_erase);
 // void free_model(void *state_ptr);
 // extern unsigned char getTokenCallback(void *, char *);
+// void llmodel_set_implementation_search_path(const char *path);
 import "C"
 import (
 	"fmt"
@@ -27,6 +28,10 @@ type Model struct {
 func New(model string, opts ...ModelOption) (*Model, error) {
 	ops := NewModelOptions(opts...)
 
+	if ops.LibrarySearchPath != "" {
+		C.llmodel_set_implementation_search_path(C.CString(ops.LibrarySearchPath))
+	}
+
 	state := C.load_model(C.CString(model), C.int(ops.Threads))
 
 	if state == nil {

gpt4all-bindings/golang/options.go

@@ -24,7 +24,8 @@ var DefaultModelOptions ModelOptions = ModelOptions{
 }
 
 type ModelOptions struct {
-	Threads int
+	Threads           int
+	LibrarySearchPath string
 }
 type ModelOption func(p *ModelOptions)
 
@@ -100,6 +101,13 @@ func SetThreads(c int) ModelOption {
 	}
 }
 
+// SetLibrarySearchPath sets the dynamic libraries used by gpt4all for the various ggml implementations.
+func SetLibrarySearchPath(t string) ModelOption {
+	return func(p *ModelOptions) {
+		p.LibrarySearchPath = t
+	}
+}
+
 // Create a new PredictOptions object with the given options.
 func NewModelOptions(opts ...ModelOption) ModelOptions {
 	p := DefaultModelOptions

gpt4all-bindings/java/.gitignore

@@ -0,0 +1,5 @@
+# Make sure native directory never gets commited to git for the project.
+/src/main/resources/native
+
+# IntelliJ project file
+*.iml

gpt4all-bindings/java/Developer_docs.md

@@ -0,0 +1,80 @@
+# Java Bindings Developer documents.
+
+This document is meant to anyone looking to build the Java bindings from source, test a build locally and perform a release.
+
+## Building locally
+
+Maven is the build tool used by the project. Maven version of 3.8 or higher is recommended. Make sure the **mvn** 
+is available on the command path.
+
+The project builds to Java version 11 target so make sure that a JDK at version 11 or newer is installed.
+
+### Setting up location of native shared libraries
+The property **native.libs.location** in pom.xml may need to be set:
+```
+    <properties>
+        ...
+        <native.libs.location>C:\Users\felix\dev\gpt4all_java_bins\release_1_1_3_Jun22_2023</native.libs.location>
+    </properties>
+```
+All the native shared libraries bundled with the Java binding jar will be copied from this location.
+The directory structure is **native/linux**, **native/macos**, **native/windows**. These directories are copied
+into the **src/main/resources** folder during the build process.
+
+For the purposes of local testing, none of these directories have to be present or just one OS type may be present.
+
+If none of the native libraries are present in **native.libs.location** the shared libraries will be searched for
+in location path set by **LLModel.LIBRARY_SEARCH_PATH** static variable in Java source code that is using the bindings.
+
+Alternately you can copy the shared libraries into the **src/resources/native/linux** before 
+you build, but note **src/main/resources/native** is on the .gitignore, so it will not be committed to sources.
+
+### Building
+
+To package the bindings jar run:
+```
+mvn package
+```
+This will build two jars. One has only the Java bindings and the other is a fat jar that will have required dependencies included as well.
+
+To package and install the Java bindings to your local maven repository run:
+```
+mvn install
+```
+
+### Using in a sample application
+
+You can check out a sample project that uses the java bindings here:
+https://github.com/felix-zaslavskiy/gpt4all-java-bindings-sample.git
+
+1. First, update the dependency of java bindings to whatever you have installed in local repository such as **1.1.4-SNAPSHOT**
+2. Second, update **Main.java** and set **baseModelPath** to the correct location of model weight files.
+
+3. To make a runnable jar run:
+```
+mvn package
+```
+
+A fat jar is also created which is easy to run from command line:
+```
+java -jar target/gpt4all-java-bindings-sample-1.0-SNAPSHOT-jar-with-dependencies.jar
+```
+
+### Publish a public release.
+
+For publishing a new version to maven central repository requires password and signing keys which F.Z. currently maintains, so
+he is responsible for making a public release.
+
+The procedure is as follows:
+
+For a snapshot release
+Run:
+```
+mvn deploy -P signing-profile
+```
+
+For a non-snapshot release
+Run:
+```
+mvn clean deploy -P signing-profile,release
+```

gpt4all-bindings/java/README.md

@@ -0,0 +1,124 @@
+# Java bindings
+
+Java bindings let you load a gpt4all library into your Java application and execute text 
+generation using an intuitive and easy to use API. No GPU is required because gpt4all executes on the CPU.
+The gpt4all models are quantized to easily fit into system RAM and use about 4 to 7GB of system RAM.
+
+## Getting Started
+You can add Java bindings into your Java project by adding the following dependency to your project:
+
+**Maven**
+```
+<dependency>
+    <groupId>com.hexadevlabs</groupId>
+    <artifactId>gpt4all-java-binding</artifactId>
+    <version>1.1.3</version>
+</dependency>
+```
+**Gradle**
+```
+implementation 'com.hexadevlabs:gpt4all-java-binding:1.1.3'
+```
+
+To add the library dependency for another build system see [Maven Central Java bindings](https://central.sonatype.com/artifact/com.hexadevlabs/gpt4all-java-binding/).
+
+To download model binary weights file use a URL such as [`https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin`](https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin).
+
+For information about other models available see the [model file list](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-chat#manual-download-of-models).
+
+### Sample code
+```java
+public class Example {
+    public static void main(String[] args) {
+
+        String prompt = "### Human:\nWhat is the meaning of life\n### Assistant:";
+
+        // Replace the hardcoded path with the actual path where your model file resides
+        String modelFilePath = "C:\\Users\\felix\\AppData\\Local\\nomic.ai\\GPT4All\\ggml-gpt4all-j-v1.3-groovy.bin";
+        
+        try (LLModel model = new LLModel(Path.of(modelFilePath))) {
+
+            // May generate up to 4096 tokens but generally stops early
+            LLModel.GenerationConfig config = LLModel.config()
+                    .withNPredict(4096).build();
+
+            // Will also stream to standard output
+            String fullGeneration = model.generate(prompt, config, true);
+
+        } catch (Exception e) {
+            // Exceptions generally may happen if the model file fails to load 
+            // for a number of reasons such as a file not found. 
+            // It is possible that Java may not be able to dynamically load the native shared library or 
+            // the llmodel shared library may not be able to dynamically load the backend 
+            // implementation for the model file you provided.
+            // 
+            // Once the LLModel class is successfully loaded into memory the text generation calls 
+            // generally should not throw exceptions.
+            e.printStackTrace(); // Printing here but in a production system you may want to take some action.
+        }
+    }
+
+}
+```
+
+For a Maven-based sample project that uses this library see this [sample project](https://github.com/felix-zaslavskiy/gpt4all-java-bindings-sample)
+
+### Additional considerations
+#### Logger warnings
+The Java bindings library may produce a warning if you don't have a SLF4J binding included in your project:
+```
+SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
+SLF4J: Defaulting to no-operation (NOP) logger implementation
+SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
+```
+The Java bindings only use logging for informational 
+purposes, so a logger is not essential to correctly use the library. You can ignore this warning if you don't have SLF4J bindings
+in your project.
+
+To add a simple logger using a Maven dependency you may use:
+```
+<dependency>
+    <groupId>org.slf4j</groupId>
+    <artifactId>slf4j-simple</artifactId>
+    <version>1.7.36</version>
+</dependency>
+```
+
+#### Loading your native libraries
+1. the Java bindings package JAR comes bundled with a native library files for Windows, macOS and Linux. These library files are 
+copied to a temporary directory and loaded at runtime. For advanced users who may want to package shared libraries into Docker containers 
+or want to use a custom build of the shared libraries and ignore the once bundled with the Java package they have option 
+to load libraries from your local directory by setting a static property to the location of library files.
+There are no guarantees of compatibility if used in such a way so be careful if you really want to do it.
+
+For example:
+```java
+class Example {
+    public static void main(String[] args) {
+        // gpt4all native shared libraries location
+        LLModel.LIBRARY_SEARCH_PATH = "C:\\Users\\felix\\gpt4all\\lib\\"; 
+        // ... use the library normally
+    }
+}
+```
+2. Not every AVX-only shared library is bundled with the JAR right now to reduce size. Only libgptj-avx is included.
+If you are running into issues please let us know using the [gpt4all project issue tracker](https://github.com/nomic-ai/gpt4all/issues).
+
+3. For Windows the native library included in jar depends on specific Microsoft C and C++ (MSVC) runtime libraries which may not be installed on your system.
+If this is the case you can easily download and install the latest x64 Microsoft Visual C++ Redistributable package from https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170
+
+4. When running Java in a Docker container it is advised to use eclipse-temurin:17-jre parent image. Alpine based parent images don't work due to the native library dependencies.
+
+## Version history
+1. Version **1.1.2**:
+   - Java bindings is compatible with gpt4ll version 2.4.6
+   - Initial stable release with the initial feature set
+2. Version **1.1.3**:
+   - Java bindings is compatible with gpt4all version 2.4.8
+   - Add static GPT4ALL_VERSION to signify gpt4all version of the bindings
+   - Add PromptIsTooLongException for prompts that are longer than context size.
+   - Replit model support to include Metal Mac hardware support.
+3. Version **1.1.4**:
+   - Java bindings is compatible with gpt4all version 2.4.11
+   - Falcon model support included.
+   

gpt4all-bindings/java/TODO.md

@@ -0,0 +1,2 @@
+1. Better Chat completions function.
+2. Chat completion that returns result in OpenAI compatible format.

gpt4all-bindings/java/pom.xml

@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.hexadevlabs</groupId>
+    <artifactId>gpt4all-java-binding</artifactId>
+    <version>1.1.4</version>
+    <packaging>jar</packaging>
+
+    <properties>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <native.libs.location>C:\Users\felix\dev\gpt4all_java_bins\release_1_1_4_July8_2023</native.libs.location>
+    </properties>
+
+    <name>${project.groupId}:${project.artifactId}</name>
+    <description>Java bindings for GPT4ALL LLM</description>
+    <url>https://github.com/nomic-ai/gpt4all</url>
+    <licenses>
+        <license>
+            <name>The Apache License, Version 2.0</name>
+            <url>https://github.com/nomic-ai/gpt4all/blob/main/LICENSE.txt</url>
+        </license>
+    </licenses>
+    <developers>
+        <developer>
+            <name>Felix Zaslavskiy</name>
+            <email>felixz@hexadevlabs.com</email>
+            <organizationUrl>https://github.com/felix-zaslavskiy/</organizationUrl>
+        </developer>
+    </developers>
+    <scm>
+        <connection>scm:git:git://github.com/nomic-ai/gpt4all.git</connection>
+        <developerConnection>scm:git:ssh://github.com/nomic-ai/gpt4all.git</developerConnection>
+        <url>https://github.com/nomic-ai/gpt4all/tree/main</url>
+    </scm>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.github.jnr</groupId>
+            <artifactId>jnr-ffi</artifactId>
+            <version>2.2.13</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.36</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>5.9.2</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-junit-jupiter</artifactId>
+            <version>5.4.0</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>5.4.0</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <distributionManagement>
+        <snapshotRepository>
+            <id>ossrh</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/snapshots</url>
+        </snapshotRepository>
+        <repository>
+            <id>ossrh</id>
+            <url>https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/</url>
+        </repository>
+    </distributionManagement>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+            </resource>
+            <resource>
+                <directory>${project.build.directory}/generated-resources</directory>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>3.0.0</version>
+                <configuration>
+                    <forkCount>0</forkCount>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-resources-plugin</artifactId>
+                <version>3.3.1</version>
+                <executions>
+                    <execution>
+                        <id>copy-resources</id>
+                        <!-- Here the phase you need -->
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/generated-resources</outputDirectory>
+                            <resources>
+                                <resource>
+                                    <directory>${native.libs.location}</directory>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <version>1.6.13</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <serverId>ossrh</serverId>
+                    <nexusUrl>https://s01.oss.sonatype.org/</nexusUrl>
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>3.5.0</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.6.0</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+         </plugins>
+
+    </build>
+
+    <profiles>
+        <profile>
+        <id>signing-profile</id>
+        <!-- activation conditions here, if any -->
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-gpg-plugin</artifactId>
+                        <version>3.1.0</version>
+                        <executions>
+                            <execution>
+                                <id>sign-artifacts</id>
+                                <phase>verify</phase>
+                                <goals>
+                                    <goal>sign</goal>
+                                </goals>
+
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+</project>

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java

@@ -0,0 +1,400 @@
+package com.hexadevlabs.gpt4all;
+
+import jnr.ffi.Pointer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public  class LLModel implements AutoCloseable {
+
+    /**
+     * Config used for how to decode LLM outputs.
+     * High temperature closer to 1 gives more creative outputs
+     * while low temperature closer to 0 produce more precise outputs.
+     * <p>
+     * Use builder to set settings you want.
+     */
+    public static class GenerationConfig extends LLModelLibrary.LLModelPromptContext {
+
+        private GenerationConfig() {
+            super(jnr.ffi.Runtime.getSystemRuntime());
+            logits_size.set(0);
+            tokens_size.set(0);
+            n_past.set(0);
+            n_ctx.set(1024);
+            n_predict.set(128);
+            top_k.set(40);
+            top_p.set(0.95);
+            temp.set(0.28);
+            n_batch.set(8);
+            repeat_penalty.set(1.1);
+            repeat_last_n.set(10);
+            context_erase.set(0.55);
+        }
+
+        public static class Builder {
+            private final GenerationConfig configToBuild;
+
+            public Builder() {
+                configToBuild = new GenerationConfig();
+            }
+
+            public Builder withNPast(int n_past) {
+                configToBuild.n_past.set(n_past);
+                return this;
+            }
+
+            public Builder withNCtx(int n_ctx) {
+                configToBuild.n_ctx.set(n_ctx);
+                return this;
+            }
+
+            public Builder withNPredict(int n_predict) {
+                configToBuild.n_predict.set(n_predict);
+                return this;
+            }
+
+            public Builder withTopK(int top_k) {
+                configToBuild.top_k.set(top_k);
+                return this;
+            }
+
+            public Builder withTopP(float top_p) {
+                configToBuild.top_p.set(top_p);
+                return this;
+            }
+
+            public Builder withTemp(float temp) {
+                configToBuild.temp.set(temp);
+                return this;
+            }
+
+            public Builder withNBatch(int n_batch) {
+                configToBuild.n_batch.set(n_batch);
+                return this;
+            }
+
+            public Builder withRepeatPenalty(float repeat_penalty) {
+                configToBuild.repeat_penalty.set(repeat_penalty);
+                return this;
+            }
+
+            public Builder withRepeatLastN(int repeat_last_n) {
+                configToBuild.repeat_last_n.set(repeat_last_n);
+                return this;
+            }
+
+            public Builder withContextErase(float context_erase) {
+                configToBuild.context_erase.set(context_erase);
+                return this;
+            }
+
+            /**
+             *
+             * @return GenerationConfig build instance of the config
+             */
+            public GenerationConfig build() {
+                return configToBuild;
+            }
+        }
+    }
+
+    /**
+     * Shortcut for making GenerativeConfig builder.
+     *
+     * @return GenerationConfig.Builder - builder that can be used to make a GenerationConfig
+     */
+    public static GenerationConfig.Builder config(){
+        return new GenerationConfig.Builder();
+    }
+
+    /**
+     * This may be set before any Model instance classes are instantiated to
+     * set where the native shared libraries are to be found.
+     * <p>
+     * This may be needed if setting library search path by standard means is not available
+     * or the libraries loaded from the temp folder bundled with the binding jar is not desirable.
+     */
+    public static String LIBRARY_SEARCH_PATH;
+
+
+    /**
+     * Generally for debugging purposes only. Will print
+     * the numerical tokens as they are generated instead of the string representations.
+     * Will also print out the processed input tokens as numbers to standard out.
+     */
+    public static boolean OUTPUT_DEBUG = false;
+
+    private static final Logger logger = LoggerFactory.getLogger(LLModel.class);
+
+    /**
+     * Which version of GPT4ALL that this binding is built for.
+     * The binding is guaranteed to work with this version of
+     * GPT4ALL native libraries. The binding may work for older
+     * versions but that is not guaranteed.
+     */
+    public static final String GPT4ALL_VERSION = "2.4.11";
+
+    protected static LLModelLibrary library;
+
+    protected Pointer model;
+
+    protected String modelName;
+
+    /**
+     * Package private default constructor, for testing purposes.
+     */
+    LLModel(){
+    }
+
+    public LLModel(Path modelPath) {
+
+        logger.info("Java bindings for gpt4all version: " + GPT4ALL_VERSION);
+
+        if(library==null) {
+
+            if (LIBRARY_SEARCH_PATH != null){
+                library = Util.loadSharedLibrary(LIBRARY_SEARCH_PATH);
+                library.llmodel_set_implementation_search_path(LIBRARY_SEARCH_PATH);
+            }  else {
+                // Copy system libraries to Temp folder
+                Path tempLibraryDirectory = Util.copySharedLibraries();
+                library = Util.loadSharedLibrary(tempLibraryDirectory.toString());
+
+                library.llmodel_set_implementation_search_path(tempLibraryDirectory.toString() );
+            }
+
+        }
+
+        // modelType = type;
+        modelName = modelPath.getFileName().toString();
+        String modelPathAbs = modelPath.toAbsolutePath().toString();
+
+        LLModelLibrary.LLModelError error = new LLModelLibrary.LLModelError(jnr.ffi.Runtime.getSystemRuntime());
+
+        // Check if model file exists
+        if(!Files.exists(modelPath)){
+            throw new IllegalStateException("Model file does not exist: " + modelPathAbs);
+        }
+
+        // Create Model Struct. Will load dynamically the correct backend based on model type
+        model = library.llmodel_model_create2(modelPathAbs, "auto", error);
+
+        if(model == null) {
+            throw new IllegalStateException("Could not load gpt4all backend :" + error.message);
+        }
+        library.llmodel_loadModel(model, modelPathAbs);
+
+        if(!library.llmodel_isModelLoaded(model)){
+            throw new IllegalStateException("The model " + modelName + " could not be loaded");
+        }
+
+    }
+
+    public void setThreadCount(int nThreads) {
+        library.llmodel_setThreadCount(this.model, nThreads);
+    }
+
+    public int threadCount() {
+        return library.llmodel_threadCount(this.model);
+    }
+
+    /**
+     * Generate text after the prompt
+     *
+     * @param prompt The text prompt to complete
+     * @param generationConfig What generation settings to use while generating text
+     * @return String The complete generated text
+     */
+    public String generate(String prompt, GenerationConfig generationConfig) {
+        return generate(prompt, generationConfig, false);
+    }
+
+    /**
+     * Generate text after the prompt
+     *
+     * @param prompt The text prompt to complete
+     * @param generationConfig What generation settings to use while generating text
+     * @param streamToStdOut Should the generation be streamed to standard output. Useful for troubleshooting.
+     * @return String The complete generated text
+     */
+    public String generate(String prompt, GenerationConfig generationConfig, boolean streamToStdOut) {
+
+        ByteArrayOutputStream bufferingForStdOutStream = new ByteArrayOutputStream();
+        ByteArrayOutputStream bufferingForWholeGeneration = new ByteArrayOutputStream();
+
+        LLModelLibrary.ResponseCallback responseCallback = getResponseCallback(streamToStdOut, bufferingForStdOutStream, bufferingForWholeGeneration);
+
+        library.llmodel_prompt(this.model,
+                prompt,
+                 (int tokenID) -> {
+                    if(LLModel.OUTPUT_DEBUG)
+                        System.out.println("token " + tokenID);
+                    return true; // continue processing
+                },
+                responseCallback,
+                (boolean isRecalculating) -> {
+                    if(LLModel.OUTPUT_DEBUG)
+                        System.out.println("recalculating");
+                    return isRecalculating; // continue generating
+                },
+                generationConfig);
+
+        return bufferingForWholeGeneration.toString(StandardCharsets.UTF_8);
+    }
+
+    /**
+     * Callback method to be used by prompt method as text is generated.
+     *
+     * @param streamToStdOut Should send generated text to standard out.
+     * @param bufferingForStdOutStream Output stream used for buffering bytes for standard output.
+     * @param bufferingForWholeGeneration Output stream used for buffering a complete generation.
+     * @return LLModelLibrary.ResponseCallback lambda function that is invoked by response callback.
+     */
+    static LLModelLibrary.ResponseCallback getResponseCallback(boolean streamToStdOut, ByteArrayOutputStream bufferingForStdOutStream, ByteArrayOutputStream bufferingForWholeGeneration) {
+        return (int tokenID, Pointer response) -> {
+
+            if(LLModel.OUTPUT_DEBUG)
+                System.out.print("Response token " + tokenID + " " );
+
+            // For all models if input sequence in tokens is longer then model context length
+            // the error is generated.
+            if(tokenID==-1){
+                throw new PromptIsTooLongException(response.getString(0, 1000, StandardCharsets.UTF_8));
+            }
+
+            long len = 0;
+            byte nextByte;
+            do{
+                try {
+                    nextByte = response.getByte(len);
+                } catch(IndexOutOfBoundsException e){
+                    // Not sure if this can ever happen but just in case
+                    // the generation does not terminate in a Null (0) value.
+                    throw new RuntimeException("Empty array or not null terminated");
+                }
+                len++;
+                if(nextByte!=0) {
+                    bufferingForWholeGeneration.write(nextByte);
+                    if(streamToStdOut){
+                        bufferingForStdOutStream.write(nextByte);
+                        // Test if Buffer is UTF8 valid string.
+                        byte[] currentBytes = bufferingForStdOutStream.toByteArray();
+                        String validString = Util.getValidUtf8(currentBytes);
+                        if(validString!=null){ // is valid string
+                            System.out.print(validString);
+                            // reset the buffer for next utf8 sequence to buffer
+                            bufferingForStdOutStream.reset();
+                        }
+                    }
+                }
+            } while(nextByte != 0);
+
+            return true; // continue generating
+        };
+    }
+
+
+    public static class ChatCompletionResponse {
+        public String model;
+        public Usage usage;
+        public List<Map<String, String>> choices;
+
+        // Getters and setters
+    }
+
+    public static class Usage {
+        public int promptTokens;
+        public int completionTokens;
+        public int totalTokens;
+
+        // Getters and setters
+    }
+
+    public ChatCompletionResponse chatCompletion(List<Map<String, String>> messages,
+                                                              GenerationConfig generationConfig) {
+        return chatCompletion(messages, generationConfig, false, false);
+    }
+
+    /**
+     * chatCompletion formats the existing chat conversation into a template to be
+     * easier to process for chat UIs. It is not absolutely necessary as generate method
+     * may be directly used to make generations with gpt models.
+     *
+     * @param messages List of Maps "role"-&gt;"user", "content"-&gt;"...", "role"-&gt; "assistant"-&gt;"..."
+     * @param generationConfig How to decode/process the generation.
+     * @param streamToStdOut Send tokens as they are calculated Standard output.
+     * @param outputFullPromptToStdOut Should full prompt built out of messages be sent to Standard output.
+     * @return ChatCompletionResponse contains stats and generated Text.
+     */
+    public ChatCompletionResponse  chatCompletion(List<Map<String, String>> messages,
+                                              GenerationConfig generationConfig, boolean streamToStdOut,
+                                              boolean outputFullPromptToStdOut) {
+        String fullPrompt = buildPrompt(messages);
+
+        if(outputFullPromptToStdOut)
+            System.out.print(fullPrompt);
+
+        String generatedText = generate(fullPrompt, generationConfig, streamToStdOut);
+
+        ChatCompletionResponse  response = new ChatCompletionResponse();
+        response.model = this.modelName;
+
+        Usage usage = new Usage();
+        usage.promptTokens = fullPrompt.length();
+        usage.completionTokens = generatedText.length();
+        usage.totalTokens = fullPrompt.length() + generatedText.length();
+        response.usage =  usage;
+
+        Map<String, String> message = new HashMap<>();
+        message.put("role", "assistant");
+        message.put("content", generatedText);
+
+        response.choices = List.of(message);
+
+        return response;
+    }
+
+    protected static String buildPrompt(List<Map<String, String>> messages) {
+        StringBuilder fullPrompt = new StringBuilder();
+
+        for (Map<String, String> message : messages) {
+            if ("system".equals(message.get("role"))) {
+                String systemMessage = message.get("content") + "\n";
+                fullPrompt.append(systemMessage);
+            }
+        }
+
+        fullPrompt.append("### Instruction: \n" +
+                "The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.\n" +
+                "### Prompt: ");
+
+        for (Map<String, String> message : messages) {
+            if ("user".equals(message.get("role"))) {
+                String userMessage = "\n" + message.get("content");
+                fullPrompt.append(userMessage);
+            }
+            if ("assistant".equals(message.get("role"))) {
+                String assistantMessage = "\n### Response: " + message.get("content");
+                fullPrompt.append(assistantMessage);
+            }
+        }
+
+        fullPrompt.append("\n### Response:");
+
+        return fullPrompt.toString();
+    }
+
+    @Override
+    public void close() throws Exception {
+        library.llmodel_model_destroy(model);
+    }
+
+}

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModelLibrary.java

@@ -0,0 +1,79 @@
+package com.hexadevlabs.gpt4all;
+
+import jnr.ffi.Pointer;
+import jnr.ffi.Struct;
+import jnr.ffi.annotations.Delegate;
+import jnr.ffi.annotations.Encoding;
+import jnr.ffi.annotations.In;
+import jnr.ffi.annotations.Out;
+import jnr.ffi.types.u_int64_t;
+
+
+/**
+ * The basic Native library interface the provides all the LLM functions.
+ */
+public interface LLModelLibrary {
+
+    interface PromptCallback {
+        @Delegate
+        boolean invoke(int token_id);
+    }
+
+    interface ResponseCallback {
+        @Delegate
+        boolean invoke(int token_id, Pointer response);
+    }
+
+    interface RecalculateCallback {
+        @Delegate
+        boolean invoke(boolean is_recalculating);
+    }
+
+    class LLModelError extends Struct {
+        public final Struct.AsciiStringRef message = new Struct.AsciiStringRef();
+        public final int32_t status = new int32_t();
+        public LLModelError(jnr.ffi.Runtime runtime) {
+            super(runtime);
+        }
+    }
+
+    class LLModelPromptContext extends Struct {
+        public final Pointer logits = new Pointer();
+        public final ssize_t logits_size = new ssize_t();
+        public final Pointer tokens = new Pointer();
+        public final ssize_t tokens_size = new ssize_t();
+        public final int32_t n_past = new int32_t();
+        public final int32_t n_ctx = new int32_t();
+        public final int32_t n_predict = new int32_t();
+        public final int32_t top_k = new int32_t();
+        public final Float top_p = new Float();
+        public final Float temp = new Float();
+        public final int32_t n_batch = new int32_t();
+        public final Float repeat_penalty = new Float();
+        public final int32_t repeat_last_n = new int32_t();
+        public final Float context_erase = new Float();
+
+        public LLModelPromptContext(jnr.ffi.Runtime runtime) {
+            super(runtime);
+        }
+    }
+
+    Pointer llmodel_model_create2(String model_path, String build_variant, @Out LLModelError llmodel_error);
+    void llmodel_model_destroy(Pointer model);
+    boolean llmodel_loadModel(Pointer model, String model_path);
+    boolean llmodel_isModelLoaded(Pointer model);
+    @u_int64_t long llmodel_get_state_size(Pointer model);
+    @u_int64_t long llmodel_save_state_data(Pointer model, Pointer dest);
+    @u_int64_t long llmodel_restore_state_data(Pointer model, Pointer src);
+
+    void llmodel_set_implementation_search_path(String path);
+
+    // ctx was an @Out ... without @Out crash
+    void llmodel_prompt(Pointer model, @Encoding("UTF-8") String prompt,
+                        PromptCallback prompt_callback,
+                        ResponseCallback response_callback,
+                        RecalculateCallback recalculate_callback,
+                        @In LLModelPromptContext ctx);
+    void llmodel_setThreadCount(Pointer model, int n_threads);
+    int llmodel_threadCount(Pointer model);
+}

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/PromptIsTooLongException.java

@@ -0,0 +1,7 @@
+package com.hexadevlabs.gpt4all;
+
+public class PromptIsTooLongException extends RuntimeException {
+    public PromptIsTooLongException(String message) {
+        super(message);
+    }
+}

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/Util.java

@@ -0,0 +1,160 @@
+package com.hexadevlabs.gpt4all;
+
+import jnr.ffi.LibraryLoader;
+import jnr.ffi.LibraryOption;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class Util {
+
+    private static final Logger logger = LoggerFactory.getLogger(Util.class);
+    private static final CharsetDecoder cs = StandardCharsets.UTF_8.newDecoder();
+
+    public static LLModelLibrary loadSharedLibrary(String librarySearchPath){
+        String libraryName = "llmodel";
+        Map<LibraryOption, Object> libraryOptions = new HashMap<>();
+        libraryOptions.put(LibraryOption.LoadNow, true); // load immediately instead of lazily (ie on first use)
+        libraryOptions.put(LibraryOption.IgnoreError, false); // calls shouldn't save last errno after call
+
+        if(librarySearchPath!=null) {
+            Map<String, List<String>> searchPaths = new HashMap<>();
+            searchPaths.put(libraryName, List.of(librarySearchPath));
+
+            return LibraryLoader.loadLibrary(LLModelLibrary.class,
+                    libraryOptions,
+                    searchPaths,
+                    libraryName
+            );
+        }else {
+
+            return LibraryLoader.loadLibrary(LLModelLibrary.class,
+                    libraryOptions,
+                    libraryName
+            );
+        }
+
+    }
+
+    /**
+     * Copy over shared library files from resource package to
+     * target Temp directory.
+     *
+     * @return Path path to the temp directory holding the shared libraries
+     */
+    public static Path copySharedLibraries() {
+        try {
+            // Identify the OS and architecture
+            String osName = System.getProperty("os.name").toLowerCase();
+            boolean isWindows = osName.startsWith("windows");
+            boolean isMac = osName.startsWith("mac os x");
+            boolean isLinux = osName.startsWith("linux");
+            if(isWindows) osName = "windows";
+            if(isMac) osName = "macos";
+            if(isLinux) osName = "linux";
+
+            //String osArch = System.getProperty("os.arch");
+
+            // Create a temporary directory
+            Path tempDirectory = Files.createTempDirectory("nativeLibraries");
+            tempDirectory.toFile().deleteOnExit();
+
+            String[] libraryNames = {
+                    "gptj-default",
+                    "gptj-avxonly",
+                    "llmodel",
+                    "mpt-default",
+                    "llamamodel-230511-default",
+                    "llamamodel-230519-default",
+                    "llamamodel-mainline-default",
+                    "llamamodel-mainline-metal",
+                    "replit-mainline-default",
+                    "replit-mainline-metal",
+                    "ggml-metal.metal",
+                    "falcon-default"
+            };
+
+            for (String libraryName : libraryNames) {
+
+                if(!isMac && (
+                        libraryName.equals("replit-mainline-metal")
+                                || libraryName.equals("llamamodel-mainline-metal")
+                                || libraryName.equals("ggml-metal.metal"))
+                ) continue;
+
+                if(isWindows){
+                    libraryName = libraryName + ".dll";
+                } else if(isMac){
+                    if(!libraryName.equals("ggml-metal.metal"))
+                        libraryName = "lib" + libraryName + ".dylib";
+                } else if(isLinux) {
+                    libraryName = "lib"+ libraryName + ".so";
+                }
+
+                // Construct the resource path based on the OS and architecture
+                String nativeLibraryPath = "/native/" + osName + "/" + libraryName;
+
+                // Get the library resource as a stream
+                InputStream in = Util.class.getResourceAsStream(nativeLibraryPath);
+                if (in == null) {
+                    throw new RuntimeException("Unable to find native library: " + nativeLibraryPath);
+                }
+
+                // Create a file in the temporary directory with the original library name
+                Path tempLibraryPath = tempDirectory.resolve(libraryName);
+
+                // Use Files.copy to copy the library to the temporary file
+                Files.copy(in, tempLibraryPath, StandardCopyOption.REPLACE_EXISTING);
+
+                // Close the input stream
+                in.close();
+            }
+
+            // Add shutdown hook to delete tempDir on JVM exit
+            // On Windows deleting dll files that are loaded into memory is not possible.
+            if(!isWindows) {
+                Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+                    try {
+                        Files.walk(tempDirectory)
+                                .sorted(Comparator.reverseOrder())
+                                .map(Path::toFile)
+                                .forEach(file -> {
+                                    try {
+                                        Files.delete(file.toPath());
+                                    } catch (IOException e) {
+                                        logger.error("Deleting temp library file", e);
+                                    }
+                                });
+                    } catch (IOException e) {
+                        logger.error("Deleting temp directory for libraries", e);
+                    }
+                }));
+            }
+
+            return tempDirectory;
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to load native libraries", e);
+        }
+    }
+
+    public static String getValidUtf8(byte[] bytes) {
+        try {
+            return cs.decode(ByteBuffer.wrap(bytes)).toString();
+        } catch (CharacterCodingException e) {
+            return null;
+        }
+    }
+}

gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/BasicTests.java

@@ -0,0 +1,155 @@
+package com.hexadevlabs.gpt4all;
+
+
+import jnr.ffi.Memory;
+import jnr.ffi.Pointer;
+import jnr.ffi.Runtime;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mockito;
+
+import org.mockito.junit.jupiter.MockitoExtension;
+
+
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.*;
+
+/**
+ * These tests only test the Java implementation as the underlying backend can't be mocked.
+ * These tests do serve the purpose of validating the java bits that do
+ * not directly have to do with the function of the underlying gp4all library.
+ */
+@ExtendWith(MockitoExtension.class)
+public class BasicTests {
+
+    @Test
+    public void simplePrompt(){
+
+        LLModel model = Mockito.spy(new LLModel());
+
+        LLModel.GenerationConfig config =
+                LLModel.config()
+                        .withNPredict(20)
+                        .build();
+
+        // The generate method will return "4"
+        doReturn("4").when( model ).generate(anyString(), eq(config), eq(true));
+
+        LLModel.ChatCompletionResponse response= model.chatCompletion(
+                    List.of(Map.of("role", "system", "content", "You are a helpful assistant"),
+                            Map.of("role", "user", "content", "Add 2+2")), config, true, true);
+
+        assertTrue( response.choices.get(0).get("content").contains("4") );
+
+        // Verifies the prompt and response are certain length.
+        assertEquals( 224 , response.usage.totalTokens );
+    }
+
+    @Test
+    public void testResponseCallback(){
+
+        ByteArrayOutputStream bufferingForStdOutStream = new ByteArrayOutputStream();
+        ByteArrayOutputStream bufferingForWholeGeneration = new ByteArrayOutputStream();
+
+        LLModelLibrary.ResponseCallback responseCallback = LLModel.getResponseCallback(false, bufferingForStdOutStream, bufferingForWholeGeneration);
+
+        // Get the runtime instance
+        Runtime runtime = Runtime.getSystemRuntime();
+
+        // Allocate memory for the byte array. Has to be null terminated
+
+        // UTF-8 Encoding of the character: 0xF0 0x9F 0x92 0xA9
+        byte[] utf8ByteArray = {(byte) 0xF0, (byte) 0x9F, (byte) 0x92, (byte) 0xA9, 0x00}; // Adding null termination
+
+        // Optional: Converting the byte array back to a String to print the character
+        String decodedString = new String(utf8ByteArray, 0, utf8ByteArray.length - 1, java.nio.charset.StandardCharsets.UTF_8);
+
+        Pointer pointer = Memory.allocateDirect(runtime, utf8ByteArray.length);
+
+        // Copy the byte array to the allocated memory
+        pointer.put(0, utf8ByteArray, 0, utf8ByteArray.length);
+
+        responseCallback.invoke(1, pointer);
+
+        String result = bufferingForWholeGeneration.toString(StandardCharsets.UTF_8);
+
+        assertEquals(decodedString, result);
+
+    }
+
+    @Test
+    public void testResponseCallbackTwoTokens(){
+
+        ByteArrayOutputStream bufferingForStdOutStream = new ByteArrayOutputStream();
+        ByteArrayOutputStream bufferingForWholeGeneration = new ByteArrayOutputStream();
+
+        LLModelLibrary.ResponseCallback responseCallback = LLModel.getResponseCallback(false, bufferingForStdOutStream, bufferingForWholeGeneration);
+
+        // Get the runtime instance
+        Runtime runtime = Runtime.getSystemRuntime();
+
+        // Allocate memory for the byte array. Has to be null terminated
+
+        // UTF-8 Encoding of the character: 0xF0 0x9F 0x92 0xA9
+        byte[] utf8ByteArray = {  (byte) 0xF0, (byte) 0x9F, 0x00}; // Adding null termination
+        byte[] utf8ByteArray2 = { (byte) 0x92, (byte) 0xA9, 0x00}; // Adding null termination
+
+        // Optional: Converting the byte array back to a String to print the character
+        Pointer pointer = Memory.allocateDirect(runtime, utf8ByteArray.length);
+
+        // Copy the byte array to the allocated memory
+        pointer.put(0, utf8ByteArray, 0, utf8ByteArray.length);
+
+        responseCallback.invoke(1, pointer);
+        // Copy the byte array to the allocated memory
+        pointer.put(0, utf8ByteArray2, 0, utf8ByteArray2.length);
+
+        responseCallback.invoke(2, pointer);
+
+        String result = bufferingForWholeGeneration.toString(StandardCharsets.UTF_8);
+
+        assertEquals("\uD83D\uDCA9", result);
+
+    }
+
+
+    @Test
+    public void testResponseCallbackExpectError(){
+
+        ByteArrayOutputStream bufferingForStdOutStream = new ByteArrayOutputStream();
+        ByteArrayOutputStream bufferingForWholeGeneration = new ByteArrayOutputStream();
+
+        LLModelLibrary.ResponseCallback responseCallback = LLModel.getResponseCallback(false, bufferingForStdOutStream, bufferingForWholeGeneration);
+
+        // Get the runtime instance
+        Runtime runtime = Runtime.getSystemRuntime();
+
+        // UTF-8 Encoding of the character: 0xF0 0x9F 0x92 0xA9
+        byte[] utf8ByteArray = {(byte) 0xF0, (byte) 0x9F, (byte) 0x92, (byte) 0xA9}; // No null termination
+
+        Pointer pointer = Memory.allocateDirect(runtime, utf8ByteArray.length);
+
+        // Copy the byte array to the allocated memory
+        pointer.put(0, utf8ByteArray, 0, utf8ByteArray.length);
+
+        Exception exception = assertThrows(RuntimeException.class, () -> responseCallback.invoke(1, pointer));
+
+        assertEquals("Empty array or not null terminated", exception.getMessage());
+
+        // With empty array
+        utf8ByteArray = new byte[0];
+        pointer.put(0, utf8ByteArray, 0, utf8ByteArray.length);
+
+        Exception exceptionN = assertThrows(RuntimeException.class, () -> responseCallback.invoke(1, pointer));
+
+        assertEquals("Empty array or not null terminated", exceptionN.getMessage());
+
+    }
+
+}

gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/Example1.java

@@ -0,0 +1,30 @@
+package com.hexadevlabs.gpt4all;
+
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * GPTJ chat completion, multiple messages
+ */
+public class Example1 {
+    public static void main(String[] args) {
+
+        // Optionally in case override to location of shared libraries is necessary
+        //LLModel.LIBRARY_SEARCH_PATH = "C:\\Users\\felix\\gpt4all\\lib\\";
+
+        try ( LLModel gptjModel = new LLModel(Path.of("C:\\Users\\felix\\AppData\\Local\\nomic.ai\\GPT4All\\ggml-gpt4all-j-v1.3-groovy.bin")) ){
+
+            LLModel.GenerationConfig config = LLModel.config()
+                    .withNPredict(4096).build();
+
+            gptjModel.chatCompletion(
+                    List.of(Map.of("role", "user", "content", "Add 2+2"),
+                            Map.of("role", "assistant", "content", "4"),
+                            Map.of("role", "user", "content", "Multiply 4 * 5")), config, true, true);
+
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/Example2.java

@@ -0,0 +1,31 @@
+package com.hexadevlabs.gpt4all;
+
+import java.nio.file.Path;
+
+/**
+ * Generation with MPT model
+ */
+public class Example2 {
+    public static void main(String[] args) {
+
+        String prompt = "### Human:\nWhat is the meaning of life\n### Assistant:";
+
+        // Optionally in case override to location of shared libraries is necessary
+        //LLModel.LIBRARY_SEARCH_PATH = "C:\\Users\\felix\\gpt4all\\lib\\";
+
+        try (LLModel mptModel = new LLModel(Path.of("C:\\Users\\felix\\AppData\\Local\\nomic.ai\\GPT4All\\ggml-mpt-7b-instruct.bin"))) {
+
+            LLModel.GenerationConfig config =
+                    LLModel.config()
+                    .withNPredict(4096)
+                    .withRepeatLastN(64)
+                    .build();
+
+            mptModel.generate(prompt, config, true);
+
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+}

gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/Example3.java

@@ -0,0 +1,33 @@
+package com.hexadevlabs.gpt4all;
+
+import jnr.ffi.LibraryLoader;
+
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * GPTJ chat completion with system message
+ */
+public class Example3 {
+    public static void main(String[] args) {
+
+        // Optionally in case override to location of shared libraries is necessary
+        //LLModel.LIBRARY_SEARCH_PATH = "C:\\Users\\felix\\gpt4all\\lib\\";
+
+        try ( LLModel gptjModel = new LLModel(Path.of("C:\\Users\\felix\\AppData\\Local\\nomic.ai\\GPT4All\\ggml-gpt4all-j-v1.3-groovy.bin")) ){
+
+            LLModel.GenerationConfig config = LLModel.config()
+                    .withNPredict(4096).build();
+
+            // String result = gptjModel.generate(prompt, config, true);
+            gptjModel.chatCompletion(
+                    List.of(Map.of("role", "system", "content", "You are a helpful assistant"),
+                            Map.of("role", "user", "content", "Add 2+2")), config, true, true);
+
+
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/Example4.java

@@ -0,0 +1,43 @@
+package com.hexadevlabs.gpt4all;
+
+import java.nio.file.Path;
+
+public class Example4 {
+
+    public static void main(String[] args) {
+
+        String prompt = "### Human:\nWhat is the meaning of life\n### Assistant:";
+        // The emoji is poop emoji. The Unicode character is encoded as surrogate pair for Java string.
+        // LLM should correctly identify it as poop emoji in the description
+        //String prompt = "### Human:\nDescribe the meaning of this emoji \uD83D\uDCA9\n### Assistant:";
+        //String prompt = "### Human:\nOutput the unicode character of smiley face emoji\n### Assistant:";
+
+        // Optionally in case override to location of shared libraries is necessary
+        //LLModel.LIBRARY_SEARCH_PATH = "C:\\Users\\felix\\gpt4all\\lib\\";
+
+        String model = "ggml-vicuna-7b-1.1-q4_2.bin";
+        //String model = "ggml-gpt4all-j-v1.3-groovy.bin";
+        //String model = "ggml-mpt-7b-instruct.bin";
+        String basePath = "C:\\Users\\felix\\AppData\\Local\\nomic.ai\\GPT4All\\";
+        //String basePath = "/Users/fzaslavs/Library/Application Support/nomic.ai/GPT4All/";
+
+        try (LLModel mptModel = new LLModel(Path.of(basePath + model))) {
+
+            LLModel.GenerationConfig config =
+                    LLModel.config()
+                            .withNPredict(4096)
+                            .withRepeatLastN(64)
+                            .build();
+
+
+            String result = mptModel.generate(prompt, config, true);
+
+            System.out.println("Code points:");
+            result.codePoints().forEach(System.out::println);
+
+
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}