Update mainline for the mul*vec and q6_k kernels.

Signed-off-by: Andriy Mulyar <andriy.mulyar@gmail.com>

.circleci/config.yml

@@ -0,0 +1,19 @@
+version: 2.1
+setup: true
+orbs:
+  path-filtering: circleci/path-filtering@0.0.1
+
+workflows:
+  version: 2.1
+  generate-config:
+    jobs:
+      - path-filtering/filter:
+          base-revision: main
+          config-path: .circleci/continue_config.yml
+          mapping: |
+            gpt4all-bindings/python/.* run-python-workflow true
+            gpt4all-bindings/typescript/.* run-ts-workflow true
+            gpt4all-bindings/csharp/.* run-csharp-workflow true
+            gpt4all-backend/.* run-chat-workflow true
+            gpt4all-chat/.* run-chat-workflow true
+            .* run-default-workflow true

.circleci/continue_config.yml

@@ -0,0 +1,993 @@
+version: 2.1
+orbs:
+  win: circleci/windows@5.0
+  python: circleci/python@1.2
+  node: circleci/node@5.1
+
+parameters:
+  run-default-workflow:
+    type: boolean
+    default: false
+  run-python-workflow:
+    type: boolean
+    default: false
+  run-chat-workflow:
+    type: boolean
+    default: false
+  run-ts-workflow:
+    type: boolean
+    default: false
+  run-csharp-workflow:
+    type: boolean
+    default: false
+
+jobs:
+  default-job:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - run: echo "CircleCI pipeline triggered"
+
+  build-gpt4all-chat-linux:
+    machine:
+      image: ubuntu-2204:2023.04.2
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - linux-qt-cache
+      - run:
+          name: Setup Linux and Dependencies
+          command: |
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
+              chmod +x qt-unified-linux-x64-4.6.0-online.run
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: linux-qt-cache
+          paths:
+            - ~/Qt
+      - run:
+          name: Build
+          command: |
+            export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
+            mkdir build
+            cd build
+            ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S ../gpt4all-chat -B .
+            ~/Qt/Tools/CMake/bin/cmake --build . --target all
+
+  build-gpt4all-chat-windows:
+    machine:
+      image: 'windows-server-2019-vs2019:2022.08.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - windows-qt-cache
+      - run:
+          name: Installing Qt
+          command: |
+            if (-not (Test-Path C:\Qt)) {
+              Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+            }
+      - save_cache:  # this is the new step to save cache
+          key: windows-qt-cache
+          paths:
+            - C:\Qt
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Build
+          command: |
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
+            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
+            $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
+            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
+            $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
+            mkdir build
+            cd build
+            & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
+              "-DCMAKE_GENERATOR:STRING=Ninja" `
+              "-DCMAKE_BUILD_TYPE=Release" `
+              "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
+              "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
+              "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
+              "-S ..\gpt4all-chat" `
+              "-B ."
+            & "C:\Qt\Tools\Ninja\ninja.exe"
+
+  build-gpt4all-chat-macos:
+    macos:
+      xcode: 14.0.0
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - restore_cache:  # this is the new step to restore cache
+          keys:
+            - macos-qt-cache_v2
+      - run:
+          name: Installing Qt
+          command: |
+            if [ ! -d ~/Qt ]; then
+              curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
+              hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
+            fi
+      - save_cache:  # this is the new step to save cache
+          key: macos-qt-cache_v2
+          paths:
+            - ~/Qt
+      - run:
+          name: Build
+          command: |
+            mkdir build
+            cd build
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
+              -DCMAKE_GENERATOR:STRING=Ninja \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
+              -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
+              -S ../gpt4all-chat \
+              -B .
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
+  build-ts-docs: 
+    docker: 
+      - image: cimg/base:stable
+    steps:
+      - checkout
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - node/install-packages:
+          pkg-manager: yarn
+          app-dir: gpt4all-bindings/typescript
+          override-ci-command: yarn install
+      - run:
+          name: build docs ts yo
+          command: |
+            cd gpt4all-bindings/typescript
+            yarn docs:build
+  build-py-docs:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - checkout 
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get -y install python3 python3-pip
+            sudo pip3 install awscli --upgrade
+            sudo pip3 install mkdocs mkdocs-material mkautodoc 'mkdocstrings[python]'
+      - run:
+          name: Make Documentation
+          command: |
+            cd gpt4all-bindings/python/
+            mkdocs build
+      - run:
+          name: Deploy Documentation
+          command: |
+            cd gpt4all-bindings/python/
+            aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat
+      - run:
+          name: Invalidate docs.gpt4all.io cloudfront
+          command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"
+
+  build-py-linux:
+    machine:
+      image: ubuntu-2204:2023.04.2
+    steps:
+      - checkout
+      - run:
+          name: Set Python Version
+          command: pyenv global 3.11.2
+      - run:
+          name: Install dependencies
+          command: |
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            sudo apt-get update
+            sudo apt-get install -y cmake build-essential vulkan-sdk
+            pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            cmake ..
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          command: |
+            cd gpt4all-bindings/python/
+            python setup.py bdist_wheel --plat-name=manylinux1_x86_64
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  build-py-macos:
+    macos:
+      xcode: "14.2.0"
+    resource_class: macos.m1.large.gen1
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            brew install cmake
+            pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            cmake .. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          command: |
+            cd gpt4all-bindings/python
+            python setup.py bdist_wheel --plat-name=macosx_10_9_universal2
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  build-py-windows:
+    executor:
+      name: win/default
+    steps:
+      - checkout
+      - run:
+          name: Install MinGW64
+          command: choco install -y mingw --force --no-progress
+      - run:
+          name: Add MinGW64 to PATH
+          command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install dependencies
+          command:
+            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+      - run:
+          name: Install Python dependencies
+          command: pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            $env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            cmake -G "MinGW Makefiles" .. -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          # TODO: As part of this task, we need to move mingw64 binaries into package.
+          # This is terrible and needs a more robust solution eventually.
+          command: |
+            cd gpt4all-bindings/python
+            cd gpt4all
+            mkdir llmodel_DO_NOT_MODIFY
+            mkdir llmodel_DO_NOT_MODIFY/build/
+            cp 'C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
+            cd ..
+            python setup.py bdist_wheel --plat-name=win_amd64
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  store-and-upload-wheels:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - setup_remote_docker
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y cmake build-essential
+            pip install setuptools wheel twine
+      - run:
+          name: Upload Python package
+          command: |
+            twine upload /tmp/workspace/*.whl --username __token__ --password $PYPI_CRED
+      - store_artifacts:
+          path: /tmp/workspace
+
+  build-bindings-backend-linux:
+    machine:
+      image: ubuntu-2204:2023.04.2
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install dependencies
+          command: |
+            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list            
+            sudo apt-get update
+            sudo apt-get install -y cmake build-essential vulkan-sdk
+      - run:
+          name: Build Libraries
+          command: |
+            cd gpt4all-backend
+            mkdir -p runtimes/build
+            cd runtimes/build
+            cmake ../..
+            cmake --build . --parallel --config Release
+            mkdir ../linux-x64
+            cp -L *.so ../linux-x64 # otherwise persist_to_workspace seems to mess symlinks
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/linux-x64/*.so
+
+  build-bindings-backend-macos:
+    macos:
+      xcode: "14.0.0"
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install dependencies
+          command: |
+            brew install cmake
+      - run:
+          name: Build Libraries
+          command: |
+            cd gpt4all-backend
+            mkdir -p runtimes/build
+            cd runtimes/build
+            cmake ../.. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake --build . --parallel --config Release
+            mkdir ../osx-x64
+            cp -L *.dylib ../osx-x64
+            cp ../../llama.cpp-mainline/*.metal ../osx-x64
+            ls ../osx-x64
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/osx-x64/*.dylib
+            - runtimes/osx-x64/*.metal
+
+  build-bindings-backend-windows:
+    executor:
+      name: win/default
+      size: large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install MinGW64
+          command: choco install -y mingw --force --no-progress
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install dependencies
+          command: |
+            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+      - run:
+          name: Build Libraries
+          command: |
+            $MinGWBin = "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
+            $Env:Path += ";$MinGwBin"
+            $Env:Path += ";C:\Program Files\CMake\bin"
+            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            cd gpt4all-backend
+            mkdir runtimes/win-x64
+            cd runtimes/win-x64
+            cmake -G "MinGW Makefiles" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON ../..
+            cmake --build . --parallel --config Release
+            cp "$MinGWBin\libgcc*.dll" .
+            cp "$MinGWBin\libstdc++*.dll" .
+            cp "$MinGWBin\libwinpthread*.dll" .
+            cp bin/*.dll .
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/win-x64/*.dll
+
+  build-bindings-backend-windows-msvc:
+    machine:
+      image: 'windows-server-2022-gui:2023.03.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - run:
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
+      - run:
+          name: Install VulkanSDK
+          command: |
+            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
+            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install dependencies
+          command: |
+            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'  
+      - run:
+          name: Build Libraries
+          command: |
+            $Env:Path += ";C:\Program Files\CMake\bin"
+            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            cd gpt4all-backend
+            mkdir runtimes/win-x64_msvc
+            cd runtimes/win-x64_msvc
+            cmake -G "Visual Studio 17 2022" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -A X64 ../..
+            cmake --build . --parallel --config Release
+            cp bin/Release/*.dll .
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - runtimes/win-x64_msvc/*.dll
+
+  build-csharp-linux:
+    docker:
+      - image: mcr.microsoft.com/dotnet/sdk:7.0-jammy # Ubuntu 22.04
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: "Prepare Native Libs"
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes/linux-x64/native
+            cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
+            ls -R runtimes
+      - restore_cache:
+          keys:
+            - gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: "Install project dependencies"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet restore Gpt4All
+      - save_cache:
+          paths:
+            - ~/.nuget/packages
+          key: gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: Build C# Project
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet build Gpt4All --configuration Release --nologo
+      - run:
+          name: "Run C# Tests"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+      - run:
+          name: Test results
+          command: |
+              cd gpt4all-bindings/csharp/Gpt4All.Tests
+              dotnet tool install -g trx2junit
+              export PATH="$PATH:$HOME/.dotnet/tools"
+              trx2junit TestResults/*.trx
+      - store_test_results:
+          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+
+  build-csharp-windows:
+    executor:
+      name: win/default
+      size: large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - gpt4all-csharp-nuget-packages-win
+      - attach_workspace:
+          at: C:\Users\circleci\workspace
+      - run:
+          name: "Prepare Native Libs"
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes\win-x64\native
+            cp C:\Users\circleci\workspace\runtimes\win-x64\*.dll runtimes\win-x64\native\
+            ls -R runtimes
+      - run:
+          name: "Install project dependencies"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet.exe restore Gpt4All
+      - save_cache:
+          paths:
+            - C:\Users\circleci\.nuget\packages
+          key: gpt4all-csharp-nuget-packages-win
+      - run:
+          name: Build C# Project
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet.exe build Gpt4All --configuration Release --nologo
+      - run:
+          name: "Run C# Tests"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet.exe test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+      - run:
+          name: Test results
+          command: |
+              cd gpt4all-bindings/csharp/Gpt4All.Tests
+              dotnet tool install -g trx2junit
+              $Env:Path += ";$Env:USERPROFILE\.dotnet\tools"
+              trx2junit TestResults/*.trx
+      - store_test_results:
+          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+
+  build-csharp-macos:
+    macos:
+      xcode: "14.0.0"
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: Install dependencies
+          command: |
+            brew install --cask dotnet-sdk
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: "Prepare Native Libs"
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes/osx/native
+            cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
+            cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
+            ls -R runtimes
+      - run:
+          name: "Install project dependencies"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet restore Gpt4All
+      - save_cache:
+          paths:
+            - ~/.nuget/packages
+          key: gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: Build C# Project
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet build Gpt4All --configuration Release --nologo
+      - run:
+          name: "Run C# Tests"
+          command: |
+            cd gpt4all-bindings/csharp
+            dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx"
+      - run:
+          name: Test results
+          command: |
+              cd gpt4all-bindings/csharp/Gpt4All.Tests
+              dotnet tool install -g trx2junit
+              export PATH="$PATH:$HOME/.dotnet/tools"
+              trx2junit TestResults/*.trx
+      - store_test_results:
+          path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults
+
+  store-and-upload-nupkgs:
+    docker:
+      - image: mcr.microsoft.com/dotnet/sdk:6.0-jammy # Ubuntu 22.04
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - checkout
+      - restore_cache:
+           keys:
+             - gpt4all-csharp-nuget-packages-nix
+      - run:
+          name: NuGet Pack
+          command: |
+            cd gpt4all-bindings/csharp
+            mkdir -p runtimes/linux-x64/native
+            cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
+            mkdir -p runtimes/win-x64/native
+            cp /tmp/workspace/runtimes/win-x64/*.dll runtimes/win-x64/native/
+            mkdir -p runtimes/osx/native
+            cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
+            cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
+            dotnet pack ./Gpt4All/Gpt4All.csproj -p:IncludeSymbols=true -p:SymbolPackageFormat=snupkg -c Release
+            dotnet nuget push ./Gpt4All/bin/Release/Gpt4All.*.nupkg -s $NUGET_URL -k $NUGET_TOKEN --skip-duplicate
+      - store_artifacts:
+          path: gpt4all-bindings/csharp/Gpt4All/bin/Release
+
+  build-nodejs-linux: 
+    docker: 
+      - image: cimg/base:stable
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - node/install-packages:
+          app-dir: gpt4all-bindings/typescript
+          pkg-manager: yarn
+      - run:
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn prebuildify -t 18.16.0 --napi
+      - run: 
+          command: |
+            mkdir -p gpt4all-backend/prebuilds/linux-x64
+            mkdir -p gpt4all-backend/runtimes/linux-x64
+            cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so gpt4all-backend/runtimes/linux-x64
+            cp gpt4all-bindings/typescript/prebuilds/linux-x64/*.node gpt4all-backend/prebuilds/linux-x64
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - prebuilds/linux-x64/*.node 
+            - runtimes/linux-x64/*-*.so
+  build-nodejs-macos: 
+    macos:
+      xcode: "14.0.0"
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - node/install-packages:
+          app-dir: gpt4all-bindings/typescript
+          pkg-manager: yarn
+      - run:
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn prebuildify -t 18.16.0 --napi
+      - run: 
+          name: "Persisting all necessary things to workspace"
+          command: |  
+            mkdir -p gpt4all-backend/prebuilds/darwin-x64
+            mkdir -p gpt4all-backend/runtimes/darwin-x64
+            cp /tmp/gpt4all-backend/runtimes/osx-x64/*-*.* gpt4all-backend/runtimes/darwin-x64
+            cp gpt4all-bindings/typescript/prebuilds/darwin-x64/*.node gpt4all-backend/prebuilds/darwin-x64
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - prebuilds/darwin-x64/*.node
+            - runtimes/darwin-x64/*-*.*
+
+  build-nodejs-windows: 
+    executor:
+      name: win/default
+      size: large
+      shell: powershell.exe -ExecutionPolicy Bypass
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - run: choco install wget -y
+      - run:
+         command: wget https://nodejs.org/dist/v18.16.0/node-v18.16.0-x86.msi -P C:\Users\circleci\Downloads\
+         shell: cmd.exe
+      - run: MsiExec.exe /i C:\Users\circleci\Downloads\node-v18.16.0-x86.msi /qn
+      - run: 
+          command: |
+              Start-Process powershell -verb runAs -Args "-start GeneralProfile"
+              nvm install 18.16.0
+              nvm use 18.16.0
+      - run: node --version 
+      - run:           
+          command: |
+            npm install -g yarn
+            cd gpt4all-bindings/typescript
+            yarn install
+      - run:
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn prebuildify -t 18.16.0 --napi 
+      - run: 
+          command: |
+            mkdir -p gpt4all-backend/prebuilds/win32-x64
+            mkdir -p gpt4all-backend/runtimes/win32-x64
+            cp /tmp/gpt4all-backend/runtimes/win-x64_msvc/*-*.dll gpt4all-backend/runtimes/win32-x64
+            cp gpt4all-bindings/typescript/prebuilds/win32-x64/*.node gpt4all-backend/prebuilds/win32-x64
+        
+      - persist_to_workspace:
+          root: gpt4all-backend
+          paths:
+            - prebuilds/win32-x64/*.node
+            - runtimes/win32-x64/*-*.dll
+
+  prepare-npm-pkg:
+    docker: 
+      - image: cimg/base:stable
+    steps:
+      - attach_workspace:
+          at: /tmp/gpt4all-backend
+      - checkout
+      - node/install:
+          install-yarn: true
+          node-version: "18.16"
+      - run: node --version
+      - run: 
+          command: |
+            cd gpt4all-bindings/typescript
+            # excluding llmodel. nodejs bindings dont need llmodel.dll
+            mkdir -p runtimes/win32-x64/native
+            mkdir -p prebuilds/win32-x64/
+            cp /tmp/gpt4all-backend/runtimes/win-x64_msvc/*-*.dll runtimes/win32-x64/native/ 
+            cp /tmp/gpt4all-backend/prebuilds/win32-x64/*.node prebuilds/win32-x64/    
+
+            mkdir -p runtimes/linux-x64/native 
+            mkdir -p prebuilds/linux-x64/
+            cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so runtimes/linux-x64/native/ 
+            cp /tmp/gpt4all-backend/prebuilds/linux-x64/*.node prebuilds/linux-x64/    
+
+            mkdir -p runtimes/darwin-x64/native
+            mkdir -p prebuilds/darwin-x64/
+            cp /tmp/gpt4all-backend/runtimes/darwin-x64/*-*.* runtimes/darwin-x64/native/
+            cp /tmp/gpt4all-backend/prebuilds/darwin-x64/*.node prebuilds/darwin-x64/    
+            
+            # Fallback build if user is not on above prebuilds
+            mv -f binding.ci.gyp binding.gyp
+
+            mkdir gpt4all-backend
+            cd ../../gpt4all-backend
+            mv llmodel.h llmodel.cpp llmodel_c.cpp llmodel_c.h sysinfo.h dlhandle.h ../gpt4all-bindings/typescript/gpt4all-backend/
+            
+      # Test install
+      - node/install-packages:
+          app-dir: gpt4all-bindings/typescript
+          pkg-manager: yarn
+          override-ci-command: yarn install
+      - run: 
+          command: | 
+            cd gpt4all-bindings/typescript
+            yarn run test
+      - run:
+          command: |
+            cd gpt4all-bindings/typescript
+            npm set //registry.npmjs.org/:_authToken=$NPM_TOKEN
+            npm publish --access public --tag alpha
+
+workflows:
+  version: 2
+  default:
+    when: << pipeline.parameters.run-default-workflow >>
+    jobs:
+      - default-job
+  build-and-test-gpt4all-chat:
+    when: << pipeline.parameters.run-chat-workflow >>
+    jobs:
+      - hold:
+          type: approval
+      - build-gpt4all-chat-linux:
+          requires:
+            - hold
+      - build-gpt4all-chat-windows:
+          requires:
+            - hold
+      - build-gpt4all-chat-macos:
+          requires:
+            - hold
+  deploy-docs:
+    when: << pipeline.parameters.run-python-workflow >>
+    jobs:
+      - build-ts-docs:
+          filters:
+            branches:
+              only:
+                - main
+      - build-py-docs:
+          filters:
+            branches:
+              only:
+                - main
+  build-py-deploy:
+    when: << pipeline.parameters.run-python-workflow >>
+    jobs:
+      - pypi-hold:
+          type: approval
+      - hold:
+          type: approval
+      - build-py-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-py-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-py-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - store-and-upload-wheels:
+          filters:
+            branches:
+              only:
+          requires:
+            - pypi-hold
+            - build-py-windows
+            - build-py-linux
+            - build-py-macos
+  build-bindings:
+    when: 
+        or:
+         - << pipeline.parameters.run-python-workflow >>
+         - << pipeline.parameters.run-csharp-workflow >>
+         - <<   pipeline.parameters.run-ts-workflow   >>
+    jobs:
+      - hold:
+          type: approval
+      - nuget-hold:
+          type: approval
+      - npm-hold:
+          type: approval
+      - build-bindings-backend-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-bindings-backend-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-bindings-backend-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+      - build-bindings-backend-windows-msvc:
+          filters:
+            branches:
+              only:
+          requires:
+            - hold
+
+      # NodeJs Jobs 
+      - prepare-npm-pkg: 
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-nodejs-linux
+            - build-nodejs-windows
+            - build-nodejs-macos
+      - build-nodejs-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-bindings-backend-linux
+      - build-nodejs-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-bindings-backend-windows-msvc
+      - build-nodejs-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - npm-hold
+            - build-bindings-backend-macos
+
+
+      # CSharp Jobs
+      - build-csharp-linux:
+          filters:
+            branches:
+              only:
+          requires:
+            - nuget-hold
+            - build-bindings-backend-linux
+      - build-csharp-windows:
+          filters:
+            branches:
+              only:
+          requires:
+            - nuget-hold
+            - build-bindings-backend-windows
+      - build-csharp-macos:
+          filters:
+            branches:
+              only:
+          requires:
+            - nuget-hold
+            - build-bindings-backend-macos
+      - store-and-upload-nupkgs:
+          filters:
+            branches:
+              only:
+          requires:
+            - nuget-hold
+            - build-csharp-windows
+            - build-csharp-linux
+            - build-csharp-macos

.codespellrc

@@ -0,0 +1,3 @@
+[codespell]
+ignore-words-list = blong, belong, afterall, som
+skip = .git,*.pdf,*.svg,*.lock

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,70 @@
+name: "\U0001F41B Bug Report"
+description: Submit a bug report to help us improve GPT4All
+labels: ["02 Bug Report"]
+body:
+  - type: markdown
+    attributes:
+      value: >
+        Thank you for taking the time to file a bug report. Before creating a new
+        issue, please make sure to take a few moments to check the issue tracker
+        for existing issues about the bug.
+
+  - type: textarea
+    id: system-info
+    attributes:
+      label: System Info
+      description: Please share your system info with us.
+      placeholder: GPT4All version, platform, python version, etc...
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: information-scripts-examples
+    attributes:
+      label: Information
+      description: "The problem arises when using:"
+      options:
+        - label: "The official example notebooks/scripts"
+        - label: "My own modified scripts"
+
+  - type: checkboxes
+    id: related-components
+    attributes:
+      label: Related Components
+      description: "Select the components related to the issue (if applicable):"
+      options:
+        - label: "backend"
+        - label: "bindings"
+        - label: "python-bindings"
+        - label: "chat-ui"
+        - label: "models"
+        - label: "circleci"
+        - label: "docker"
+        - label: "api"
+
+  - type: textarea
+    id: reproduction
+    validations:
+      required: true
+    attributes:
+      label: Reproduction
+      description: |
+        Please provide a [code sample](https://stackoverflow.com/help/minimal-reproducible-example) that reproduces the problem you ran into. It can be a Colab link or just a code snippet.
+        If you have code snippets, error messages, stack traces please provide them here as well.
+        Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting
+        Avoid screenshots when possible, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code.
+
+      placeholder: |
+        Steps to reproduce the behavior:
+
+          1.
+          2.
+          3.
+
+  - type: textarea
+    id: expected-behavior
+    validations:
+      required: true
+    attributes:
+      label: Expected behavior
+      description: "A clear and concise description of what you would expect to happen."

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,2 @@
+blank_issues_enabled: false
+version: 2.1

.github/ISSUE_TEMPLATE/documentation.yml

@@ -0,0 +1,19 @@
+name: Documentation
+description: Report an issue related to the GPT4All documentation.
+title: "DOC: <Please write a comprehensive title after the 'DOC: ' prefix>"
+labels: [03 - Documentation]
+
+body:
+- type: textarea
+  attributes: 
+    label: "Issue with current documentation:"
+    description: >
+      Please make sure to leave a reference to the document/code you're
+      referring to.
+
+- type: textarea
+  attributes:
+    label: "Idea or request for content:"
+    description: >
+      Please describe as clearly as possible what topics you think are missing
+      from the current documentation.

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -0,0 +1,30 @@
+name: "\U0001F680 Feature Request"
+description: Submit a proposal/request for a new GPT4All feature
+labels: ["02 Feature Request"]
+body:
+  - type: textarea
+    id: feature-request
+    validations:
+      required: true
+    attributes:
+      label: Feature request
+      description: |
+        A clear and concise description of the feature proposal. Please provide links to any relevant GitHub repos, papers, or other resources if relevant.
+
+  - type: textarea
+    id: motivation
+    validations:
+      required: true
+    attributes:
+      label: Motivation
+      description: |
+        Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too.
+
+  - type: textarea
+    id: contribution
+    validations:
+      required: true
+    attributes:
+      label: Your contribution
+      description: |
+        Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/nomic-ai/gpt4all/blob/main/CONTRIBUTING.md)

.github/ISSUE_TEMPLATE/other.yml

@@ -0,0 +1,18 @@
+name: Other Issue
+description: Raise an issue that wouldn't be covered by the other templates.
+title: "Issue: <Please write a comprehensive title after the 'Issue: ' prefix>"
+labels: [04 - Other]
+
+body:
+  - type: textarea
+    attributes:
+      label: "Issue you'd like to raise."
+      description: >
+        Please describe the issue you'd like to raise as clearly as possible.
+        Make sure to include any relevant links or references.
+
+  - type: textarea
+    attributes:
+      label: "Suggestion:"
+      description: >
+        Please outline a suggestion to improve the issue here.

.github/pull_request_template.md

@@ -0,0 +1,19 @@
+## Describe your changes
+
+## Issue ticket number and link
+
+## Checklist before requesting a review
+- [ ] I have performed a self-review of my code.
+- [ ] If it is a core feature, I have added thorough tests.
+- [ ] I have added thorough documentation for my code.
+- [ ] I have tagged PR with relevant project labels. I acknowledge that a PR without labels may be dismissed.
+- [ ] If this PR addresses a bug, I have provided both a screenshot/video of the original bug and the working solution.
+
+## Demo
+<!-- Screenshots or video of new or updated code changes !-->
+
+### Steps to Reproduce
+<!-- Steps to reproduce demo !-->
+
+## Notes
+<!-- Any other relevant information to include about PR !-->

.github/workflows/close_issues.yml

@@ -0,0 +1,33 @@
+# This workflow will close issues that do not have labels or additional comments.
+# Trigger manually.
+
+name: "Close Issues"
+on:
+  workflow_dispatch:
+
+jobs:
+  close_issues:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Close issues without label or comment
+      uses: actions/github-script@v3
+      with:
+        github-token: ${{secrets.GITHUB_TOKEN}}
+        script: |
+          const repo = context.repo;
+          let page = 1;
+          let issues = [];
+          while (true) {
+            const result = await github.issues.listForRepo({...repo, per_page: 100, page: page});
+            if (result.data.length === 0) break;
+            issues = issues.concat(result.data);
+            page += 1;
+          }
+          for (let { number } of issues) {
+            const issueData = await github.issues.get({...repo, issue_number: number});
+            const comments = await github.issues.listComments({...repo, issue_number: number});
+            if (issueData.data.labels.length === 0 && comments.data.length < 1) {
+              await github.issues.update({...repo, issue_number: number, state: 'closed'});
+              await github.issues.createComment({...repo, issue_number: number, body: 'Issue closed as it does not have any labels or comments.'});
+            }
+          }

.github/workflows/codespell.yml

@@ -0,0 +1,19 @@
+---
+name: Codespell
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  codespell:
+    name: Check for spelling errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Codespell
+        uses: codespell-project/actions-codespell@v2

.gitignore

@@ -1,3 +1,9 @@
+*.arrow
+squad_*
+*sbert_embedded*
+*.pkl
+ckpts*
+.deepspeed_env
 *.jsonl
 *tar.gz
 ckpts**
@@ -166,4 +172,15 @@ cython_debug/
 
 # vs code
 .vscode
-*.bin
+*.bin
+
+.DS_Store
+
+# gpt4all-chat
+CMakeLists.txt.user
+gpt4all-chat/models/*
+build_*
+build-*
+
+# IntelliJ
+.idea/

.gitmodules

@@ -1,6 +1,9 @@
-[submodule "transformers"]
-	path = transformers
-	url = https://github.com/huggingface/transformers.git
-[submodule "peft"]
-	path = peft
-	url = https://github.com/huggingface/peft.git
+[submodule "llama.cpp-230519"]
+	path = gpt4all-backend/llama.cpp-230519
+	url = https://github.com/ggerganov/llama.cpp.git
+[submodule "llama.cpp-230511"]
+	path = gpt4all-backend/llama.cpp-230511
+	url = https://github.com/nomic-ai/llama.cpp
+[submodule "llama.cpp-mainline"]
+	path = gpt4all-backend/llama.cpp-mainline
+	url = https://github.com/nomic-ai/llama.cpp.git

CONTRIBUTING.md

@@ -0,0 +1,91 @@
+# Contributing
+
+When contributing to this repository, please first discuss the change you wish to make via issue,
+email, or any other method with the owners of this repository before making a change. 
+
+Please note we have a code of conduct, please follow it in all your interactions with the project.
+
+## Pull Request Process
+
+1. Ensure any install or build dependencies are removed before the end of the layer when doing a build.
+2. Make sure Pull Request is tagged with appropriate project identifiers and has a clear description of contribution.
+3. Any new or updated code must have documentation and preferably tests included with Pull Request.
+4. Significant feature or code changes should provide a short video or screenshot demo.
+4. Fill out relevant parts of Pull Request template. 
+4. Pull requests must have sign-off from one other developer. Reach out to a repository owner once your
+   code is ready to be merged into `main`.
+
+## Code of Conduct
+
+### Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+### Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+### Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+### Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+### Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at support@nomic.ai. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+### Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2023 Nomic, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LICENSE_SOM.txt

@@ -0,0 +1,30 @@
+Software for Open Models License (SOM)
+Version 1.0 dated August 30th, 2023
+
+This license governs use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
+
+This license is intended to encourage open release of models created, modified, processed, or otherwise used via the Software under open licensing terms, and should be interpreted in light of that intent.
+
+1. Definitions
+The “Licensor” is the person or entity who is making the Software available under this license. “Software” is the software made available by Licensor under this license.
+A “Model” is the output of a machine learning algorithm, and excludes the Software.
+“Model Source Materials” must include the Model and model weights, and may include any input data, input data descriptions, documentation or training descriptions for the Model.
+“Open Licensing Terms” means: (a) any open source license approved by the Open Source Initiative, or (b) any other terms that make the Model Source Materials publicly available free of charge, and allow recipients to use, modify and distribute the Model Source Materials. Terms described in (b) may include reasonable restrictions such as non-commercial or non-production limitations, or require use in compliance with law.
+
+2. Grant of Rights. Subject to the conditions and limitations in section 3:
+(A) Copyright Grant. Licensor grants you a non-exclusive, worldwide, royalty-free copyright license to copy, modify, and distribute the Software and any modifications of the Software you create under this license. The foregoing license includes without limitation the right to create, modify, and use Models using this Software.
+
+(B) Patent Grant. Licensor grants you a non-exclusive, worldwide, royalty-free license, under any patents owned or controlled by Licensor, to make, have made, use, sell, offer for sale, import, or otherwise exploit the Software.  No license is granted to patent rights that are not embodied in the operation of the Software in the form provided by Licensor.
+
+3. Conditions and Limitations
+(A) Model Licensing and Access. If you use the Software to create, modify, process, or otherwise use any Model, including usage to create inferences with a Model, whether or not you make the Model available to others, you must make that Model Source Materials publicly available under Open Licensing Terms. 
+
+(B) No Re-Licensing. If you redistribute the Software, or modifications to the Software made under the license granted above, you must make it available only under the terms of this license. You may offer additional terms such as warranties, maintenance and support, but You, and not Licensor, are responsible for performing such terms.
+
+(C) No Trademark License. This license does not grant you rights to use the Licensor’s name, logo, or trademarks.
+
+(D) If you assert in writing a claim against any person or entity alleging that the use of the Software infringes any patent, all of your licenses to the Software under Section 2 end automatically as of the date you asserted the claim.
+
+(E) If you distribute any portion of the Software, you must retain all copyright, patent, trademark, and attribution notices that are present in the Software, and you must include a copy of this license.
+
+(F) The Software is licensed “as-is.” You bear the entire risk of using it. Licensor gives You no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws that this license cannot change. To the extent permitted under your local laws, the Licensor disclaims and excludes the implied warranties of merchantability, fitness for a particular purpose and non-infringement. To the extent this disclaimer is unlawful, you, and not Licensor, are responsible for any liability.

README.md

@@ -1,229 +1,100 @@
 <h1 align="center">GPT4All</h1>
-<p align="center">Demo, data and code to train an assistant-style large language model with ~800k GPT-3.5-Turbo Generations based on LLaMa</p>
+
+<p align="center">Open-source assistant-style large language models that run locally on your CPU</p>
+
+<p align="center"><strong>New</strong>: Now with Nomic Vulkan Universal GPU support. <a href="https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan">Learn more</a>.</p>
 
 <p align="center">
-<a href="https://s3.amazonaws.com/static.nomic.ai/gpt4all/2023_GPT4All_Technical_Report.pdf">:green_book: Technical Report</a>
+<a href="https://gpt4all.io">GPT4All Website</a>
 </p>
 
-
 <p align="center">
-<a href="https://github.com/nomic-ai/pyllamacpp">:snake: Official Python Bindings</a>
+<a href="https://docs.gpt4all.io">GPT4All Documentation</a>
 </p>
 
 <p align="center">
 <a href="https://discord.gg/mGZE39AS3e">Discord</a>
 </p>
 
+<p align="center">
+<a href="https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html">🦜️🔗 Official Langchain Backend</a> 
+</p>
+
+<p align="center">
+GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
+</p>
+
+<p align="center">
+  <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
+</p>
+<p align="center">
+Run on an M1 macOS Device (not sped up!)
+</p>
+
+## GPT4All: An ecosystem of open-source on-edge large language models.
+GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
+
+Learn more in the [documentation](https://docs.gpt4all.io).
+
+The goal is simple - be the best instruction tuned assistant-style language model that any person or enterprise can freely use, distribute and build on.
+
+A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models. 
 
 
-![gpt4all-lora-demo](https://user-images.githubusercontent.com/13879686/228352356-de66ca7a-df70-474e-b929-2e3656165051.gif)
+### Chat Client
+Run any GPT4All model natively on your home desktop with the auto-updating desktop chat client. See <a href="https://gpt4all.io">GPT4All Website</a> for a full list of open-source models you can run with this powerful desktop application.
 
-Run on M1 Mac (not sped up!)
+Direct Installer Links:
 
-# Try it yourself
+* [macOS](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
 
-Here's how to get started with the CPU quantized GPT4All model checkpoint:
+* [Windows](https://gpt4all.io/installers/gpt4all-installer-win64.exe)
 
-1. Download the `gpt4all-lora-quantized.bin` file from [Direct Link](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized.bin) or [[Torrent-Magnet]](https://tinyurl.com/gpt4all-lora-quantized).
-2. Clone this repository, navigate to `chat`, and place the downloaded file there.
-3. Run the appropriate command for your OS:
-   - M1 Mac/OSX: `cd chat;./gpt4all-lora-quantized-OSX-m1`
-   - Linux: `cd chat;./gpt4all-lora-quantized-linux-x86`
-   - Windows (PowerShell): `cd chat;./gpt4all-lora-quantized-win64.exe`
-   - Intel Mac/OSX: `cd chat;./gpt4all-lora-quantized-OSX-intel`
+* [Ubuntu](https://gpt4all.io/installers/gpt4all-installer-linux.run)
 
-For custom hardware compilation, see our [llama.cpp](https://github.com/zanussbaum/gpt4all.cpp) fork.
+Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io/)
 
------------
+### Chat Client building and running
 
-[Secret Unfiltered Checkpoint](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-unfiltered-quantized.bin) - [[Torrent]](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-unfiltered-quantized.bin.torrent)
+* Follow the visual instructions on the chat client [build_and_run](gpt4all-chat/build_and_run.md) page
 
-This model had all refusal to answer responses removed from training. Try it with:
-- `cd chat;./gpt4all-lora-quantized-OSX-m1 -m gpt4all-lora-unfiltered-quantized.bin`
+### Bindings
 
------------
-Note: the full model on GPU (16GB of RAM required) performs much better in our qualitative evaluations.
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/java">:computer: Official Java Bindings</a>
 
-# Python Client
-## CPU Interface
-To run GPT4all in python, see the new [official Python bindings](https://github.com/nomic-ai/pyllamacpp).
+### Integrations
 
-The old bindings are still available but now deprecated. They will not work in a notebook environment.
-To get running using the python client with the CPU interface, first install the [nomic client](https://github.com/nomic-ai/nomic) using `pip install nomic`
-Then, you can use the following script to interact with GPT4All:
-```
-from nomic.gpt4all import GPT4All
-m = GPT4All()
-m.open()
-m.prompt('write me a story about a lonely computer')
-```
+* 🗃️ [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
 
-## GPU Interface
-There are two ways to get up and running with this model on GPU.
-The setup here is slightly more involved than the CPU model.
-1. clone the nomic client [repo](https://github.com/nomic-ai/nomic) and run `pip install .[GPT4All]` in the home dir.
-2. run `pip install nomic` and install the additional deps from the wheels built [here](https://github.com/nomic-ai/nomic/tree/main/bin)
+## Contributing
+GPT4All welcomes contributions, involvement, and discussion from the open source community!
+Please see CONTRIBUTING.md and follow the issues, bug reports, and PR markdown templates.
 
-Once this is done, you can run the model on GPU with a script like the following:
-```
-from nomic.gpt4all import GPT4AllGPU
-m = GPT4AllGPU(LLAMA_PATH)
-config = {'num_beams': 2,
-          'min_new_tokens': 10,
-          'max_length': 100,
-          'repetition_penalty': 2.0}
-out = m.generate('write me a story about a lonely computer', config)
-print(out)
-```
-Where LLAMA_PATH is the path to a Huggingface Automodel compliant LLAMA model.
-Nomic is unable to distribute this file at this time.
-We are working on a GPT4All that does not have this limitation right now.
+Check project discord, with project owners, or through existing issues/PRs to avoid duplicate work.
+Please make sure to tag all of the above with relevant project identifiers or your contribution could potentially get lost.
+Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 
-You can pass any of the [huggingface generation config params](https://huggingface.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig) in the config.
+## Technical Reports
 
-# Roadmap
-## Short Term
- - <span style="color:green">(IN PROGRESS)</span> Train a GPT4All model based on GPTJ to alleviate llama distribution issues.
- - <span style="color:green">(IN PROGRESS)</span> Create improved CPU and GPU interfaces for this model.
- - <span style="color:red">(NOT STARTED)</span> Integrate llama.cpp bindings
- - <span style="color:red">(NOT STARTED)</span> Create a good conversational chat interface for the model.
- - <span style="color:red">(NOT STARTED)</span> Allow users to opt in and submit their chats for subsequent training runs
+<p align="center">
+<a href="https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf">:green_book: Technical Report 3: GPT4All Snoozy and Groovy </a>
+</p>
 
-## Medium Term
- - <span style="color:red">(NOT STARTED)</span> Integrate GPT4All with [Atlas](https://atlas.nomic.ai) to allow for document retrieval.
-   - BLOCKED by GPT4All based on GPTJ
- - <span style="color:red">(NOT STARTED)</span> Integrate GPT4All with Langchain.
- - <span style="color:green">(IN PROGRESS)</span> Build easy custom training scripts to allow users to fine tune models.
+<p align="center">
+<a href="https://static.nomic.ai/gpt4all/2023_GPT4All-J_Technical_Report_2.pdf">:green_book: Technical Report 2: GPT4All-J </a>
+</p>
 
-## Long Term
- - <span style="color:red">(NOT STARTED)</span> Allow anyone to curate training data for subsequent GPT4All releases using Atlas.
- - <span style="color:green">(IN PROGRESS)</span> Democratize AI. 
+<p align="center">
+<a href="https://s3.amazonaws.com/static.nomic.ai/gpt4all/2023_GPT4All_Technical_Report.pdf">:green_book: Technical Report 1: GPT4All</a>
+</p>
 
-# Reproducibility
-
-Trained LoRa Weights:
-- gpt4all-lora (four full epochs of training):  https://huggingface.co/nomic-ai/gpt4all-lora
-- gpt4all-lora-epoch-2 (three full epochs of training) https://huggingface.co/nomic-ai/gpt4all-lora-epoch-2
-
-Raw Data:
-- [Training Data Without P3](https://huggingface.co/datasets/nomic-ai/gpt4all_prompt_generations)
-  - Explorer: https://atlas.nomic.ai/map/gpt4all_data_clean_without_p3
-- [Full Dataset with P3](https://huggingface.co/datasets/nomic-ai/gpt4all_prompt_generations_with_p3)
-  - Explorer: https://atlas.nomic.ai/map/gpt4all_data_clean
-
-We are not distributing a LLaMa 7B checkpoint.
-
-You can reproduce our trained model by doing the following:
-
-## Setup
-
-Clone the repo
-
-```
-git clone --recurse-submodules https://github.com/nomic-ai/gpt4all.git
-git submodule update --init
-```
-
-Setup the environment
-
-```
-python -m pip install -r requirements.txt
-
-cd transformers
-pip install -e . 
-
-cd ../peft
-pip install -e .
-```
-
-## Training
-
-```bash
-accelerate launch --dynamo_backend=inductor --num_processes=8 --num_machines=1 --machine_rank=0 --deepspeed_multinode_launcher standard --mixed_precision=bf16  --use_deepspeed --deepspeed_config_file=configs/deepspeed/ds_config.json train.py --config configs/train/finetune-7b.yaml
-```
-
-## Generate
-
-```bash
-python generate.py --config configs/generate/generate.yaml --prompt "Write a script to reverse a string in Python"
-```
-
-## Need Help?
-
-Join the <a href="https://discord.gg/kvmy6dQB"> Discord </a> and ask for help in `#gpt4all-help`
-
-# Sample Generations
-
-### Provide instructions for the given exercise. Leg Raises
->Leg Raises
->1. Stand with your feet shoulder-width apart and your knees slightly bent.
->2. Raise your arms straight out in front of you.
->3. Slowly bend your knees and raise your heels off the ground.
->4. Hold this position for a count of 3, then slowly lower your heels back to the ground.
->5. Repeat the exercise 10 times.
->6. Do not let your knees go beyond your toes.
->7. Keep your back straight and your abdominal muscles engaged.
->8. Do not let your hips sag or your back arch.
->9. Keep your arms straight and your elbows locked.
->10. Do not let your head drop or your shoulders rise.
->11. Breathe normally throughout the exercise.
->12. Rest for 30 seconds before repeating the exercise.
->13. Gradually increase the number of repetitions as you become stronger.
-
-
-### A color description has been provided. Find the CSS code associated with that color. A light red color with a medium light shade of pink
-
->The CSS code associated with the color provided is: #FF6347
-
-
-### Come up with an interesting idea for a new movie plot. Your plot should be described with a title and a summary.
-
->1. The title of your movie plot should be "The Last Stand". 2. The summary of your movie plot should be: A group of mercenaries are hired to protect a small town from a dangerous criminal who is on the run. However, the town's sheriff is not willing to let them take over and decides to take matters into his own hands.
-
-### Reverse a string in python.
-> In Python, you can reverse a list or tuple by using the reversed() function on it. Here's an example of how to use this method with strings: 
->    ```python
->   my_string = "Hello World" # Define your original string here
->   reversed_str = my_string[::-1]   # Use a slice and negative index for reverse order
->   print(reversed_str)     # Output will be reversed version of the inputted string.
-
-### List 10 dogs.
->1. Labrador Retriever
->2. Golden Retriever
->3. Beagle
->4. Poodle
->5. German Shepherd
->6. Bulldog
->7. Rottweiler
->8. Boxer
->9. Shih Tzu
->10. Pug
-
-### Write me a poem about the fall of Julius Ceasar into a ceasar salad in iambic pentameter. 
->The fall of Julius Ceasar into a ceasar salad
->In iambic pentameter,
->The mighty Roman emperor
->Fell into a salad of lettuce and croutons,
->His empire crumbling around him,
->As he was devoured by the hungry diners.
->The once mighty emperor
->Was now just a salad topping,
->His legacy reduced to a mere garnish,
->As he was consumed by the masses.
->The fall of Julius Ceasar into a ceasar salad,
->In iambic pentameter,
->Was a tragic end to a great man,
->Who had once ruled the world with an iron fist.
->But now, he was just a salad topping,
->His legacy reduced to a mere garnish,
->As he was consumed by the masses.
-    
-### What is a three word topic describing the following keywords: baseball, football, soccer: 
->Sports, athletics, games
-    
 ## Citation
 
-If you utilize this reposistory, models or data in a downstream project, please consider citing it with:
+If you utilize this repository, models or data in a downstream project, please consider citing it with:
 ```
 @misc{gpt4all,
   author = {Yuvanesh Anand and Zach Nussbaum and Brandon Duderstadt and Benjamin Schmidt and Andriy Mulyar},

configs/eval/generate.yaml

@@ -1,15 +0,0 @@
-# model/tokenizer
-model_name: # update with llama 7b 
-tokenizer_name: # update with llama 7b
-lora: true
-lora_path: "nomic-ai/gpt4all-lora"
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?

configs/eval/generate_baseline.yaml

@@ -1,17 +0,0 @@
-# model/tokenizer
-model_name: # update with llama model name
-tokenizer_name: # update with llama model name
-lora: true
-lora_path: "tloen/alpaca-lora-7b"
-
-
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?

configs/eval/generate_full.yaml

@@ -1,14 +0,0 @@
-# model/tokenizer
-model_name: # update
-tokenizer_name: # update
-lora_path: "no-lora"
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?

configs/eval/generate_large_2.yaml

@@ -1,15 +0,0 @@
-# model/tokenizer
-model_name: # update
-tokenizer_name: # update
-lora: true
-lora_path: # update
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?

data.py

@@ -1,113 +0,0 @@
-import glob
-import torch
-from datasets import load_dataset, concatenate_datasets
-import os
-from torch.utils.data import DataLoader
-from transformers import DefaultDataCollator
-
-
-
-def tokenize_inputs(config, tokenizer, examples):
-    max_length = config["max_length"]
-    input_ids = torch.full((len(examples["prompt"]), max_length), tokenizer.pad_token_id)
-    # ignore bos
-    newline_tokens = tokenizer("\n", return_tensors="pt")["input_ids"][0, 1:]
-
-    out = {"labels": [], "attention_mask": []}
-    for i, (prompt, response) in enumerate(zip(examples["prompt"], examples["response"])):
-        input_tokens = tokenizer(prompt, truncation=True, max_length=max_length // 2, return_tensors="pt")["input_ids"].squeeze()
-        input_len = len(input_tokens)
-
-        # plus one since we remove bos from response
-        # but we subtract one since we want to add eos token
-        remaining_tokens = max_length - input_len - len(newline_tokens) + 1
-        # remove bos
-        target_tokens = tokenizer(response, truncation=True, max_length=remaining_tokens, return_tensors="pt")["input_ids"].squeeze()[1:]
-
-        input_ids[i, :input_len] = input_tokens
-        # add newline between prompt and response
-        newline_plus_inputs = input_len + len(newline_tokens)
-        input_ids[i, input_len: newline_plus_inputs] = newline_tokens
-
-        # add target tokens, remove bos
-        input_ids[i, newline_plus_inputs: newline_plus_inputs + len(target_tokens)] = target_tokens
-        # add eos token, enforce stopping if we don't truncate 
-        # we don't want long code to stop generating if truncated during training
-        if newline_plus_inputs + len(target_tokens) < max_length:
-            input_ids[i, newline_plus_inputs + len(target_tokens)] = tokenizer.eos_token_id
-
-        labels = input_ids[i].clone()
-        labels[: newline_plus_inputs] = -100
-        labels[labels == tokenizer.pad_token_id] = -100
-        # to debug this, can set all values == -100 to the pad token, then assert that tokenizer.decode(labels, skip_special_tokens=True).strip() == response
-
-        attention_mask = input_ids[i].ne(tokenizer.pad_token_id).int()
-
-        out["labels"].append(labels)
-        out["attention_mask"].append(attention_mask)
-
-    out["input_ids"] = input_ids
-
-    out = {k: torch.stack(v) if isinstance(v, list) else v for k, v in out.items()}
-
-    return out
-
-
-def load_data(config, tokenizer):
-    dataset_path = config["dataset_path"]
-
-    if os.path.exists(dataset_path):
-        # check if path is a directory
-        if os.path.isdir(dataset_path):
-            files = glob.glob(os.path.join(dataset_path, "*_clean.jsonl"))
-        else:
-            files = [dataset_path]
-
-        print(f"Reading files {files}")
-
-        dataset = load_dataset("json", data_files=files, split="train")
-
-    else:
-        dataset = load_dataset(dataset_path,split='train')
-
-    dataset = dataset.train_test_split(test_size=.05, seed=config["seed"])
-
-    train_dataset, val_dataset = dataset["train"], dataset["test"]
-
-    if config["streaming"] is False:
-        kwargs = {"num_proc": config["num_proc"]}
-    else:
-        kwargs = {}
-
-    # tokenize inputs and return labels and attention mask
-    train_dataset = train_dataset.map(
-        lambda ele: tokenize_inputs(config, tokenizer, ele),
-        batched=True,
-        remove_columns=["source", "prompt"],
-        **kwargs
-    )
-    val_dataset = val_dataset.map(
-        lambda ele: tokenize_inputs(config, tokenizer, ele), 
-        batched=True,
-        remove_columns=["source", "prompt"],
-        **kwargs
-    )
-
-    train_dataset = train_dataset.with_format("torch")
-    val_dataset = val_dataset.with_format("torch")
-
-    # create dataloader with default data collator since we already have labels
-
-    train_dataloader = DataLoader(
-        train_dataset,
-        collate_fn=DefaultDataCollator(),
-        batch_size=config["batch_size"],
-    )
-
-    val_dataloader = DataLoader(
-        val_dataset,
-        collate_fn=DefaultDataCollator(),
-        batch_size=config["batch_size"],
-    )
-
-    return train_dataloader, val_dataloader

eval_data/user_oriented_instructions.jsonl

@@ -1,252 +0,0 @@
-{"id": "user_oriented_task_0", "motivation_app": "Grammarly", "instruction": "The sentence you are given might be too wordy, complicated, or unclear. Rewrite the sentence and make your writing clearer by keeping it concise. Whenever possible, break complex sentences into multiple sentences and eliminate unnecessary words.", "instances": [{"input": "If you have any questions about my rate or if you find it necessary to increase or decrease the scope for this project, please let me know.", "output": "If you have any questions about my rate or find it necessary to increase or decrease this project's scope, please let me know."}]}
-{"id": "user_oriented_task_1", "motivation_app": "Grammarly", "instruction": "Analyze the word choice, phrasing, punctuation, and capitalization in the given email. How may the writer of this email sound to the reader? These tones include Disheartening, Accusatory, Worried, Curious, Surprised, Disapproving, Unassuming, Formal, Assertive, Confident, Appreciative, Concerned, Sad, Informal, Regretful, Encouraging, Egocentric, Joyful, Optimistic, and Excited.", "instances": [{"input": "Hi Jen, \nI hope you're well. Can we catch up today? I'd appreciate your input on my presentation for tomorrow's meeting. I'd especially love it if you could double-check the sales numbers with me. There's a coffee in it for you!", "output": "Confident"}]}
-{"id": "user_oriented_task_2", "motivation_app": "Grammarly", "instruction": "Rewrite the given text and correct grammar, spelling, and punctuation errors.", "instances": [{"input": "If you'd told me year ago that today I would finish a marathon, I would of laughed. Your support had a huge affect on me!", "output": "If you'd told me a year ago that today I would finish a marathon, I would have laughed. Your support had a huge effect on me!"}]}
-{"id": "user_oriented_task_3", "motivation_app": "Google Scholar", "instruction": "You are given a paper citation, convert it to the requested citation style.", "instances": [{"input": "Chicago: Vaswani, Ashish, Shazeer, Noam, Parmar, Niki, Uszkoreit, Jakob, Jones, Llion, Gomez, Aidan N., Kaiser, Lukasz, and Illia Polosukhin. \"Attention Is All You Need.\" arXiv, (2017). https://doi.org/10.48550/arXiv.1706.03762.\nMLA:", "output": "Vaswani, Ashish, et al. \"Attention Is All You Need.\" arXiv, 2017,  https://doi.org/10.48550/arXiv.1706.03762."}]}
-{"id": "user_oriented_task_4", "motivation_app": "Grammarly", "instruction": "Desk jobs require writing a lot of emails, so it isn't surprising we get tired of repeating ourselves. Come up with several synonyms for the given word.", "instances": [{"input": "Sincerely", "output": "Best regards, All the best, Cheers, Best"}]}
-{"id": "user_oriented_task_5", "motivation_app": "Gmail", "instruction": "If you could help me write an email to my friends inviting them to dinner on Friday, it would be greatly appreciated.", "instances": [{"input": "", "output": "Hi there,\n\nI hope you're all doing well. I'm inviting you over for dinner on Friday night. Please let me know if you can make it. I'll be cooking your favorite dishes!\n\nLooking forward to seeing you,"}]}
-{"id": "user_oriented_task_6", "motivation_app": "Gmail", "instruction": "The tone of the email should be changed and rewritten in a more professional manner.", "instances": [{"input": "Subject: Asking for Vacation Time\nHi [Employer], \nI hope all is well. I am writing to request a day off on [date]. I have already taken care of my work responsibilities for that day and will be available to answer any questions or concerns you may have. \nThank you for your time, \n[Your name]", "output": "Subject: Request for Vacation Time\nHello [Employer],\nI am writing to request vacation time from work. I would like to take [Number of Days] off, starting on [Date]. I will be sure to complete all of my work before taking my vacation, and I will be available by email if anything comes up while I'm away. Thank you for your consideration.\n[Your name]"}]}
-{"id": "user_oriented_task_7", "motivation_app": "Twitter", "instruction": "Write a social media post about the call for collaboration on a crowdsourcing project in a persuasive way.", "instances": [{"input": "", "output": "Looking for collaborators on a new crowdsourcing project! This is a great opportunity to get involved in something innovative and impactful. If you're interested, please let me know."}]}
-{"id": "user_oriented_task_8", "motivation_app": "IMDB", "instruction": "My favorite witty review of The Rings of Power series is this:", "instances": [{"input": "", "output": "The Rings of Power series is one of the best things to come out of Middle Earth since, well, the Ring itself! If you're a fan of fantasy, adventure, and intrigue, then this is definitely the series for you. The characters are richly drawn and the plot is twisty and full of surprises. I can't recommend it highly enough!"}]}
-{"id": "user_oriented_task_9", "motivation_app": "you.com (YouWrite)", "instruction": "A helpful writing exercise is writing a story without using adjectives or adverbs. This will help you learn to use other words and phrases to describe what you are trying to say. Here are some examples:", "instances": [{"input": "", "output": "The trees were silhouetted against the sky. The wind was blowing. The leaves were rustling. The grass was waving. The flowers were nodding. The birds were singing. The bees were buzzing. The squirrels were running. The rabbits were hopping. The deer were grazing. The wolves were howling. The moon was rising. The stars were shining."}]}
-{"id": "user_oriented_task_10", "motivation_app": "Google Search", "instruction": "Come up with some search queries on google about coding stuff.", "instances": [{"input": "", "output": "- binary search\n- pop in javascript\n- audio chunking algorithm"}]}
-{"id": "user_oriented_task_11", "motivation_app": "StackOverflow", "instruction": "Discover the most relevant StackOverflow questions for your search term. And output a list of their URLs.", "instances": [{"input": "remove item in array", "output": "1- https://stackoverflow.com/questions/5767325/how-can-i-remove-a-specific-item-from-an-array\n2- https://stackoverflow.com/questions/19544452/remove-last-item-from-array\n3- https://stackoverflow.com/questions/15292278/how-do-i-remove-an-array-item-in-typescript\n4- https://stackoverflow.com/questions/3954438/how-to-remove-item-from-array-by-value\n5- https://stackoverflow.com/questions/40462369/remove-item-from-stored-array-in-angular-2\n6- https://stackoverflow.com/questions/16994212/remove-an-item-from-array-using-underscorejs"}]}
-{"id": "user_oriented_task_12", "motivation_app": "GeeksforGeeks", "instruction": "You are given an algorithm or a search query about coding. Look for a code snippet related to the search query.", "instances": [{"input": "Python loop", "output": "#Print each fruit in a fruit list:\nfruits = [\"apple\", \"banana\", \"cherry\"]\nfor x in fruits:\n   print(x)"}]}
-{"id": "user_oriented_task_13", "motivation_app": "GeeksforGeeks", "instruction": "You are provided with a description of the code, specifications, or prompts. Your help is needed in automating code generation to speed up the development process.", "instances": [{"input": "python function to determine whether a number is prime.", "output": "def is_prime(n):\n      if n == 1:\n           return False\n      for i in range(2, int(n**0.5)+1):\n           if n % i == 0:\n                return False\n      return True"}]}
-{"id": "user_oriented_task_14", "motivation_app": "Google Search", "instruction": "When you're stuck on a coding problem, the best place to look for help is in the documentation. Read the docs carefully and see if there's a section that covers your specific issue. Often, the answer will be right there in front of you. If not, try searching for key terms related to your problem. Come up with the useful links about the given search term.", "instances": [{"input": "sql find duplicate values", "output": "- https://iteration-utilities.readthedocs.io/en/latest/generated/duplicates.html\n- https://orange3-text.readthedocs.io/en/latest/widgets/duplicatedetection.html\n- https://beets.readthedocs.io/en/v1.3.14/plugins/duplicates.html\n- https://doma.readthedocs.io/en/latest/sql/\n- https://koalas.readthedocs.io/en/latest/reference/api/databricks.koalas.DataFrame.drop_duplicates.html\n- https://sqlparse.readthedocs.io/en/latest/analyzing/"}]}
-{"id": "user_oriented_task_15", "motivation_app": "MS Excel", "instruction": "We have entered the home supplies budget in the following table. Calculate the last column with the given function and rewrite the completed table.\nNote that, columns in a row are separated using a comma, and rows are separated using a \"|\".", "instances": [{"input": "Item, Qty, Price, Subtotal | mango, 5, 500, ? | apple, 4, 150, ? | oil, 2, 1500, ?  \nFunction: column2 * column 3", "output": "Item, Qty, Price, Subtotal | mango, 5, 500, 2500 | apple, 4, 150, 600 | oil, 2, 1500, 3000"}]}
-{"id": "user_oriented_task_16", "motivation_app": "MS Excel", "instruction": "Please write the Excel function name associated with each description.", "instances": [{"input": "- Returns the number of days between two dates\n- Returns the starting position of a text string within another text string.\n- Returns the number in the middle of the set of given numbers", "output": "- DAYS\n- FIND\n- MEDIAN"}]}
-{"id": "user_oriented_task_17", "motivation_app": "MS Word", "instruction": "What are the steps to using random (yet coherent) text as filler content within your Word document?", "instances": [{"input": "", "output": "You can quickly add several paragraphs to your document by using =rand(). These steps will guide you through using it:\nPut your Microsoft Word cursor (not your mouse cursor) where you would like random text to appear. It is necessary to place the cursor at the beginning of a line. The cursor just needs to be at the beginning, not on a blank line.\n2- Type =rand()and press Enter. You will now see random text added to your document."}]}
-{"id": "user_oriented_task_18", "motivation_app": "MS Powerpoint", "instruction": "Design a soothing pastel color palette for your slides. Pastel colors generally come across as pretty and delicate, so you\u2019ll want to make sure your presentation calls for a similar mood. Choose up to five colors or color codes.", "instances": [{"input": "", "output": "Color codes: #FDB3AE  #CAE4E2  #FBDF74"}]}
-{"id": "user_oriented_task_19", "motivation_app": "MS Excel", "instruction": "I bought two shirts from the store and each one cost me $10. I also bought a pair of pants and they cost me $20. Then, I bought a dress and it cost me $30. Also,  I bought a pair of shoes and they cost me $40. Lastly, I bought a jacket and it cost me $50.\n\nMake a table chart showing items, the number, and the price of these items. Separate columns with \"|\".", "instances": [{"input": "", "output": "Item | Number | Price\n\nShirts | 2 | $10\nPants | 1 | $20\nDress | 1 | $30\nShoes | 1 | $40\nJacket | 1 | $50"}]}
-{"id": "user_oriented_task_20", "motivation_app": "ludwig.guru", "instruction": "Find sentences from reliable sources such as the Guardian or Forbes that contain the exact match for the given sentence or phrase. Also, mention the source of each sentence.", "instances": [{"input": "There are great options", "output": "- Stella McCartney's faux fur has fashionable, luxury and ethical credentials and there are great options at Dries Van Noten too. (Independent)\n- And in the capital, Funchal, there are great options for self-caterers; I stayed in a beach house in the old town, one of a collection of seafront villas (funchalcottages.co.uk). (The Guardian - Travel)\n- City by city around the world, there are great options for shoppers to bring home not just special goods, but special memories. (Forbes)\n- Yes, there are great options out there that aren't owned by any big company (yet), but Twitter has an opportunity here to act as an app Switzerland, and attract some ad business while it's at it. (TechCrunch)"}]}
-{"id": "user_oriented_task_21", "motivation_app": "ludwig.guru", "instruction": "During writing, we added an asterisk for the word that did not come to mind. You will need to provide several examples to demonstrate all the words that can be used in the sentence instead of the asterisk.", "instances": [{"input": "we * to know", "output": "1- We need to know.\n2- We want to know.\n3- we wanted to know.\n4- We deserve to know why.\n5- We ought to know."}]}
-{"id": "user_oriented_task_22", "motivation_app": "ludwig.guru", "instruction": "Think of alternatives and paraphrases for the underlined word.", "instances": [{"input": "what we have _expected", "output": "+ what we have projected \n+ what we have anticipated\n+ what we have waited\n+ what we have hoped for\n+ what we have accustomed"}]}
-{"id": "user_oriented_task_23", "motivation_app": "LinkedIn", "instruction": "Design a skill assessment\u00a0questioner for R (Programming Language).", "instances": [{"input": "", "output": "Q1. How does a matrix differ from a data frame?\nQ2. What value does this statement return?\nunclass(as.Date(\"1971-01-01\"))\nQ3. What do you use to take an object such as a data frame out of the workspace?\nQ4. What function joins two or more column vectors to form a data frame?\nQ5. What is the value of y in this code?\nx <- NA\ny <- x/1"}]}
-{"id": "user_oriented_task_24", "motivation_app": "LinkedIn", "instruction": "Write a template for First-Person LinkedIn profile summary.", "instances": [{"input": "", "output": "[Degree] in [Area of Study] from [School], I entered the [Industry] world to explore my passion for ____, ____, and ____. Currently, I work as a [Job Title] for [Company], where I'm focused on ____, ____, and ____."}]}
-{"id": "user_oriented_task_25", "motivation_app": "LinkedIn", "instruction": "Write a LinkedIn post to announce that you have accepted a new job offer.", "instances": [{"input": "", "output": "\u201cI\u2019m excited beyond words to share with you my decision to accept the role of Marketing Director at the XYZ Company!\nI couldn\u2019t have had this opportunity if not for the help of my friend, [name], who shared this job opportunity with me, and my former boss, [boss\u2019s name], for her stellar recommendation and guidance.\nI happily look forward to starting this new journey and growing professionally with my new family\u2014the marketing rock stars of XYZ Company.\u201d"}]}
-{"id": "user_oriented_task_26", "motivation_app": "LinkedIn", "instruction": "Make a list of desirable Skills for software engineers to add to LinkedIn.", "instances": [{"input": "", "output": "Test-Driven Development\nObject-Oriented Programming (OOP)\nMVC\nREST API\nTechnical Writing\nDebugging"}]}
-{"id": "user_oriented_task_27", "motivation_app": "LinkedIn", "instruction": "You will be asked to answer the following question as part of the LinkedIn Skill assessments.", "instances": [{"input": "You find that your project has a tag and branch both named push-notifications, which causes confusion when trying to print out given reference. How can you specify which branch you want to look at?", "output": "use git show refs/head/push-notifications"}]}
-{"id": "user_oriented_task_28", "motivation_app": "Amazon", "instruction": "Take the title of the product and extract its attributes. The attributes in this case refer to the characteristics of the products, such as their brand and color. Your answer must a list following the format of \"attribute: value\".", "instances": [{"input": "SAMSUNG 980 PRO SSD 2TB PCIe NVMe Gen 4 Gaming M.2 Internal Solid State Hard Drive Memory Card, Maximum Speed, Thermal Control, MZ-V8P2T0B", "output": "Brand: Samsung, Style: 980 PRO, Capacity: 2TB, Hard Disk Interface: NVMe, Hard Disk Description: Solid State Hard Drive"}]}
-{"id": "user_oriented_task_29", "motivation_app": "Amazon", "instruction": "You should write a Product Description that will persuade the buyer that the product is beneficial.", "instances": [{"input": "Flipslide Game, Electronic Handheld Game | Flip, Slide, and Match the Colors to Beat the Clock - 4 Game Modes - Multiplayer Fun", "output": "Flipside is an addictive, multi-player puzzle game of skill! Flip, slide & match the colors to beat the clock. Four thrilling game modes including, Speed Mode, Multiplayer Mode, Level Mode and Memory mode make challenging yourself or playing with friends a fun distraction for everyone! Play at home, on the road, or wherever you'd like! Recommended for ages 8+."}]}
-{"id": "user_oriented_task_30", "motivation_app": "Amazon", "instruction": "What other Amazon products might interest someone who visited the given product?", "instances": [{"input": "Zeroll Zerolon Hardcoat Anodized Commercial Ice Cream Scoop with Unique Liquid Filled Heat Conductive Handle Easy Release Made in USA, 1.5-Ounce, Black", "output": "- Ice Scoop, Fashion Ice Cream Scoop, Premium Stainless Steel Cookie Scoop, Dog Food Scoop, Sturdy Flour Scoop, Utility Candy Scoop, Dishwasher Safe (Silver/8oz/9 Inch)\n- AmazonCommercial Non-Stick Heat Resistant Silicone Spatula Set, 2 Small & 2 Large Spatulas, Multicolor, Pack of 4\n- Winco Acrylic 4-Hole Ice Cream Cone Stand,Clear,Medium\n- Zeroll 1065FS-ZT Original Zelato Tubmate Aluminum Gelato Spade for Leveling Tubs Packing and Hand-Mixing, Self-Defrosting Fluid-Filled Handle Longer-Lasting Thicker Blade , 9-Inch, Gray"}]}
-{"id": "user_oriented_task_31", "motivation_app": "Amazon", "instruction": "Compare the given item with similar products based on its properties such as price, rating, etc. For product comparison, use a table and separate the columns with \"|\".", "instances": [{"input": "Item: iBayam Journal Planner Pens Colored Pens Fine Point Markers Fine Tip Drawing Pens Porous Fineliner Pen for Journaling Writing Note Taking Calendar Coloring Art Office Back to School Supplies, 18 Color", "output": "Product | this item |\u00a0Fineliner Color Pens Set, 0.38mm Fine Tip Pens, Porous Fine Point Makers Drawing Pen, Perfect for Writing in Bullet Journal and Planner, 24 Assorted Colors |\u00a0iBayam Fineliner Pens, 24 Bright Colors Fine Point Pens Colored Pens for Journaling Note Taking Writing Drawing Coloring Planner Calendar, Office School Teacher Classroom Fine Tip Marker Pens Supplies |\u00a024 colors paint pens for journaling, Colored pens for note taking 0.4mm fineliner cute fine point smooth writing pens, Fast dry felt tip pens agenda office school teacher art markers drawing supplies\nCustomer Rating | 4.7 out of 5 stars (87557) |\u00a0 4.5 out of 5 stars (2099) | 4.7 out of 5 stars (8666) | 4.5 out of 5 stars (396) \nPrice | $6.39 | $7.66 | $8.99 | $10.99\nSold By | Qi US | ai-natebok | Qi US | MingNor\nColor | Black, Grey, Red, Blue, Magenta, Pink, Purple, Violet, Pale Yellow, Yellow, Orange, Raw Sienna, Sap Green, C Green, O Green, Lake Blue, Burnt Sienna, Crimson | 24 Color | 24 Colors | 24 colors\nItem Dimensions | 6.6 x 6.2 x 0.6 inches | 6.3 x 5.8 x 0.6 inches | 6.6 x 6.2 x 0.6 inches | 6.22 x 0.31 x 0.31 inches\nPoint Type | Fine | Extra Fine | Fine | Fine"}]}
-{"id": "user_oriented_task_32", "motivation_app": "Amazon", "instruction": "Write a  review based on the given information.", "instances": [{"input": "- Product: Persil Discs Laundry Detergent Pacs, Original Scent, High Efficiency (HE) Compatible, Laundry Soap, 62 Count\n- Sentiment: Positive", "output": "I love this brand of soap it's always gotten my kids' clothes clean. When I'm done washing with the soap my kids is closed look feel and smell and feel clean and I like that that's what I want is to be able to wash my kids' clothes and not stress that it didn't come out clean or it's still kind of stinks or it just didn't seem like it washed very well. And would this pod of soap it's really easy to use I just put two pods in the wash and if it's a big wash and my kids' clothes are really dirty I just use three and it gets my kids's clothes washed the first time and I really like that. I think it's definitely worth the money. I am glad that I purchased it. It works so well. I love using it. It's a really good buy for me and for getting my family's clothes clean especially my kids' clothes clean. Great buy!"}]}
-{"id": "user_oriented_task_33", "motivation_app": "Netflix", "instruction": "Write a short description about the given movie or series.", "instances": [{"input": "The Witcher (2019)", "output": "Geralt of Rivia, a mutated monster-hunter for hire, journeys toward his destiny in a turbulent world where people often prove more wicked than beasts."}]}
-{"id": "user_oriented_task_34", "motivation_app": "Netflix", "instruction": "Think of some series or movies that would be enjoyable for someone who liked the given series.", "instances": [{"input": "Squid Game", "output": "- 3%\n- Clickbait\n- Circle\n- Hellbound\n- Sweet Home"}]}
-{"id": "user_oriented_task_35", "motivation_app": "Netflix", "instruction": "Give examples of popular shows and movies in the genre.", "instances": [{"input": "Genre: Crime TV Shows", "output": "Breaking Bad, Peaky Blinders, Arrow, Money Heist, Better Call Saul, DAHMER, The Blacklist, Lucifer, NCIS"}]}
-{"id": "user_oriented_task_36", "motivation_app": "Netflix", "instruction": "In what genres does the given movie or series fall?", "instances": [{"input": "Manifest", "output": "Mystery, Sci-Fi, Drama"}]}
-{"id": "user_oriented_task_37", "motivation_app": "Netflix", "instruction": "Give a brief description of the given category of movies and shows.", "instances": [{"input": "Period Dramas", "output": "Want to escape the contemporary world? Explore these historical dramas and shows from the time that have magnificent art and costume design, lots of drama, and a lot of history."}]}
-{"id": "user_oriented_task_38", "motivation_app": "Overleaf", "instruction": "Provide an example of how a table of contents can be generated automatically in a LATEX document. Make sure your example contains the necessary commands.", "instances": [{"input": "", "output": "To create the table of contents is straightforward, the command \\tableofcontents does the job. Sections, subsections and chapters are included in the table of contents. To manually add entries, for example when you want an unnumbered section, use the command \\addcontentsline as shown in the following example:\n\n\\documentclass{article}\n\\usepackage{blindtext}\n\\usepackage{titlesec}\n\\title{Sections and Chapters}\n\\author{Gubert Farnsworth}\n\\date{ }\n\\begin{document}\n\\maketitle\n\\tableofcontents\n\\section{Introduction}\nThis is the first section.\n\\blindtext\n\\addcontentsline{toc}{section}{Unnumbered Section}\n\\section*{Unnumbered Section}\n\\blindtext\n\\section{Second Section}\n\\blindtext\n\\end{document}"}]}
-{"id": "user_oriented_task_39", "motivation_app": "Overleaf", "instruction": "Imagine you have a FEN notation of a chessboard. How can you draw a board to show this state of the chessboard in LATEX? Create a LATEX code that includes all the required commands and packages.", "instances": [{"input": "", "output": "The following example uses the setfen key in a \\chessboard command to demonstrate a FEN example taken from the skak package documentation:\n\n\\documentclass{article}\n\\usepackage{xskak}\n\\begin{document}\n\\newchessgame\n\\chessboard[setfen=r5k1/1b1p1ppp/p7/1p1Q4/2p1r3/PP4Pq/BBP2b1P/R4R1K w - - 0 20]\n\\end{document}"}]}
-{"id": "user_oriented_task_40", "motivation_app": "Overleaf", "instruction": "What is the output of the given LATEX code?", "instances": [{"input": "\\documentclass{article}\n\\begin{document}\nFirst document. This is a simple example, with no \nextra parameters or packages included.\n\\end{document}", "output": "First document. This is a simple example, with no extra parameters or packages included."}]}
-{"id": "user_oriented_task_41", "motivation_app": "Overleaf", "instruction": "You should find a few LATEX templates appropriate for the task at hand and list the links to them.", "instances": [{"input": "Homework Assignment", "output": "- https://www.overleaf.com/latex/templates/gijss-homework-template/xrhhfgqcfbft\n- https://www.overleaf.com/latex/templates/homework-solutions/xykkzgnbzmxp\n- https://www.overleaf.com/latex/templates/ocen-cfd-article-template/mtpnnssgzjzw\n- https://www.overleaf.com/latex/templates/pitt-state-physics-homework-template/wdsxknmntnxk"}]}
-{"id": "user_oriented_task_42", "motivation_app": "Overleaf", "instruction": "Write a simple guide for uploading the bibliography database\n on Overleaf.", "instances": [{"input": "", "output": "If you already have a bibliography file that you need to use in an Overleaf project, you can upload it. Click the Upload icon located on top of the left panel. A dialog box will appear, you can either drag and drop the .bib file(s) or choose select from your computer to upload file(s) from your device. Once the upload process is finished you can include the .bib file in your main .tex document."}]}
-{"id": "user_oriented_task_43", "motivation_app": "Spotify", "instruction": "Choose Spotify playlists that match the given mood. Provide a link to these playlists for easy access.", "instances": [{"input": "Feeling Good", "output": "- https://open.spotify.com/playlist/37i9dQZF1DX9XIFQuFvzM4\n- https://open.spotify.com/playlist/37i9dQZF1DXcEKFjZJYZcc"}]}
-{"id": "user_oriented_task_44", "motivation_app": "Spotify", "instruction": "Curate a Spotify playlist based on the vibe. Publish this playlist as a song list.", "instances": [{"input": "Vibe: coding on weekend", "output": "- Candles by Jon Hopkins\n- Train from Paris by Mindtrap\n- Intro by The xx\n- Take Me Back by WYS\n- Riverline by Tor\n- The Dream by SwuM"}]}
-{"id": "user_oriented_task_45", "motivation_app": "Spotify", "instruction": "How would you describe the meaning of this lyrics? Describe some of the metaphors.", "instances": [{"input": "My lover\u2019s got humor\nShe\u2019s the giggle at a funeral\nKnows everybody\u2019s disapproval\nI should\u2019ve worshipped her sooner", "output": "Instead of using a straight-forward line like, \u201cshe is funny,\u201d the writer opts for the metaphoric version, \u201cshe\u2019s the giggle at a funeral.\u201d This is a powerful metaphor stating that the writer's lover is so full of humor that even in really sad moments, she finds a way to lighten the moment."}]}
-{"id": "user_oriented_task_46", "motivation_app": "Spotify", "instruction": "Suggest some Audiobooks for First-Time Listeners. You can attract them by choosing popular works.", "instances": [{"input": "", "output": "+ Harry Potter and the Sorcerer's Stone By J.K. Rowling\n+ The Vanishing Half by Brit Bennett\n+ The Fellowship of the Ring by J.R.R. Tolkien\n+ And Then There Were None by Agatha Christie.\n+ The Woman in Cabin 10 by Ruth Ware.\n+ The Hitchhiker's Guide to the Galaxy by Douglas Adams.\n+ Spinning Silver by Naomi Novik.\n+ Good Omens by Terry Pratchett and Neil Gaiman.\n+ Charlotte's Web by E. B."}]}
-{"id": "user_oriented_task_47", "motivation_app": "Spotify", "instruction": "Make a list of the most popular podcasts.", "instances": [{"input": "", "output": "+ Top Podcasts\n+ The Joe Rogan Experience\n+ Distractible\n+ Call Her Daddy\n+ Huberman Lab\n+ Anything Goes with Emma Chamberlain\n+ Stuff You Should Know\n+ The Psychology of your 20\u2019s\n+ On Purpose with Jay Shetty\n+ Armchair Expert\n+ Archetypes\n+ Rachel Maddow Presents: Ultra\n+ The Mel Robbins Podcast\n+ Lex Fridman Podcast\n+ The Daily\n+ Think Fast, Talk Smart: Communication Techniques\n+ TED Talks Daily"}]}
-{"id": "user_oriented_task_48", "motivation_app": "Reddit", "instruction": "You should take the time to read the situation and problem carefully and give the best advice you can.", "instances": [{"input": "I'm roadtripping down New Zealand with a friend and he is driving me insane. He hasn't enjoyed all the tours we've been on and doesn't want to chat about ANYTHING. We're a week into our trip and this is the quietest I've ever been when hanging out with another person as he rebuffs all my attempts at conversation. He wears his airpods in the car and around the accommodation - not that I've kept trying to talk to him. Honestly, he is bringing some seriously bad vibes to everything and it's ruining my trip.\nWe don't have anything booked in yet for the South Island, so I'm planning to ditch him in Wellington at the ferry. Probably book my own car in Picton and just relax alone for the rest of the trip. (Just thinking about it is such sharp relief that it makes me feel emotional. I'll pay whatever price I have to just to get away from him.)\nObviously telling him he is ruining my trip with his negative energy isn't the best way to approach this.\nAny advice for the best approach so we can split amicably? Or at least not have an argument that ends with very hurt feelings?", "output": "I would be as honest as you can while being nice. Next time you\u2019re both in the hotel or wtv just say \u201chey man, I was thinking about when we get to X\u2014it seems like we like to travel really differently and I think it\u2019s stressing us both out, so how do you feel about splitting up after that stop?\u201d"}]}
-{"id": "user_oriented_task_49", "motivation_app": "CNN News", "instruction": "You are given a description that provides a set of facts or a scenario. It is up to you to craft a story from these facts and scenarios. The missing pieces must be filled in with imaginative but logical information.", "instances": [{"input": "Ten European football teams \u2013 the Netherlands, England, Belgium, Denmark, France, Germany, Norway, Sweden, Switzerland and Wales \u2013 will participate in a season-long \u201cOneLove\u201d campaign promoting inclusion and opposing discrimination.", "output": "Every country except Sweden and Norway has qualified for the 2022 World Cup in Qatar and each captain of these eight nations will wear a distinctive OneLove armband \u2013 which features a heart containing colors from all backgrounds \u2013 during the tournament.\nThe Netherlands FA, which is spearheading the campaign, chose the colors to represent all heritages, backgrounds, genders and sexual identities; the armband will be worn in Qatar where same-sex relationships are a criminal offense.\nSweden and Norway will participate in the initiative during the upcoming Nations League matches, while England will also wear black armbands during both its UEFA Nations League matches to mark the death of Queen Elizabeth II.\n\u201cThis is an important message which suits the game of football: on the field everybody is equal and this should be the case in every place in society. With the OneLove band we express this message,\u201d said Virgil van Dijk, the Netherlands captain.\n\u201cOn behalf of the Dutch team I have been wearing this band for quite a while now. It is good to see that other countries are joining this initiative.\u201d\nOneLove was founded in the Netherlands in 2020 to emphasize that all football fans have at least one thing in common \u2013 their love of football \u2013 and to speak out against any form of discrimination.\nAs well as focusing on public messaging, the initiative has also developed to offer diversity training to grassroots club.\n\u201cOur love of football unites us all. No matter where you come from, what you look like and who you love. Football is there for everyone and our sport must stand up for the people across the world who face discrimination and exclusion,\u201d said Germany captain Manuel Neuer.\n\u201cI am proud to be sending out this message with my colleagues from the other national teams. Every single voice counts.\u201d\nIn June, England\u2019s captain Harry Kane revealed that he had discussed taking a collective stance regarding human rights in Qatar with Denmark\u2019s Christian Eriksen and France captain Hugo Lloris.\n\u201cI am honored to join my fellow national team captains in supporting the important OneLove campaign,\u201d he said on Tuesday.\n\u201cAs captains we may all be competing against each other on the pitch, but we stand together against all forms of discrimination. This is even more relevant at a time when division is common in society. Wearing the armband together on behalf of our teams will send a clear message when the world is watching.\u201d"}]}
-{"id": "user_oriented_task_50", "motivation_app": "Google Search", "instruction": "Based on the given query, suggest some related search queries.", "instances": [{"input": "learning french", "output": "learning french for beginners\nearning french online\nlearning french free\nfrench language course\nlearning french review\nhow to learn french on your own\nbest way to learn french online"}]}
-{"id": "user_oriented_task_51", "motivation_app": "Google Scholar", "instruction": "Please suggest a few papers to consider based on the search term given. The names of the papers should be listed.", "instances": [{"input": "Topic: scaling law + machine learning", "output": "- Learning to limit data collection via scaling laws: A computational interpretation for the legal principle of data minimization\n- Data and parameter scaling laws for neural machine translation\n- Explaining neural scaling laws\n- Scaling laws for autoregressive generative modeling\n- Scaling laws for neural language models"}]}
-{"id": "user_oriented_task_52", "motivation_app": "Gmail", "instruction": "Prepare an email signature template for the given person or company.", "instances": [{"input": "a graduate student of computer science", "output": "[Your name]\n[Your degree] in Computer Science\n[Your university]\n\nWebsite: [Your website]\nORCID ID: [Your ORCID ID]"}]}
-{"id": "user_oriented_task_53", "motivation_app": "yelp", "instruction": "React properly to reviews from your customers. In your responses, you can highlight your business' policies and customer service.", "instances": [{"input": "Another repeat visit and no disappointment from us. Great place for breakfast or brunch - and you won't need to eat again for awhile. The restaurant is close to the Tenderloin so be aware of your surroundings.\nWe enjoyed the Succotash Omelet (at least my wife enjoyed it), the Pulled Pork Benedict and the Hangtown Fry. Portions are large and pricing is fair. I wish we could have managed a Beignet Flight!\nDefinitely a spot to seek out. It can be crowed at times so be patient. Well worth any wait.", "output": "Hi [customer's name], thank you so much for the wonderful feedback. We\u2019re happy to hear that you enjoyed your experience with us and that you love [restauran't name] so much! We\u2019re beyond thankful for your support and can\u2019t wait to see you back again soon! Thanks!"}]}
-{"id": "user_oriented_task_54", "motivation_app": "Twitter", "instruction": "Write a good Twitter Bio. Try to cover all the provided main points in your generated bio.", "instances": [{"input": "Name: Jane\nMain points:\n- Architect\n- Interested in Tech\nWritten in first person\nUse casual tone", "output": "I like to think of myself as a kind of architect. I'm interested in the tech side of things, and I love coming up with new ideas for how to make things better."}]}
-{"id": "user_oriented_task_55", "motivation_app": "Twitter", "instruction": "We need to write a thematic tweet to share company news and facts. Please take a careful look at the facts and details provided and create a tweet based on them.", "instances": [{"input": "- news: Applications for summer internships are open now\n- Start with a thematic tie-in to Halloween", "output": "Halloween is right around the corner, and we're celebrating with some exciting news: applications are now open for our summer internships!\nIf you want to work with us this summer, we'd love to hear from you. We're looking for people who are passionate about [company name], and committed to helping us grow. If that sounds like you, check out all the details on our website: [website address]."}]}
-{"id": "user_oriented_task_56", "motivation_app": "Indeed", "instruction": "A job description is a document that clearly states the requirements, duties, responsibilities, and skills required to perform a specific role. You need to write a job description based on the given information. You should describe responsibilities and duties in your job description in a logical order. Remember that potential employees often get their first impression of your company from your job description. So, the job description should convey the attractiveness of your company.", "instances": [{"input": "Company: [Company name]\nJob: Software Engineer\nJob Responsibilities:\nReceive and perform code reviews with other engineers.\nWrite unit, integration, and end-to-end tests to verify functionality using automated testing frameworks such as Pytest.\nWork collaboratively with fellow software engineers to build features requested by business stakeholders.\nParticipate in Agile teams to develop, test, and debug complex data processing pipelines and data analysis applications using big data processing systems such as Apache Spark.\nDiagnose, debug, and perform root cause analysis of issues and defects.\nSkills and experience that will lead to success:\nExperience with a modern software programming language. We use Python, and Pyspark extensively.\nExperience with cloud systems like AWS or Azure\nExperience writing SQL queries \nStrong written and verbal communication skills\nEagerness to work in a highly-collaborative environment\nSome visualization experience such as Tableau or PowerBI is helpful", "output": "Software Engineer\n[company name] is looking for a self-motivated software engineer to join our team. We're a team of engineers and data scientists who work with large-scale data processing systems such as Apache Spark and Amazon Redshift to analyze data in the cloud.\nYou'll be responsible for receiving and performing code reviews with other engineers, writing unit, integration, and end-to-end tests to verify functionality using automated testing frameworks such as Pytest, working collaboratively with fellow engineers to build features requested by business stakeholders, participating in Agile teams to develop, test and debug complex data processing pipelines and data analysis applications using big data processing systems such as Apache Spark.\nIn addition to being able to diagnose, debug, and perform root cause analysis of issues and defects when needed, we're looking for someone with some visualization experience (Tableau or PowerBI) who can bring their own ideas about how best use these tools on a project.\nSkills that will lead to success include: Experience with Python programming languages; experience with cloud systems like AWS or Azure; experience writing SQL queries; strong written communication skills; eagerness to work in a highly collaborative environment;"}]}
-{"id": "user_oriented_task_57", "motivation_app": "Gmail", "instruction": "Considering the reason for the rejection, write a polite rejection letter for rejecting a candidate.", "instances": [{"input": "Reason: You came in second", "output": "Dear [Candidate Name],\n\nThank you for applying to [company name]! We're sorry to inform you that we're going to be passing on your application.\nWe were impressed with your initiative and would love to have had the opportunity to work with you. At this time, however, we're not able to offer any positions at our company.\nPlease accept our sincerest apologies for passing on your application, and know that we remain open to working with you in the future!"}]}
-{"id": "user_oriented_task_58", "motivation_app": "Blogger", "instruction": "We've already decided on the title and topic for our blog post. I would appreciate it if you wrote an introduction paragraph to attract the readers' attention.", "instances": [{"input": "Blog title: Healthy Apple Muffins\nBlog Topic: recipe and tips for cooking a good apple muffin", "output": "Let me tell you this, apple muffins are extremely easy to make, and they are totally worth the effort. If you've never been much of a cook or baker before, I can promise you that making these will change your mind. And if they don't get the job done, at least they'll taste good!"}]}
-{"id": "user_oriented_task_59", "motivation_app": "Blogger", "instruction": "Design an outline for a blog post based on the given information and list the sections accordingly.", "instances": [{"input": "This blog post is going to about making a list of last-minute gift ideas. Also, the tone of the post is going to be relaxed and casual.", "output": "Section: Gift card to a favorite restaurant or store\nSection: Movie tickets\nSection: A massage or mani/pedi\nSection: A fun game console\nSection: Books\nSection: Candles, flameless candles\nSection: Portable battery charger for phones/tablets\nSection: Water filter pitcher\nSection: Personalized items with photos/artwork\nSection: Favorite team gear (hat, sweatshirt)\nTakeaway: Don't stress about out running out of time to buy, make a gift."}]}
-{"id": "user_oriented_task_60", "motivation_app": "Blogger", "instruction": "Write a section for a blog post and try to cover all of the provided information about this section in your text.", "instances": [{"input": "Blog Topic: 7 Fall Fashion Trends Worth Taking on Your Trip\nSection Title: Trusty Jeans\nMain point of the section: jeans can be worn all year and look good with everything .", "output": "Jeans are one of the most iconic fashion items for a reason \u2013 they look great with everything and can be worn year-round. While this Fall, the trends lean toward a baggier fit, bringing along your favorite, trusty jeans is a must no matter where you\u2019re headed. If you can\u2019t get on board with the baggy jean trend, at least opt for a straight leg and ditch the skinnies."}]}
-{"id": "user_oriented_task_61", "motivation_app": "Instagram", "instruction": "Create a detailed caption for an Instagram post about a discount. A proper caption should explain the product, offer the discount, and tell the user how to claim it.", "instances": [{"input": "Product: Golang course\nProduct description: teaching the foundational skills for building cloud services, web applications, DevOps tools, and backend development\nDiscount: 50%\nHow to claim: Use the coupon code: GOLANG", "output": "We've just launched the Golang course. In this course you will learn the foundational skills needed for high-paying careers involving building cloud services, web applications, DevOps tools, and backend development. Upon completion of the course, you will be able to build a web server that end-users can use to interact and consume as a REST API.\nLearning outcomes\nAs a graduate you\u2019ll be able to:\n1- Articulate the advantages and disadvantages of Go compared to similar languages.\n2- Design a business case for when to develop a project in Go.\n3- Set up a Go environment.\n4- Program a basic backend using Go.\nCheck out the Golang course syllabus to view the course project.\n\nEnroll this week and save 50% with our new course special. Use the coupon code: GOLANG at checkout to save 50% on your subscription to the Golang course. Start learning soon\u2014this special promotion is available only until [date]."}]}
-{"id": "user_oriented_task_62", "motivation_app": "Instagram", "instruction": "Create a captivating Instagram caption based on the provided details. Try to think of as many captions as you can.", "instances": [{"input": "I am sharing images of the first snow of the year taken from a window in an office", "output": "- It's snowing! Here's the first snow of the year from our office windows.\n- Let the photos do the talking. The first snow of the year is just around the corner.\n- I know it's not snowing outside, but just seeing the first flakes of the year from our office window is enough to make me feel like winter has finally arrived. :D\n- First snow of the year and I'm in a good mood. Because I get to do my favorite thing: Look at pictures of snow on Instagram\n- The first snow of the year and I'm getting cozy in my office chair. \ud83d\ude0d\n- Snowpocalypse: the sweet, delicate joy of first snowfall.\n- Snow day: here's to a great start to the week!\n- Snowing in my office and I am not even mad.\n- It's a snowy day in the office. I'm wearing my long johns, drinking hot chocolate, and binge-watching Game of Thrones. What are you doing?"}]}
-{"id": "user_oriented_task_63", "motivation_app": "Instagram", "instruction": "You are given a topic for an Instagram post. Help the post reach a broader audience by suggesting hashtags related to the post.", "instances": [{"input": "Another episode of women in science is out now", "output": "#womeninSTEM #WomenScience #science #research #stempodcast"}]}
-{"id": "user_oriented_task_64", "motivation_app": "Instagram", "instruction": "Use appropriate emojis to convey the meaning in the given text.", "instances": [{"input": "Cute cat\nplayful\nSoft", "output": "\ud83d\udc31 Cute cat\n\ud83d\ude1b playful\n\ud83d\udc9b Soft"}]}
-{"id": "user_oriented_task_65", "motivation_app": "YouTube", "instruction": "The topic of a YouTube post has been described and based on the information, you need to write a hook for starting the post. A catchy hook will keep your readers interested so they keep reading. It would be great if you could come up with as many hooks as you can.", "instances": [{"input": "A video showing how to make a tasty cup of coffee", "output": "- How do you make a great cup of coffee? We asked a professional.\n- Real life stories of the greatest coffee shops in the world.\n- You'll definitely want a cup of this after watching it so be sure not to miss this.\n- Coffee is something most people drink every day. This video tells you everything about it.\n- Watch this video on 7 habits to make you a better barista."}]}
-{"id": "user_oriented_task_66", "motivation_app": "YouTube", "instruction": "Using the provided topic as a starting point, brainstorm ideas for videos that can be made about it on YouTube.", "instances": [{"input": "In spite of the fact that procrastination feels bad to you, why do you do it?", "output": "+ The reason why you procrastinate\n+ A documentary about procrastination\n+ How to quit procrastination\n+ Recording people talking about their procrastination habits\n+ Interview people while they are procrastinating\n+ People who have failed to accomplish their New Year's resolutions explain why"}]}
-{"id": "user_oriented_task_67", "motivation_app": "YouTube", "instruction": "You should choose a YouTube video title based on the video's content. A video's title tells viewers what to expect from it. It should be direct, honest, and clear. The title of the video needs to capture the attention of viewers, so do not use an unclear or ambiguous one.", "instances": [{"input": "A research study has been conducted to determine if exercise really can \"boost\" your metabolism.", "output": "You Will Not Believe The Results Of A Recent Study On Exercise And The Metabolism!"}]}
-{"id": "user_oriented_task_68", "motivation_app": "YouTube", "instruction": "Choosing a name for your product or business YouTube channel is an important part of the process. Based on the description of the product or business, you should come up with some interesting names. Take some time to brainstorm your ideas.", "instances": [{"input": "Here you will find videos and content that will help students prepare for the application process to graduate schools as well as how to apply to graduate schools", "output": "GradSchoolPrep, GoGradSchool, GradSchoolVidz, Apply2Gradschool"}]}
-{"id": "user_oriented_task_69", "motivation_app": "merriam-webster.com", "instruction": "Find the answer that best describes the underlined SAT word. Select the correct option and explain the meaning of the underlined word.", "instances": [{"input": "Despite the _cacophony, the student tried to study. \nA. Loud sounds\nB. Difficult subject\nC. Late hour\nD. Low lighting", "output": "A. Loud sounds, Although it comes from the Greek for \u201cbad sound,\u201d cacophony can now also mean \u201can incongruous or chaotic mixture\u201d of other things, like colors or smells."}]}
-{"id": "user_oriented_task_70", "motivation_app": "merriam-webster.com", "instruction": "Find synonyms for the given word. You need to write down how the provided synonyms differ from the original word in terms of meaning, usage, etc.", "instances": [{"input": "adversity", "output": "- mischance:The words mischance and adversity are synonyms, but do differ in nuance. Specifically, mischance applies especially to a situation involving no more than slight inconvenience or minor annoyance.\n- misfortune: The meanings of misfortune and adversity largely overlap; however, misfortune may apply to either the incident or conjunction of events that is the cause of an unhappy change of fortune or to the ensuing state of distress.\n- mishap: The synonyms mishap and adversity are sometimes interchangeable, but mishap applies to an often trivial instance of bad luck."}]}
-{"id": "user_oriented_task_71", "motivation_app": "merriam-webster.com", "instruction": "Discuss the origins and history of the word that has been given to you.", "instances": [{"input": "oblique", "output": "Middle English oblique, oblike, borrowed from Anglo-French oblic, oblique, borrowed from Latin obl\u012bquus \"slanting, transverse,\" from ob- \"against, facing\" + -l\u012bquus, of uncertain meaning and origin"}]}
-{"id": "user_oriented_task_72", "motivation_app": "Yelp", "instruction": "Give the provided brand a motto to use as a part of its marketing strategies.", "instances": [{"input": "The Ice cream shop is a small, quaint shop located in the heart of the historic district. They offer a variety of ice cream flavors, as well as Italian ice, gelato, and sorbet. The shop has a few tables and chairs, but most of their business is to-go. They offer a loyalty program where you can get a free scoop of ice cream after 10 purchases.", "output": "- It's a scoop of friendly!\n- Come in and have a treat!\n- For a sweet treat, stop by and see us."}]}
-{"id": "user_oriented_task_73", "motivation_app": "Gmail", "instruction": "A confirmation email should be written appropriately for the situation.", "instances": [{"input": "A meeting has been scheduled, and the sender expects the other to review the slides.", "output": "Hi [name],\n\nI hope you're having a great week so far. I wanted to follow up on our meeting and let you know that we've scheduled a time for you and your team to come in next week. I'll have some slides prepared for you to review, and if there are any questions, I'd be happy to answer them! Just email me back at [email] when you have a moment.\n\nThanks so much for all of your help with this project\u2014we're excited to get started!"}]}
-{"id": "user_oriented_task_74", "motivation_app": "Gmail", "instruction": "You need to write an email to negotiate your salary.", "instances": [{"input": "", "output": "Dear [hiring manager's name],\n\nI wanted to take a moment to thank you for the offer. I'm honored that you would consider me for this position, and I'm excited to be working with you.\nI know that it's not easy to get a sense of how much salary should be offered in a given situation, so I wanted to explain why I believe my skill set is worth more than the offered salary. \nMy experience as a [position title] will prepare me well for this role, which requires extensive project management skills, as well as strong written communication skills and a deep understanding of the company's product offerings. These are all areas where I've already built up substantial experience over the past few years and am confident that they would serve me well in your company. \n\nI hope we can reach an agreement soon!"}]}
-{"id": "user_oriented_task_75", "motivation_app": "Gmail", "instruction": "Write an email to attendees as a reminder that the event is coming up.", "instances": [{"input": "", "output": "Hello, [name]!\n\nWe're excited to announce the [event name] event that you signed up for. The location is [location], and the date and time is [date and time].\n\nIf you have any questions or concerns about the event, please let us know by emailing us at [email]."}]}
-{"id": "user_oriented_task_76", "motivation_app": "Github", "instruction": "It would be helpful if you could suggest an acronym for the repository.", "instances": [{"input": "A python framework for creating, editing, and invoking Noisy Intermediate Scale Quantum (NISQ) circuits.", "output": "Cirq"}]}
-{"id": "user_oriented_task_77", "motivation_app": "Github", "instruction": "Write a step-by-step guide for resolving GitHub merge conflicts.", "instances": [{"input": "", "output": "1- Under your repository name, click Pull requests.\n2- In the \"Pull Requests\" list, click the pull request with a merge conflict that you'd like to resolve.\n3- Near the bottom of your pull request, click Resolve conflicts.\n4- Decide if you want to keep only your branch's changes, keep only the other branch's changes, or make a brand new change, which may incorporate changes from both branches. Delete the conflict markers <<<<<<<, =======, >>>>>>> and make the changes you want in the final merge.\n5- If you have more than one merge conflict in your file, scroll down to the next set of conflict markers and repeat steps four and five to resolve your merge conflict.\n6- Once you've resolved all the conflicts in the file, click Mark as resolved.\n7- If you have more than one file with a conflict, select the next file you want to edit on the left side of the page under \"conflicting files\" and repeat steps four through seven until you've resolved all of your pull request's merge conflicts.\n8- Once you've resolved all your merge conflicts, click Commit merge. This merges the entire base branch into your head branch.\n9- If prompted, review the branch that you are committing to.\nIf the head branch is the default branch of the repository, you can choose either to update this branch with the changes you made to resolve the conflict, or to create a new branch and use this as the head branch of the pull request.\nIf you choose to create a new branch, enter a name for the branch.\nIf the head branch of your pull request is protected you must create a new branch. You won't get the option to update the protected branch.\nClick Create branch and update my pull request or I understand, continue updating BRANCH. The button text corresponds to the action you are performing.\n10- To merge your pull request, click Merge pull request. For more information about other pull request merge options, see \"Merging a pull request.\""}]}
-{"id": "user_oriented_task_78", "motivation_app": "Github", "instruction": "One or more jobs can be run automatically by a workflow. The workflows are defined in the repository as YAML files and will run if an event occurs in the repository, as well as manually, or on a certain schedule.\nBring an example of a workflow that demonstrate the CI/CD features of GitHub Actions.", "instances": [{"input": "", "output": "Using scripts to test your code on a runner is a common example. When this workflow is triggered, it automatically runs a script."}]}
-{"id": "user_oriented_task_79", "motivation_app": "IMDB", "instruction": "The movie tagline is the film's advertising slogan, usually printed on posters prior to the official release. It is a short description or comment on a movie to capture the essence of the movie, and ultimately make you watch the movie. Here is an example:", "instances": [{"input": "Movie: Joker (2019)", "output": "Put on a happy face."}]}
-{"id": "user_oriented_task_80", "motivation_app": "IMDB", "instruction": "You are given some reviews for a movie. Each video might have mentioned some negative or positive points about the movie. Read them carefully and extract the positive and negative points mentioned in all of the reviews. Then, make a list of these points and summarize them as positive points and negative points.", "instances": [{"input": "Review 1: I'm not sure if it's just the fact that Ed Norton really grates on me in this film, but I never really bought into the idea of Fight Club being some kind of cultural phenomenon. Yes it's an innovative story, told via a strong script, engaging characters and intense cinematography. But is it really worthy of such high praise? Personally, I find it starts to fall apart halfway through and actually becomes quite tedious towards the end. Everything just becomes a little bit childish and contrived as the story loses touch with its base elements and the rage against the machine vibe takes over. I have huge respect for the way this film was made but, much like The Matrix, I feel like people attach way more importance to this movie than it deserves.\nReview 2: The film tells the story of men who cannot adapt to the social order in today's world, under the leadership of Tyler Durden, to discover their natural instinct, masculinity, and to escape from their unhappy lives.\nReview 3: Despite a good theme, great acting and important messages that this movie convey in an unorthodox way, I think it fails to connect the audience with the storyline and leaves him in a world of confusion. Although, majority of reviews find this movie entertaining and interesting, yet I would choose to be a minority that believes that this movie is extremely overrated.\nReview 4: Is creating chaos, explosions and membership in a militant sect the way to get rid of material shackles ?! The visual effects are fascinating and the story is compelling until the second half. Unfortunately, it doesn't end well.", "output": "Positive points: \n- Innovative story\n- Strong script\n- Engaging characters\n- Intense cinematography\n- Good theme\n- Great acting\n- Important messages\n- Fascinating visual effects \n\nNegative points: \n- Ed Norton grates on the viewer\n- Starts to fall apart halfway through\n- Childish and contrived towards the end\n- Loses touch with its base elements\n- World of confusion\n- Overrated"}]}
-{"id": "user_oriented_task_81", "motivation_app": "IMDB", "instruction": "Give a brief summary of the movie or series' plot.", "instances": [{"input": "Breaking Bad (TV Series 2008\u20132013)", "output": "Walter H. White is a chemistry genius, but works as a chemistry teacher in an Albequerque, New Mexico high school. His life drastically changes when he's diagnosed with stage III terminal lung cancer, and given a short amount of time left to live: a mere matter of months. To ensure his handicapped son and his pregnant wife have a financial future, Walt uses his chemistry background to create and sell the world's finest crystal methamphetamine. To sell his signature \"blue meth,\" he teams up with Jesse Pinkman, a former student of his. The meth makes them very rich very quickly, but it attracts the attention of his DEA brother in law Hank. As Walt and Jesse's status in the drug world escalates, Walt becomes a dangerous criminal and Jesse becomes a hot-headed salesman. Hank is always hot on his tail, and it forces Walt to come up with new ways to cover his tracks."}]}
-{"id": "user_oriented_task_82", "motivation_app": "IMDB", "instruction": "Decide whether children can watch the given movie. Mention what makes it inappropriate if you think they can't.", "instances": [{"input": "The Dark Knight (2008)", "output": "No, children should not watch The Dark Knight. The movie is rated PG-13 for intense sequences of violence and action, some menace, and brief language."}]}
-{"id": "user_oriented_task_83", "motivation_app": "IMDB", "instruction": "Write a short analysis of the cinematography in the movie.", "instances": [{"input": "Dead Poets Society", "output": "'Dead Poets Society' uses film techniques such as camera angles and camera techniques to help emphasise certain emotions between each character. The Film techniques are used to help the audience understand the theme, character, emotion and setting of a specific scene. All movies use Film Techniques to help the audience join in the experience and share the emotions that occurring in the film. Film techniques used in this film help shape meaning and give a better understanding of the relationship between the characters in the film. Although this film does not only use film techniques to emphasize the relationship of only characters, it uses it to emphasizes the relationship of characters with the themes of the film such as, poetry, carpe diem (seize the day), The school etc."}]}
-{"id": "user_oriented_task_84", "motivation_app": "tripadvisor.com", "instruction": "Please let me know your thoughts on the given place and why you think it deserves to be visited.", "instances": [{"input": "Barcelona, Spain", "output": "There\u2019s nowhere in the world like Barcelona. From days spent strolling the Gothic Quarter to afternoons at the beach and nights partying \u2018til dawn, the city is magnetic. While tapas and bustling mercados are worth every bit of attention they get in guidebooks, Barcelona\u2019s true crown jewel is the work of Antoni Gaud\u00ed. His iconic creations, like the jaw-dropping Basilica of La Sagrada Familia, are instantly recognizable and have fueled millions of travel dreams. You won\u2019t be the only tourist here (especially during the summer), but even amid the crowds you\u2019ll find your own Mediterranean magic any time of year."}]}
-{"id": "user_oriented_task_85", "motivation_app": "tripadvisor.com", "instruction": "Consider the best time of year to visit the given city, and provide your reasons for choosing that time.", "instances": [{"input": "Sydney, Australia", "output": "March to May: The best times to visit Sydney are between March and May and September and November. You\u2019ll find more comfortable temperatures and fewer crowds \u2014 which means you can likely nab better prices on airfare, too.\nBut if you want to warm up with the masses from winter back home, the average daily temperatures in January can hit highs of 30\u00b0C/86\u00b0F and lows around 20\u00b0C/67\u00b0F."}]}
-{"id": "user_oriented_task_86", "motivation_app": "tripadvisor.com", "instruction": "Make a questionnaire to help hotel guests write hotel reviews.", "instances": [{"input": "", "output": "1- Your overall rating of this property:\n2- Title of your review:\n3- Summarize your visit or highlight an interesting detail:\n4- Your Review:\n5- What sort of trip was this? Business/Couples/Family/Friends/Solo\n6- When did you travel?\n7- Could you say a little more about it? (optional)\nWe'd love your opinion! Anything you can share will help other travelers choose their perfect hotel. Thanks.\nHotel Style & Amenities: \nDoes this hotel have laundry service? Yes/No/Not Sure\nDoes this hotel have a game room? Yes/No/Not Sure\nDoes this hotel have a sun terrace? Yes/No/Not Sure\nIs the front desk of the hotel staffed 24-hours a day? Yes/No/Not Sure\n8-  How expensive is this hotel? Budget/Mid-range/Luxury\n9- Add a tip to help travelers choose a good room PE.g., best views, quieter floors, accessibility, etc.)\n10- Do you have photos to share? (optional)"}]}
-{"id": "user_oriented_task_87", "motivation_app": "tripadvisor.com", "instruction": "Make a list of the top places in the U.S. to visit at the given time.", "instances": [{"input": "November", "output": "1- For mild temperatures and local brews: San Diego\n2- For the last gasp of fall foliage: Manchester, VT\n3- For unexpected art and mountain views: Las Vegas\n4- For ice skating and family fun: Chicago\n5- For a waterfront Thanksgiving experience: Key West, FL\n6- For great food and ghost tours: Savannah, GA"}]}
-{"id": "user_oriented_task_88", "motivation_app": "tripadvisor.com", "instruction": "Name the top cities in France that should not be missed. Include the best aspects of each place as well.", "instances": [{"input": "", "output": "1- Paris, Best for: Historic sights and romantic vibes\n2- Nice, Best for: Architecture and dining\n3- Lyon, Best for: Dining and live music\n4- Bordeaux, Best for: Wine and antiques\n5- Marseille, Best for: Museums and nightlife\n6- Strasbourg, Best for: Historic architecture and dining"}]}
-{"id": "user_oriented_task_89", "motivation_app": "StackOverflow", "instruction": "Answer the following question.", "instances": [{"input": "How do I compare strings in Java?\nI've been using the == operator in my program to compare all my strings so far. However, I ran into a bug, changed one of them into .equals() instead, and it fixed the bug.\nIs == bad? When should it and should it not be used? What's the difference?", "output": "== tests for reference equality (whether they are the same object).\n.equals() tests for value equality (whether they are logically \"equal\").\nObjects.equals() checks for null before calling .equals() so you don't have to (available as of JDK7, also available in Guava).\nConsequently, if you want to test whether two strings have the same value you will probably want to use Objects.equals().\nYou almost always want to use Objects.equals(). In the rare situation where you know you're dealing with interned strings, you can use ==."}]}
-{"id": "user_oriented_task_90", "motivation_app": "StackOverflow", "instruction": "You are given a question and two possible answers. The task is to evaluate each answer and determine if it is correct. If incorrect, explain why.", "instances": [{"input": "Is there a way of reading one single character from the user input? For instance, they press one key at the terminal and it is returned (sort of like getch()). I know there's a function in Windows for it, but I'd like something that is cross-platform.\nAnswer 1: Here's a link to the ActiveState Recipes site that says how you can read a single character in Windows, Linux and OSX :\nhttps://code.activestate.com/recipes/134892/\nAnswer 2: if you just want to hold the screen so you can see the result on the terminal just write `input()` at the end of the code and it will hold the screen", "output": "Answer 1 is correct.\nAnswer 2 is incorrect because it does not answer the question. The user is asking how to read a single character."}]}
-{"id": "user_oriented_task_91", "motivation_app": "StackOverflow", "instruction": "You are given two questions. Check if they are duplicates.", "instances": [{"input": "Question1: I am trying to understand the advantages of multiprocessing over threading. I know that multiprocessing gets around the Global Interpreter Lock, but what other advantages are there, and can threading not do the same thing?\nQuestion2: I am learning how to use the threading and the multiprocessing modules in Python to run certain operations in parallel and speed up my code.\nI am finding this hard (maybe because I don't have any theoretical background about it) to understand what the difference is between a threading.Thread() object and a multiprocessing.Process() one.\nAlso, it is not entirely clear to me how to instantiate a queue of jobs and having only 4 (for example) of them running in parallel, while the other wait for resources to free before being executed.\nI find the examples in the documentation clear, but not very exhaustive; as soon as I try to complicate things a bit, I receive a lot of weird errors (like a method that can't be pickled, and so on).\nSo, when should I use the threading and multiprocessing modules?\nCan you link me to some resources that explain the concepts behind these two modules and how to use them properly for complex tasks?", "output": "Yes, they are duplicates."}]}
-{"id": "user_oriented_task_92", "motivation_app": "Leetcode", "instruction": "Design a programming problem related to the subject that has been given to you. Use some examples and constraints to improve your question.", "instances": [{"input": "Dynamic Programming", "output": "Given an integer array nums, return the length of the longest strictly increasing subsequence.\n\n Example 1:\nIn: nums = [10,9,2,5,3,7,101,18]\nOut: 4\nExplanation: The longest increasing subsequence is [2,3,7,101], therefore the length is 4.\n\nExample 2:\nIn: nums = [0,1,0,3,2,3]\nOut: 4\n\nExample 3:\nIn: nums = [7,7,7,7,7,7,7]\nOut: 1\n \nConstraints:\n1 <= nums.length <= 2500\n-104 <= nums[i] <= 104\n \nFollow up: Can you come up with an algorithm that runs in O(n log(n)) time complexity?"}]}
-{"id": "user_oriented_task_93", "motivation_app": "Leetcode", "instruction": "Provide a Java solution to the following problem.", "instances": [{"input": "Given an integer n, return true if it is a power of three. Otherwise, return false.\nAn integer n is a power of three, if there exists an integer x such that n == 3^x.\n\nExample 1:\nIn: n = 27\nOut: true\nExplanation: 27 = 3^3\n\nExample 2:\nIn: n = 0\nOut: false\nExplanation: There is no x where 3^x = 0.\n\nExample 3:\nIn: n = -1\nOut: false\nExplanation: There is no x where 3x = (-1).\n \nConstraints:\n-231 <= n <= 231 - 1\nFollow up: Could you solve it without loops/recursion?", "output": "public class Solution {\n    public boolean isPowerOfThree(int n) {\n        return (Math.log10(n) / Math.log10(3)) % 1 == 0;\n    }\n}"}]}
-{"id": "user_oriented_task_94", "motivation_app": "Leetcode", "instruction": "Think of topics that are most common in classic interview questions for a job in computer science.", "instances": [{"input": "", "output": "+ Array\n+ Strings\n+ Linked List\n+ Trees\n+ Sorting and Searching\n+ Dynamic Programming\n+ Design\n+ Math"}]}
-{"id": "user_oriented_task_95", "motivation_app": "Leetcode", "instruction": "You will need to guide this person through the scenario they have described for the interview.", "instances": [{"input": "Finally! After a while of shooting in the dark and frantically fiddling with sample inputs on the whiteboard, you've came up with an algorithm for solving the coding question your interviewer gave you.\nWhew. Such a relief to have a clear path forward. To not be flailing anymore.\nNow you're cruising, getting ready to code up your solution.\nWhen suddenly, your interviewer throws you a curve ball.\n\"What if we thought of the problem this way?\"\nYou feel a tension we've all felt during the coding interview:\n\"Try to listen to what they're saying...but don't lose your train of thought...ugh, I can't do both!\"", "output": "Your interviewer probably won't fault you for going down the wrong path at first. They've seen really smart engineers do the same thing. They understand it's because you only have a partial view of the maze.\nThey might have let you go down the wrong path for a bit to see if you could keep your thinking organized without help. But now they want to rush you through the part where you discover the dead end and double back. Not because they don't believe you can manage it yourself. But because they want to make sure you have enough time to finish the question.\nBut here's something they will fault you for: failing to listen to them. Nobody wants to work with an engineer who doesn't listen.\nSo when you find yourself in that crucial coding interview moment, when you're torn between holding your train of thought and considering the idea your interviewer is suggesting...remember this:\nListening to your interviewer is the most important thing.\nTake what they're saying and run with it. Think of the next steps that follow from what they're saying.\nEven if it means completely leaving behind the path you were on. Trust the route your interviewer is pointing you down.\nBecause they can see the whole maze."}]}
-{"id": "user_oriented_task_96", "motivation_app": "Leetcode", "instruction": "You are given a programming problem and its implementation. Analyze the problem and implementation and explain the algorithm and approach used to solve the problem.", "instances": [{"input": "Table: Person\n| Column Name | Type     |\n| personId          | int         |\n| lastName         | varchar |\n| firstName         | varchar |\npersonId is the primary key column for this table.\nThis table contains information about the ID of some persons and their first and last names.\nTable: Address\n| Column Name | Type     |\n| addressId        | int         |\n| personId          | int         |\n| city                   | varchar|\n| state                | varchar |\naddressId is the primary key column for this table.\nEach row of this table contains information about the city and state of one person with ID = PersonId.\nWrite an SQL query to report the first name, last name, city, and state of each person in the Person table. If the address of a personId is not present in the Address table, report null instead.\nReturn the result table in any order.\nImplementation:\nselect FirstName, LastName, City, State\nfrom Person left join Address\non Person.PersonId = Address.PersonId\n;", "output": "Since the PersonId in table Address is the foreign key of table Person, we can join these two tables to get the address information of a person.\nConsidering there might be no address information for every person, we should use outer join instead of the default inner join."}]}
-{"id": "user_oriented_task_97", "motivation_app": "Wikipedia", "instruction": "The given article should be checked for any scientific errors and corrected if necessary.", "instances": [{"input": "Zinc is an essential mineral that is required for good health. It is involved in many biochemical processes in the body and is necessary for the proper functioning of the immune system. Excess zink is not harmful as it is excreted in the urine.", "output": "Although zinc is an essential requirement for good health, excess zinc can be harmful. Excessive absorption of zinc suppresses copper and iron absorption. The free zinc ion in solution is highly toxic to plants, invertebrates, and even vertebrate fish. The Free Ion Activity Model is well-established in the literature, and shows that just micromolar amounts of the free ion kills some organisms. A recent example showed 6 micromolar killing 93% of all Daphnia in water.\nThe free zinc ion is a powerful Lewis acid up to the point of being corrosive. Stomach acid contains hydrochloric acid, in which metallic zinc dissolves readily to give corrosive zinc chloride. Swallowing a post-1982 American one cent piece (97.5% zinc) can cause damage to the stomach lining through the high solubility of the zinc ion in the acidic stomach."}]}
-{"id": "user_oriented_task_98", "motivation_app": "Wikipedia", "instruction": "Summarize the article you have been given in a brief manner.", "instances": [{"input": "Mathematics and art are related in a variety of ways. Mathematics has itself been described as an art motivated by beauty. Mathematics can be discerned in arts such as music, dance, painting, architecture, sculpture, and textiles. This article focuses, however, on mathematics in the visual arts.\nMathematics and art have a long historical relationship. Artists have used mathematics since the 4th century BC when the Greek sculptor Polykleitos wrote his Canon, prescribing proportions conjectured to have been based on the ratio 1:\u221a2 for the ideal male nude. Persistent popular claims have been made for the use of the golden ratio in ancient art and architecture, without reliable evidence. In the Italian Renaissance, Luca Pacioli wrote the influential treatise De divina proportione (1509), illustrated with woodcuts by Leonardo da Vinci, on the use of the golden ratio in art. Another Italian painter, Piero della Francesca, developed Euclid's ideas on perspective in treatises such as De Prospectiva Pingendi, and in his paintings. The engraver Albrecht D\u00fcrer made many references to mathematics in his work Melencolia I. In modern times, the graphic artist M. C. Escher made intensive use of tessellation and hyperbolic geometry, with the help of the mathematician H. S. M. Coxeter, while the De Stijl movement led by Theo van Doesburg and Piet Mondrian explicitly embraced geometrical forms. Mathematics has inspired textile arts such as quilting, knitting, cross-stitch, crochet, embroidery, weaving, Turkish and other carpet-making, as well as kilim. In Islamic art, symmetries are evident in forms as varied as Persian girih and Moroccan zellige tilework, Mughal jali pierced stone screens, and widespread muqarnas vaulting.", "output": "This article discusses the relationship between mathematics and art, with a focus on visual arts. It describes how artists have used mathematics since ancient times, and how mathematics has inspired various textile arts."}]}
-{"id": "user_oriented_task_99", "motivation_app": "Wikipedia", "instruction": "Improve the article using your knowledge of the topic in order to make it more informative.", "instances": [{"input": "The \"fair trade\" movement, also known as the \"trade justice\" movement, promotes the use of labour, environmental and social standards for the production of commodities, particularly those exported from the Third and Second Worlds to the First World. Such ideas have also sparked a debate on whether trade itself should be codified as a human right.", "output": "The \"fair trade\" movement, also known as the \"trade justice\" movement, promotes the use of labour, environmental and social standards for the production of commodities, particularly those exported from the Third and Second Worlds to the First World. Such ideas have also sparked a debate on whether trade itself should be codified as a human right.\nImporting firms voluntarily adhere to fair trade standards or governments may enforce them through a combination of employment and commercial law. Proposed and practiced fair trade policies vary widely, ranging from the common prohibition of goods made using slave labour to minimum price support schemes such as those for coffee in the 1980s. Non-governmental organizations also play a role in promoting fair trade standards by serving as independent monitors of compliance with labeling requirements. As such, it is a form of Protectionism."}]}
-{"id": "user_oriented_task_100", "motivation_app": "Wikipedia", "instruction": "Make the article available in a second language by translating it into it.", "instances": [{"input": "Dentistry, also known as dental medicine and oral medicine, is the branch of medicine focused on the teeth, gums, and mouth. It consists of the study, diagnosis, prevention, management, and treatment of diseases, disorders, and conditions of the mouth, most commonly focused on dentition (the development and arrangement of teeth) as well as the oral mucosa. Dentistry may also encompass other aspects of the craniofacial complex including the temporomandibular joint. The practitioner is called a dentist.\nTranslate to French:", "output": "La dentisterie, \u00e9galement connue sous le nom de m\u00e9decine dentaire et de m\u00e9decine buccale, est la branche de la m\u00e9decine ax\u00e9e sur les dents, les gencives et la bouche. Il consiste en l'\u00e9tude, le diagnostic, la pr\u00e9vention, la gestion et le traitement des maladies, troubles et affections de la bouche, le plus souvent ax\u00e9s sur la dentition (le d\u00e9veloppement et la disposition des dents) ainsi que sur la muqueuse buccale.\nLa dentisterie peut \u00e9galement englober d'autres aspects du complexe craniofacial, y compris l'articulation temporo-mandibulaire. Le praticien est appel\u00e9 dentiste."}]}
-{"id": "user_oriented_task_101", "motivation_app": "Wikipedia", "instruction": "An evaluation of the article's quality should be carried out. In order to do so, you should review the quality of the writing and the explanation of the topic.", "instances": [{"input": "The 20th century saw a revolution in music listening as the radio gained popularity worldwide and new media and technologies were developed to record, edit and distribute music. Music performances became increasingly visual with the broadcast and recording of performances.\n20th-century music brought new freedom and wide experimentation with new musical styles and forms that challenged the accepted rules of music of earlier periods. The invention of musical amplification and electronic instruments, especially the synthesizer, in the mid-20th century revolutionized classical and popular music, and accelerated the development of new forms of music.", "output": "The article is well-written and researched, and provides a good overview of the topic."}]}
-{"id": "user_oriented_task_102", "motivation_app": "Wikipedia", "instruction": "Please answer the following question based on the information provided in the article.", "instances": [{"input": "Development studies is an interdisciplinary branch of social science. Development studies is offered as a specialized master's degree in a number of reputed universities around the world. It has grown in popularity as a subject of study since the early 1990s, and has been most widely taught and researched in developing countries and countries with a colonial history, such as the UK, where the discipline originated.  Students of development studies often choose careers in international organisations such as the United Nations, World Bank, non-governmental organisations (NGOs), media and journalism houses, private sector development consultancy firms, corporate social responsibility (CSR) bodies and research centers.\nWhat are some popular careers for students of development studies?", "output": "Some popular careers for students of development studies include working for international organisations such as the United Nations, World Bank, or NGOs, as well as working in media, journalism, private sector development consultancy, or corporate social responsibility."}]}
-{"id": "user_oriented_task_103", "motivation_app": "Coursera", "instruction": "Design a syllabus for the given course. Students should be given a list of the chapters with brief explanations of each chapter's purpose.", "instances": [{"input": "Programming for Everybody (Getting Started with Python)", "output": "- Chapter One - Why we Program?\nThese are the course-wide materials as well as the first part of Chapter One where we explore what it means to write programs. We finish Chapter One and have the quiz and first assignment in the third week of the class. Throughout the course you may want to come back and look at these materials. This section should not take you an entire week.\n- Installing and Using Python\nIn this module you will set things up so you can write Python programs. Not all activities in this module are required for this class so please read the \"Using Python in this Class\" material for details.\n- Chapter One: Why We Program (continued)\nIn the first chapter we try to cover the \"big picture\" of programming so you get a \"table of contents\" of the rest of the book. Don't worry if not everything makes perfect sense the first time you hear it. This chapter is quite broad and you would benefit from reading the chapter in the book in addition to watching the lectures to help it all sink in. You might want to come back and re-watch these lectures after you have funished a few more chapters.\n- Chapter Two: Variables and Expressions\nIn this chapter we cover how a program uses the computer's memory to store, retrieve and calculate information.\n- Chapter Three: Conditional Code\nIn this section we move from sequential code that simply runs one line of code after another to conditional code where some steps are skipped. It is a very simple concept - but it is how computer software makes \"choices\".\n- Chapter Four: Functions\nThis is a relatively short chapter. We will learn about what functions are and how we can use them. The programs in the first chapters of the book are not large enough to require us to develop functions, but as the book moves into more and more complex programs, functions will be an essential way for us to make sense of our code.\n- Chapter Five: Loops and Iteration\nLoops and iteration complete our four basic programming patterns. Loops are the way we tell Python to do something over and over. Loops are the way we build programs that stay with a problem until the problem is solved."}]}
-{"id": "user_oriented_task_104", "motivation_app": "Coursera", "instruction": "List the personality traits that are required to be successful in the given job.", "instances": [{"input": "Social Media Marketer", "output": "- Engaging with people\n- Thinking creatively\n- Following trends"}]}
-{"id": "user_oriented_task_105", "motivation_app": "Coursera", "instruction": "Please list the courses that someone interested in the first course might like.", "instances": [{"input": "Bitcoin and Cryptocurrency Technologies", "output": "- Blockchain 360: A State of the Art for Professionals\n- Blockchain Scalability and its Foundations in Distributed Systems\n- Blockchain Business Models\n- Financial Technology (Fintech) Innovations Specialization"}]}
-{"id": "user_oriented_task_106", "motivation_app": "Coursera", "instruction": "Come up with the courses that one is supposed to take in order to be an expert in a given field.", "instances": [{"input": "Graphic Design", "output": "1- Fundamentals of Graphic Design\n2- Introduction to Typography\n3- Introduction to Imagemaking\n4- Ideas from the History of Graphic Design\n5- Brand New Brand"}]}
-{"id": "user_oriented_task_107", "motivation_app": "National Geographic", "instruction": "Create a daily itinerary based on the given information.", "instances": [{"input": "Our family is looking for a 9-day Morocco trip that has light to moderate activity levels.", "output": "1- DAY 1: MARRAKECH\nArrive in time to meet your fellow travelers for a welcome gathering this evening. Your arrival transfer is included.\n2- DAY 2: MARRAKECH/OUARZAZATE\nWind past red slopes and green terraces as we make our way south through the stunning valleys of Morocco\u2019s High Atlas Mountains, stopping to enjoy a picnic lunch beside a sparkling river. Arrive in Ouarzazate, known as the \u201cgateway to the Sahara,\u201d and step behind the scenes at a film studio to see how the surrounding dunes have starred on the silver screen.\n3- DAY 3: OUARZAZATE/MERZOUGA\nToday, travel between jagged peaks and palm oases toward the desert town of Merzouga. Settle into our hotel near the incredible Erg Chebbi, a sweeping sea of dunes formed by wind-blown sand. As the sun sets, savour a traditional Amazigh dinner, then spread a blanket on the sand for an evening of desert stargazing.\n4- DAY 4: MERZOUGA\nThis morning, hop aboard a 4x4 and ride across the dunes to discover fossils left behind by an ancient sea, and be entranced by Gnaoua musicians in a local village. Return to the hotel for free time to cool off in the pool. Enjoy a tasty Amazigh dinner, then gather around a glowing bonfire for a storytelling session.\n5- DAY 5: MERZOUGA/DADES GORGE\nVenture into the rugged landscape of the Dades Gorge, a dramatic and colourful valley carved by the waters of the Dades River. Visit a local village for lunch and enjoy a special dish\u2014traditional Amazigh pizza! Wind through the valley on a guided walk as you learn about the amazing geology behind the weird and wonderful rock formations of the Gorge. This evening, enjoy free time to relax at the hotel.\n6- DAY 6: DADES GORGE/A\u00cfT BEN HADDOU\nDrive through the Ounila Valley to the ancient fortified village of A\u00eft Ben Haddou, a UNESCO World Heritage site where clay houses crowd together in a traditional ksar behind defensive walls. Explore alleyways twisting between historic structures, and stand guard atop the clay brick ramparts, taking in views of the green oasis around this imposing fort. Entertain your taste buds at a hands-on tagine cooking class, followed by a delicious dinner.\n7- DAY 7: A\u00cfT BEN HADDOU/MARRAKECH\nJourney through the mountains to Marrakech, known as the \u201cred city\u201d for its walls, houses, and many mosques built from red clay. After checking into your hotel and resting up, head to a local henna workshop to have your hands painted in this ancient temporary tattooing art. Then, explore the lively Djemaa el Fna, an ancient marketplace still used as the main square in Marrakech. After, enjoy free time to discover more of Marrakech.\n8- DAY 8: MARRAKECH \nWake up as the call to prayer echoes across the city. After breakfast, set out on a guided walk through the narrow, maze-like streets of the medina, lined with colorful shops brimming with rugs, spices, and much more. Spend the rest of the day exploring Marrakech on your own, or opt to explore the Marjorelle Garden. In the evening, gather for a farewell dinner in a traditional riad, a grand house with an elegant interior courtyard.\n9- DAY 9: MARRAKECH\nDepart at any time."}]}
-{"id": "user_oriented_task_108", "motivation_app": "National Geographic", "instruction": "Give students tips on how to keep their nerves under control during class presentations.", "instances": [{"input": "", "output": "Everyone gets a little nervous when speaking in front of a group. That\u2019s normal. But the more preparation you\u2019ve done\u2014meaning plenty of researching, organizing, and rehearsing\u2014the more confident you\u2019ll be. Preparation is the key. And if you make a mistake or stumble over your words, just regroup and keep going. Nobody\u2019s perfect, and nobody expects you to be."}]}
-{"id": "user_oriented_task_109", "motivation_app": "National Geographic", "instruction": "You can easily make your holidays more eco-friendly by following these simple steps:", "instances": [{"input": "", "output": "- If you prefer to buy a tree, look for local growers with advice on where to recycle. To complete the natural look, make ornaments from collected and dried foliage.\n- Another way to be environmentally friendly during the holidays is to use candles made with soy, ethical beeswax, and coconut wax instead of those made with petroleum-based paraffin wax. LED lights are also an environmentally friendly option among artificial lights.\n-Another way to reduce waste is to use brown paper grocery bags or furoshiki, which is swaddling gifts in reusable squares of cloth, rather than wrapping paper. \n- Instead of throwing away the excess food, find groups that redistribute it at epa.gov by searching for \u201creduce wasted food.\u201d What\u2019s no longer edible might be compostable."}]}
-{"id": "user_oriented_task_110", "motivation_app": "instructables", "instruction": "Think of an idea to help you win a challenge given to you. Write the steps you need to take to develop the idea as well.", "instances": [{"input": "What can you make by recycling and repurposing some old, previously used materials? We want to see it!\nProjects must incorporate some kind of used materials - take that old used thing, and turn it into something new and different!\nE-waste is a huge problem. What can you make using old electronics and other salvageable electronic materials? \nWe have a special judges prize for our favorite Instructable that incorporates or is related to reusing e-waste.\nBe smart and be safe; note that potentially deadly projects (like Lichtenberg/fractal wood burning machines) will not be allowed to be published or submitted to this contest.", "output": "This power booster recycles a 'dead' battery to power a 1.8 volt red LED light. It's mainly made from recycled parts.\nWhen a 1.5 volt battery is new it will probably have more than 1.5 volts in it. As the battery is used the voltage will decrease until the voltage is not enough to run the application ( E.g. a red LED) and will be called 'dead' . But a Dead battery still has some power remaining in it - just not enough voltage. The trick is to use some of that remaining power to increase the voltage in a circuit. The trade off being there will be a reduction in current.\nHere is the required steps:\n1- Connect a switch to a battery holder. A soldiering iron and hot glue gun assisted with this.\n2- Make a small pyramid above the battery holder using 4 bamboo skewer sticks secured with hot glue.\n3- Soldier on a 1.8 volt red LED at the top of the mini Christmas tree.\n4- Test the LED glows with a new battery (1.6 volts)\n5- Test the LED doesn't glow with a dead battery (1.2 volts)\n6- Add in the buzzer circuit, along with a switch.\n7- Add in the windings circuit - to touch the striker of the buzzer a little copper foil was added.\n8- Insert a dead battery (1.2 volt) with the buzzer circuit turned switch off - the LED will not glow but when the buzzer circuit is switched on the Red LED lights up."}]}
-{"id": "user_oriented_task_111", "motivation_app": "instructables", "instruction": "There are many creative projects we can build at home that are related to the given theme, so let's list some of them.", "instances": [{"input": "Theme: Circuits", "output": "1- Build a UV Level Monitoring Budgie - Using IoT and Weather Data APIs\n2- DIY Weather Station With ESP32\n3- Modular Display Clock\n4- Overengineered Fume Extractor Project"}]}
-{"id": "user_oriented_task_112", "motivation_app": "instructables", "instruction": "Make a list of the materials that will be required to build the given tool.", "instances": [{"input": "Coffee Darkness Meter", "output": "- Arduino Nano (or Uno)\n- 0.96\" OLED display\n- TSL2561 lux and IR sensor on a brake-out board (I used Adafruit's)\n- ON/OFF toggle switch\n- Momentary push button\n- 9V battery and battery clip\n- 2x IR LEDs\n- 2x 100ohm resistors\n- 2x 10K resistors\n- 2x 2.2K resistors\n- wire for connecting\n- case for everything (3D printed)"}]}
-{"id": "user_oriented_task_113", "motivation_app": "instructables", "instruction": "Provide a detailed explanation of the steps that are required in order to construct the given project.", "instances": [{"input": "Steam Engine Using an Empty Beer Can", "output": "Step 1: Close Drink Opening\nTo build up steam the beverage can needs to act as a pressure vessel and so needs to be completely sealed, except where the steam will come out of the jet nozzles. It also needs to rotate on itself. Bend up the outer tab into a L shape using some pilers and then bend it forward so a piece of string can hang from it over the center of the can. This will allow the can to rotate on itself. Close the pouring opening by using a knife or other thin object, bending the opening back up so it almost where it was before the beverage can was opened.\nStep 2: Nozzel Jets\nMake a couple of holes in the side of the can, about half way up, opposite each other using a small drill. The diameter of the holes should be about the same as the thin plastic tubing to be used as jet nozzles. Enlarge the holes so they are elliptical allowing the nozzle tubes to lie parallel against the cans side.\nStep 3: Making a Pressure Vessel\nSeal the top of the opened can using super glue and baking soda to make a small pressure vessel. Add some super glue to the closed opening and then sprinkle on some baking soda - they will combine to make a plastic. Keep adding super glue and baking soda until the top is sealed.\nStep 4: Attaching the Nozzle Jets\nFit in the thin plastic tubes to the holes drilled in the sides of the can and seal with super glue and baking soda. Try to have the tubes pointing horizontal in the direction you want to rotate the can when the steam exits the nozzles.\nStep 5: Creating Steam\nAdd a little water into the can though one of the nozzles. Just add maybe a teaspoon or so. \nStep 6: In Action\nThe can was suspended from a tripod over the methylated sprits flame. After a couple of minutes enough steam had built up to rotate the can."}]}
-{"id": "user_oriented_task_114", "motivation_app": "instructables", "instruction": "Estimate the cost of supplies required to create the given project and calculate the total cost.", "instances": [{"input": "DIY Water Chiller\nRequired items are: Ice-maker, Icebox/cooler, Temperature regulator, Aquarium pump x2, 12V power supply", "output": "Ice-maker ($90)\nIcebox/cooler ($17)\nTemperature regulator ($16)\nAquarium pump x2 ($13)\n12V power supply ($8)\nTotal cost = $144"}]}
-{"id": "user_oriented_task_115", "motivation_app": "Socratic by Google", "instruction": "Answer the following literature question as accurately as possible based on your knowledge of literature for high school students.", "instances": [{"input": "What does the green light symbolize at the end of the book?", "output": "The green light symbolizes several different things to different characters throughout the book, but overall, it stands for the inaccessibility of the American Dream. At the start of the novel, the light symbolizes Gatsby's unwavering love for Daisy. He stands at the end of the dock gazing across the bay, reaching for the light, waiting for Daisy to come back to him. His hope, like the eternal green light, is unfaltering. It isn't until Daisy admits later that she had never really noticed the light that the facade began to slip. Nick, the narrator, starts to recognize it as a symbol of Gatsby's unhealthy desperation rather than his love. This is parallel to Gatsby's experience of the American Dream and money. Gatsby is unbelievably rich at the beginning of the novel. This solidifies his belief in the American Dream, though he still reaches for the acceptance of those who are wealthier and more popular. Nick believes money is not an object to Gatsby, though he has no idea how he made all of it. As the story unfolds and the mystery and mystique around Gatsby fade, so does Gatsby and Nick's blind belief and trust in the American Dream. After Gatsby is murdered, the green light is symbolic of the ways that money and affluence ruin the lives of those without. It is also symbolic of the way that the American dream ruins the lives of those eternally reaching for it."}]}
-{"id": "user_oriented_task_116", "motivation_app": "Wysa", "instruction": "Develop a mental exercise that can help people manage their anxiety and explain how it works.", "instances": [{"input": "", "output": "Submodalities\nGiving a physical form to your anxiety can help your mind feel more in control of it. In this exercise, we will visualize what anxiety looks and feels like, and learn to reduce its intensity. Here are the questions that you need to answer:\n1- Everyone feels anxiety differently. Where in your body do you feel it?\n2- Now close your eyes and picture your anxiety. Try to give it color, shape, and temperature. What does it feel like?\n3- Does it move, or is it in one place? If it moves, tell how it is moving.\n4- Now for the good part, close your eyes and try changing this picture. Change how it moves. Slow it done shrink its size. Try to move it away from your body. Imagine it fading. Keep doing this until you feel like you can change how it feels. Changing which submodality helped you curb your anxiety? Was it its shape, size, or something else?\n5- Do you feel better now?"}]}
-{"id": "user_oriented_task_117", "motivation_app": "Wysa", "instruction": "Give me an idea of the mental exercises you do to inspire yourself.", "instances": [{"input": "", "output": "1- Identify your strengths\n2- Understand you motivation style\n3- Build worthiness\n4- Dream ahead"}]}
-{"id": "user_oriented_task_118", "motivation_app": "Weather", "instruction": "Make a list of outdoor activities that are suitable for the given weather condition.", "instances": [{"input": "Temperature: 57 F, Showers", "output": "- Take a walk in the rain\n- Jump in puddles\n- Put on your own musical in the rain\n- Paint with Mud\n- Build a shelter \n- Make Nature Boats \n- Create a Dam"}]}
-{"id": "user_oriented_task_119", "motivation_app": "Weather", "instruction": "Please provide us with a list of the best conservation starter questions related to the weather.", "instances": [{"input": "", "output": "1. Beautiful day, isn't it?\n2. Can you believe all of this rain we've been having?\n3. It looks like it's going to storm.\n4. We couldn't ask for a nicer day, could we?\n5. How about this weather?\n6. It's so nice out today, isn't it?\n7. The weather's nice today, right?\n8. It\u2019s freezing today! Hopefully it doesn\u2019t snow.\n9. Wow, it\u2019s really hot/cold for this time of year.\n10. It\u2019s really been pouring all day, huh?"}]}
-{"id": "user_oriented_task_120", "motivation_app": "Weather", "instruction": "In relation to the given weather scenario, give some tips on how to adjust the travel plans with it.", "instances": [{"input": "a sudden temperature change", "output": "- Make sure that you have a good variety of options, and pack for the weather of your destination.\n- Rapid temperature changes can have major impacts on your body. Make sure you are aware of your particular responses to temperature change and pack accordingly. \n- Make sure to pack ointments, creams and lotions to help keep your skin healthy while you are on the go. \n- With extreme heat, you may experience rash, fainting, headaches, and fatigue as well as nausea, sweating, and loss of coordination. Be particularly aware of heat exhaustion and heat stroke as these are serious medical conditions that can have serious consequences. In cold climates, be aware of frostbite and hypothermia, intense shivering, goosebumps, difficulty using your hands, and false sense of warmth can indicate a developing cold illness."}]}
-{"id": "user_oriented_task_121", "motivation_app": "Notion", "instruction": "Write a to-do list based on the given information.", "instances": [{"input": "Daily tasks in the pharmacy", "output": "- Police sidewalk and entrance for litter \n- Assess window displays to make sure they are not outdated or faded\n- Clean all doors and remove outdated and unnecessary signs \n- Fill bags at each checkout area \n- Clean and straighten checkout counters \n- Tidy up seating/waiting area \n- Face (pull product forward) in all departments \n- Check one department a day for short and outdated product \n- Empty all trash receptacles\n- Vacuum (when the store is closed or only if customer traffic allows for it during store hours)"}]}
-{"id": "user_oriented_task_122", "motivation_app": "Notion", "instruction": "Design a template table for keeping track of all subscriptions.", "instances": [{"input": "", "output": "Service | Category | Status | Billing | Pricing | Renewal Date | Next Renewal | Monthly Costs | Yearly Costs | Nptes"}]}
-{"id": "user_oriented_task_123", "motivation_app": "Notion", "instruction": "Plan an agenda for the daily standup meetings. Assign a time duration to each topic on the agenda.", "instances": [{"input": "", "output": "[10 min] Status check\n- Current and upcoming work\n- Have spare bandwidth\n- Any blockers?\n[5 min] Big project status + general updates\n[5 min] Discuss blockers, otherwise social time\n[2 min] Add follow Ups"}]}
-{"id": "user_oriented_task_124", "motivation_app": "Doulingo", "instruction": "Answer the following question.", "instances": [{"input": "How do you say \"good evening\" in French.", "output": "bonsoir"}]}
-{"id": "user_oriented_task_125", "motivation_app": "Doulingo", "instruction": "What sound does this make?", "instances": [{"input": "\u597d", "output": "h\u01ceo"}]}
-{"id": "user_oriented_task_126", "motivation_app": "Doulingo", "instruction": "Give a grammar tip on how to end a German verb.", "instances": [{"input": "", "output": "Verbs have different endings depending on the subject of the sentence. Lucky for you er and sie use the same ending.\nSubject | Ending\nich        | e\ndu         | st\ner / sie  | t"}]}
-{"id": "user_oriented_task_127", "motivation_app": "Doulingo", "instruction": "This is a test for the reading ability of French. Read the given story carefully, and then answer the question in the end.", "instances": [{"input": "Lucie est \u00e0 la maison avec sa petite-fille, Lin.\nLucie: Oh, non! Nous n'avons pas de pain!\nLin: Ah! Tu vas au supermarch\u00e9?\nLucie: Oui, je veux une baguette pour mon sandwich.\nDo Lin and Lucie have a lot of bread?", "output": "No, that's wrong."}]}
-{"id": "user_oriented_task_128", "motivation_app": "Tasty", "instruction": "Come up with healthy and easy dinner ideas for weeknights.", "instances": [{"input": "", "output": "- Chicken & Veggie Stir-Fry\n- Classic Chicken Noodle Soup\n- Low-Carb Eggplant Lasagna\n- Avocado Lime Salmon\n- Zesty One-Pot Shrimp Pasta\n- Grilled Filet Mignon Street Tacos\n- Bruschetta Pasta"}]}
-{"id": "user_oriented_task_129", "motivation_app": "Tasty", "instruction": "Provide a cooking hack for improving the flavor of the given food.", "instances": [{"input": "popcorn", "output": "Everyone knows butter on popcorn is delicious. But supposedly adding some soy sauce to popcorn butter makes for a next-level popcorn topper. According to several sites, like Food52, the addition of salty soy sauce to creamy butter brings out the savory, cheesy, and nutty flavors in popcorn."}]}
-{"id": "user_oriented_task_130", "motivation_app": "Tasty", "instruction": "Make a list of snacks and foods to serve as party snacks on a game day!", "instances": [{"input": "", "output": "- Classic Chocolate Cake\n- Buffalo Chicken Sliders\n- Smoky Oven-Fried Chicken\n- Classic Hot Crab Dip For A Crowd\n- Potato Crust Breakfast Pizza\n- Muffin Tin Deviled Eggs"}]}
-{"id": "user_oriented_task_131", "motivation_app": "Tasty", "instruction": "Describe how to prepare the given food in your own words. Note down the ingredients you will need and the steps you will take to prepare them.", "instances": [{"input": "Chewy Chocolate Chip Cookies", "output": "Ingredients (for 12 cookies):\n\u00bd cup granulated sugar(100 g)\n\u00be cup brown sugar(165 g), packed\n1 teaspoon salt\n\u00bd cup unsalted butter(115 g), melted\n1 egg\n1 teaspoon vanilla extract\n1 \u00bc cups all-purpose flour(155 g)\n\u00bd teaspoon baking soda\n4 oz milk or semi-sweet chocolate chunks(110 g)\n4 oz dark chocolate chunk(110 g), or your preference\nPreparation:\nIn a large bowl, whisk together the sugars, salt, and butter until a paste forms with no lumps.\nWhisk in the egg and vanilla, beating until light ribbons fall off the whisk and remain for a short while before falling back into the mixture.\nSift in the flour and baking soda, then fold the mixture with a spatula (Be careful not to overmix, which would cause the gluten in the flour to toughen resulting in cakier cookies).\nFold in the chocolate chunks, then chill the dough for at least 30 minutes. For a more intense toffee-like flavor and deeper color, chill the dough overnight. The longer the dough rests, the more complex its flavor will be.\nPreheat oven to 350\u00b0F (180\u00b0C). Line a baking sheet with parchment paper.\nScoop the dough with an ice-cream scoop onto a parchment paper-lined baking sheet, leaving at least 4 inches (10 cm) of space between cookies and 2 inches (5 cm) of space from the edges of the pan so that the cookies can spread evenly.\nBake for 12-15 minutes, or until the edges have started to barely brown.\nCool completely before serving."}]}
-{"id": "user_oriented_task_132", "motivation_app": "Tasty", "instruction": "Suggest a recipe for a vegan dessert.", "instances": [{"input": "", "output": "Honey Lime Fruit Salad\nIngredients: (for 4 servings)\n\u00bd lb fresh strawberry(225 g), quartered\n2 kiwis, peeled and diced\n2 mangoes, diced\n2 bananas, sliced\n\u00bd lb fresh blueberry(225 g)\n2 tablespoons honey\n1 lime, juiced\n\nPreparation:\nPlace sliced fruits in a large bowl.\nIn a small bowl, mix honey and lime juice. Pour syrup over the fruit and mix."}]}
-{"id": "user_oriented_task_133", "motivation_app": "Sudoku", "instruction": "Design a medium-level sudoku puzzle.", "instances": [{"input": "", "output": "|  |  |  |  |  |6|1|4|9|\n|  |8|  |  |1|3|  |  |  |\n|  |  |6|4|2|  |7|  |  |\n|7|  |  |9|  |  |4|3|  |\n|3|6|  |  |  |  |5|9|1|\n|  |9|5|  |8|1|  |  |7|\n|  |  |  |6|  |2|9|  |5|\n|1|  |  |5|4|  |8|6|  |\n|6|5|2|  |  |  |  |7|  |"}]}
-{"id": "user_oriented_task_134", "motivation_app": "Sudoku", "instruction": "Solve the given Sudoku puzzle.", "instances": [{"input": "The Sudoku puzzle is:\n|1|  |5|  |  |  |  |8|  |\n|  |  |  |  |  |3|  |6|4|\n|8|3|4|6|7|  |9|  |  |\n|2|9|1|  |  |8|3|7|6|\n|  |  |6|  |  |9|8|1|  |\n|  |  |  |  |  |2|  |  |  |\n|  |2|  |  |9|7|6|  |  |\n|5|4|  |  |  |  |  |9|8|\n|6|1|  |  |  |5|4|3|7|", "output": "The solved Sudoku puzzle is:\n|1|6|5|9|2|4|7|8|9|\n|9|7|2|8|5|3|1|6|4|\n|8|3|4|6|7|1|9|2|5|\n|2|9|1|5|4|8|3|7|6|\n|4|5|6|7|3|9|8|1|2|\n|7|8|3|1|6|2|5|4|9|\n|3|2|8|4|9|7|6|5|1|\n|5|4|7|3|1|6|2|9|8|\n|6|1|9|2|8|5|4|3|7|"}]}
-{"id": "user_oriented_task_135", "motivation_app": "Goodreads", "instruction": "Think of a motivational quote that you have read in a book. Try to keep it positive and sweet.", "instances": [{"input": "", "output": "\u201cBelieve in yourself. You are braver than you think, more talented than you know, and capable of more than you imagine.\u201d\n\u2015 Roy T. Bennett, The Light in the Heart"}]}
-{"id": "user_oriented_task_136", "motivation_app": "Goodreads", "instruction": "Choose the best books from the given genre.", "instances": [{"input": "Crime & Mystery", "output": "1- The Girl with the Dragon Tattoo\n2- And Then There Were None\n3- Angels & Demons\n4- Rebecca\n5- In Cold Blood\n6- The Godfather\n7- The Lovely Bones\n8- Gone Girl\n9- The Name of the Rose\n10- Shutter Island"}]}
-{"id": "user_oriented_task_137", "motivation_app": "Goodreads", "instruction": "Give a brief description of the book that you are given.", "instances": [{"input": "Matilda", "output": "Matilda is a little girl who is far too good to be true. At age five-and-a-half she's knocking off double-digit multiplication problems and blitz-reading Dickens. Even more remarkably, her classmates love her even though she's a super-nerd and the teacher's pet. But everything is not perfect in Matilda's world...\nFor starters she has two of the most idiotic, self-centered parents who ever lived. Then there's the large, busty nightmare of a school principal, Miss (\"The\") Trunchbull, a former hammer-throwing champion who flings children at will, and is approximately as sympathetic as a bulldozer. Fortunately for Matilda, she has the inner resources to deal with such annoyances: astonishing intelligence, saintly patience, and an innate predilection for revenge."}]}
-{"id": "user_oriented_task_138", "motivation_app": "Goodreads", "instruction": "In the event that a person enjoyed the book that was given, please suggest other books that they might also like.", "instances": [{"input": "The Great Gatsby", "output": "To Kill a Mockingbird, The Catcher in the Rye, 1984, Animal Farm, Pride and Prejudice, The Hunger Games, Of Mice and Men, Lord of the Flies"}]}
-{"id": "user_oriented_task_139", "motivation_app": "Goodreads", "instruction": "Indicate the genre of the book to which it belongs.", "instances": [{"input": "Love in the Time of Cholera", "output": "Fiction"}]}
-{"id": "user_oriented_task_140", "motivation_app": "Wolfram alpha", "instruction": "Solve this equation.", "instances": [{"input": "x^3 - 4x^2 + 6x - 24 = 0", "output": "x = 4, x= -i * sqrt(6), x= i * sqrt(6)"}]}
-{"id": "user_oriented_task_141", "motivation_app": "Wolfram alpha", "instruction": "Prove the given statement using your information.", "instances": [{"input": "\"If x is a positive integer or a solution to x+3>4, then x>0 and x>12.\"", "output": "To determine if it's true, let's look first at the assumptions. We are assuming that either x is a positive integer, or that it solves the inequality x+3>4.\nNext let's consider the conclusion. We are concluding that x must satisfy both inequalities x>0 and x>12. If we look more closely, we see that once we satisfy the second inequality, the first is redundant. (If x>12, then it must already be larger than zero.)\nNow, in order for this statement to be true, we need that if x solves either of the assumptions, then it must solve x>12. Well, the first assumption is that x is a positive integer, which means that x\u22651, so in this case the conclusion holds. The second assumption is that x+3>4, or equivalently, that x>1, which means the conclusion holds as well."}]}
-{"id": "user_oriented_task_142", "motivation_app": "Wolfram alpha", "instruction": "Based on the facts that have been provided, prove the following statement.", "instances": [{"input": "Statement: \"For every integer k, k^2 + 2k + 1 \u2265 0\"\nFact: The square of any real number is non-negative.", "output": "Let k be an integer. Then (k + 1)^2 \u2265 0 because the square\nof any real number is non-negative. But (k+1)^2 = k^2+2k+1. So,\nby combining these two equations, we find that k^2 + 2k + 1 \u2265 0."}]}
-{"id": "user_oriented_task_143", "motivation_app": "Wolfram alpha", "instruction": "Verify the correctness of the given statement.", "instances": [{"input": "\"For all integers j and k, if j and k are odd, then jk is odd.\"", "output": "True"}]}
-{"id": "user_oriented_task_144", "motivation_app": "Wolfram alpha", "instruction": "Provide a formula for computing the nth term in the given sequence", "instances": [{"input": "5, 14, 23, 32, 41, ...", "output": "an = 9*n - 4"}]}
-{"id": "user_oriented_task_145", "motivation_app": "ESPN", "instruction": "Create a table listing all games that meet the specified criteria in the National Football League. Use the season, local time, game, and score as columns of the table.", "instances": [{"input": "Ravens home games in 2011", "output": "season | local time | game | score\n2011-12 | Sun, Jan 15 | Texans (11-7) at Ravens (13-4) | 13 - 20\n2011-12 | Sat, Dec 24 | Browns (4-11) at Ravens (11-4) | 14 - 20\n2011-12 | Sun, Dec 11 | Colts (0-13) at Ravens (10-3) | 10 - 24\n2011-12 | Thu, Nov 24 | 49ers (9-2) at Ravens (8-3) | 6 - 16\n2011-12 | Sun, Nov 20 | Bengals (6-4) at Ravens (7-3) | 24 - 31\n2011-12 | Sun, Oct 30 | Cardinals (1-6) at Ravens (5-2) | 27 - 30\n2011-12 | Sun, Oct 16 | Texans (3-3) at Ravens (4-1) | 14 - 29\n2011-12 | Sun, Oct 2 | Jets (2-2) at Ravens (3-1) | 17 - 34\n2011-12 | Sun, Sep 11 | Steelers (0-1) at Ravens (1-0) | 7 - 35"}]}
-{"id": "user_oriented_task_146", "motivation_app": "ESPN", "instruction": "Using a table, compare the career overviews of the given players in Major League Baseball. Use \"|\" for separating the columns in the table.", "instances": [{"input": "Derek Jeter, Albert Pujols", "output": "| D. Jeter | A. Pujols\ngames played | 2747 | 2746\ngames started | 2734 | 2704\nbatting average | .310 | .296\nhome runs | 260 | 703\nruns batted in | 1311 | 2218"}]}
-{"id": "user_oriented_task_147", "motivation_app": "(Wolfram alpha)?", "instruction": "You will be tested on your knowledge of classic witticisms and aphorisms by completing the given aphorism. Write the original quote as the answer.", "instances": [{"input": "two things are infinite", "output": "The universe and human stupidity; and I'm not sure about the universe. (attributed to Albert Einstein)"}]}
-{"id": "user_oriented_task_148", "motivation_app": "IMDB", "instruction": "A list of all movies that meet the criteria given should be compiled.", "instances": [{"input": "movies directed by Spike Lee by release date", "output": "| | release date\n1 | American Utopia | 10/09/2020\n2 | Da 5 Bloods | 12/06/2020\n3 | BlacKkKlansman | 10/08/2018\n4 | Pass Over | 20/04/2018\n5 | Rodney King | 28/04/2017"}]}
-{"id": "user_oriented_task_149", "motivation_app": "(Wolfram alpha)?", "instruction": "Using a given amount, determine an appropriate tip.", "instances": [{"input": "14% tip on $47.50", "output": "amount of tip = $6.65 \namount with tip = $54.15"}]}
-{"id": "user_oriented_task_150", "motivation_app": "Wordle", "instruction": "Identify all words that match the pattern given.", "instances": [{"input": "H_AR_", "output": "haart, heard, hears, heart, hoard, hoary"}]}
-{"id": "user_oriented_task_151", "motivation_app": "merriam-webster.com", "instruction": "Come up with words that rhyme with the given word.", "instances": [{"input": "instruct", "output": "abduct, bucked, chucked, conduct, construct, deconstruct, deduct, destruct, ducked, duct, induct, obstruct, plucked, reconstruct, shucked, sucked, trucked, tucked"}]}
-{"id": "user_oriented_task_152", "motivation_app": "(Wolfram alpha)?", "instruction": "Look for poems that mention the given object or character. The names of the poems and their poets should be written.", "instances": [{"input": "santa claus", "output": "Santeclaus by William Gilley\nA Visit From St. Nicholas by Clement Clarke Moore"}]}
-{"id": "user_oriented_task_153", "motivation_app": "Messenger", "instruction": "Respond to the last text in the chat using the correct emojis to convey your feelings.", "instances": [{"input": "+ Hey, how did your exams go?\n- They were alright. I'm just glad they're over.\n+ Yeah, I know how you feel. I'm just glad I don't have to worry about them anymore.", "output": "- \ud83d\ude0c\ud83d\ude0a"}]}
-{"id": "user_oriented_task_154", "motivation_app": "Messenger", "instruction": "For answering the chat, think of a funny response you can use.", "instances": [{"input": "+ how are you?", "output": "- I still haven\u2019t figured it out."}]}
-{"id": "user_oriented_task_155", "motivation_app": "Messenger", "instruction": "The last message in the chat is a partial response. You should complete and rewrite it.", "instances": [{"input": "+ What are going to do now?\n- I don't know. I had to give up my dream of being a mathematician.\n+ Why", "output": "+ Why is that? What happened?"}]}
-{"id": "user_oriented_task_156", "motivation_app": "Messenger", "instruction": "Change the response to have a more empathic tone in the chat.", "instances": [{"input": "+ How are you doing?\n- Fine, I had a bunch of problems today.\n+ What kind of problems?", "output": "+ Oh, Is there anything I can help you with?"}]}
-{"id": "user_oriented_task_157", "motivation_app": "Messenger", "instruction": "Currently, there is a chat in which people are arranging a meeting in the future. Retrieve the meeting information from the conversation.", "instances": [{"input": "Person 1: Hey, when are you free? I'd like to meet up.\nPerson 2: I'm free tomorrow afternoon.\nPerson 1: Great, what time?\nPerson 2: How about 3 pm?\nPerson 1: I have another meeting at that time. what about 5 pm?\nPerson 2: That works. Where should we meet?\nPerson 1: How about the coffee shop on Main Street?\nPerson 2: Sounds good. I've heard they have great coffee there. See you tomorrow!", "output": "Date: Tomorrow\nTime: 5 pm\nPlace: Coffee shop on Main Street"}]}
-{"id": "user_oriented_task_158", "motivation_app": "Messenger", "instruction": "The given text must be classified as offensive or not offensive.", "instances": [{"input": "You acted really crazy last night.", "output": "offensive"}]}
-{"id": "user_oriented_task_159", "motivation_app": "https://cohere.ai/", "instruction": "Give a brief summary of the intention of the dialogue that just happened.", "instances": [{"input": "Customer: Hi there, I'm looking for a new phone.\nAI: Hi! What type of phone are you looking for?\nCustomer: I'm not sure. Maybe something with a good camera?\nAI: We have a few phones with great cameras. Would you like to see some options?\nCustomer: Yeah, that would be great.", "output": "A customer wants to buy a new phone."}]}
-{"id": "user_oriented_task_160", "motivation_app": "https://cohere.ai/", "instruction": "Describe the content of the article in a brief manner.", "instances": [{"input": "A study published earlier this year by Zee and her team examined the role of light in sleep for healthy adults in their 20s. Sleeping for only one night with a dim light, such as a TV set with the sound off, raised the blood sugar and heart rate of the young people during the sleep lab experiment. An elevated heart rate at night has been shown in prior studies to be a risk factor for future heart disease and early death, while higher blood sugar levels are a sign of insulin resistance, which can ultimately lead to type 2 diabetes.", "output": "A study finds that sleeping with a light pollution may increase your health risks."}]}
-{"id": "user_oriented_task_161", "motivation_app": "https://cohere.ai/", "instruction": "Correct the transcription of an excerpt containing errors.", "instances": [{"input": "I got got charged interest on ly credit card but I paid my pull balance one day due date. I not missed a pavement year yet. Man you reverse the interest charge?", "output": "I was charged interest on my credit card but I paid my balance on the due date. I haven't missed a payment year yet. Can you reverse the interest charge?"}]}
-{"id": "user_oriented_task_162", "motivation_app": "https://cohere.ai/", "instruction": "Take a look at the contract and extract the parties of the agreement from it.", "instances": [{"input": "As of the 13th day of December 2021, this Music Recording Agreement (\"Agreement\") is made between Good Kid, a Toronto-based musical group (\"Artist\"), and Universal Music Group, a record label with license number 545345 (\"Record Label\"). The Artist and the Recording Label may be referred to in this Agreement individually as \"Parties\" and collectively as \"Parties.\" Work under this Agreement will begin on March 15, 2022, and will last for the duration of the Agreement.", "output": "Artist: Good Kid\nRecording Label: Universal Music Group"}]}
-{"id": "user_oriented_task_163", "motivation_app": "Gmail", "instruction": "On the basis of the subject of the email, determine whether the email should be treated as spam or not.", "instances": [{"input": "Dermatologists don't like her!", "output": "Spam"}]}
-{"id": "user_oriented_task_164", "motivation_app": "Twitter", "instruction": "Classify the tweet into one of the following three categories: event, demo, or news. Event is a happening, often social or public, that people attend or are aware of. Demo is a  public showing or presentation of something, typically to promote or advertise it. And, news is information about recent events or happenings, especially as reported by news media.", "instances": [{"input": "Using this tutorial, you can analyze and classify a user\u2019s review as either positive or negative using sentiment analysis features in an existing Next.js app. We show you how to leverage language AI into your applications. Check it out \u2193", "output": "demo"}]}
-{"id": "user_oriented_task_165", "motivation_app": "CNN News", "instruction": "Give the news title a category. Pick a category from the list of News & Buzz, Travel, Style, Arts & Culture, Politics, Tech, and Science & Health.", "instances": [{"input": "The #Banksy Exhibit in Cambridge, MA is absolutely terrific.", "output": "Arts & Culture"}]}
-{"id": "user_oriented_task_166", "motivation_app": "https://cohere.ai/", "instruction": "Classify the questions in the FAQ into Finding policy details, Change account settings, Filing a claim and viewing status, or Cancelling coverage.", "instances": [{"input": "Could you deposit money into my account rather than mailing me a physical cheque?", "output": "Change account settings"}]}
-{"id": "user_oriented_task_167", "motivation_app": "Quora", "instruction": "Choose an appealing title for your post.", "instances": [{"input": "The typical avocado is over 300 calories from the oil in it. That\u2019s the amount of calories in a large candy bar. If you get enough exercise to eat a large candy bar every day without gaining weight, it wouldn\u2019t be a problem to eat an avocado every day. Other wise you should probably eat them sparingly.", "output": "What will happen if you eat an avocado everyday?"}]}
-{"id": "user_oriented_task_168", "motivation_app": "Quora", "instruction": "Give some examples of what people usually say in the given social situation.", "instances": [{"input": "when someone arrives safely", "output": "- \u201cGlad you made it safe and sound.\u201d\n- \u201cGood to see you got here unscathed.\u201d\n- \u201cSo, bullet proof vests actually work!\u201d"}]}
-{"id": "user_oriented_task_169", "motivation_app": "Indeed", "instruction": "You will be required to write a few questions from the given company's interview section.", "instances": [{"input": "Kohl's", "output": "- Why do you want to work at kohls? \n- If you were helping a customer and another interrupted you, what would you do? \n- What\u2019s your availability? \n- What do you think is the most important aspect of working in a retail store? \n- What\u2019s a reasonable reason for not showing up at work?"}]}
-{"id": "user_oriented_task_170", "motivation_app": "Indeed", "instruction": "Based on the information provided, you need to estimate the average summary for the given job.", "instances": [{"input": "Data entry clerk in United States", "output": "$17.58 Per hour"}]}
-{"id": "user_oriented_task_171", "motivation_app": "Indeed", "instruction": "Describe the responsibilities of the given job.", "instances": [{"input": "Security Officer", "output": "A security officer maintains a safe environment on behalf of their employers. They ensure the staff, customers, visitors and business premises are secure. They patrol perimeters, use surveillance equipment to monitor activity and perform security checks required for visitors to the site."}]}
-{"id": "user_oriented_task_172", "motivation_app": "Indeed", "instruction": "Provide a list of the skills that may help you find the job.", "instances": [{"input": "Crew Members", "output": "Baking\nBasic math\nBilingual\nCash handling\nCashiering\nCommunication skills\nComputer literacy\nComputer skills\nConstruction\nCooking\nCustomer service"}]}
-{"id": "user_oriented_task_173", "motivation_app": "Google Calendar", "instruction": "Schedule the meetings based on the given informqation.", "instances": [{"input": "Person1: available from 9-11 am, meeting duration: 30 min \nPerson2: available from 1-3 pm, meeting duration: 90 min\nPerson3: available from 2-3:30 pm, meeting duration: 60 min", "output": "Person1: 9-9:30 am, Person2: 1-2:30 pm, Person3: 2:30-3:30 pm"}]}
-{"id": "user_oriented_task_174", "motivation_app": "Google Map", "instruction": "Find the directions between the given locations.", "instances": [{"input": "From: Rowan Avenue Elementary School, 600 S Rowan Ave, Los Angeles, CA 90023, United States\nTo: Calvary Cemetery and Mortuary, 4201 Whittier Blvd, Los Angeles, CA 90023, United States", "output": "Head north on S Rowan Ave toward E 6th St. Turn right onto Whittier Blvd. Destination will be on the right."}]}
-{"id": "user_oriented_task_175", "motivation_app": "Explainpaper", "instruction": "Give a brief explanation of the requested part of the paper.", "instances": [{"input": "Paper: The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles, by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature. We show that the Transformer generalizes well to other tasks by applying it successfully to English constituency parsing both with large and limited training data.\nExplain: The dominant sequence transduction models", "output": "The dominant sequence transduction models are the most common models used to translate between two languages. These models are based on complex neural networks that include an encoder and a decoder. The best performing models also connect the encoder and decoder through an attention mechanism."}]}
-{"id": "user_oriented_task_176", "motivation_app": "Explainpaper", "instruction": "Answer the question about the paper after reading it carefully.", "instances": [{"input": "To evaluate if the Transformer can generalize to other tasks we performed experiments on English constituency parsing. This task presents specific challenges: the output is subject to strong structural constraints and is significantly longer than the input. Furthermore, RNN sequence-to-sequence models have not been able to attain state-of-the-art results in small-data regimes.\nQuestion: What is English constituency parsing?", "output": "English constituency parsing is a task in natural language processing where the goal is to analyze the syntactic structure of a sentence in English. This can be used to understand the meaning of a sentence, or to generate new sentences with similar meaning."}]}
-{"id": "user_oriented_task_177", "motivation_app": "Meetup", "instruction": "Recommend some helpful activities or conversation starters to use at meetups to break the ice.", "instances": [{"input": "", "output": "- \u201cLet me tell you something very interesting\u2026\u201d (nature, history, ecology, language ,geography etc)\n- Joke (neutral not abrasive)\n- ask for help with something\n- ask open ended questions"}]}
-{"id": "user_oriented_task_178", "motivation_app": "Markdown", "instruction": "Convert the given HTML into Markdown format.", "instances": [{"input": "<code>Use `code` in your Markdown file.</code>", "output": "``Use `code` in your Markdown file.``"}]}
-{"id": "user_oriented_task_179", "motivation_app": "Semantic Scholar", "instruction": "Extract the method that has been used in the research from the abstract of the paper. Give the exact sentences from the abstract that discuss the method.", "instances": [{"input": "We present a state-of-the-art speech recognition system developed using end-to-end deep learning. Our architecture is significantly simpler than traditional speech systems, which rely on laboriously engineered processing pipelines; these traditional systems also tend to perform poorly when used in noisy environments. In contrast, our system does not need hand-designed components to model background noise, reverberation, or speaker variation, but instead directly learns a function that is robust to such effects. We do not need a phoneme dictionary, nor even the concept of a \"phoneme.\" Key to our approach is a well-optimized RNN training system that uses multiple GPUs, as well as a set of novel data synthesis techniques that allow us to efficiently obtain a large amount of varied data for training. Our system, called Deep Speech, outperforms previously published results on the widely studied Switchboard Hub5'00, achieving 16.0% error on the full test set. Deep Speech also handles challenging noisy environments better than widely used, state-of-the-art commercial speech systems.Collapse", "output": "We do not need a phoneme dictionary, nor even the concept of a \"phoneme.\" Key to our approach is a well-optimized RNN training system that uses multiple GPUs, as well as a set of novel data synthesis techniques that allow us to efficiently obtain a large amount of varied data for training."}]}
-{"id": "user_oriented_task_180", "motivation_app": "Semantic Scholar", "instruction": "You need to come up with the names of some of the papers that relate to the paper you are given.", "instances": [{"input": "RoMQA: A Benchmark for Robust, Multi-evidence, Multi-answer Question Answering", "output": "+ Question Answering Infused Pre-training of General-Purpose Contextualized Representations\n+ Zero-Shot Dialogue State Tracking via Cross-Task Transfer\n+ Adapting Language Models for Zero-shot Learning by Meta-tuning on Dataset and Prompt Collections\n+ HotpotQA: A Dataset for Diverse, Explainable Multi-hop Question Answering\n+ FewshotQA: A simple framework for few-shot learning of question answering tasks using pre-trained text-to-text models\n+ MKQA: A Linguistically Diverse Benchmark for Multilingual Open Domain Question Answering\n+ FewCLUE: A Chinese Few-shot Learning Evaluation Benchmark\n+ Multilingual Transfer Learning for QA Using Translation as Data Augmentation\n+ Towards Zero Shot Conditional Summarization with Adaptive Multi-task Fine-Tuning"}]}
-{"id": "user_oriented_task_181", "motivation_app": "Semantic Scholar", "instruction": "Create a short, concise summary of the paper based on its abstract.", "instances": [{"input": "Few-shot learning (FSL) is one of the key future steps in machine learning and raises a lot of attention. In this paper, we focus on the FSL problem of dialogue understanding, which contains two closely related tasks: intent detection and slot filling. Dialogue understanding has been proven to benefit a lot from jointly learning the two sub-tasks. However, such joint learning becomes challenging in the few-shot scenarios: on the one hand, the sparsity of samples greatly magnifies the difficulty of modeling the connection between the two tasks; on the other hand, how to jointly learn multiple tasks in the few-shot setting is still less investigated. In response to this, we introduce FewJoint, the first FSL benchmark for joint dialogue understanding. FewJoint provides a new corpus with 59 different dialogue domains from real industrial API and a code platform to ease FSL experiment set-up, which are expected to advance the research of this field. Further, we find that insufficient performance of the few-shot setting often leads to noisy sharing between two sub-task and disturbs joint learning. To tackle this, we guide slot with explicit intent information and propose a novel trust gating mechanism that blocks low-confidence intent information to ensure high quality sharing. Besides, we introduce a Reptile-based meta-learning strategy to achieve better generalization in unseen few-shot domains. In the experiments, the proposed method brings significant improvements on two datasets and achieve new state-of-the-art performance.", "output": "This paper introduces FewJoint, the first FSL benchmark for joint dialogue understanding, and guides slot with explicit intent information and proposes a novel trust gating mechanism that blocks low-confidence intent information to ensure high quality sharing."}]}
-{"id": "user_oriented_task_182", "motivation_app": "https://abcnotation.com/", "instruction": "You will need to compose the ABC notation for a given song.", "instances": [{"input": "The South Wind", "output": "X:435\nT:Southwind\nR:WALTZ\nM:3/4\nL:1/4\nQ:1/4=68\nK:G\n|:d/c/|\"G\" B3/2 A/2 G|Bcd|\"D\" (A3 |A2) d/c/|\"G\"B3/2 A/2 G|\"C\" EDE|\"G\"(G3|G2) d/c/|!\n|\"G\" B3/2 A/2 G|Bcd|\"D\" A3/2 (3G/F/G/|\"D\"  A2 d/c/| \"G\" B3/2 A/ G| \"C\" EDE | \"G\" (G3|G) B d|!\n\"G\" g2 g| \"C\" g3/2 f/ e| \"G\" (d3|\"D\"d2) d/c/|\"G\" B3/2 A/ G| B c d| \"D\" (A3|A2) d| \"C\" g2 g|!\n\"C\" g3/2  f/ e| \"G\" d3| \"G\" d2 c| \"G\" B3/2 A/ G| \"D\" A/ c3/2 F| \"G\" (G3|G2):||"}]}
-{"id": "user_oriented_task_183", "motivation_app": "Scribd", "instruction": "Categorize the given magazine based on its content into one of the categories: Politics, Entertainment News, Tech News, Business News, Parenting & Kids, Health & Healing, Cooking, Food & Wine, and Arts & Literature,", "instances": [{"input": "Inc.", "output": "Business News"}]}
-{"id": "user_oriented_task_184", "motivation_app": "Gmail", "instruction": "Decide if the given email belongs to the Promotions or Social category.", "instances": [{"input": "Subject: You have an invitation\nEmail: Hi, I'd like to join your LinkedIn network.", "output": "Social"}]}
-{"id": "user_oriented_task_185", "motivation_app": "Netflix", "instruction": "The story of a movie has been described using many emojis. You need to guess the name of the movie based on the emojis.", "instances": [{"input": "\ud83e\uddd9\u200d\u2642\ufe0f\ud83d\udeaa\ud83d\udcda\ud83e\udd85\u26a1", "output": "Harry Potter"}]}
-{"id": "user_oriented_task_186", "motivation_app": "Netflix", "instruction": "Come up with an interesting idea for a new movie plot. Your plot should be described with a title and a summary.", "instances": [{"input": "", "output": "The Cat burglars. \nA group of criminals specializes in stealing from high-end residences. They use their knowledge of cats to help them break in and avoid detection. However, one member of the group is a police officer working undercover to bring them down."}]}
-{"id": "user_oriented_task_187", "motivation_app": "Grammarly", "instruction": "Change the first person to the third person in the given sentence. The meaning should be kept, but you can paraphrase it or expand it in order to have a better pose.", "instances": [{"input": "We were recently able to increase the amount of stock we hold with the same supplier thereby reducing our risk.", "output": "The company was recently able to increase the amount of stock it holds with the same supplier thereby reducing its risk."}]}
-{"id": "user_oriented_task_188", "motivation_app": "Grammarly", "instruction": "You should capitalize the sentence according to the guide.", "instances": [{"input": "Guide: Every other letter alternates between lower case and upper case.\nSentence: A giant spider blocks your path.", "output": "a gIaNt SpIdEr BlOcKs YoUr PaTh."}]}
-{"id": "user_oriented_task_189", "motivation_app": "merriam-webster.com", "instruction": "Create alliterations by finding synonyms for words in the given sentence.", "instances": [{"input": "David wears a hat everyday.", "output": "David dons a derby daily."}]}
-{"id": "user_oriented_task_190", "motivation_app": "GeeksforGeeks", "instruction": "Convert the given description to a bash command.", "instances": [{"input": "read the first 10 lines from f", "output": "f.readlines(10)"}]}
-{"id": "user_oriented_task_191", "motivation_app": "Notion", "instruction": "Please prepare a table containing a summary of the information of the participants at the conference.", "instances": [{"input": "The conference was called to discuss how to improve the quality of education in schools. The main speaker was Professor Crew Purdie (cpurdie@email.com), who spoke about the importance of setting high standards for students and teachers. He said that schools should be places where students can learn and be challenged. Other speakers included Teacher Oliver Malachi (oliver@email.com) and Parent Kolby Reese (kolbyreese82@email.com). The teacher talked about the need for teachers to be well-prepared and to have high expectations for their students. The parent spoke about the need for parents to be involved in their children's education. There was also a discussion about the importance of funding for schools.", "output": "Name | Email | Occupation\nCrew Purdie | cpurdie@email.com | Professor\nOliver Malachi | oliver@email.com | Teacher\nKolby Reese | kolbyreese82@email.com | Parent"}]}
-{"id": "user_oriented_task_192", "motivation_app": "Coursera", "instruction": "List the concepts that should be learned before approaching the given complex concept.", "instances": [{"input": "Deep Learning", "output": "- Linear Algebra\n- Calculus\n- Statistics\n- Probability\n- Programming\n- Machine Learning"}]}
-{"id": "user_oriented_task_193", "motivation_app": "Grammarly", "instruction": "Given a sentence that may be ambiguous, rewrite the sentence using less ambiguous words.", "instances": [{"input": "The chicken is ready to eat.", "output": "The chicken is going to eat."}]}
-{"id": "user_oriented_task_194", "motivation_app": "Twitter", "instruction": "Assess the sentiment of the given tweet and categorize it as Positive, Negative, or Neutral.", "instances": [{"input": "\"This is the best cheat-sheet paper that I've found in a LONG time.\"", "output": "Positive"}]}
-{"id": "user_oriented_task_195", "motivation_app": "GeeksforGeeks", "instruction": "You are given a Python command and are required to convert it into Ruby.", "instances": [{"input": "print(\"Hello World!\")", "output": "puts \"Hello World!\""}]}
-{"id": "user_oriented_task_196", "motivation_app": "StackOverflow", "instruction": "You are given a sentence in first-order logic. Translate it into natural language.", "instances": [{"input": "\u2203x\u2200y(soldier(x)\u2192general(x,y))", "output": "There is someone (x) such that if if he is a soldier, then he is general of everyone."}]}
-{"id": "user_oriented_task_197", "motivation_app": "Google Search", "instruction": "You are given a search query and a document. Classify whether the document is relevant to the search query or not relevant.", "instances": [{"input": "Search: why sky is blue\nDocument: The Short Answer: Sunlight reaches Earth's atmosphere and is scattered in all directions by all the gases and particles in the air. Blue light is scattered more than the other colors because it travels as shorter, smaller waves. This is why we see a blue sky most of the time.", "output": "relevant"}]}
-{"id": "user_oriented_task_198", "motivation_app": "Quora", "instruction": "Create a list of subtopics for the given topic.", "instances": [{"input": "Music theory", "output": "melody, rhythm, counterpoint, harmony, form, tonal systems, scales, tuning, intervals, consonance, dissonance, durational proportions, the acoustics of pitch systems, composition, performance, orchestration, ornamentation, improvisation, electronic sound production"}]}
-{"id": "user_oriented_task_199", "motivation_app": "Netflix", "instruction": "Summarize the movie in a snarky way. Try to explain the movie in just one sentence.", "instances": [{"input": "The Shining", "output": "A family's first Airbnb experience goes very wrong."}]}
-{"id": "user_oriented_task_200", "motivation_app": "merriam-webster.com", "instruction": "Enter the words that satisfy the given condition.", "instances": [{"input": "5 Countries that Start with S", "output": "Switzerland, Spain, Senegal, Sweden, Sudan"}]}
-{"id": "user_oriented_task_201", "motivation_app": "Tasty", "instruction": "Provide a name for the dish given the ingredients and instructions.", "instances": [{"input": "INGREDIENTS:\n2 (5 oz) cans Bumble Bee\u00ae Solid White Albacore Tuna, drained\n1 avocado\n2 Tbsp Sriracha\n1 Tbsp Dijon mustard\n2 to 3 Tbsp celery, chopped\n2 Tbsp red onion, chopped\n2 green onions, chopped\n1 Tbsp fresh cilantro, chopped\nSalt and pepper, to taste\n2 heaping cups leafy green lettuce\n1 cup matchstick carrots\n4 (10 inch) whole wheat tortillas\nINSTRUCTIONS:\nIn a medium bowl, mash together tuna and avocado until combined. Add in the rest of the ingredients through the salt and pepper, mixing well.\nTo assemble, top each tortilla with a 1/2 cup leafy greens, 1/4 cup matchstick carrots and divide the tuna mixture evenly among the wraps. Tightly roll up the tortilla, slice and enjoy!", "output": "Tuna & Avocado Wrap"}]}
-{"id": "user_oriented_task_202", "motivation_app": "Google Meet", "instruction": "Summarize a meeting from the given list of bullet points. Be sure to convert shorthand into a first-hand account.", "instances": [{"input": "Rose: Analyze data and presents findings\nJohn: propose new idea\nJane: appointed to head project\nTom: need more time to fix software bug", "output": "At the meeting, Rose analyzed data and presented her findings. Then, John proposed a new idea. Moreover, Jane was appointed to head the project. And, Tom said he needs more time to fix a software bug."}]}
-{"id": "user_oriented_task_203", "motivation_app": "Spotify", "instruction": "List a few popular songs from the given album.", "instances": [{"input": "Back To Black", "output": "- Rehab\n- Just Friends\n- Tears Dry on Their Own\n- Wake Up Alone\n- Addicted"}]}
-{"id": "user_oriented_task_204", "motivation_app": "merriam-webster.com", "instruction": "Invent a new word based on a description of the word.", "instances": [{"input": "The feeling of knowing, while you are still in the moment, that something occurring will be remembered forever.", "output": "mnemonty"}]}
-{"id": "user_oriented_task_205", "motivation_app": "w3schools", "instruction": "Given a programming language and the name of a function, write a command to show how to use the function.", "instances": [{"input": "Language: Python\nFunction: input", "output": "username = input(\"Enter username:\")"}]}
-{"id": "user_oriented_task_206", "motivation_app": "Reddit", "instruction": "Provide a pleasant compliment based on the quality you have been given.", "instances": [{"input": "Truthfulness", "output": "Everything would be better if more people were like you."}]}
-{"id": "user_oriented_task_207", "motivation_app": "Overleaf", "instruction": "Create a brief resume introduction using the following information.", "instances": [{"input": "name: Mark\nnationality: Italian\nlanguage: English\nattributes: creative, goal-oriented, motivated\nskills: software engineering, content marketing", "output": "Mark is a creative and goal-oriented software engineer and content marketer. He has a strong background in software engineering and is skilled in content marketing. He is a motivated individual who always looks for new opportunities to learn and grow."}]}
-{"id": "user_oriented_task_208", "motivation_app": "Amazon", "instruction": "Make a list of adjectives that can be used to describe the given brand.", "instances": [{"input": "a creative tech startup", "output": "1. Innovative\n2. Creative\n3. Disruptive\n4. cutting-edge\n5. revolutionary\n6. game-changing\n7. industry-leading\n8. trailblazing\n9. groundbreaking\n10. world-changing"}]}
-{"id": "user_oriented_task_209", "motivation_app": "tripadvisor.com", "instruction": "Someone gives you some information about where they want to go and what they enjoy seeing and doing. Describe a proper road trip for them and find the best stops and sctivities in the way.", "instances": [{"input": "I'm looking for a west coast road trip. My goal is to start near San Diego and travel to Los Angeles.", "output": "Begin near the city of San Diego and make the first stop in coastal La Jolla. This upscale little village is a great a place for an overnight, a day of sunbathing at La Jolla Shores Beach, or a cocktail with a view.\nJust 20 minutes north, stop into beachy Encinitas to window shop the boutiques or sample fish tacos. Walk through the coastal meditation garden at the Self-Realization Fellowship or join surfers in the line-up at one of the fantastic local beaches here\nJust up the coast is cute little San Juan Capistrano, home to one of California\u2019s favorite missions. In spring, more than 6,000 swallows migrate here from Argentina! As you approach LA, don\u2019t miss swank Laguna Hills. Home to the one-of-a-kind Pageant of the Masters art event, Laguna is brimming with artists and charm. It\u2019s a great place for an overnight. Plus, Crystal Cove Beach in nearby Newport Beach is one of California\u2019s finest!\nAfter that, the choice is yours: Disney lovers should head directly to Disneyland in Anaheim to meet Mickey. To join the beautiful people and snap a photo of an iconic California scene, head to Santa Monica Pier and walk the boardwalk or ride the world\u2019s only solar-powered Ferris wheel. A San Diego to LA road trip is the perfect way to enjoy all things Southern California."}]}
-{"id": "user_oriented_task_210", "motivation_app": "StackOverflow", "instruction": "Regex is a set of characters that define a search pattern. This kind of pattern is usually used by string-searching algorithms to find or replace strings or to validate inputs. Convert the natural english text to regex.", "instances": [{"input": "Match a valid ip address", "output": "^(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])\\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])$"}]}
-{"id": "user_oriented_task_211", "motivation_app": "sth related to real estate?", "instruction": "Write an engaging and well-written property listing description for selling a house. Address of the house and some of the details are given to you. Fill in the information gap with hallucinations if needed.", "instances": [{"input": "Property Address: 412 Monterey Ave, Capitola, CA 95010\nParking: 3 cars + a finished single car garage\nDetails:\n- open floorplan\n- window with views to park/mountains\n- Kitchen with shaker cabinetry and white Calcatta quartz counters", "output": "Amazing designer updates to this beach chich home that is just a quick 3-minute stroll to Capitola Village! As you enter this light-filled home you will love the open floorplan and huge picture window with views to park/mountains! Kitchen with shaker cabinetry and white Calcatta quartz counters new stainless appliances (gas stove). Master bedroom with own access to yard! Beautiful bathrooms w/designer finishes and custom, subway & picket tile! New composition roof and gutters, new indoor and outdoor paint, new dual pane windows, new wide plank washed oak laminate floors, new doors, new fencing, forced air heat + LOTS more! Home sits high above Monterey Ave with alley access and parking for 3 cars + a finished single car garage! The indoor/outdoor living is ideal from this sunny locale with front, side and backyards that are super sunny with pathways and perfect for gardening or entertaining! Stroll to beach, Gayles bakery, parks, schools & churches from this adorable beach home!"}]}
-{"id": "user_oriented_task_212", "motivation_app": "Redfin", "instruction": "Given a real estate listing description, extract the information and details of the property from the text. The details consist of address, basic house attributes, etc.", "instances": [{"input": "WALK TO THE BEACH FROM CAPITOLA KNOLLS! Single-level upstairs condo with your own large private deck overlooking nature. Adorable home that is move-in ready and waiting for you to put your own personal touches on. Gorgeous and well-maintained landscaped park-like setting with lush green grasses with a community pool and spa to enjoy. Desirable location...walk to Capitola Village, Gayle's Bakery, restaurants, shopping, schools or jump on Hwy 1 right around the corner.", "output": "-Address: \"Capitola\"\n-Type of home: Condo\n-Location: \"walk to Capitola Village, Gayle's Bakery, restaurants, shopping, and schools or jump on Hwy 1 right around the corner.\"\n-House attributes: \"Single-level upstairs, large private deck overlooking nature, move-in ready home\"\n-Outdoor details: \"well-maintained landscaped park-like setting with lush green grasses with a Community pool and spa\""}]}
-{"id": "user_oriented_task_213", "motivation_app": "Redfin", "instruction": "Review the property description and tell us what is good and what is bad about it and list its key takeaways.", "instances": [{"input": "There's real magic in combining inherently compelling Victorian architecture with a modern aesthetic. Add in an extraordinary multi-level garden & the result is the undeniable elegance of 444 29th St. The front of the home is light-filled w/ a seamless connection between living, dining & kitchen. Newly updated kitchen w/ quartz counters & upgraded appliances. A lg dining rm has a bay of windows &, when paired w/ the adjacent living rm, provides an ideal space to entertain. 2 lg bdrms on this level are at the rear. Both w/ lg closets & walk out to the garden. Stylishly remodeled full bath on this level. 3rd bdrm/2nd full bath are located on the ground floor - perfect as a guest rm, home office, fitness area, etc. The enormous garden occupies an extra deep lot & is beautifully landscaped & terraced. Laundry/storage rm, 1-car garage pkg, Tesla solar panels. Prime, flat part of Noe Valley - short walk to all that Noe has to offer. Steps from the J-Church & convenient to shuttles/freeways.", "output": "1- Really engaging opening hook \u2013 very unique wording used here that captures the buyers' attention\n2- Good use of search engine optimization (SEO) keywords, for example using the address within the description, as the address is often what a buyer will search\n3- Use of too many short forms may make it harder for buyers to understand and read. It\u2019s often better to improve the quality of the content vs maximizing the quantity \u2013 in this case it may have made more sense to remove some pieces and reduce the use of acronyms"}]}
-{"id": "user_oriented_task_214", "motivation_app": "Redfin", "instruction": "List some of the top real estate marketing words to add value to the listing and engage more potential buyers.", "instances": [{"input": "", "output": "1. Luxurious\n2. New\n3. Spacious\n4. Captivating\n5. Move-In-Ready\n6. Landscaped\n7. Beautiful\n8. Character\n9. Custom\n10. Pet-Friendly"}]}
-{"id": "user_oriented_task_215", "motivation_app": "Amazon", "instruction": "Suggest some product bundles that are often purchased with a given product.", "instances": [{"input": "MacBook Air", "output": "MacBook Air, MacBook Air sleeve, MacBook Air case, MacBook Air external drive"}]}
-{"id": "user_oriented_task_216", "motivation_app": "Jira", "instruction": "Create a template in markdown to create scope for Jira tickets. Members should use this template as a checklist to ensure they have included all the necessary information when creating a ticket.", "instances": [{"input": "", "output": "# TO SCOPE THIS TICKET:\n[] create a descriptive title\n[] complete TBDs below\n[] assign epic to this ticket\n[] create relevant linking relations\n[] delete this section\n# DELETE ME AND ABOVE!\n\n## \ud83e\uddd1 Story\nWe need to TBDTASK from TBDRESOURCE in order for TBDUSER to TBDACTION\n\n## \ud83d\udd28 Acceptance Criteria\n- AC1\n- AC2\n\n## \ud83d\udcda Resources\n- [PLANNINGDOC1](WWWDOTEXAMPLEDOTCOM)\n- [FIGMADOC1](WWWDOTEXAMPLEDOTCOM)\n- [TECHDOCUMENTATION1](WWWDOTEXAMPLEDOTCOM)\n- [TECHDOCUMENTATION2](WWWDOTEXAMPLEDOTCOM)"}]}
-{"id": "user_oriented_task_217", "motivation_app": "Jira", "instruction": "As part of software engineering, user stories describe what the user wants the system to do, while acceptance criteria describe the conditions a specific user story must meet. Create basic acceptance criteria based on the given user story.", "instances": [{"input": "User story: As a user, I want to use a search field to type a city, name, or street, so that I could find matching hotel options.", "output": "- The search field is placed on the top bar\n- Search starts once the user clicks \u201cSearch\u201d\n- The field contains a placeholder with a grey-colored text: \u201cWhere are you going?\u201d\n- The placeholder disappears once the user starts typing\n- Search is performed if a user types in a city, hotel name, street, or all combined\nSearch is in English, French, German, and Ukrainian\nThe user can\u2019t type more than 200 symbols\nThe search doesn\u2019t support special symbols (characters). If the user has typed a special symbol, show the warning message: \u201cSearch input cannot contain  special symbols.\u201d"}]}
-{"id": "user_oriented_task_218", "motivation_app": "Jira", "instruction": "A user story is an informal, general explanation of a software feature written from the perspective of the end user or customer. Write a user story for a given software.", "instances": [{"input": "Gmail", "output": "As a user, I want to be able to recover the password to my account, so that I will be able to access my account in case I forgot the password."}]}
-{"id": "user_oriented_task_219", "motivation_app": "Jira", "instruction": "Write a Jira ticket for the given task.", "instances": [{"input": "New Employee onboarding", "output": "Title: Please help our new employee get settled.\nPrepare a desk, chair, and computer for them to use.\nCreate their company account, email, and temporary password.\nGrant access to company documents."}]}
-{"id": "user_oriented_task_220", "motivation_app": "Google Docs", "instruction": "Paraphrase the given text in an academic style.", "instances": [{"input": "Lots of papers have been published on this topic.", "output": "A considerable amount of literature has been published on this topic."}]}
-{"id": "user_oriented_task_221", "motivation_app": "Google Sheet", "instruction": "Make a list of the pros and cons of the given decision.", "instances": [{"input": "Implementing a Remote Working Policy", "output": "Pros: \n- Allowing employees to work from home can reduce their stressful commutes and increase their productivity. \n- Remote working provides greater flexibility, particularly for working parents and caregivers who have other responsibilities. \n- It's possible to hire highly creative people who live further away if they don't have to come into the office every day.\n- People who aren't distracted by office noises and regular interruptions will perform better. \n- Team members' communication skills will improve, as they will learn to communicate more effectively using email and IM, for example. \n\nCons:\n- It may be more difficult for team members to build meaningful, productive, creative relationships with one another.\n- The organization would have to budget for equipment that team members can take home.\n- The organization may not be able to keep track of how people spend their time. \n- The company could lose control of its data, because equipment and documents are removed from the office.\n- It will be harder to schedule face-to-face meetings and provide constructive feedback."}]}
-{"id": "user_oriented_task_222", "motivation_app": "Google Docs", "instruction": "Include important study notes and key points that someone should know about the given subject.", "instances": [{"input": "history of the USA", "output": "1. The United States of America was founded in 1776.\n2. The United States Constitution was ratified in 1788.\n3. The United States Declaration of Independence was signed in 1776.\n4. The United States Bill of Rights was ratified in 1791.\n5. The United States of America is a federal republic."}]}
-{"id": "user_oriented_task_223", "motivation_app": "Yelp", "instruction": "Based on the given keywords, write a review of a restaurant.", "instances": [{"input": "Name: Pizzeria\ngood prices, quiet place, tasty, nice staff", "output": "I had a great experience at Pizzeria. The prices were good, it was a quiet place, and the food was tasty. The staff was also very nice."}]}
-{"id": "user_oriented_task_224", "motivation_app": "Google Map", "instruction": "Convert natural language into a list of turn-by-turn directions.", "instances": [{"input": "Go west on I-10 until you hit the 101 then take it north. You will eventually see the Hollywood sign.", "output": "1. Go west on I-10.\n2. Take the 101 north.\n3. You will eventually see the Hollywood sign."}]}
-{"id": "user_oriented_task_225", "motivation_app": "Play Store", "instruction": "Provide an ESRB rating for the following game.", "instances": [{"input": "This is an action-adventure game in which players help Kratos and his son on a dangerous quest. Players explore various realms and engage in frenetic hand-to-hand combat against human-like raiders and fantastical creatures (e.g., centaurs, trolls, dragons). Players use axes and chained blades to battle enemies, often resulting in large blood-splatter effects and dismemberment. Players can perform finishing attacks that depict close-up impalement via bladed and hand-held weapons; repeated axe strikes to a creature's neck results in decapitation. The words \u201cf**k\u201d and \u201csh*t\u201d are heard in the game.", "output": "Mature 17+"}]}
-{"id": "user_oriented_task_226", "motivation_app": "MS Powerpoint", "instruction": "A color description has been provided. Find the CSS code associated with that color.", "instances": [{"input": "A light red color with a medium light shade of pink", "output": "#DB7093"}]}
-{"id": "user_oriented_task_227", "motivation_app": "Leetcode", "instruction": "Identify and fix bugs in the given code and rewrite it", "instances": [{"input": "for i in range(10)\n    print(Answer is:)\n    print(i)", "output": "for i in range(10):\n    print(\"Answer is:\")\n    print(i)"}]}
-{"id": "user_oriented_task_228", "motivation_app": "Google Docs", "instruction": "Make a list of common phrases for the given section of the paper.", "instances": [{"input": "Introduction", "output": "1- ... is an important component of ...\n2- Central to the discipline of ... is ...\n3- Recent developments in ... have shown the need for ...\n4- The study build on ... and contributes to ...\n5- Until now, has only been applied to ..."}]}
-{"id": "user_oriented_task_229", "motivation_app": "Netflix", "instruction": "We have described a scene from a movie that we have seen in the past. Find out what movie it is.", "instances": [{"input": "A choice has to be made between picking the red pill or the blue pill", "output": "Matrix"}]}
-{"id": "user_oriented_task_230", "motivation_app": "Facebook", "instruction": "Take the product description and write a creative ad for it.", "instances": [{"input": "Document Cameras allow teachers to display documents, books, or other materials for their students to see.", "output": "Looking for a way to make your lessons more engaging? Check out our document cameras! With our cameras, you can display documents, books, or other materials for your students to see. This is a great way to add more visual elements to your lessons and make them more exciting for your students."}]}
-{"id": "user_oriented_task_231", "motivation_app": "Google Sheets", "instruction": "Translate the natural language description into a Google Sheets formula.", "instances": [{"input": "if C7 is equal to \"U.S.A.\" or \"Canada\" or \"Europe\" or \"The Americas\"", "output": "IF (OR(C7 = \"U.S.A.\"; C7 = \"Canada\"); \"The Americas\"; \"Europe\")"}]}
-{"id": "user_oriented_task_232", "motivation_app": "Twitter", "instruction": "You are given a tweet and you should decide whether it's offensive or not.", "instances": [{"input": "She has certainly taken some heat for being such an....well idiot.", "output": "Offensive"}]}
-{"id": "user_oriented_task_233", "motivation_app": "Workout", "instruction": "For a quick and efficient office workout, suggest a short routine.", "instances": [{"input": "", "output": "Desk pushups: 20 reps.\nSeated knee-to-chest: 10 reps.\nSquats: 10 reps.\nDesk plank: 30 seconds.\nSeated flutter kicks: 10 reps on each side.\nRomanian deadlifts: 10 reps.\nJumping jacks: 20 reps."}]}
-{"id": "user_oriented_task_234", "motivation_app": "Yelp", "instruction": "Predict how many stars the author will give to the restaurant from a Yelp review.", "instances": [{"input": "The reviews were great, but honestly i felt it was just ok. Seemed like a typical tourist spot were the food was mediocre. The service was ok, not great, but not the worst.\nThe bread was very good. I ordered the muscle fries. They had a sweet Vidalia onion flavor. No need for fries bc they were soggy wo much flavor. My boyfriend ordered a pesto chicken pasta and it was again mediocre.", "output": "3 stars"}]}
-{"id": "user_oriented_task_235", "motivation_app": "Yelp", "instruction": "Categorize the Business into one of the Restaurants, Home Services, Auto Services and Miscellaneous based on its Specialties.", "instances": [{"input": "Call us at 650-636-4884 or visit our website to receive a quote. This shop specializes in New Tires and General Auto Repair. We carry all tires in-house and have a wide range to fit any budget or car specifics. If you are unsure what tires you need there are experts on hand and available to help you pick the best tire for your needs. We also carry commercial vehicle tires and can serve a wide array of fleets.", "output": "Auto Services"}]}
-{"id": "user_oriented_task_236", "motivation_app": "Grammarly", "instruction": "Notify me of any suggestions you may have for making the text more grammatically correct.", "instances": [{"input": "This is the second time I've been here, and the vibes, and the food were exquisite.", "output": "It appears that you have an unnecessary comma in a compound subject. Consider removing it."}]}
-{"id": "user_oriented_task_237", "motivation_app": "Tasty", "instruction": "Using the ingredients that are available, create a dinner recipe.", "instances": [{"input": "Kale + almond butter + apple cider vinegar + agave nectar + tamari", "output": "Gently pat the kale leaves dry.\nDe-stem the kale and tear into pieces of desired size. Add the kale to a large bowl.\nAdd the almond butter, apple cider vinegar, liquid aminos, and agave nectar to a small bowl and whisk together for 30 seconds.\nPour the dressing over the kale leaves, roll up your sleeves, and massage the kale with your hands for 2-3 minutes. The more you massage the kale, the more tender and flavorful the leaves will become.*\nTransfer the kale to one large or two small salad bowls and top with desired garnishes."}]}
-{"id": "user_oriented_task_238", "motivation_app": "Amazon", "instruction": "Categorize the given product into one of Electronics, Computers, Smart Home, or Arts & Crafts departments.", "instances": [{"input": "Google Nest Learning Thermostat", "output": "Smart Home"}]}
-{"id": "user_oriented_task_239", "motivation_app": "Workout", "instruction": "Provide instructions for the given exercise.", "instances": [{"input": "Leg Raises", "output": "To start toning your abs by hitting up the lower abdominal muscles, Riggins suggests 30 seconds of leg raises. Here's how: Turn on your back with legs straight and your feet and ankles together. Raise your legs up and down in a vertical position from your body, while keeping your belly button. Slowly bring legs back down, but if that is too difficult, tuck your knees. (And be careful not to strain your lower back!)"}]}
-{"id": "user_oriented_task_240", "motivation_app": "Grammarly", "instruction": "Rewrite the text and correct the spelling errors.", "instances": [{"input": "It solves problems comon and uniqe to every team.", "output": "It solves problems common and unique to every team."}]}
-{"id": "user_oriented_task_241", "motivation_app": "merriam-webster.com", "instruction": "Write down antonyms for the given word.", "instances": [{"input": "laureating", "output": "discrediting, humiliating, dishonoring, shaming"}]}
-{"id": "user_oriented_task_242", "motivation_app": "w3schools", "instruction": "Write the HTML syntax for the required action.", "instances": [{"input": "adding an image to the page", "output": "<img src=\"url\" alt=\"alternatetext\">"}]}
-{"id": "user_oriented_task_243", "motivation_app": "w3schools", "instruction": "Identify the programming language used to write the given code.", "instances": [{"input": "if (20 > 18) {\n  printf(\"20 is greater than 18\");\n}", "output": "C"}]}
-{"id": "user_oriented_task_244", "motivation_app": "merriam-webster.com", "instruction": "Decide which part of speech the underlined word belongs to.", "instances": [{"input": "school will _keep through the winter", "output": "verb"}]}
-{"id": "user_oriented_task_245", "motivation_app": "merriam-webster.com", "instruction": "Define what the underlined word means for kids.", "instances": [{"input": "_keep a promise", "output": "to be faithful to : Fulfill"}]}
-{"id": "user_oriented_task_246", "motivation_app": "Grammarly", "instruction": "Expand the given sentence and give it more details and depth.", "instances": [{"input": "It changed my life.", "output": "The experience changed the course of my life forever."}]}
-{"id": "user_oriented_task_247", "motivation_app": "Airbnb", "instruction": "Extract the address of the property from the \"About this space\" section of Airbnb.", "instances": [{"input": "Tucked in the foothills of the quaint historic mining town of Grass Valley, CA this funky and spacious chalet invites you to experience the great outdoors with family, friends and pets. Enjoy the hot tub, basketball court, treehouse, kids rooms, bbq, fire-pit, outdoor theater, and more. One hour from Tahoe and 10 minutes to town, you\u2019re never short of things to explore.", "output": "Grass Valley, California, United States"}]}
-{"id": "user_oriented_task_248", "motivation_app": "Strava", "instruction": "Design a simple training challenge on Strava that everyone could attend.", "instances": [{"input": "", "output": "This month, push yourself to log 30 hours of whatever activities get you stoked to move. Carve out time for yourself and your health and get out there for less than one hour a day \u2013 or go long on the weekends to mix it up. It doesn\u2019t always have to be your most all-out effort, yoga or walks count for just as much. Commit to consistency to earn this time-intensive trophy."}]}
-{"id": "user_oriented_task_249", "motivation_app": "Telegram", "instruction": "Suggest some names for a friendly group in telegram.", "instances": [{"input": "", "output": "1. The Conversation Club \n2. The Chatterbox \n3. F.R.I.E.N.D.S \n4. The Chatty Collective \n5. The Tea Talkers \n6. The Support Squad \n7. The Cheerful Chums"}]}
-{"id": "user_oriented_task_250", "motivation_app": "Google Search", "instruction": "Write what the pronunciation of the given word sounds like. Follow the \"Google pronunciation dictionary\" scheme for phonetic spelling.", "instances": [{"input": "interpretations", "output": "in\u00b7tr\u00b7pruh\u00b7tay\u00b7shnz"}]}
-{"id": "user_oriented_task_251", "motivation_app": "Reddit", "instruction": "Explain the meaning of the given phrase in simple terms. Use an example if possible. It would be helpful if you could give an example.", "instances": [{"input": "\"With a little give in them\"", "output": "It means with some space to move. It's a metaphor. \"Give\" is a noun here. It's meaning is similar to\"flexibility\". If something has no \"give\" it's tight and rigid. It's the opposite of flexible and loose. For example, you might use it to talk about clothes: \"These jeans are so tight. There's no give in them. I can't move.\""}]}

gpt4all-api/.gitignore

@@ -0,0 +1,112 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+app/__pycache__/
+gpt4all_api/__pycache__/
+gpt4all_api/app/api_v1/__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# VS Code
+.vscode/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+*.lock
+*.cache

gpt4all-api/.isort.cfg

@@ -0,0 +1,7 @@
+[settings]
+known_third_party=geopy,nltk,np,numpy,pandas,pysbd,fire,torch
+
+line_length=120
+include_trailing_comma=True
+multi_line_output=3
+use_parentheses=True

gpt4all-api/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2023 Nomic, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

gpt4all-api/README.md

@@ -0,0 +1,87 @@
+# GPT4All REST API
+This directory contains the source code to run and build docker images that run a FastAPI app
+for serving inference from GPT4All models. The API matches the OpenAI API spec.
+
+## Tutorial
+
+The following tutorial assumes that you have checked out this repo and cd'd into it.
+
+### Starting the app
+
+First change your working directory to `gpt4all/gpt4all-api`.
+
+Now you can build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
+```bash
+DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
+```
+
+Then, start the backend with:
+
+```bash
+docker compose up --build
+```
+
+This will run both the API and locally hosted GPU inference server. If you want to run the API without the GPU inference server, you can run:
+
+```bash
+docker compose up --build gpt4all_api
+```
+
+To run the API with the GPU inference server, you will need to include environment variables (like the `MODEL_ID`). Edit the `.env` file and run
+```bash
+docker compose --env-file .env up --build
+```
+
+
+#### Spinning up your app
+Run `docker compose up` to spin up the backend. Monitor the logs for errors in-case you forgot to set an environment variable above.
+
+
+#### Development
+Run
+
+```bash
+docker compose up --build
+```
+and edit files in the `api` directory. The api will hot-reload on changes.
+
+You can run the unit tests with
+
+```bash
+make test
+```
+
+#### Viewing API documentation
+
+Once the FastAPI ap is started you can access its documentation and test the search endpoint by going to:
+```
+localhost:80/docs
+```
+
+This documentation should match the OpenAI OpenAPI spec located at https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+#### Running inference
+```python
+import openai
+openai.api_base = "http://localhost:4891/v1"
+
+openai.api_key = "not needed for a local LLM"
+
+
+def test_completion():
+    model = "gpt4all-j-v1.3-groovy"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Completion.create(
+        model=model,
+        prompt=prompt,
+        max_tokens=50,
+        temperature=0.28,
+        top_p=0.95,
+        n=1,
+        echo=True,
+        stream=False
+    )
+    assert len(response['choices'][0]['text']) > len(prompt)
+    print(response)
+```

gpt4all-api/docker-compose.gpu.yaml

@@ -0,0 +1,24 @@
+version: "3.8"
+
+services:
+  gpt4all_gpu:
+    image: ghcr.io/huggingface/text-generation-inference:0.9.3
+    container_name: gpt4all_gpu
+    restart: always #restart on error (usually code compilation from save during bad state)
+    environment:
+      - HUGGING_FACE_HUB_TOKEN=token
+      - USE_FLASH_ATTENTION=false
+      - MODEL_ID=''
+      - NUM_SHARD=1
+    command: --model-id $MODEL_ID --num-shard $NUM_SHARD
+    volumes:
+      - ./:/data
+    ports:
+      - "8080:80"
+    shm_size: 1g
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: [gpu]

gpt4all-api/docker-compose.yaml

@@ -0,0 +1,19 @@
+version: "3.8"
+
+services:
+  gpt4all_api:
+    image: gpt4all_api
+    container_name: gpt4all_api
+    restart: always #restart on error (usually code compilation from save during bad state)
+    ports:
+      - "4891:4891"
+    environment:
+      - APP_ENVIRONMENT=dev
+      - WEB_CONCURRENCY=2
+      - LOGLEVEL=debug
+      - PORT=4891
+      - model=ggml-mpt-7b-chat.bin
+      - inference_mode=cpu
+    volumes:
+      - './gpt4all_api/app:/app'
+    command: ["/start-reload.sh"]

gpt4all-api/gpt4all_api/Dockerfile.buildkit

@@ -0,0 +1,23 @@
+# syntax=docker/dockerfile:1.0.0-experimental
+FROM tiangolo/uvicorn-gunicorn:python3.11
+
+ARG MODEL_BIN=ggml-mpt-7b-chat.bin
+
+# Put first so anytime this file changes other cached layers are invalidated.
+COPY gpt4all_api/requirements.txt /requirements.txt
+
+RUN pip install --upgrade pip
+
+# Run various pip install commands with ssh keys from host machine.
+RUN --mount=type=ssh pip install -r /requirements.txt && \
+  rm -Rf /root/.cache && rm -Rf /tmp/pip-install*
+
+# Finally, copy app and client.
+COPY gpt4all_api/app /app
+
+RUN mkdir -p /models
+
+# Include the following line to bake a model into the image and not have to download it on API start.
+RUN wget -q --show-progress=off https://gpt4all.io/models/${MODEL_BIN} -P /models \
+  && md5sum /models/${MODEL_BIN}
+

gpt4all-api/gpt4all_api/README.md

@@ -0,0 +1 @@
+# FastAPI app for serving GPT4All models

gpt4all-api/gpt4all_api/app/api_v1/api.py

@@ -0,0 +1,9 @@
+from api_v1.routes import chat, completions, engines, health
+from fastapi import APIRouter
+
+router = APIRouter()
+
+router.include_router(chat.router)
+router.include_router(completions.router)
+router.include_router(engines.router)
+router.include_router(health.router)

gpt4all-api/gpt4all_api/app/api_v1/events.py

@@ -0,0 +1,29 @@
+import logging
+
+from api_v1.settings import settings
+from fastapi import HTTPException
+from fastapi.responses import JSONResponse
+from starlette.requests import Request
+
+log = logging.getLogger(__name__)
+
+
+startup_msg_fmt = """
+ Starting up GPT4All API
+"""
+
+
+async def on_http_error(request: Request, exc: HTTPException):
+    return JSONResponse({'detail': exc.detail}, status_code=exc.status_code)
+
+
+async def on_startup(app):
+    startup_msg = startup_msg_fmt.format(settings=settings)
+    log.info(startup_msg)
+
+
+def startup_event_handler(app):
+    async def start_app() -> None:
+        await on_startup(app)
+
+    return start_app

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -0,0 +1,61 @@
+import logging
+import time
+from typing import Dict, List
+
+from api_v1.settings import settings
+from fastapi import APIRouter, Depends, Response, Security, status
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+class ChatCompletionMessage(BaseModel):
+    role: str
+    content: str
+
+
+class ChatCompletionRequest(BaseModel):
+    model: str = Field(..., description='The model to generate a completion from.')
+    messages: List[ChatCompletionMessage] = Field(..., description='The model to generate a completion from.')
+
+
+class ChatCompletionChoice(BaseModel):
+    message: ChatCompletionMessage
+    index: int
+    finish_reason: str
+
+
+class ChatCompletionUsage(BaseModel):
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+
+
+class ChatCompletionResponse(BaseModel):
+    id: str
+    object: str = 'text_completion'
+    created: int
+    model: str
+    choices: List[ChatCompletionChoice]
+    usage: ChatCompletionUsage
+
+
+router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
+
+
+@router.post("/completions", response_model=ChatCompletionResponse)
+async def chat_completion(request: ChatCompletionRequest):
+    '''
+    Completes a GPT4All model response.
+    '''
+
+    return ChatCompletionResponse(
+        id='asdf',
+        created=time.time(),
+        model=request.model,
+        choices=[{}],
+        usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+    )

gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py

@@ -0,0 +1,215 @@
+import json
+from typing import List, Dict, Iterable, AsyncIterable
+import logging
+import time
+from typing import Dict, List, Union, Optional
+from uuid import uuid4
+import aiohttp
+import asyncio
+from api_v1.settings import settings
+from fastapi import APIRouter, Depends, Response, Security, status, HTTPException
+from fastapi.responses import StreamingResponse
+from gpt4all import GPT4All
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+class CompletionRequest(BaseModel):
+    model: str = Field(settings.model, description='The model to generate a completion from.')
+    prompt: Union[List[str], str] = Field(..., description='The prompt to begin completing from.')
+    max_tokens: int = Field(None, description='Max tokens to generate')
+    temperature: float = Field(settings.temp, description='Model temperature')
+    top_p: Optional[float] = Field(settings.top_p, description='top_p')
+    top_k: Optional[int] = Field(settings.top_k, description='top_k')
+    n: int = Field(1, description='How many completions to generate for each prompt')
+    stream: bool = Field(False, description='Stream responses')
+    repeat_penalty: float = Field(settings.repeat_penalty, description='Repeat penalty')
+
+
+class CompletionChoice(BaseModel):
+    text: str
+    index: int
+    logprobs: float
+    finish_reason: str
+
+
+class CompletionUsage(BaseModel):
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+
+
+class CompletionResponse(BaseModel):
+    id: str
+    object: str = 'text_completion'
+    created: int
+    model: str
+    choices: List[CompletionChoice]
+    usage: CompletionUsage
+
+
+class CompletionStreamResponse(BaseModel):
+    id: str
+    object: str = 'text_completion'
+    created: int
+    model: str
+    choices: List[CompletionChoice]
+
+
+router = APIRouter(prefix="/completions", tags=["Completion Endpoints"])
+
+def stream_completion(output: Iterable, base_response: CompletionStreamResponse):
+    """
+    Streams a GPT4All output to the client.
+
+    Args:
+        output: The output of GPT4All.generate(), which is an iterable of tokens.
+        base_response: The base response object, which is cloned and modified for each token.
+
+    Returns:
+        A Generator of CompletionStreamResponse objects, which are serialized to JSON Event Stream format.
+    """
+    for token in output:
+        chunk = base_response.copy()
+        chunk.choices = [dict(CompletionChoice(
+            text=token,
+            index=0,
+            logprobs=-1,
+            finish_reason=''
+        ))]
+        yield f"data: {json.dumps(dict(chunk))}\n\n"
+
+async def gpu_infer(payload, header):
+    async with aiohttp.ClientSession() as session:
+        try:
+            async with session.post(
+                settings.hf_inference_server_host, headers=header, data=json.dumps(payload)
+            ) as response:
+                resp = await response.json()
+            return resp
+
+        except aiohttp.ClientError as e:
+            # Handle client-side errors (e.g., connection error, invalid URL)
+            logger.error(f"Client error: {e}")
+        except aiohttp.ServerError as e:
+            # Handle server-side errors (e.g., internal server error)
+            logger.error(f"Server error: {e}")
+        except json.JSONDecodeError as e:
+            # Handle JSON decoding errors
+            logger.error(f"JSON decoding error: {e}")
+        except Exception as e:
+            # Handle other unexpected exceptions
+            logger.error(f"Unexpected error: {e}")
+
+@router.post("/", response_model=CompletionResponse)
+async def completions(request: CompletionRequest):
+    '''
+    Completes a GPT4All model response.
+    '''
+    if settings.inference_mode == "gpu":
+        params = request.dict(exclude={'model', 'prompt', 'max_tokens', 'n'})
+        params["max_new_tokens"] = request.max_tokens
+        params["num_return_sequences"] = request.n
+
+        header = {"Content-Type": "application/json"}
+        if isinstance(request.prompt, list):
+            tasks = []
+            for prompt in request.prompt:
+                payload = {"parameters": params}
+                payload["inputs"] = prompt
+                task = gpu_infer(payload, header)
+                tasks.append(task)
+            results = await asyncio.gather(*tasks)
+
+            choices = []
+            for response in results:
+                scores = response["scores"] if "scores" in response else -1.0
+                choices.append(
+                    dict(
+                        CompletionChoice(
+                            text=response["generated_text"], index=0, logprobs=scores, finish_reason='stop'
+                        )
+                    )
+                )
+
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=choices,
+                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+            )
+
+        else:
+            payload = {"parameters": params}
+            # If streaming, we need to return a StreamingResponse
+            payload["inputs"] = request.prompt
+
+            resp = await gpu_infer(payload, header)
+
+            output = resp["generated_text"]
+            # this returns all logprobs
+            scores = resp["scores"] if "scores" in resp else -1.0
+
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[dict(CompletionChoice(text=output, index=0, logprobs=scores, finish_reason='stop'))],
+                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+            )
+
+    else:
+
+        if request.model != settings.model:
+            raise HTTPException(status_code=400,
+                                detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
+
+        if isinstance(request.prompt, list):
+            if len(request.prompt) > 1:
+                raise HTTPException(status_code=400, detail="Can only infer one inference per request in CPU mode.")
+            else:
+                request.prompt = request.prompt[0]
+
+        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+        output = model.generate(prompt=request.prompt,
+                                max_tokens=request.max_tokens,
+                                streaming=request.stream,
+                                top_k=request.top_k,
+                                top_p=request.top_p,
+                                temp=request.temperature,
+                                )
+
+        # If streaming, we need to return a StreamingResponse
+        if request.stream:
+            base_chunk = CompletionStreamResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[]
+            )
+            return StreamingResponse((response for response in stream_completion(output, base_chunk)),
+                                     media_type="text/event-stream")
+        else:
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[dict(CompletionChoice(
+                    text=output,
+                    index=0,
+                    logprobs=-1,
+                    finish_reason='stop'
+                ))],
+                usage={
+                    'prompt_tokens': 0,  # TODO how to compute this?
+                    'completion_tokens': 0,
+                    'total_tokens': 0
+                }
+            )

gpt4all-api/gpt4all_api/app/api_v1/routes/embeddings.py

@@ -0,0 +1,65 @@
+from typing import List, Union
+from fastapi import APIRouter
+from api_v1.settings import settings
+from gpt4all import Embed4All
+from pydantic import BaseModel, Field
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+class EmbeddingRequest(BaseModel):
+    model: str = Field(
+        settings.model, description="The model to generate an embedding from."
+    )
+    input: Union[str, List[str], List[int], List[List[int]]] = Field(
+        ..., description="Input text to embed, encoded as a string or array of tokens."
+    )
+
+
+class EmbeddingUsage(BaseModel):
+    prompt_tokens: int = 0
+    total_tokens: int = 0
+
+
+class Embedding(BaseModel):
+    index: int = 0
+    object: str = "embedding"
+    embedding: List[float]
+
+
+class EmbeddingResponse(BaseModel):
+    object: str = "list"
+    model: str
+    data: List[Embedding]
+    usage: EmbeddingUsage
+
+
+router = APIRouter(prefix="/embeddings", tags=["Embedding Endpoints"])
+
+embedder = Embed4All()
+
+
+def get_embedding(data: EmbeddingRequest) -> EmbeddingResponse:
+    """
+    Calculates the embedding for the given input using a specified model.
+
+    Args:
+        data (EmbeddingRequest): An EmbeddingRequest object containing the input data
+        and model name.
+
+    Returns:
+        EmbeddingResponse: An EmbeddingResponse object encapsulating the calculated embedding,
+        usage info, and the model name.
+    """
+    embedding = embedder.embed(data.input)
+    return EmbeddingResponse(
+        data=[Embedding(embedding=embedding)], usage=EmbeddingUsage(), model=data.model
+    )
+
+
+@router.post("/", response_model=EmbeddingResponse)
+def embeddings(data: EmbeddingRequest):
+    """
+    Creates a GPT4All embedding
+    """
+    return get_embedding(data)

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -0,0 +1,40 @@
+import logging
+from typing import Dict, List
+
+from api_v1.settings import settings
+from fastapi import APIRouter, Depends, Response, Security, status
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+class ListEnginesResponse(BaseModel):
+    data: List[Dict] = Field(..., description="All available models.")
+
+
+class EngineResponse(BaseModel):
+    data: List[Dict] = Field(..., description="All available models.")
+
+
+router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
+
+
+@router.get("/", response_model=ListEnginesResponse)
+async def list_engines():
+    '''
+    List all available GPT4All models from
+    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json
+    '''
+    raise NotImplementedError()
+    return ListEnginesResponse(data=[])
+
+
+@router.get("/{engine_id}", response_model=EngineResponse)
+async def retrieve_engine(engine_id: str):
+    ''' '''
+
+    raise NotImplementedError()
+    return EngineResponse()

gpt4all-api/gpt4all_api/app/api_v1/routes/health.py

@@ -0,0 +1,13 @@
+import logging
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+
+log = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/health", tags=["Health"])
+
+
+@router.get('/', response_class=JSONResponse)
+async def health_check():
+    """Runs a health check on this instance of the API."""
+    return JSONResponse({'status': 'ok'}, headers={'Access-Control-Allow-Origin': '*'})

gpt4all-api/gpt4all_api/app/api_v1/settings.py

@@ -0,0 +1,19 @@
+from pydantic import BaseSettings
+
+
+class Settings(BaseSettings):
+    app_environment = 'dev'
+    model: str = 'ggml-mpt-7b-chat.bin'
+    gpt4all_path: str = '/models'
+    inference_mode: str = "cpu"
+    hf_inference_server_host: str = "http://gpt4all_gpu:80/generate"
+    sentry_dns: str = None
+
+    temp: float = 0.18
+    top_p: float = 1.0
+    top_k: int = 50
+    repeat_penalty: float = 1.18
+
+
+
+settings = Settings()

gpt4all-api/gpt4all_api/app/docs.py

@@ -0,0 +1,3 @@
+desc = 'GPT4All API'
+
+endpoint_paths = {'health': '/health'}

gpt4all-api/gpt4all_api/app/main.py

@@ -0,0 +1,84 @@
+import logging
+import os
+
+import docs
+from api_v1 import events
+from api_v1.api import router as v1_router
+from api_v1.settings import settings
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.logger import logger as fastapi_logger
+from starlette.middleware.cors import CORSMiddleware
+
+logger = logging.getLogger(__name__)
+
+app = FastAPI(title='GPT4All API', description=docs.desc)
+
+# CORS Configuration (in-case you want to deploy)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["GET", "POST", "OPTIONS"],
+    allow_headers=["*"],
+)
+
+logger.info('Adding v1 endpoints..')
+
+# add v1
+app.include_router(v1_router, prefix='/v1')
+app.add_event_handler('startup', events.startup_event_handler(app))
+app.add_exception_handler(HTTPException, events.on_http_error)
+
+
+@app.on_event("startup")
+async def startup():
+    global model
+    if settings.inference_mode == "cpu":
+        logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
+        from gpt4all import GPT4All
+
+        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
+
+    else:
+        # is it possible to do this once the server is up?
+        ## TODO block until HF inference server is up.
+        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
+
+
+
+@app.on_event("shutdown")
+async def shutdown():
+    logger.info("Shutting down API")
+
+
+if settings.sentry_dns is not None:
+    import sentry_sdk
+
+    def traces_sampler(sampling_context):
+        if 'health' in sampling_context['transaction_context']['name']:
+            return False
+
+    sentry_sdk.init(
+        dsn=settings.sentry_dns, traces_sample_rate=0.1, traces_sampler=traces_sampler, send_default_pii=False
+    )
+
+# This is needed to get logs to show up in the app
+if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
+    gunicorn_error_logger = logging.getLogger("gunicorn.error")
+    gunicorn_logger = logging.getLogger("gunicorn")
+
+    root_logger = logging.getLogger()
+    fastapi_logger.setLevel(gunicorn_logger.level)
+    fastapi_logger.handlers = gunicorn_error_logger.handlers
+    root_logger.setLevel(gunicorn_logger.level)
+
+    uvicorn_logger = logging.getLogger("uvicorn.access")
+    uvicorn_logger.handlers = gunicorn_error_logger.handlers
+else:
+    # https://github.com/tiangolo/fastapi/issues/2019
+    LOG_FORMAT2 = (
+        "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
+    )
+    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -0,0 +1,59 @@
+"""
+Use the OpenAI python API to test gpt4all models.
+"""
+from typing import List, get_args
+
+import openai
+
+openai.api_base = "http://localhost:4891/v1"
+
+openai.api_key = "not needed for a local LLM"
+
+
+def test_completion():
+    model = "ggml-mpt-7b-chat.bin"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Completion.create(
+        model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
+    )
+    assert len(response['choices'][0]['text']) > len(prompt)
+
+def test_streaming_completion():
+    model = "ggml-mpt-7b-chat.bin"
+    prompt = "Who is Michael Jordan?"
+    tokens = []
+    for resp in openai.Completion.create(
+            model=model,
+            prompt=prompt,
+            max_tokens=50,
+            temperature=0.28,
+            top_p=0.95,
+            n=1,
+            echo=True,
+            stream=True):
+        tokens.append(resp.choices[0].text)
+
+    assert (len(tokens) > 0)
+    assert (len("".join(tokens)) > len(prompt))
+
+
+def test_batched_completion():
+    model = "ggml-mpt-7b-chat.bin"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Completion.create(
+        model=model, prompt=[prompt] * 3, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
+    )
+    assert len(response['choices'][0]['text']) > len(prompt)
+    assert len(response['choices']) == 3
+
+
+def test_embedding():
+    model = "ggml-all-MiniLM-L6-v2-f16.bin"
+    prompt = "Who is Michael Jordan?"
+    response = openai.Embedding.create(model=model, input=prompt)
+    output = response["data"][0]["embedding"]
+    args = get_args(List[float])
+
+    assert response["model"] == model
+    assert isinstance(output, list)
+    assert all(isinstance(x, args) for x in output)

gpt4all-api/gpt4all_api/requirements.txt

@@ -0,0 +1,12 @@
+aiohttp>=3.6.2
+aiofiles
+pydantic>=1.4.0,<2.0.0
+requests>=2.24.0
+ujson>=2.0.2
+fastapi>=0.95.0
+Jinja2>=3.0
+gpt4all>=1.0.0
+pytest
+openai
+black
+isort

gpt4all-api/makefile

@@ -0,0 +1,46 @@
+ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+APP_NAME:=gpt4all_api
+PYTHON:=python3.8
+SHELL := /bin/bash
+
+all: dependencies
+
+fresh: clean dependencies
+
+testenv: clean_testenv test_build
+	docker compose -f docker-compose.yaml up --build
+
+testenv_gpu: clean_testenv test_build
+	docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up --build
+
+testenv_d: clean_testenv test_build
+	docker compose up --build -d
+
+test:
+	docker compose exec $(APP_NAME) pytest -svv --disable-warnings -p no:cacheprovider /app/tests
+
+test_build:
+    DOCKER_BUILDKIT=1 docker build -t $(APP_NAME) --progress plain -f $(APP_NAME)/Dockerfile.buildkit .
+
+clean_testenv:
+	docker compose down -v
+
+fresh_testenv: clean_testenv testenv
+
+venv:
+	if [ ! -d $(ROOT_DIR)/env ]; then $(PYTHON) -m venv $(ROOT_DIR)/env; fi
+
+dependencies: venv
+	source $(ROOT_DIR)/env/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
+
+clean: clean_testenv
+	# Remove existing environment
+	rm -rf $(ROOT_DIR)/env;
+	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
+
+
+black:
+	source $(ROOT_DIR)/env/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
+
+isort:
+	source $(ROOT_DIR)/env/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)

gpt4all-backend/CMakeLists.txt

@@ -0,0 +1,146 @@
+cmake_minimum_required(VERSION 3.16)
+set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+if(APPLE)
+  option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
+  if(BUILD_UNIVERSAL)
+    # Build a Universal binary on macOS
+    # This requires that the found Qt library is compiled as Universal binaries.
+    set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
+  else()
+    # Build for the host architecture on macOS
+    if(NOT CMAKE_OSX_ARCHITECTURES)
+      set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
+    endif()
+  endif()
+endif()
+
+# Include the binary directory for the generated header file
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+
+set(LLMODEL_VERSION_MAJOR 0)
+set(LLMODEL_VERSION_MINOR 4)
+set(LLMODEL_VERSION_PATCH 0)
+set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
+project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
+set(BUILD_SHARED_LIBS ON)
+
+# Check for IPO support
+include(CheckIPOSupported)
+check_ipo_supported(RESULT IPO_SUPPORTED OUTPUT IPO_ERROR)
+if (NOT IPO_SUPPORTED)
+    message(WARNING "Interprocedural optimization is not supported by your toolchain! This will lead to bigger file sizes and worse performance: ${IPO_ERROR}")
+else()
+    message(STATUS "Interprocedural optimization support detected")
+endif()
+
+if(NOT APPLE)
+  set(LLAMA_KOMPUTE YES)
+endif()
+
+include(llama.cpp.cmake)
+
+set(BUILD_VARIANTS default avxonly)
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    set(BUILD_VARIANTS ${BUILD_VARIANTS} metal)
+endif()
+
+set(CMAKE_VERBOSE_MAKEFILE ON)
+
+# Go through each build variant
+foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
+    # Determine flags
+    if (BUILD_VARIANT STREQUAL avxonly)
+        set(GPT4ALL_ALLOW_NON_AVX NO)
+    else()
+        set(GPT4ALL_ALLOW_NON_AVX YES)
+    endif()
+    set(LLAMA_AVX2 ${GPT4ALL_ALLOW_NON_AVX})
+    set(LLAMA_F16C ${GPT4ALL_ALLOW_NON_AVX})
+    set(LLAMA_FMA  ${GPT4ALL_ALLOW_NON_AVX})
+
+    if (BUILD_VARIANT STREQUAL metal)
+        set(LLAMA_METAL YES)
+    else()
+        set(LLAMA_METAL NO)
+    endif()
+
+    # Include GGML
+    set(LLAMA_K_QUANTS YES)
+    include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
+
+    # Function for preparing individual implementations
+    function(prepare_target TARGET_NAME BASE_LIB)
+        set(TARGET_NAME ${TARGET_NAME}-${BUILD_VARIANT})
+        message(STATUS "Configuring model implementation target ${TARGET_NAME}")
+        # Link to ggml/llama
+        target_link_libraries(${TARGET_NAME}
+            PRIVATE ${BASE_LIB}-${BUILD_VARIANT})
+        # Let it know about its build variant
+        target_compile_definitions(${TARGET_NAME}
+            PRIVATE GGML_BUILD_VARIANT="${BUILD_VARIANT}")
+        # Enable IPO if possible
+# FIXME: Doesn't work with msvc reliably. See https://github.com/nomic-ai/gpt4all/issues/841
+#        set_property(TARGET ${TARGET_NAME}
+#                     PROPERTY INTERPROCEDURAL_OPTIMIZATION ${IPO_SUPPORTED})
+    endfunction()
+
+    # Add each individual implementations
+    add_library(llamamodel-mainline-${BUILD_VARIANT} SHARED
+        llamamodel.cpp llmodel_shared.cpp)
+    target_compile_definitions(llamamodel-mainline-${BUILD_VARIANT} PRIVATE
+        LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+    prepare_target(llamamodel-mainline llama-mainline)
+
+    add_library(replit-mainline-${BUILD_VARIANT} SHARED
+    replit.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+    target_compile_definitions(replit-mainline-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+    prepare_target(replit-mainline llama-mainline)
+
+    if (NOT LLAMA_METAL)
+# FIXME: These need to be forward ported to latest ggml
+#        add_library(gptj-${BUILD_VARIANT} SHARED
+#            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+#        prepare_target(gptj ggml-230511)
+
+        add_library(falcon-${BUILD_VARIANT} SHARED
+            falcon.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(falcon-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+        prepare_target(falcon llama-mainline)
+# FIXME: These need to be forward ported to latest ggml
+#        add_library(mpt-${BUILD_VARIANT} SHARED
+#            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+#        prepare_target(mpt ggml-230511)
+
+        add_library(bert-${BUILD_VARIANT} SHARED
+            bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+        prepare_target(bert llama-mainline)
+
+        add_library(starcoder-${BUILD_VARIANT} SHARED
+            starcoder.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(starcoder-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+        prepare_target(starcoder llama-mainline)
+    endif()
+endforeach()
+
+add_library(llmodel
+    llmodel.h llmodel.cpp llmodel_shared.cpp
+    llmodel_c.h llmodel_c.cpp
+    dlhandle.h
+)
+target_link_libraries(llmodel PRIVATE ggml-mainline-default)
+target_compile_definitions(llmodel PRIVATE GGML_BUILD_VARIANT="default")
+target_compile_definitions(llmodel PRIVATE LIB_FILE_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}")
+
+set_target_properties(llmodel PROPERTIES
+                              VERSION ${PROJECT_VERSION}
+                              SOVERSION ${PROJECT_VERSION_MAJOR})
+
+set(COMPONENT_NAME_MAIN ${PROJECT_NAME})
+set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/install)

gpt4all-backend/README.md

@@ -0,0 +1,42 @@
+# GPT4ALL Backend
+This directory contains the C/C++ model backend used by GPT4All for inference on the CPU. This backend acts as a universal library/wrapper for all models that the GPT4All ecosystem supports. Language bindings are built on top of this universal library. The native GPT4all Chat application directly uses this library for all inference.
+
+# What models are supported by the GPT4All ecosystem?
+
+Currently, there are three different model architectures that are supported:
+
+1. GPTJ - Based off of the GPT-J architecture with examples found [here](https://huggingface.co/EleutherAI/gpt-j-6b)
+2. LLAMA - Based off of the LLAMA architecture with examples found [here](https://huggingface.co/models?sort=downloads&search=llama)
+3. MPT - Based off of Mosaic ML's MPT architecture with examples found [here](https://huggingface.co/mosaicml/mpt-7b)
+
+# Why so many different architectures? What differentiates them?
+
+One of the major differences is license. Currently, the LLAMA based models are subject to a non-commercial license, whereas the GPTJ and MPT base models allow commercial usage. In the early advent of the recent explosion of activity in open source local models, the llama models have generally been seen as performing better, but that is changing quickly. Every week - even every day! - new models are released with some of the GPTJ and MPT models competitive in performance/quality with LLAMA. What's more, there are some very nice architectural innovations with the MPT models that could lead to new performance/quality gains.
+
+# How does GPT4All make these models available for CPU inference?
+
+By leveraging the ggml library written by Georgi Gerganov and a growing community of developers. There are currently multiple different versions of this library. The original github repo can be found [here](https://github.com/ggerganov/ggml), but the developer of the library has also created a LLAMA based version [here](https://github.com/ggerganov/llama.cpp). Currently, this backend is using the latter as a submodule.
+
+# Does that mean GPT4All is compatible with all llama.cpp models and vice versa?
+
+Unfortunately, no for three reasons:
+
+1. The upstream [llama.cpp](https://github.com/ggerganov/llama.cpp) project has introduced [a compatibility breaking](https://github.com/ggerganov/llama.cpp/commit/b9fd7eee57df101d4a3e3eabc9fd6c2cb13c9ca1) re-quantization method recently. This is a breaking change that renders all previous models (including the ones that GPT4All uses) inoperative with newer versions of llama.cpp since that change.
+2. The GPT4All backend has the llama.cpp submodule specifically pinned to a version prior to this breaking change.
+3. The GPT4All backend currently supports MPT based models as an added feature. Neither llama.cpp nor the original ggml repo support this architecture as of this writing, however efforts are underway to make MPT available in the ggml repo which you can follow [here.](https://github.com/ggerganov/ggml/pull/145)
+
+# What is being done to make them more compatible?
+
+A few things. Number one, we are maintaining compatibility with our current model zoo by way of the submodule pinning. However, we are also exploring how we can update to newer versions of llama.cpp without breaking our current models. This might involve an additional magic header check or it could possibly involve keeping the currently pinned submodule and also adding a new submodule with later changes and differienting them with namespaces or some other manner. Investigations continue.
+
+# What about GPU inference?
+
+In newer versions of llama.cpp, there has been some added support for NVIDIA GPU's for inference. We're investigating how to incorporate this into our downloadable installers.
+
+# Ok, so bottom line... how do I make my model on Hugging Face compatible with GPT4All ecosystem right now?
+
+1. Check to make sure the Hugging Face model is available in one of our three supported architectures
+2. If it is, then you can use the conversion script inside of our pinned llama.cpp submodule for GPTJ and LLAMA based models
+3. Or if your model is an MPT model you can use the conversion script located directly in this backend directory under the scripts subdirectory 
+
+# Check back for updates as we'll try to keep this updated as things change!

gpt4all-backend/bert_impl.h

@@ -0,0 +1,44 @@
+#ifndef BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of bert.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef BERT_H
+#define BERT_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <memory>
+#include "llmodel.h"
+
+struct BertPrivate;
+class Bert : public LLModel {
+public:
+    Bert();
+    ~Bert();
+
+    bool supportsEmbedding() const override { return true; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+    std::vector<float> embedding(const std::string &text) override;
+
+private:
+    std::unique_ptr<BertPrivate> d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // BERT_H

gpt4all-backend/dlhandle.h

@@ -0,0 +1,104 @@
+#ifndef DLHANDLE_H
+#define DLHANDLE_H
+#ifndef _WIN32
+#include <string>
+#include <stdexcept>
+#include <utility>
+#include <dlfcn.h>
+
+
+
+class Dlhandle {
+    void *chandle;
+
+public:
+    class Exception : public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
+    Dlhandle() : chandle(nullptr) {}
+    Dlhandle(const std::string& fpath, int flags = RTLD_LAZY | RTLD_LOCAL) {
+        chandle = dlopen(fpath.c_str(), flags);
+        if (!chandle) {
+            throw Exception("dlopen(\""+fpath+"\"): "+dlerror());
+        }
+    }
+    Dlhandle(const Dlhandle& o) = delete;
+    Dlhandle(Dlhandle&& o) : chandle(o.chandle) {
+        o.chandle = nullptr;
+    }
+    ~Dlhandle() {
+        if (chandle) dlclose(chandle);
+    }
+
+    auto operator =(Dlhandle&& o) {
+        chandle = std::exchange(o.chandle, nullptr);
+    }
+
+    bool is_valid() const {
+        return chandle != nullptr;
+    }
+    operator bool() const {
+        return is_valid();
+    }
+
+    template<typename T>
+    T* get(const std::string& fname) const {
+        auto fres = reinterpret_cast<T*>(dlsym(chandle, fname.c_str()));
+        return (dlerror()==NULL)?fres:nullptr;
+    }
+    auto get_fnc(const std::string& fname) const {
+        return get<void*(...)>(fname);
+    }
+};
+#else
+#include <string>
+#include <exception>
+#include <stdexcept>
+#ifndef NOMINMAX
+    #define NOMINMAX
+#endif
+#include <windows.h>
+#include <libloaderapi.h>
+
+
+
+class Dlhandle {
+    HMODULE chandle;
+
+public:
+    class Exception : public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
+    Dlhandle() : chandle(nullptr) {}
+    Dlhandle(const std::string& fpath) {
+        chandle = LoadLibraryExA(fpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
+        if (!chandle) {
+            throw Exception("dlopen(\""+fpath+"\"): Error");
+        }
+    }
+    Dlhandle(const Dlhandle& o) = delete;
+    Dlhandle(Dlhandle&& o) : chandle(o.chandle) {
+        o.chandle = nullptr;
+    }
+    ~Dlhandle() {
+        if (chandle) FreeLibrary(chandle);
+    }
+
+    bool is_valid() const {
+        return chandle != nullptr;
+    }
+
+    template<typename T>
+    T* get(const std::string& fname) const {
+        return reinterpret_cast<T*>(GetProcAddress(chandle, fname.c_str()));
+    }
+    auto get_fnc(const std::string& fname) const {
+        return get<void*(...)>(fname);
+    }
+};
+#endif
+#endif // DLHANDLE_H

gpt4all-backend/falcon.cpp

@@ -0,0 +1,985 @@
+#include "ggml.h"
+#define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#include "falcon_impl.h"
+#include "llama.h"
+#include "llama-util.h"
+#include "utils.h"
+#include "llmodel_shared.h"
+
+#include <cassert>
+#include <cinttypes>
+#include <iostream>
+#include <sstream>
+
+namespace {
+const char *modelType_ = "Falcon";
+}
+
+// commented out 40B support as it presently would require forking ggml/llama.cpp
+// can re-add once mainline ggml supports it
+
+#define FALCON_MAGIC 0x67676a74
+
+// default hparams (Falcon 7B)
+struct falcon_hparams {
+    int32_t n_vocab = 65024;
+    int32_t n_embd  = 4544;
+    int32_t n_head  = 71;
+    int32_t n_head_kv = 1;
+    int32_t n_layer = 32;
+    int32_t falcon_version = 7; // 7 for Falcon-7B, 40 for Falcon-40B
+    int32_t ftype   = 1;
+    int32_t n_ctx   = 2048;
+};
+
+struct falcon_layer {
+    // normalization
+    struct ggml_tensor* input_layernorm;
+    struct ggml_tensor* input_layernorm_b;
+    //struct ggml_tensor* attention_norm;    // Falcon-40B only
+    //struct ggml_tensor* attention_norm_b;  // Falcon-40B only
+
+    // attention
+    struct ggml_tensor* query_key_value;
+    struct ggml_tensor* wo;
+
+    // ff
+    struct ggml_tensor* ffn_up;
+    struct ggml_tensor* ffn_down;
+};
+
+struct falcon_model {
+    falcon_hparams hparams;
+
+    struct ggml_tensor* tok_embeddings;
+    struct ggml_tensor* output_norm;
+    struct ggml_tensor* output_norm_b;
+    struct ggml_tensor* lm_head;
+
+    std::vector<falcon_layer> layers;
+
+    // key + value memory
+    llm_kv_cache kv_self;
+
+    struct ggml_context* ctx;
+    std::map<std::string, struct ggml_tensor*> tensors;
+
+    llm_buffer eval_buf;
+    llm_buffer work_buf;
+    llm_buffer scr0_buf;
+    llm_buffer scr1_buf;
+};
+
+static bool kv_cache_init(
+        const struct falcon_hparams & hparams,
+              struct llm_kv_cache & cache,
+                         ggml_type   wtype,
+                               int   n_ctx) {
+    const int n_embd  = hparams.n_embd;
+    const int dim_head  = n_embd / hparams.n_head;
+    const int dim_kv = dim_head * hparams.n_head_kv;
+    const int n_layer = hparams.n_layer;
+
+    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
+    const int64_t n_elements = dim_kv * n_mem;
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+    struct ggml_init_params params;
+    params.mem_size   = cache.buf.size;
+    params.mem_buffer = cache.buf.addr;
+    params.no_alloc   = false;
+
+    cache.ctx = ggml_init(params);
+    if (!cache.ctx) {
+        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
+        return false;
+    }
+
+    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+    return true;
+}
+
+// load the model's weights from a file
+bool falcon_model_load(const std::string & fname, falcon_model & model, gpt_vocab & vocab, size_t *mem_req) {
+    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+    if (mem_req) {
+        *mem_req = 0;
+    }
+
+    auto fin = std::ifstream(fname, std::ios::binary);
+    if (!fin) {
+        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
+        return false;
+    }
+
+    // verify magic
+    {
+        uint32_t magic;
+        fin.read((char *) &magic, sizeof(magic));
+        if (magic != FALCON_MAGIC) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
+            return false;
+        }
+    }
+
+    uint32_t format_version;
+    fin.read((char *) &format_version, sizeof(format_version));
+
+    // load hparams
+    {
+        auto & hparams = model.hparams;
+
+        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
+        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
+        fin.read((char *) &hparams.n_head_kv, sizeof(hparams.n_head_kv));
+        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
+        fin.read((char *) &hparams.falcon_version, sizeof(hparams.falcon_version));
+        fin.read((char *) &hparams.ftype,   sizeof(hparams.ftype));
+
+        if (hparams.falcon_version != 7) { // && hparams.falcon_version != 40) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad Falcon version: %d)\n", __func__, fname.c_str(), hparams.falcon_version);
+            return false;
+        }
+
+        const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
+
+        printf("%s: n_vocab   = %d\n", __func__, hparams.n_vocab);
+        printf("%s: n_embd    = %d\n", __func__, hparams.n_embd);
+        printf("%s: n_head    = %d\n", __func__, hparams.n_head);
+        printf("%s: n_head_kv = %d\n", __func__, hparams.n_head_kv);
+        printf("%s: n_layer   = %d\n", __func__, hparams.n_layer);
+        printf("%s: ftype     = %d\n", __func__, hparams.ftype);
+        printf("%s: qntvr     = %d\n", __func__, qntvr);
+
+        hparams.ftype %= GGML_QNT_VERSION_FACTOR;
+    }
+
+    // load vocab
+    {
+        const int32_t n_vocab = model.hparams.n_vocab;
+
+        std::string word;
+        std::vector<char> buf(128);
+
+        for (int i = 0; i < n_vocab; i++) {
+            uint32_t len;
+            fin.read((char *) &len, sizeof(len));
+
+            buf.resize(len);
+            fin.read((char *) buf.data(), len);
+            word.assign(buf.data(), len);
+
+            uint32_t dummy;
+            fin.read((char *) &dummy, sizeof(dummy));
+
+            vocab.token_to_id[word] = i;
+            vocab.id_to_token[i] = word;
+        }
+    }
+
+    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
+    // in order to save memory and also to speed up the computation
+    ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
+    if (wtype == GGML_TYPE_COUNT) {
+        fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
+                __func__, fname.c_str(), model.hparams.ftype);
+        return false;
+    }
+
+    auto & ctx = model.ctx;
+
+    size_t ctx_size = 0;
+
+    {
+        const auto& hparams = model.hparams;
+
+        const int n_embd = hparams.n_embd;
+        const int n_head = hparams.n_head;
+        const int n_head_kv = hparams.n_head_kv;
+        const int n_layer = hparams.n_layer;
+        const int n_ctx = hparams.n_ctx;
+        const int n_ff = 4 * model.hparams.n_embd;
+        const int n_vocab = hparams.n_vocab;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // tok_embeddings
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm_b
+        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // lm_head
+
+        // if (hparams.version == 40) { // Falcon-40B
+        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm
+        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm_b
+        // }
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm_b
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * (n_head_kv * 2 + n_head) * head_dim);  // query_key_value
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_embd);  // wo
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_ff);  // ffn_up
+        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_ff * n_embd);  // ffn_down
+ 
+        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
+    }
+
+    if (mem_req) {
+        const int n_embd  = model.hparams.n_embd;
+        const int dim_head  = n_embd / model.hparams.n_head;
+        const int dim_kv = dim_head * model.hparams.n_head_kv;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = dim_kv * n_mem;
+        size_t kv_cache_size = 2u*n_elements*ggml_type_size(wtype) + 2_MiB;
+        *mem_req = ctx_size + kv_cache_size;
+        return false;
+    }
+
+    // create the ggml context
+    {
+        struct ggml_init_params params = {
+            .mem_size   = ctx_size,
+            .mem_buffer = NULL,
+            .no_alloc   = false,
+        };
+
+        model.ctx = ggml_init(params);
+        if (!model.ctx) {
+            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
+            return false;
+        }
+    }
+
+    // prepare memory for the weights
+    {
+        const auto& hparams = model.hparams;
+
+        const int n_embd = hparams.n_embd;
+        const int n_head = hparams.n_head;
+        const int n_head_kv = hparams.n_head_kv;
+        const int n_layer = hparams.n_layer;
+        const int n_ff = 4 * model.hparams.n_embd;
+        const int n_vocab = hparams.n_vocab;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        model.layers.resize(n_layer);
+
+        model.tok_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
+
+        model.output_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.output_norm_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
+
+        // map by name
+        model.tensors["transformer.word_embeddings.weight"] =
+            model.tok_embeddings;
+
+        model.tensors["transformer.ln_f.weight"] = model.output_norm;
+        model.tensors["transformer.ln_f.bias"] = model.output_norm_b;
+        model.tensors["lm_head.weight"] = model.lm_head;
+
+        for (int i = 0; i < n_layer; ++i) {
+            auto& layer = model.layers[i];
+
+            layer.input_layernorm =
+                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.input_layernorm_b =
+                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+            // if (hparams.version == 40) { // for Falcon-40B only
+            //     layer.attention_norm =
+            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            //     layer.attention_norm_b =
+            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            // }
+
+            // query_key_value shape for config.multi_query == True:
+            layer.query_key_value = ggml_new_tensor_2d(
+                ctx, wtype, n_embd, (n_head_kv * 2 + n_head) * head_dim);
+            layer.wo = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
+
+            layer.ffn_up = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);
+            layer.ffn_down = ggml_new_tensor_2d(ctx, wtype, n_ff, n_embd);
+
+            // map by name
+            // if (hparams.version == 40) {
+            //     // Falcon-40B:
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_mlp.weight"] = layer.input_layernorm;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_mlp.bias"] = layer.input_layernorm_b;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_attn.weight"] = layer.attention_norm;
+            //     model.tensors["transformer.h." + std::to_string(i) +
+            //                   ".ln_attn.bias"] = layer.attention_norm_b;
+            // } else {
+            // Falcon-7B:
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".input_layernorm.weight"] = layer.input_layernorm;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".input_layernorm.bias"] = layer.input_layernorm_b;
+            //}
+
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".self_attention.query_key_value.weight"] =
+                layer.query_key_value;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".self_attention.dense.weight"] = layer.wo;
+
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".mlp.dense_h_to_4h.weight"] = layer.ffn_up;
+            model.tensors["transformer.h." + std::to_string(i) +
+                          ".mlp.dense_4h_to_h.weight"] = layer.ffn_down;
+        }
+    }
+
+    // key + value memory
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_layer = hparams.n_layer;
+        const int n_ctx   = hparams.n_ctx;
+        const int n_head_kv = hparams.n_head_kv;
+        const int head_dim = hparams.n_embd / hparams.n_head;
+
+        const int64_t n_mem      = n_layer*n_ctx;
+        const int64_t n_elements = head_dim*n_mem;
+
+        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F32, model.hparams.n_ctx)) {
+            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
+            ggml_free(ctx);
+            return false;
+        }
+        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
+
+        printf("%s: memory_size = %8.2f MB, n_mem = %" PRId64 "\n", __func__, memory_size/1024.0/1024.0, n_mem);
+    }
+
+    // load weights
+    {
+        int n_tensors = 0;
+        size_t total_size = 0;
+
+        printf("%s: ", __func__);
+
+        while (true) {
+            int32_t n_dims;
+            int32_t length;
+            int32_t ttype;
+
+            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
+            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
+
+            if (fin.eof()) {
+                break;
+            }
+
+            int32_t nelements = 1;
+            int32_t ne[2] = { 1, 1 };
+            for (int i = 0; i < n_dims; ++i) {
+                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+                nelements *= ne[i];
+            }
+
+            std::string name(length, 0);
+            fin.read(&name[0], length);
+            fin.seekg(-static_cast<ptrdiff_t>(fin.tellg()) & 31, std::ios_base::cur);
+
+            if (model.tensors.find(name.data()) == model.tensors.end()) {
+                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
+                return false;
+            }
+
+            auto tensor = model.tensors[name.data()];
+            if (ggml_nelements(tensor) != nelements) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
+                return false;
+            }
+
+            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%5d, %5d], expected [%5d, %5d]\n",
+                        __func__, name.data(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
+                return false;
+            }
+
+            // for debugging
+            if (0) {
+                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
+            }
+
+            const size_t bpe = ggml_type_size(ggml_type(ttype));
+
+            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
+                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
+                return false;
+            }
+
+            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
+
+            total_size += ggml_nbytes(tensor);
+            if (++n_tensors % 8 == 0) {
+                printf(".");
+                fflush(stdout);
+            }
+        }
+
+        printf(" done\n");
+
+        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
+    }
+
+    fin.close();
+
+    model.eval_buf.resize(1280u * 1024 * 1024);
+    model.scr0_buf.resize(256u * 1024 * 1024);
+    model.scr1_buf.resize(256u * 1024 * 1024);
+    return true;
+}
+
+
+// evaluate the transformer
+//
+//   - model:     the model
+//   - n_threads: number of threads to use
+//   - n_past:    the context size so far
+//   - embd_inp:  the embeddings of the tokens in the context
+//   - embd_w:    the predicted logits for the next token
+//
+bool falcon_eval(
+        falcon_model & model,
+        const int n_threads,
+        const int n_past,
+        const std::vector<gpt_vocab::id> & embd_inp,
+              std::vector<float>         & embd_w,
+              size_t                     & mem_per_token) {
+    const int N = embd_inp.size();
+
+    const auto & hparams = model.hparams;
+
+    const int n_embd  = hparams.n_embd;
+    const int n_layer = hparams.n_layer;
+    const int n_ctx   = hparams.n_ctx;
+    const int n_head  = hparams.n_head;
+    const int n_head_kv = hparams.n_head_kv;
+    const int n_vocab = hparams.n_vocab;
+    const int version = hparams.falcon_version;
+    const size_t head_dim = n_embd / n_head;
+
+   struct ggml_init_params eval_ctx_params = {
+        .mem_size = model.eval_buf.size,
+        .mem_buffer = model.eval_buf.addr,
+        .no_alloc = false,
+    };
+
+    struct ggml_context * ctx0 = ggml_init(eval_ctx_params);
+    struct ggml_cgraph gf = {};
+
+    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
+
+    // wte
+    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.tok_embeddings, embd);
+    struct ggml_tensor* repeat_dummy = ggml_new_tensor_3d(ctx0, inpL->type, head_dim, N + n_past, n_head);
+
+    ggml_type wtype = GGML_TYPE_F32;
+    const int sizeof_wtype = ggml_type_sizef(wtype);
+
+    for (int il = 0; il < n_layer; ++il) {
+        struct ggml_tensor * cur;
+        struct ggml_tensor * layernorm_output;
+
+        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
+        // self-attention
+        {
+            layernorm_output = ggml_norm(ctx0, inpL);
+
+            layernorm_output = ggml_add(ctx0,
+                    ggml_mul(ctx0,
+                        ggml_repeat(ctx0, model.layers[il].input_layernorm, layernorm_output),
+                        layernorm_output),
+                    ggml_repeat(ctx0, model.layers[il].input_layernorm_b, layernorm_output));
+
+            // if (version == 40) { // Falcon-40B only
+            //     cur = ggml_norm(ctx0, inpL);
+
+            //     cur = ggml_add(ctx0,
+            //             ggml_mul(ctx0,
+            //                 ggml_repeat(ctx0, model.layers[il].attention_norm, cur),
+            //                 cur),
+            //             ggml_repeat(ctx0, model.layers[il].attention_norm_b, cur));
+            // }
+            // else {
+            cur = layernorm_output;
+            // }
+
+            // compute QKV
+
+            cur = ggml_mul_mat(ctx0, model.layers[il].query_key_value, cur);
+
+            // Note that the strides for Kcur, Vcur are set up so that the
+            // resulting views are misaligned with the tensor's storage
+            // (by applying the K/V offset we shift the tensor's original
+            // view to stick out behind the viewed QKV tensor's allocated
+            // memory, so to say). This is ok because no actual accesses
+            // happen to that out-of-range memory, but it can require some
+            // trickery when trying to accurately dump these views for
+            // debugging.
+
+            struct ggml_tensor * Qcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                0);
+
+            struct ggml_tensor * Kcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head_kv, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                head_dim * n_head * sizeof_wtype);
+
+            struct ggml_tensor * Vcur = ggml_view_3d(
+                ctx0, cur, head_dim, n_head_kv, N,
+                head_dim * sizeof_wtype,
+                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
+                head_dim * (n_head + n_head_kv) * sizeof_wtype);
+
+            // using mode = 2 for neox mode
+            Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, head_dim, 2, n_ctx);
+            Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, head_dim, 2, n_ctx);
+
+            // store key and value to memory
+            {
+                struct ggml_tensor* k = ggml_view_1d(
+                    ctx0, model.kv_self.k, N * n_head_kv * head_dim,
+                    (ggml_element_size(model.kv_self.k) * n_head_kv * head_dim) *
+                        (il * n_ctx + n_past));
+                struct ggml_tensor* v = ggml_view_1d(
+                    ctx0, model.kv_self.v, N * n_head_kv * head_dim,
+                    (ggml_element_size(model.kv_self.v) * n_head_kv * head_dim) *
+                        (il * n_ctx + n_past));
+
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
+            }
+
+            struct ggml_tensor * K = ggml_permute(
+                ctx0,
+                ggml_view_3d(
+                    ctx0,
+                    model.kv_self.k,
+                    head_dim, n_head_kv, n_past + N,
+                    head_dim * sizeof_wtype,
+                    head_dim * n_head_kv * sizeof_wtype,
+                    il * n_ctx * ggml_element_size(model.kv_self.k) * n_head_kv * head_dim),
+                0, 2, 1, 3);
+
+            // K * Q
+
+            // changed from repeat2 back to repeat, will not support 40B!
+            K = ggml_cont(ctx0, ggml_repeat(ctx0, K, repeat_dummy));
+
+            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
+
+            // KQ_scaled = KQ / sqrt(n_embd/n_head)
+            struct ggml_tensor * KQ_scaled =
+                ggml_scale_inplace(ctx0,
+                        KQ,
+                        ggml_new_f32(ctx0, 1.0f/sqrt(float(head_dim)))
+                        );
+
+            // KQ_masked = mask_past(KQ_scaled)
+            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
+
+            // KQ = soft_max(KQ_masked)
+            struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
+
+            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
+            struct ggml_tensor* V = ggml_permute(
+                ctx0,
+                ggml_view_3d(
+                    ctx0,
+                    model.kv_self.v,
+                    head_dim, n_head_kv, n_past + N,
+                    head_dim * sizeof_wtype,
+                    head_dim * n_head_kv * sizeof_wtype,
+                    il * n_ctx * ggml_element_size(model.kv_self.v) * n_head_kv * head_dim),
+                0, 2, 1, 3);
+
+            // changed from repeat2 back to repeat, will not support 40B!
+            V = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_repeat(ctx0, V, repeat_dummy)));
+
+            // KQV = transpose(V) * KQ_soft_max
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
+
+            // KQV_merged = KQV.permute(0, 2, 1, 3)
+            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
+
+            // cur = KQV_merged.contiguous().view(n_embd, N)
+            cur = ggml_cpy(ctx0,
+                    KQV_merged,
+                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+
+            // projection
+            {
+                cur = ggml_mul_mat(ctx0,
+                        model.layers[il].wo,
+                        cur);
+            }
+        }
+
+        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
+
+        struct ggml_tensor* inpFF = layernorm_output;
+        struct ggml_tensor* attn_out = ggml_cpy(
+            ctx0, cur, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+
+        {
+            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_up, inpFF);
+            cur = ggml_gelu(ctx0, cur);
+            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_down, cur);
+        }
+
+        cur = ggml_add(ctx0, cur, attn_out);
+        cur = ggml_add(ctx0, cur, inpL);
+        // input for next layer
+        inpL = cur;
+    }
+
+    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
+    // norm
+    {
+        inpL = ggml_norm(ctx0, inpL);
+
+        // inpL = ln_f_g*inpL + ln_f_b
+        inpL = ggml_add(ctx0,
+                ggml_mul(ctx0,
+                    ggml_repeat(ctx0, model.output_norm, inpL),
+                    inpL),
+                ggml_repeat(ctx0, model.output_norm_b, inpL));
+    }
+
+    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
+
+    // lm_head
+    {
+        inpL = ggml_mul_mat(ctx0, model.lm_head, inpL);
+
+        //inpL = ggml_add(ctx0,
+        //        ggml_repeat(ctx0, model.lmh_b, inpL),
+        //        inpL);
+    }
+
+    // logits -> probs
+    //inpL = ggml_soft_max_inplace(ctx0, inpL);
+
+    // run the computation
+    ggml_build_forward_expand(&gf, inpL);
+    ggml_graph_compute_g4a(model.work_buf, &gf, n_threads);
+  
+
+    //if (n_past%100 == 0) {
+    //    ggml_graph_print   (&gf);
+    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
+    //}
+
+    //embd_w.resize(n_vocab*N);
+    //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
+
+    // return result for just the last token
+    embd_w.resize(n_vocab);
+    memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
+
+    if (mem_per_token == 0) {
+        mem_per_token = ggml_used_mem(ctx0)/N;
+    }
+    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
+
+    ggml_free(ctx0);
+
+    return true;
+}
+
+
+#define MAX_RNG_STATE 64*1024
+size_t falcon_get_state_size(const falcon_model &model) {
+    const size_t s_rng_size        = sizeof(size_t);
+    const size_t s_rng             = MAX_RNG_STATE;
+    const size_t s_kv_size         = sizeof(size_t);
+    const size_t s_kv_ntok         = sizeof(int);
+    const size_t s_kv              = model.kv_self.buf.size;
+    const size_t s_total = (
+        + s_rng_size
+        + s_rng
+        + s_kv_size
+        + s_kv_ntok
+        + s_kv
+    );
+    return s_total;
+}
+
+size_t falcon_copy_state_data(const falcon_model &model, const std::mt19937 &rng, uint8_t *dest)
+{
+    uint8_t * out = dest;
+    // copy rng
+    {
+        std::stringstream rng_ss;
+        rng_ss << rng;
+
+        const size_t rng_size = rng_ss.str().size();
+        char rng_buf[MAX_RNG_STATE];
+
+        memset(&rng_buf[0], 0, MAX_RNG_STATE);
+        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
+
+        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
+        memcpy(out, &rng_buf[0], MAX_RNG_STATE); out += MAX_RNG_STATE;
+    }
+
+    // copy kv cache
+    {
+        const size_t kv_size = model.kv_self.buf.size;
+        const int    kv_ntok = model.kv_self.n;
+
+        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
+        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
+
+        if (kv_size) {
+            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
+        }
+    }
+
+    const size_t written  = out - dest;
+    assert(written == falcon_get_state_size(model));
+    fflush(stdout);
+    return written;
+}
+
+size_t falcon_set_state_data(falcon_model *model, std::mt19937 *rng, const uint8_t *src)
+{
+    const uint8_t * in = src;
+
+    // set rng
+    {
+        size_t rng_size;
+        char   rng_buf[MAX_RNG_STATE];
+
+        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
+        memcpy(&rng_buf[0], in, MAX_RNG_STATE); in += MAX_RNG_STATE;
+
+        std::stringstream rng_ss;
+        rng_ss.str(std::string(&rng_buf[0], rng_size));
+        rng_ss >> *rng;
+
+        assert(rng_ss.fail() == false);
+    }
+
+    // set kv cache
+    {
+        size_t kv_size;
+        int kv_ntok;
+
+        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
+        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
+
+        if (kv_size) {
+            assert(model->kv_self.buf.size == kv_size);
+
+            void * k_data = model->kv_self.k->data; // remember data pointers
+            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
+
+            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
+
+            model->kv_self.k->data = k_data; // restore correct data pointers
+            model->kv_self.v->data = v_data;
+
+        }
+
+        model->kv_self.n = kv_ntok;
+    }
+
+    const size_t nread    = in - src;
+    assert(nread == falcon_get_state_size(*model));
+    fflush(stdout);
+    return nread;
+}
+
+struct FalconPrivate {
+    const std::string modelPath;
+    bool modelLoaded;
+    gpt_vocab vocab;
+    falcon_model *model = nullptr;
+    int64_t n_threads = 0;
+    size_t mem_per_token = 0;
+    std::mt19937 rng;
+};
+
+Falcon::Falcon() : d_ptr(new FalconPrivate) {
+    d_ptr->model = new falcon_model;
+    d_ptr->model->ctx = nullptr;
+    d_ptr->modelLoaded = false;
+}
+
+Falcon::~Falcon() {
+    if(d_ptr->model->ctx) {
+        ggml_free(d_ptr->model->ctx);
+        d_ptr->model->ctx = nullptr;
+    }
+    delete d_ptr->model;
+}
+
+bool Falcon::loadModel(const std::string &modelPath)
+{
+    std::mt19937 rng(time(NULL));
+    d_ptr->rng = rng;
+
+    // load the model
+    if (!falcon_model_load(modelPath, *d_ptr->model, d_ptr->vocab, nullptr)) {
+        std::cerr << "FALCON ERROR: failed to load model from " <<  modelPath;
+        return false;
+    }
+
+    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    d_ptr->modelLoaded = true;
+    fflush(stdout);
+    return true;
+}
+
+bool Falcon::isModelLoaded() const
+{
+    return d_ptr -> modelLoaded;
+}
+
+size_t Falcon::requiredMem(const std::string &modelPath)
+{
+    falcon_model dummy_model;
+    gpt_vocab dummy_vocab;
+    size_t mem_req;
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    falcon_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
+    return mem_req;
+}
+
+size_t Falcon::stateSize() const
+{
+    return falcon_get_state_size(*d_ptr->model);
+}
+
+size_t Falcon::saveState(uint8_t *dest) const
+{
+    return falcon_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
+}
+
+size_t Falcon::restoreState(const uint8_t *src)
+{
+    return falcon_set_state_data(d_ptr->model, &d_ptr->rng, src);
+}
+
+void Falcon::setThreadCount(int32_t n_threads)
+{
+    d_ptr->n_threads = n_threads;
+}
+
+int32_t Falcon::threadCount() const
+{
+    return d_ptr->n_threads;
+}
+
+std::vector<LLModel::Token> Falcon::tokenize(PromptContext &, const std::string &str) const
+{
+    return ::gpt_tokenize(d_ptr->vocab, str);
+}
+
+LLModel::Token Falcon::sampleToken(PromptContext &promptCtx) const
+{
+    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
+    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
+        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
+        n_prev_toks,
+        promptCtx.logits,
+        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
+        promptCtx.repeat_penalty,
+        d_ptr->rng);
+}
+
+std::string Falcon::tokenToString(Token id) const
+{
+    return d_ptr->vocab.id_to_token[id];
+}
+
+bool Falcon::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
+{
+    // determine the required inference memory per token:
+    static bool initialized = false;
+    if (!initialized) {
+        falcon_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
+            d_ptr->mem_per_token);
+        initialized = true;
+    }
+
+    return falcon_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
+}
+
+int32_t Falcon::contextLength() const
+{
+    return d_ptr->model->hparams.n_ctx;
+}
+
+const std::vector<LLModel::Token> &Falcon::endTokens() const
+{
+    static const std::vector<LLModel::Token> out = { 11 };
+    return out;
+}
+
+#if defined(_WIN32)
+#define DLL_EXPORT __declspec(dllexport)
+#else
+#define DLL_EXPORT __attribute__ ((visibility ("default")))
+#endif
+
+extern "C" {
+DLL_EXPORT bool is_g4a_backend_model_implementation() {
+    return true;
+}
+
+DLL_EXPORT const char *get_model_type() {
+    return modelType_;
+}
+
+DLL_EXPORT const char *get_build_variant() {
+    return GGML_BUILD_VARIANT;
+}
+
+DLL_EXPORT bool magic_match(std::istream& f) {
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    uint32_t version = 0;
+    f.read(reinterpret_cast<char*>(&version), sizeof(version));
+    if (magic != FALCON_MAGIC) {
+         return false; 
+    }
+    falcon_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    // we're matching the file format of existing pre-converted models
+    // compatible with ctransformers llama.cpp based format, which also
+    // unfortunately shares its magic number what llama uses, so we now
+    // differentiate by n_vocab
+    // give some wiggle room over the max to allow for finetunes that expand the
+    // vocabulary
+    if (!(hparams.n_vocab >= 65024 && hparams.n_vocab <= 65100)) {
+        return false;
+    }
+    if (hparams.falcon_version != 7) {
+        return false;
+    }
+    return true;
+}
+
+DLL_EXPORT LLModel *construct() {
+    return new Falcon;
+}
+}

gpt4all-backend/falcon_impl.h

@@ -0,0 +1,42 @@
+#ifndef FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of falcon.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef FALCON_H
+#define FALCON_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <memory>
+#include "llmodel.h"
+
+struct FalconPrivate;
+class Falcon : public LLModel {
+public:
+    Falcon();
+    ~Falcon();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    std::unique_ptr<FalconPrivate> d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // Falcon_H

gpt4all-backend/gptj.cpp

@@ -0,0 +1,975 @@
+#define GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#include "gptj_impl.h"
+
+#include "utils.h"
+#include "llmodel_shared.h"
+
+#include <cassert>
+#include <cinttypes>
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+#include <fstream>
+#include <map>
+#include <string>
+#include <vector>
+#include <iostream>
+#if defined(_WIN32) && defined(_MSC_VER)
+    #define WIN32_LEAN_AND_MEAN
+    #ifndef NOMINMAX
+        #define NOMINMAX
+    #endif
+    #include <windows.h>
+    #include <io.h>
+    #include <stdio.h>
+#else
+    #include <unistd.h>
+#endif
+#include <sstream>
+#include <unordered_set>
+#include <ggml.h>
+
+
+namespace {
+const char *modelType_ = "GPT-J";
+}
+
+// default hparams (GPT-J 6B)
+struct gptj_hparams {
+    int32_t n_vocab = 50400;
+    int32_t n_ctx   = 2048;
+    int32_t n_embd  = 4096;
+    int32_t n_head  = 16;
+    int32_t n_layer = 28;
+    int32_t n_rot   = 64;
+    int32_t f16     = 1;
+};
+
+struct gptj_layer {
+    // normalization
+    struct ggml_tensor * ln_1_g;
+    struct ggml_tensor * ln_1_b;
+
+    // attention
+    struct ggml_tensor * c_attn_q_proj_w;
+    struct ggml_tensor * c_attn_k_proj_w;
+    struct ggml_tensor * c_attn_v_proj_w;
+
+    struct ggml_tensor * c_attn_proj_w;
+
+    // ff
+    struct ggml_tensor * c_mlp_fc_w;
+    struct ggml_tensor * c_mlp_fc_b;
+
+    struct ggml_tensor * c_mlp_proj_w;
+    struct ggml_tensor * c_mlp_proj_b;
+};
+
+struct gptj_model {
+    gptj_hparams hparams;
+
+    // normalization
+    struct ggml_tensor * ln_f_g;
+    struct ggml_tensor * ln_f_b;
+
+    struct ggml_tensor * wte; // position embedding
+
+    struct ggml_tensor * lmh_g; // language model head
+    struct ggml_tensor * lmh_b; // language model bias
+
+    std::vector<gptj_layer> layers;
+
+    // key + value memory
+    struct llm_kv_cache kv_self;
+
+    //
+    struct ggml_context * ctx;
+    std::map<std::string, struct ggml_tensor *> tensors;
+
+    llm_buffer eval_buf;
+    llm_buffer scr0_buf;
+    llm_buffer scr1_buf;
+
+    ~gptj_model() {
+        if (ctx) {
+            ggml_free(ctx);
+        }
+    }
+};
+
+static bool kv_cache_init(
+        const struct gptj_hparams & hparams,
+              struct llm_kv_cache & cache,
+                         ggml_type   wtype,
+                               int   n_ctx) {
+    const int n_embd  = hparams.n_embd;
+    const int n_layer = hparams.n_layer;
+
+    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
+    const int64_t n_elements = n_embd*n_mem;
+
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+
+    struct ggml_init_params params;
+    params.mem_size   = cache.buf.size;
+    params.mem_buffer = cache.buf.addr;
+    params.no_alloc   = false;
+
+    cache.ctx = ggml_init(params);
+
+    if (!cache.ctx) {
+        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
+        return false;
+    }
+
+    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+
+    return true;
+}
+
+// load the model's weights from a stream
+bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
+    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+    if(mem_req != nullptr) {
+        *mem_req = 0;
+    }
+
+    // verify magic
+    {
+        uint32_t magic;
+        fin.read((char *) &magic, sizeof(magic));
+        if (magic != 0x67676d6c) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
+            return false;
+        }
+    }
+
+    // load hparams
+    {
+        auto & hparams = model.hparams;
+
+        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
+        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
+        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
+        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
+        fin.read((char *) &hparams.n_rot,   sizeof(hparams.n_rot));
+        fin.read((char *) &hparams.f16,     sizeof(hparams.f16));
+
+        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
+        printf("%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
+        printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
+        printf("%s: n_head  = %d\n", __func__, hparams.n_head);
+        printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
+        printf("%s: n_rot   = %d\n", __func__, hparams.n_rot);
+        printf("%s: f16     = %d\n", __func__, hparams.f16);
+    }
+
+    // load vocab
+    {
+        int32_t n_vocab = 0;
+        fin.read((char *) &n_vocab, sizeof(n_vocab));
+
+        if (n_vocab != model.hparams.n_vocab) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
+                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
+            return false;
+        }
+
+        std::string word;
+        for (int i = 0; i < n_vocab; i++) {
+            uint32_t len;
+            fin.read((char *) &len, sizeof(len));
+
+            word.resize(len);
+            fin.read((char *) word.data(), len);
+
+            vocab.token_to_id[word] = i;
+            vocab.id_to_token[i] = word;
+        }
+    }
+
+    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
+    // in order to save memory and also to speed up the computation
+    ggml_type wtype = GGML_TYPE_COUNT;
+    switch (model.hparams.f16) {
+        case 0: wtype = GGML_TYPE_F32;  break;
+        case 1: wtype = GGML_TYPE_F16;  break;
+        case 2: wtype = GGML_TYPE_Q4_0; break;
+        case 3: wtype = GGML_TYPE_Q4_1; break;
+        case 5: wtype = GGML_TYPE_Q4_2; break;
+        default:
+                {
+                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
+                            __func__, fname.c_str(), model.hparams.f16);
+                    return false;
+                }
+    }
+
+    auto & ctx = model.ctx;
+
+    size_t ctx_size = 0;
+
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_ctx   = hparams.n_ctx;
+        const int n_vocab = hparams.n_vocab;
+
+        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_g
+        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_b
+
+        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype); // wte
+
+        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype);         // lmh_g
+        ctx_size +=        n_vocab*ggml_type_sizef(GGML_TYPE_F32); // lmh_b
+
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_g
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_b
+
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_q_proj_w
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_k_proj_w
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_v_proj_w
+
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_proj_w
+
+        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_fc_w
+        ctx_size += n_layer*(       4*n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_fc_b
+
+        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_proj_w
+        ctx_size += n_layer*(         n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_proj_b
+
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_k
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_v
+
+        ctx_size += (5 + 10*n_layer)*256; // object overhead
+
+        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
+    }
+
+    if (mem_req != nullptr) {
+        *mem_req += ctx_size;
+        const int n_embd  = model.hparams.n_embd;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = n_embd*n_mem;
+
+        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+        return false;
+    }
+
+
+    // create the ggml context
+    {
+        struct ggml_init_params params = {
+            .mem_size   = ctx_size,
+            .mem_buffer = NULL,
+            .no_alloc = false
+        };
+
+        model.ctx = ggml_init(params);
+        if (!model.ctx) {
+            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
+            return false;
+        }
+    }
+
+    // prepare memory for the weights
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_vocab = hparams.n_vocab;
+
+        model.layers.resize(n_layer);
+
+        model.wte    = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
+
+        model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+        model.lmh_g  = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
+        model.lmh_b  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab);
+
+        // map by name
+        model.tensors["transformer.wte.weight"] = model.wte;
+
+        model.tensors["transformer.ln_f.weight"] = model.ln_f_g;
+        model.tensors["transformer.ln_f.bias"]   = model.ln_f_b;
+
+        model.tensors["lm_head.weight"] = model.lmh_g;
+        model.tensors["lm_head.bias"]   = model.lmh_b;
+
+        for (int i = 0; i < n_layer; ++i) {
+            auto & layer = model.layers[i];
+
+            layer.ln_1_g          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
+            layer.ln_1_b          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
+
+            layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+
+            layer.c_attn_proj_w   = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+
+            layer.c_mlp_fc_w      = ggml_new_tensor_2d(ctx, wtype,           n_embd, 4*n_embd);
+            layer.c_mlp_fc_b      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
+
+            layer.c_mlp_proj_w    = ggml_new_tensor_2d(ctx, wtype,         4*n_embd,   n_embd);
+            layer.c_mlp_proj_b    = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
+
+            // map by name
+            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.weight"]          = layer.ln_1_g;
+            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.bias"]            = layer.ln_1_b;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.q_proj.weight"]   = layer.c_attn_q_proj_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.k_proj.weight"]   = layer.c_attn_k_proj_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.v_proj.weight"]   = layer.c_attn_v_proj_w;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.out_proj.weight"] = layer.c_attn_proj_w;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.weight"]     = layer.c_mlp_fc_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.bias"]       = layer.c_mlp_fc_b;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.weight"]    = layer.c_mlp_proj_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.bias"]      = layer.c_mlp_proj_b;
+        }
+    }
+
+    // key + value memory
+    {
+        const auto & hparams = model.hparams;
+        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
+            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
+            ggml_free(ctx);
+            return false;
+        }
+
+        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
+        printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
+    }
+
+    // load weights
+    {
+        int n_tensors = 0;
+        size_t total_size = 0;
+
+        printf("%s: ", __func__);
+
+        while (true) {
+            int32_t n_dims;
+            int32_t length;
+            int32_t ftype;
+
+            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
+            fin.read(reinterpret_cast<char *>(&ftype),  sizeof(ftype));
+
+            if (fin.eof()) {
+                break;
+            }
+
+            int32_t nelements = 1;
+            int32_t ne[2] = { 1, 1 };
+            for (int i = 0; i < n_dims; ++i) {
+                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+                nelements *= ne[i];
+            }
+
+            std::string name(length, 0);
+            fin.read(&name[0], length);
+
+            if (model.tensors.find(name.data()) == model.tensors.end()) {
+                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
+                return false;
+            }
+
+            auto tensor = model.tensors[name.data()];
+            if (ggml_nelements(tensor) != nelements) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
+                return false;
+            }
+
+            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
+                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
+                return false;
+            }
+
+            if (0) {
+                static const char * ftype_str[] = { "f32", "f16", "q4_0", "q4_1", };
+                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ftype_str[ftype], ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
+            }
+
+            size_t bpe = 0;
+
+            switch (ftype) {
+                case 0: bpe = ggml_type_size(GGML_TYPE_F32);  break;
+                case 1: bpe = ggml_type_size(GGML_TYPE_F16);  break;
+                case 2: bpe = ggml_type_size(GGML_TYPE_Q4_0); assert(ne[0] % 64 == 0); break;
+                case 3: bpe = ggml_type_size(GGML_TYPE_Q4_1); assert(ne[0] % 64 == 0); break;
+                default:
+                        {
+                            fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype);
+                            return false;
+                        }
+            };
+
+            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
+                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
+                return false;
+            }
+
+            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
+
+            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
+            total_size += ggml_nbytes(tensor);
+            if (++n_tensors % 8 == 0) {
+                printf(".");
+                fflush(stdout);
+            }
+        }
+
+        printf(" done\n");
+
+        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
+    }
+
+    model.scr0_buf.resize(256u * 1024 * 1024);
+    model.scr1_buf.resize(256u * 1024 * 1024);
+
+    return true;
+}
+
+// load the model's weights from a file path
+bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) {
+
+    auto fin = std::ifstream(fname, std::ios::binary);
+    if (!fin) {
+        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
+        return false;
+    }
+
+    bool loaded = gptj_model_load(fname, fin, model, vocab);
+    fin.close();
+    return loaded;
+}
+
+// evaluate the transformer
+//
+//   - model:     the model
+//   - n_threads: number of threads to use
+//   - n_past:    the context size so far
+//   - embd_inp:  the embeddings of the tokens in the context
+//   - embd_w:    the predicted logits for the next token
+//
+// The GPT-J model requires about 16MB of memory per input token.
+//
+bool gptj_eval(
+        gptj_model & model,
+        const int n_threads,
+        const int n_past,
+        const std::vector<gpt_vocab::id> & embd_inp,
+              std::vector<float>         & embd_w,
+              size_t                     & mem_per_token) {
+    const int N = embd_inp.size();
+
+    const auto & hparams = model.hparams;
+
+    const int n_embd  = hparams.n_embd;
+    const int n_layer = hparams.n_layer;
+    const int n_ctx   = hparams.n_ctx;
+    const int n_head  = hparams.n_head;
+    const int n_vocab = hparams.n_vocab;
+    const int n_rot   = hparams.n_rot;
+
+    const size_t init_buf_size = 1024_MiB;
+    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
+        model.eval_buf.resize(init_buf_size);
+
+    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
+        const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
+        printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.eval_buf.size, buf_size_new);
+
+        // reallocate
+        model.eval_buf.resize(buf_size_new);
+        if (model.eval_buf.addr == nullptr) {
+            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
+            return false;
+        }
+    }
+
+    struct ggml_init_params params = {
+        .mem_size   = model.eval_buf.size,
+        .mem_buffer = model.eval_buf.addr,
+        .no_alloc = false
+    };
+
+    struct ggml_context * ctx0 = ggml_init(params);
+    struct ggml_cgraph gf = {};
+    gf.n_threads = n_threads;
+
+    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
+
+    // wte
+    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
+
+    for (int il = 0; il < n_layer; ++il) {
+        struct ggml_tensor * cur;
+        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+        // norm
+        {
+            cur = ggml_norm(ctx0, inpL);
+
+            // cur = ln_1_g*cur + ln_1_b
+            cur = ggml_add(ctx0,
+                    ggml_mul(ctx0,
+                        ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),
+                        cur),
+                    ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));
+        }
+
+        struct ggml_tensor * inpSA = cur;
+
+        // self-attention
+        {
+            struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur);
+            struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur);
+            struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur);
+
+            // store key and value to memory
+            {
+                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
+                struct ggml_tensor * v = ggml_view_1d(ctx0, model.kv_self.v, N*n_embd, (ggml_element_size(model.kv_self.v)*n_embd)*(il*n_ctx + n_past));
+
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
+            }
+
+            // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
+            struct ggml_tensor * Q =
+                ggml_permute(ctx0,
+                        ggml_rope(ctx0,
+                            ggml_cpy(ctx0,
+                                Qcur,
+                                ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),
+                            n_past, n_rot, 0),
+                        0, 2, 1, 3);
+
+            // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
+            struct ggml_tensor * K =
+                ggml_permute(ctx0,
+                        ggml_rope(ctx0,
+                            ggml_reshape_3d(ctx0,
+                                ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
+                                n_embd/n_head, n_head, n_past + N),
+                            n_past, n_rot, 1),
+                        0, 2, 1, 3);
+
+            // K * Q
+            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
+
+            // KQ_scaled = KQ / sqrt(n_embd/n_head)
+            struct ggml_tensor * KQ_scaled =
+                ggml_scale(ctx0,
+                        KQ,
+                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
+                        );
+
+            // KQ_masked = mask_past(KQ_scaled)
+            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past);
+
+            // KQ = soft_max(KQ_masked)
+            struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
+
+            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
+            struct ggml_tensor * V_trans =
+                ggml_cpy(ctx0,
+                        ggml_permute(ctx0,
+                            ggml_reshape_3d(ctx0,
+                                ggml_view_1d(ctx0, model.kv_self.v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd),
+                                n_embd/n_head, n_head, n_past + N),
+                            1, 2, 0, 3),
+                        ggml_new_tensor_3d(ctx0, model.kv_self.v->type, n_past + N, n_embd/n_head, n_head));
+
+            // KQV = transpose(V) * KQ_soft_max
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);
+
+            // KQV_merged = KQV.permute(0, 2, 1, 3)
+            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
+
+            // cur = KQV_merged.contiguous().view(n_embd, N)
+            cur = ggml_cpy(ctx0,
+                    KQV_merged,
+                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+
+            // projection (no bias)
+            cur = ggml_mul_mat(ctx0,
+                    model.layers[il].c_attn_proj_w,
+                    cur);
+        }
+
+        struct ggml_tensor * inpFF = cur;
+
+        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
+        // feed-forward network
+        // this is independent of the self-attention result, so it could be done in parallel to the self-attention
+        {
+            // note here we pass inpSA instead of cur
+            cur = ggml_mul_mat(ctx0,
+                    model.layers[il].c_mlp_fc_w,
+                    inpSA);
+
+            cur = ggml_add(ctx0,
+                    ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),
+                    cur);
+
+            // GELU activation
+            cur = ggml_gelu(ctx0, cur);
+
+            // projection
+            // cur = proj_w*cur + proj_b
+            cur = ggml_mul_mat(ctx0,
+                    model.layers[il].c_mlp_proj_w,
+                    cur);
+
+            cur = ggml_add(ctx0,
+                    ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),
+                    cur);
+        }
+
+        // self-attention + FF
+        cur  = ggml_add(ctx0, cur, inpFF);
+
+        // input for next layer
+        inpL = ggml_add(ctx0, cur, inpL);
+    }
+
+    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
+    // norm
+    {
+        inpL = ggml_norm(ctx0, inpL);
+
+        // inpL = ln_f_g*inpL + ln_f_b
+        inpL = ggml_add(ctx0,
+                ggml_mul(ctx0,
+                    ggml_repeat(ctx0, model.ln_f_g, inpL),
+                    inpL),
+                ggml_repeat(ctx0, model.ln_f_b, inpL));
+    }
+
+    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
+
+    // lm_head
+    {
+        inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL);
+
+        inpL = ggml_add(ctx0,
+                ggml_repeat(ctx0, model.lmh_b, inpL),
+                inpL);
+    }
+
+    // logits -> probs
+    //inpL = ggml_soft_max(ctx0, inpL);
+
+    // run the computation
+    ggml_build_forward_expand(&gf, inpL);
+    ggml_graph_compute       (ctx0, &gf);
+
+    //if (n_past%100 == 0) {
+    //    ggml_graph_print   (&gf);
+    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
+    //}
+
+    //embd_w.resize(n_vocab*N);
+    //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
+
+    // return result for just the last token
+    embd_w.resize(n_vocab);
+    memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
+
+    if (mem_per_token == 0) {
+        mem_per_token = ggml_used_mem(ctx0)/N;
+    }
+    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
+
+    ggml_free(ctx0);
+
+    return true;
+}
+
+#define GPTJ_MAX_RNG_STATE 64*1024
+
+size_t gptj_get_state_size(const gptj_model &model)
+{
+    // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state.
+    // for reference, std::mt19937(1337) serializes to 6701 bytes.
+    const size_t s_rng_size        = sizeof(size_t);
+    const size_t s_rng             = GPTJ_MAX_RNG_STATE;
+    const size_t s_kv_size         = sizeof(size_t);
+    const size_t s_kv_ntok         = sizeof(int);
+    const size_t s_kv              = model.kv_self.buf.size;
+    const size_t s_total = (
+        + s_rng_size
+        + s_rng
+        + s_kv_size
+        + s_kv_ntok
+        + s_kv
+    );
+    fflush(stdout);
+    return s_total;
+}
+
+size_t gptj_copy_state_data(const gptj_model &model, const std::mt19937 &rng, uint8_t *dest)
+{
+    uint8_t * out = dest;
+    fflush(stdout);
+    // copy rng
+    {
+        std::stringstream rng_ss;
+        rng_ss << rng;
+
+        const size_t rng_size = rng_ss.str().size();
+        char rng_buf[GPTJ_MAX_RNG_STATE];
+
+        memset(&rng_buf[0], 0, GPTJ_MAX_RNG_STATE);
+        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
+
+        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
+        memcpy(out, &rng_buf[0], GPTJ_MAX_RNG_STATE); out += GPTJ_MAX_RNG_STATE;
+    }
+
+    // copy kv cache
+    {
+        const size_t kv_size = model.kv_self.buf.size;
+        const int    kv_ntok = model.kv_self.n;
+
+        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
+        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
+
+        if (kv_size) {
+            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
+        }
+    }
+
+    const size_t written  = out - dest;
+    assert(written == gptj_get_state_size(model));
+    fflush(stdout);
+    return written;
+}
+
+size_t gptj_set_state_data(gptj_model *model, std::mt19937 *rng, const uint8_t *src)
+{
+    const uint8_t * in = src;
+
+    // set rng
+    {
+        size_t rng_size;
+        char   rng_buf[GPTJ_MAX_RNG_STATE];
+
+        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
+        memcpy(&rng_buf[0], in, GPTJ_MAX_RNG_STATE); in += GPTJ_MAX_RNG_STATE;
+
+        std::stringstream rng_ss;
+        rng_ss.str(std::string(&rng_buf[0], rng_size));
+        rng_ss >> *rng;
+
+        assert(rng_ss.fail() == false);
+    }
+
+    // set kv cache
+    {
+        size_t kv_size;
+        int kv_ntok;
+
+        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
+        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
+
+        if (kv_size) {
+            assert(model->kv_self.buf.size == kv_size);
+
+            void * k_data = model->kv_self.k->data; // remember data pointers
+            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
+
+            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
+
+            model->kv_self.k->data = k_data; // restore correct data pointers
+            model->kv_self.v->data = v_data;
+
+        }
+
+        model->kv_self.n = kv_ntok;
+    }
+
+    const size_t nread    = in - src;
+    assert(nread == gptj_get_state_size(*model));
+    fflush(stdout);
+    return nread;
+}
+
+struct GPTJPrivate {
+    const std::string modelPath;
+    bool modelLoaded;
+    gpt_vocab vocab;
+    gptj_model *model = nullptr;
+    int64_t n_threads = 0;
+    size_t mem_per_token = 0;
+    std::mt19937 rng;
+};
+
+GPTJ::GPTJ()
+    : d_ptr(new GPTJPrivate) {
+    d_ptr->model = new gptj_model;
+    d_ptr->model->ctx = nullptr;
+    d_ptr->modelLoaded = false;
+}
+
+size_t GPTJ::requiredMem(const std::string &modelPath) {
+    gptj_model dummy_model;
+    gpt_vocab dummy_vocab;
+    size_t mem_req;
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    gptj_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
+    return mem_req;
+}
+
+bool GPTJ::loadModel(const std::string &modelPath) {
+    std::mt19937 rng(time(NULL));
+    d_ptr->rng = rng;
+
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+
+    // load the model
+    if (!gptj_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab)) {
+        std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
+        return false;
+    }
+
+    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    d_ptr->modelLoaded = true;
+    fflush(stdout);
+    return true;
+}
+
+void GPTJ::setThreadCount(int32_t n_threads) {
+    d_ptr->n_threads = n_threads;
+}
+
+int32_t GPTJ::threadCount() const
+{
+    return d_ptr->n_threads;
+}
+
+GPTJ::~GPTJ()
+{
+    delete d_ptr->model;
+}
+
+bool GPTJ::isModelLoaded() const
+{
+    return d_ptr->modelLoaded;
+}
+
+size_t GPTJ::stateSize() const
+{
+    return gptj_get_state_size(*d_ptr->model);
+}
+
+size_t GPTJ::saveState(uint8_t *dest) const
+{
+    return gptj_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
+}
+
+size_t GPTJ::restoreState(const uint8_t *src)
+{
+    return gptj_set_state_data(d_ptr->model, &d_ptr->rng, src);
+}
+
+std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &, const std::string &str) const
+{
+    return ::gpt_tokenize(d_ptr->vocab, str);
+}
+
+LLModel::Token GPTJ::sampleToken(PromptContext &promptCtx) const
+{
+    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
+    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
+        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
+        n_prev_toks,
+        promptCtx.logits,
+        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
+        promptCtx.repeat_penalty,
+        d_ptr->rng);
+}
+
+std::string GPTJ::tokenToString(Token id) const
+{
+    return d_ptr->vocab.id_to_token[id];
+}
+
+bool GPTJ::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
+{
+    // determine the required inference memory per token:
+    static bool initialized = false;
+    if (!initialized) {
+        gptj_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
+            d_ptr->mem_per_token);
+        initialized = true;
+    }
+
+    return gptj_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
+}
+
+int32_t GPTJ::contextLength() const
+{
+    return d_ptr->model->hparams.n_ctx;
+}
+
+const std::vector<LLModel::Token> &GPTJ::endTokens() const
+{
+    static const std::vector<LLModel::Token> fres = {50256};
+    return fres;
+}
+
+#if defined(_WIN32)
+#define DLL_EXPORT __declspec(dllexport)
+#else
+#define DLL_EXPORT __attribute__ ((visibility ("default")))
+#endif
+
+extern "C" {
+DLL_EXPORT bool is_g4a_backend_model_implementation() {
+    return true;
+}
+
+DLL_EXPORT const char *get_model_type() {
+    return modelType_;
+}
+
+DLL_EXPORT const char *get_build_variant() {
+    return GGML_BUILD_VARIANT;
+}
+
+DLL_EXPORT bool magic_match(std::istream& f) {
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    gptj_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    if (!(hparams.n_vocab >= 50300 && hparams.n_vocab <= 50400)) {
+        return false; // not a gptj.
+    }
+    return magic == 0x67676d6c;
+}
+
+DLL_EXPORT LLModel *construct() {
+    return new GPTJ;
+}
+}

gpt4all-backend/gptj_impl.h

@@ -0,0 +1,41 @@
+#ifndef GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of gptj.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef GPTJ_H
+#define GPTJ_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include "llmodel.h"
+
+struct GPTJPrivate;
+class GPTJ : public LLModel {
+public:
+    GPTJ();
+    ~GPTJ();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    GPTJPrivate *d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // GPTJ_H

gpt4all-backend/llama.cpp.cmake

@@ -0,0 +1,601 @@
+#
+# Copyright (c) 2023 Nomic, Inc. All rights reserved.
+#
+# This software is licensed under the terms of the Software for Open Models License (SOM),
+# version 1.0, as detailed in the LICENSE_SOM.txt file. A copy of this license should accompany
+# this software. Except as expressly granted in the SOM license, all rights are reserved by Nomic, Inc.
+#
+
+cmake_minimum_required(VERSION 3.12) # Don't bump this version for no reason
+
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
+    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
+endif()
+
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+
+if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+    set(LLAMA_STANDALONE ON)
+
+    # configure project version
+    # TODO
+else()
+    set(LLAMA_STANDALONE OFF)
+endif()
+
+if (EMSCRIPTEN)
+    set(BUILD_SHARED_LIBS_DEFAULT OFF)
+
+    option(LLAMA_WASM_SINGLE_FILE "llama: embed WASM inside the generated llama.js" ON)
+else()
+    if (MINGW)
+        set(BUILD_SHARED_LIBS_DEFAULT OFF)
+    else()
+        set(BUILD_SHARED_LIBS_DEFAULT ON)
+    endif()
+endif()
+
+
+#
+# Option list
+#
+# some of the options here are commented out so they can be set "dynamically" before calling include_ggml()
+
+# general
+option(LLAMA_STATIC                 "llama: static link libraries"                          OFF)
+option(LLAMA_NATIVE                 "llama: enable -march=native flag"                      OFF)
+option(LLAMA_LTO                    "llama: enable link time optimization"                  OFF)
+
+# debug
+option(LLAMA_ALL_WARNINGS           "llama: enable all compiler warnings"                   ON)
+option(LLAMA_ALL_WARNINGS_3RD_PARTY "llama: enable all compiler warnings in 3rd party libs" OFF)
+option(LLAMA_GPROF                  "llama: enable gprof"                                   OFF)
+
+# sanitizers
+option(LLAMA_SANITIZE_THREAD        "llama: enable thread sanitizer"                        OFF)
+option(LLAMA_SANITIZE_ADDRESS       "llama: enable address sanitizer"                       OFF)
+option(LLAMA_SANITIZE_UNDEFINED     "llama: enable undefined sanitizer"                     OFF)
+
+# instruction set specific
+#option(LLAMA_AVX                    "llama: enable AVX"                                     ON)
+#option(LLAMA_AVX2                   "llama: enable AVX2"                                    ON)
+#option(LLAMA_AVX512                 "llama: enable AVX512"                                  OFF)
+#option(LLAMA_AVX512_VBMI            "llama: enable AVX512-VBMI"                             OFF)
+#option(LLAMA_AVX512_VNNI            "llama: enable AVX512-VNNI"                             OFF)
+#option(LLAMA_FMA                    "llama: enable FMA"                                     ON)
+# in MSVC F16C is implied with AVX2/AVX512
+#if (NOT MSVC)
+#    option(LLAMA_F16C               "llama: enable F16C"                                    ON)
+#endif()
+
+# 3rd party libs
+option(LLAMA_ACCELERATE             "llama: enable Accelerate framework"                    ON)
+option(LLAMA_OPENBLAS               "llama: use OpenBLAS"                                   OFF)
+#option(LLAMA_CUBLAS                 "llama: use cuBLAS"                                     OFF)
+#option(LLAMA_CLBLAST                "llama: use CLBlast"                                    OFF)
+#option(LLAMA_METAL                  "llama: use Metal"                                      OFF)
+#option(LLAMA_K_QUANTS               "llama: use k-quants"                                   ON)
+set(LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor")
+set(LLAMA_CUDA_DMMV_X "32" CACHE STRING "llama: x stride for dmmv CUDA kernels")
+set(LLAMA_CUDA_DMMV_Y "1" CACHE STRING  "llama: y block size for dmmv CUDA kernels")
+
+#
+# Compile flags
+#
+
+set(CMAKE_C_STANDARD 11)
+set(CMAKE_C_STANDARD_REQUIRED true)
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+
+if (NOT MSVC)
+    if (LLAMA_SANITIZE_THREAD)
+        add_compile_options(-fsanitize=thread)
+        link_libraries(-fsanitize=thread)
+    endif()
+
+    if (LLAMA_SANITIZE_ADDRESS)
+        add_compile_options(-fsanitize=address -fno-omit-frame-pointer)
+        link_libraries(-fsanitize=address)
+    endif()
+
+    if (LLAMA_SANITIZE_UNDEFINED)
+        add_compile_options(-fsanitize=undefined)
+        link_libraries(-fsanitize=undefined)
+    endif()
+endif()
+
+if (APPLE AND LLAMA_ACCELERATE)
+    find_library(ACCELERATE_FRAMEWORK Accelerate)
+    if (ACCELERATE_FRAMEWORK)
+        message(STATUS "Accelerate framework found")
+
+        add_compile_definitions(GGML_USE_ACCELERATE)
+        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${ACCELERATE_FRAMEWORK})
+    else()
+        message(WARNING "Accelerate framework not found")
+    endif()
+endif()
+
+if (LLAMA_OPENBLAS)
+    if (LLAMA_STATIC)
+        set(BLA_STATIC ON)
+    endif()
+
+    set(BLA_VENDOR OpenBLAS)
+    find_package(BLAS)
+    if (BLAS_FOUND)
+        message(STATUS "OpenBLAS found")
+
+        add_compile_definitions(GGML_USE_OPENBLAS)
+        add_link_options(${BLAS_LIBRARIES})
+        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} openblas)
+
+        # find header file
+        set(OPENBLAS_INCLUDE_SEARCH_PATHS
+            /usr/include
+            /usr/include/openblas
+            /usr/include/openblas-base
+            /usr/local/include
+            /usr/local/include/openblas
+            /usr/local/include/openblas-base
+            /opt/OpenBLAS/include
+            $ENV{OpenBLAS_HOME}
+            $ENV{OpenBLAS_HOME}/include
+            )
+        find_path(OPENBLAS_INC NAMES cblas.h PATHS ${OPENBLAS_INCLUDE_SEARCH_PATHS})
+        add_compile_options(-I${OPENBLAS_INC})
+    else()
+        message(WARNING "OpenBLAS not found")
+    endif()
+endif()
+
+if (LLAMA_KOMPUTE)
+    add_compile_definitions(VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
+    find_package(Vulkan COMPONENTS glslc REQUIRED)
+    find_program(glslc_executable NAMES glslc HINTS Vulkan::glslc)
+    if (NOT glslc_executable)
+        message(FATAL_ERROR "glslc not found")
+    endif()
+
+    set(LLAMA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp-mainline)
+
+    function(compile_shader)
+      set(options)
+      set(oneValueArgs)
+      set(multiValueArgs SOURCES)
+      cmake_parse_arguments(compile_shader "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+      foreach(source ${compile_shader_SOURCES})
+        get_filename_component(OP_FILE ${source} NAME)
+        set(spv_file ${CMAKE_CURRENT_BINARY_DIR}/${OP_FILE}.spv)
+        add_custom_command(
+            OUTPUT ${spv_file}
+            DEPENDS ${LLAMA_DIR}/${source}
+              ${LLAMA_DIR}/kompute/common.comp
+              ${LLAMA_DIR}/kompute/op_getrows.comp
+              ${LLAMA_DIR}/kompute/op_mul_mv_q_n.comp
+            COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
+            COMMENT "Compiling ${source} to ${source}.spv"
+        )
+
+        get_filename_component(RAW_FILE_NAME ${spv_file} NAME)
+        set(FILE_NAME "shader${RAW_FILE_NAME}")
+        string(REPLACE ".comp.spv" ".h" HEADER_FILE ${FILE_NAME})
+        string(TOUPPER ${HEADER_FILE} HEADER_FILE_DEFINE)
+        string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}")
+        set(OUTPUT_HEADER_FILE "${HEADER_FILE}")
+        message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}")
+        if(CMAKE_GENERATOR MATCHES "Visual Studio") 
+            add_custom_command(
+              OUTPUT ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              DEPENDS ${spv_file} xxd
+              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}/xxd"
+            )
+        else()
+            add_custom_command(
+              OUTPUT ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              DEPENDS ${spv_file} xxd
+              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
+            )
+        endif()
+      endforeach()
+    endfunction()
+
+    if (EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
+        message(STATUS "Kompute found")
+        add_subdirectory(${LLAMA_DIR}/kompute)
+
+        # Compile our shaders
+        compile_shader(SOURCES
+          kompute/op_scale.comp
+          kompute/op_add.comp
+          kompute/op_addrow.comp
+          kompute/op_mul.comp
+          kompute/op_mulrow.comp
+          kompute/op_silu.comp
+          kompute/op_relu.comp
+          kompute/op_gelu.comp
+          kompute/op_softmax.comp
+          kompute/op_norm.comp
+          kompute/op_rmsnorm.comp
+          kompute/op_diagmask.comp
+          kompute/op_mul_mat_f16.comp
+          kompute/op_mul_mat_q4_0.comp
+          kompute/op_mul_mat_q4_1.comp
+          kompute/op_mul_mat_q6_k.comp
+          kompute/op_getrows_f16.comp
+          kompute/op_getrows_q4_0.comp
+          kompute/op_getrows_q4_1.comp
+          kompute/op_getrows_q6_k.comp
+          kompute/op_rope.comp
+          kompute/op_cpy_f16_f16.comp
+          kompute/op_cpy_f16_f32.comp
+          kompute/op_cpy_f32_f16.comp
+          kompute/op_cpy_f32_f32.comp
+        )
+
+        # Create a custom target for our generated shaders
+        add_custom_target(generated_shaders DEPENDS
+          shaderop_scale.h
+          shaderop_add.h
+          shaderop_addrow.h
+          shaderop_mul.h
+          shaderop_mulrow.h
+          shaderop_silu.h
+          shaderop_relu.h
+          shaderop_gelu.h
+          shaderop_softmax.h
+          shaderop_norm.h
+          shaderop_rmsnorm.h
+          shaderop_diagmask.h
+          shaderop_mul_mat_f16.h
+          shaderop_mul_mat_q4_0.h
+          shaderop_mul_mat_q4_1.h
+          shaderop_mul_mat_q6_k.h
+          shaderop_getrows_f16.h
+          shaderop_getrows_q4_0.h
+          shaderop_getrows_q4_1.h
+          shaderop_getrows_q6_k.h
+          shaderop_rope.h
+          shaderop_cpy_f16_f16.h
+          shaderop_cpy_f16_f32.h
+          shaderop_cpy_f32_f16.h
+          shaderop_cpy_f32_f32.h
+        )
+
+        # Create a custom command that depends on the generated_shaders
+        add_custom_command(
+            OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp
+            COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp
+            DEPENDS generated_shaders
+            COMMENT "Ensuring shaders are generated before compiling ggml-vulkan.cpp"
+        )
+
+        # Add the stamp to the main sources to ensure dependency tracking
+        set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml-vulkan.cpp ${LLAMA_DIR}/ggml-vulkan.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp)
+        add_compile_definitions(GGML_USE_KOMPUTE)
+        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute)
+        set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CMAKE_BINARY_DIR})
+    else()
+        message(WARNING "Kompute not found")
+    endif()
+endif()
+
+if (LLAMA_ALL_WARNINGS)
+    if (NOT MSVC)
+        set(c_flags
+            -Wall
+            -Wextra
+            -Wpedantic
+            -Wcast-qual
+            -Wdouble-promotion
+            -Wshadow
+            -Wstrict-prototypes
+            -Wpointer-arith
+        )
+        set(cxx_flags
+            -Wall
+            -Wextra
+            -Wpedantic
+            -Wcast-qual
+            -Wno-unused-function
+            -Wno-multichar
+        )
+    else()
+        # todo : msvc
+    endif()
+
+    add_compile_options(
+            "$<$<COMPILE_LANGUAGE:C>:${c_flags}>"
+            "$<$<COMPILE_LANGUAGE:CXX>:${cxx_flags}>"
+    )
+
+endif()
+
+if (MSVC)
+    add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
+
+    if (BUILD_SHARED_LIBS)
+        set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+    endif()
+endif()
+
+if (LLAMA_LTO)
+    include(CheckIPOSupported)
+    check_ipo_supported(RESULT result OUTPUT output)
+    if (result)
+        set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+    else()
+        message(WARNING "IPO is not supported: ${output}")
+    endif()
+endif()
+
+# Architecture specific
+# TODO: probably these flags need to be tweaked on some architectures
+#       feel free to update the Makefile for your architecture and send a pull request or issue
+message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
+if (NOT MSVC)
+    if (LLAMA_STATIC)
+        add_link_options(-static)
+        if (MINGW)
+            add_link_options(-static-libgcc -static-libstdc++)
+        endif()
+    endif()
+    if (LLAMA_GPROF)
+        add_compile_options(-pg)
+    endif()
+    if (LLAMA_NATIVE)
+        add_compile_options(-march=native)
+    endif()
+endif()
+
+function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
+    message(STATUS "Configuring ggml implementation target llama${SUFFIX} in ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}")
+
+    #
+    # Build libraries
+    #
+
+    set(GGML_CUBLAS_USE NO)
+    if (LLAMA_CUBLAS)
+        cmake_minimum_required(VERSION 3.17)
+
+        find_package(CUDAToolkit)
+        if (CUDAToolkit_FOUND)
+            set(GGML_CUBLAS_USE YES)
+            message(STATUS "cuBLAS found")
+
+            enable_language(CUDA)
+
+            set(GGML_SOURCES_CUDA ${DIRECTORY}/ggml-cuda.cu ${DIRECTORY}/ggml-cuda.h)
+
+            if (LLAMA_STATIC)
+                set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static)
+            else()
+                set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart CUDA::cublas CUDA::cublasLt)
+            endif()
+
+        else()
+            message(WARNING "cuBLAS not found")
+        endif()
+    endif()
+
+    set(GGML_CLBLAST_USE NO)
+    if (LLAMA_CLBLAST)
+        find_package(CLBlast)
+        if (CLBlast_FOUND)
+            set(GGML_CLBLAST_USE YES)
+            message(STATUS "CLBlast found")
+
+            set(GGML_OPENCL_SOURCE_FILE ggml-opencl.cpp)
+            if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/${GGML_OPENCL_SOURCE_FILE})
+                set(GGML_OPENCL_SOURCE_FILE ggml-opencl.c)
+            endif()
+
+            set(GGML_OPENCL_SOURCES ${DIRECTORY}/${GGML_OPENCL_SOURCE_FILE} ${DIRECTORY}/ggml-opencl.h)
+
+            set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} clblast)
+        else()
+            message(WARNING "CLBlast not found")
+        endif()
+    endif()
+
+    set(GGML_SOURCES_QUANT_K )
+    set(GGML_METAL_SOURCES )
+    if (LLAMA_K_QUANTS)
+        set(GGML_SOURCES_QUANT_K
+            ${DIRECTORY}/k_quants.h
+            ${DIRECTORY}/k_quants.c)
+
+        if (LLAMA_METAL)
+            find_library(FOUNDATION_LIBRARY         Foundation              REQUIRED)
+            find_library(METAL_FRAMEWORK            Metal                   REQUIRED)
+            find_library(METALKIT_FRAMEWORK         MetalKit                REQUIRED)
+            find_library(METALPERFORMANCE_FRAMEWORK MetalPerformanceShaders REQUIRED)
+
+            set(GGML_METAL_SOURCES ${DIRECTORY}/ggml-metal.m ${DIRECTORY}/ggml-metal.h)
+            # get full path to the file
+            #add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
+
+            # copy ggml-metal.metal to bin directory
+            configure_file(${DIRECTORY}/ggml-metal.metal bin/ggml-metal.metal COPYONLY)
+
+            set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS}
+                ${FOUNDATION_LIBRARY}
+                ${METAL_FRAMEWORK}
+                ${METALKIT_FRAMEWORK}
+                ${METALPERFORMANCE_FRAMEWORK}
+            )
+        endif()
+    endif()
+
+    add_library(ggml${SUFFIX} OBJECT
+                ${DIRECTORY}/ggml.c
+                ${DIRECTORY}/ggml.h
+                ${DIRECTORY}/ggml-alloc.c
+                ${DIRECTORY}/ggml-alloc.h
+                ${GGML_SOURCES_QUANT_K}
+                ${GGML_SOURCES_CUDA}
+                ${GGML_METAL_SOURCES}
+                ${GGML_OPENCL_SOURCES}
+                ${GGML_SOURCES_KOMPUTE})
+
+    if (LLAMA_K_QUANTS)
+        target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_K_QUANTS)
+    endif()
+
+    if (LLAMA_METAL AND GGML_METAL_SOURCES)
+        target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)
+    endif()
+    target_include_directories(ggml${SUFFIX} PUBLIC ${DIRECTORY})
+    target_compile_features(ggml${SUFFIX} PUBLIC c_std_11) # don't bump
+
+    if (BUILD_SHARED_LIBS)
+        set_target_properties(ggml${SUFFIX} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+    endif()
+
+    if (WITH_LLAMA)
+        # Backwards compatibility with old llama.cpp versions
+        set(LLAMA_UTIL_SOURCE_FILE llama-util.h)
+        if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
+            set(LLAMA_UTIL_SOURCE_FILE llama_util.h)
+        endif()
+
+        add_library(llama${SUFFIX} STATIC
+                    ${DIRECTORY}/llama.cpp
+                    ${DIRECTORY}/llama.h
+                    ${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
+
+        if (LLAMA_METAL AND GGML_METAL_SOURCES)
+            target_compile_definitions(llama${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)
+        endif()
+        target_include_directories(llama${SUFFIX} PUBLIC ${DIRECTORY})
+        target_compile_features(llama${SUFFIX} PUBLIC cxx_std_11) # don't bump
+
+        if (BUILD_SHARED_LIBS)
+            set_target_properties(llama${SUFFIX} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+            target_compile_definitions(llama${SUFFIX} PRIVATE LLAMA_SHARED LLAMA_BUILD)
+        endif()
+    endif()
+
+    if (GGML_SOURCES_CUDA)
+        message(STATUS "GGML CUDA sources found, configuring CUDA architecture")
+        set_property(TARGET ggml${SUFFIX} PROPERTY CUDA_ARCHITECTURES OFF)
+        set_property(TARGET ggml${SUFFIX} PROPERTY CUDA_SELECT_NVCC_ARCH_FLAGS "Auto")
+        if (WITH_LLAMA)
+            set_property(TARGET llama${SUFFIX} PROPERTY CUDA_ARCHITECTURES OFF)
+        endif()
+    endif()
+
+    if (GGML_CUBLAS_USE)
+        target_compile_definitions(ggml${SUFFIX} PRIVATE
+            GGML_USE_CUBLAS
+            GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X}
+            GGML_CUDA_DMMV_Y=${LLAMA_CUDA_DMMV_Y})
+        if (WITH_LLAMA)
+            target_compile_definitions(llama${SUFFIX} PRIVATE
+                GGML_USE_CUBLAS
+                GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X}
+                GGML_CUDA_DMMV_Y=${LLAMA_CUDA_DMMV_Y})
+        endif()
+    endif()
+    if (GGML_CLBLAST_USE)
+        if (WITH_LLAMA)
+            target_compile_definitions(llama${SUFFIX} PRIVATE GGML_USE_CLBLAST)
+        endif()
+        target_compile_definitions(ggml${SUFFIX} PRIVATE GGML_USE_CLBLAST)
+    endif()
+
+    if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
+        message(STATUS "ARM detected")
+        if (MSVC)
+            # TODO: arm msvc?
+        else()
+            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
+                target_compile_options(ggml${SUFFIX} PRIVATE -mcpu=native)
+            endif()
+            # TODO: armv6,7,8 version specific flags
+        endif()
+    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$")
+        message(STATUS "x86 detected")
+        if (MSVC)
+            if (LLAMA_AVX512)
+                target_compile_options(ggml${SUFFIX} PRIVATE
+                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX512>
+                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
+                # MSVC has no compile-time flags enabling specific
+                # AVX512 extensions, neither it defines the
+                # macros corresponding to the extensions.
+                # Do it manually.
+                if (LLAMA_AVX512_VBMI)
+                    target_compile_definitions(ggml${SUFFIX} PRIVATE
+                        $<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>
+                        $<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
+                endif()
+                if (LLAMA_AVX512_VNNI)
+                    target_compile_definitions(ggml${SUFFIX} PRIVATE
+                        $<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>
+                        $<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
+                endif()
+            elseif (LLAMA_AVX2)
+                target_compile_options(ggml${SUFFIX} PRIVATE
+                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX2>
+                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
+            elseif (LLAMA_AVX)
+                target_compile_options(ggml${SUFFIX} PRIVATE
+                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX>
+                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
+            endif()
+        else()
+            if (LLAMA_F16C)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mf16c)
+            endif()
+            if (LLAMA_FMA)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mfma)
+            endif()
+            if (LLAMA_AVX)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx)
+            endif()
+            if (LLAMA_AVX2)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx2)
+            endif()
+            if (LLAMA_AVX512)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512f)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512bw)
+            endif()
+            if (LLAMA_AVX512_VBMI)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512vbmi)
+            endif()
+            if (LLAMA_AVX512_VNNI)
+                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512vnni)
+            endif()
+        endif()
+    else()
+        # TODO: support PowerPC
+        message(STATUS "Unknown architecture")
+    endif()
+
+    target_link_libraries(ggml${SUFFIX} PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
+    if (WITH_LLAMA)
+        target_link_libraries(llama${SUFFIX} PRIVATE ggml${SUFFIX} ${LLAMA_EXTRA_LIBS})
+    endif()
+endfunction()

gpt4all-backend/llamamodel.cpp

@@ -0,0 +1,411 @@
+#define LLAMAMODEL_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#include "llamamodel_impl.h"
+
+#include <cassert>
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+#include <fstream>
+#include <map>
+#include <string>
+#include <vector>
+#include <iostream>
+#if defined(_WIN32) && defined(_MSC_VER)
+    #define WIN32_LEAN_AND_MEAN
+    #ifndef NOMINMAX
+        #define NOMINMAX
+    #endif
+    #include <windows.h>
+    #include <io.h>
+    #include <stdio.h>
+#else
+    #include <unistd.h>
+#endif
+#include <random>
+#include <thread>
+#include <unordered_set>
+
+#include <llama.h>
+#include <ggml.h>
+
+#ifdef GGML_USE_KOMPUTE
+#include "ggml-vulkan.h"
+#endif
+
+namespace {
+const char *modelType_ = "LLaMA";
+}
+
+struct gpt_params {
+    int32_t seed          = -1;   // RNG seed
+    int32_t n_keep        = 0;    // number of tokens to keep from initial prompt
+#if LLAMA_DATE <= 230511
+    int32_t n_parts       = -1;   // amount of model parts (-1 = determine from model dimensions)
+#endif
+
+#if LLAMA_DATE >= 230519
+    // sampling parameters
+    float   tfs_z         = 1.0f; // 1.0 = disabled
+    float   typical_p     = 1.0f; // 1.0 = disabled
+#endif
+
+    std::string prompt = "";
+
+    bool memory_f16        = true;  // use f16 instead of f32 for memory kv
+
+    bool use_mmap          = true;  // use mmap for faster loads
+    bool use_mlock         = false; // use mlock to keep model in memory
+};
+
+#if LLAMA_DATE >= 230519
+static int llama_sample_top_p_top_k(
+        llama_context *ctx,
+        const llama_token *last_n_tokens_data,
+        int last_n_tokens_size,
+        int top_k,
+        float top_p,
+        float temp,
+        float repeat_penalty) {
+    auto logits = llama_get_logits(ctx);
+    auto n_vocab = llama_n_vocab(ctx);
+    // Populate initial list of all candidates
+    std::vector<llama_token_data> candidates;
+    candidates.reserve(n_vocab);
+    for (int token_id = 0; token_id < n_vocab; token_id++) {
+        candidates.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f});
+    }
+    llama_token_data_array candidates_p = {candidates.data(), candidates.size(), false};
+    // Sample repeat penalty
+    llama_sample_repetition_penalty(nullptr, &candidates_p, last_n_tokens_data, last_n_tokens_size, repeat_penalty);
+    // Temperature sampling
+    llama_sample_top_k(ctx, &candidates_p, top_k, 1);
+    llama_sample_tail_free(ctx, &candidates_p, 1.0f, 1);
+    llama_sample_typical(ctx, &candidates_p, 1.0f, 1);
+    llama_sample_top_p(ctx, &candidates_p, top_p, 1);
+    llama_sample_temperature(ctx, &candidates_p, temp);
+    return llama_sample_token(ctx, &candidates_p);
+}
+#endif
+
+struct LLamaPrivate {
+    const std::string modelPath;
+    bool modelLoaded;
+    llama_context *ctx = nullptr;
+    llama_context_params params;
+    int64_t n_threads = 0;
+};
+
+LLamaModel::LLamaModel()
+    : d_ptr(new LLamaPrivate) {
+    d_ptr->modelLoaded = false;
+}
+
+// default hparams (LLaMA 7B)
+struct llama_file_hparams {
+    uint32_t n_vocab = 32000;
+    uint32_t n_embd  = 4096;
+    uint32_t n_mult  = 256;
+    uint32_t n_head  = 32;
+    uint32_t n_layer = 32;
+    uint32_t n_rot   = 64;
+    enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16;
+};
+
+size_t LLamaModel::requiredMem(const std::string &modelPath) {
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    fin.seekg(0, std::ios_base::end);
+    size_t filesize = fin.tellg();
+    fin.seekg(0, std::ios_base::beg);
+    uint32_t magic = 0;
+    fin.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    if (magic != 0x67676a74) return 0;
+    uint32_t version = 0;
+    fin.read(reinterpret_cast<char*>(&version), sizeof(version));
+    llama_file_hparams hparams;
+    fin.read(reinterpret_cast<char*>(&hparams.n_vocab), sizeof(hparams.n_vocab));
+    fin.read(reinterpret_cast<char*>(&hparams.n_embd), sizeof(hparams.n_embd));
+    fin.read(reinterpret_cast<char*>(&hparams.n_head), sizeof(hparams.n_head));
+    fin.read(reinterpret_cast<char*>(&hparams.n_layer), sizeof(hparams.n_layer));
+    fin.read(reinterpret_cast<char*>(&hparams.n_rot), sizeof(hparams.n_rot));
+    fin.read(reinterpret_cast<char*>(&hparams.ftype), sizeof(hparams.ftype));
+    const size_t n_ctx = 2048;
+    const size_t kvcache_element_size = 2; // fp16
+    const size_t est_kvcache_size = hparams.n_embd * hparams.n_layer * 2u * n_ctx * kvcache_element_size;
+    return filesize + est_kvcache_size;
+}
+
+bool LLamaModel::loadModel(const std::string &modelPath)
+{
+    // load the model
+    d_ptr->params = llama_context_default_params();
+
+    gpt_params params;
+    d_ptr->params.n_ctx      = 2048;
+    d_ptr->params.seed       = params.seed;
+    d_ptr->params.f16_kv     = params.memory_f16;
+    d_ptr->params.use_mmap   = params.use_mmap;
+#if defined (__APPLE__)
+    d_ptr->params.use_mlock  = true;
+#else
+    d_ptr->params.use_mlock  = params.use_mlock;
+#endif
+#if LLAMA_DATE <= 230511
+    d_ptr->params.n_parts  = params.n_parts;
+#endif
+#ifdef GGML_USE_METAL
+    std::cerr << "llama.cpp: using Metal" << std::endl;
+    // metal always runs the whole model if n_gpu_layers is not 0, at least
+    // currently
+    d_ptr->params.n_gpu_layers = 1;
+#endif
+#ifdef GGML_USE_KOMPUTE
+    if (ggml_vk_has_device()) {
+        // vulkan always runs the whole model if n_gpu_layers is not 0, at least
+        // currently
+        d_ptr->params.n_gpu_layers = 1;
+    }
+#endif
+
+    d_ptr->ctx = llama_init_from_file(modelPath.c_str(), d_ptr->params);
+    if (!d_ptr->ctx) {
+#ifdef GGML_USE_KOMPUTE
+        // Explicitly free the device so next load it doesn't use it
+        ggml_vk_free_device();
+#endif
+        std::cerr << "LLAMA ERROR: failed to load model from " <<  modelPath << std::endl;
+        return false;
+    }
+
+#ifdef GGML_USE_KOMPUTE
+    if (ggml_vk_has_device()) {
+        std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
+    }
+#endif
+
+    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    d_ptr->modelLoaded = true;
+    fflush(stderr);
+    return true;
+}
+
+void LLamaModel::setThreadCount(int32_t n_threads) {
+    d_ptr->n_threads = n_threads;
+}
+
+int32_t LLamaModel::threadCount() const {
+    return d_ptr->n_threads;
+}
+
+LLamaModel::~LLamaModel()
+{
+    if (d_ptr->ctx) {
+        llama_free(d_ptr->ctx);
+    }
+}
+
+bool LLamaModel::isModelLoaded() const
+{
+    return d_ptr->modelLoaded;
+}
+
+size_t LLamaModel::stateSize() const
+{
+    return llama_get_state_size(d_ptr->ctx);
+}
+
+size_t LLamaModel::saveState(uint8_t *dest) const
+{
+    return llama_copy_state_data(d_ptr->ctx, dest);
+}
+
+size_t LLamaModel::restoreState(const uint8_t *src)
+{
+    // const_cast is required, see: https://github.com/ggerganov/llama.cpp/pull/1540
+    return llama_set_state_data(d_ptr->ctx, const_cast<uint8_t*>(src));
+}
+
+std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str) const
+{
+    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
+    std::vector<LLModel::Token> fres(str.size()+4);
+    auto fres_len = llama_tokenize(d_ptr->ctx, str.c_str(), fres.data(), fres.size(), useBOS);
+    fres.resize(fres_len);
+    return fres;
+}
+
+std::string LLamaModel::tokenToString(Token id) const
+{
+    return llama_token_to_str(d_ptr->ctx, id);
+}
+
+LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
+{
+    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
+    return llama_sample_top_p_top_k(d_ptr->ctx,
+        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
+        n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
+        promptCtx.repeat_penalty);
+}
+
+bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
+{
+    // When we recalculate context we could have erased the original BOS token... we need to replace it
+    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
+    if (useBOS) {
+        std::vector<int32_t> myTokens;
+        myTokens.push_back(llama_token_bos());
+        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
+        ctx.n_past += 1;
+        return llama_eval(d_ptr->ctx, myTokens.data(), myTokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+    } else
+        return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+}
+
+int32_t LLamaModel::contextLength() const
+{
+    return llama_n_ctx(d_ptr->ctx);
+}
+
+const std::vector<LLModel::Token> &LLamaModel::endTokens() const
+{
+    static const std::vector<LLModel::Token> fres = {llama_token_eos()};
+    return fres;
+}
+
+#if defined(GGML_USE_KOMPUTE)
+#include "ggml-vulkan.h"
+#endif
+
+std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryRequired)
+{
+#if defined(GGML_USE_KOMPUTE)
+    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(memoryRequired);
+
+    std::vector<LLModel::GPUDevice> devices;
+    for(const auto& vkDevice : vkDevices) {
+        LLModel::GPUDevice device;
+        device.index = vkDevice.index;
+        device.type = vkDevice.type;
+        device.heapSize = vkDevice.heapSize;
+        device.name = vkDevice.name;
+        device.vendor = vkDevice.vendor;
+
+        devices.push_back(device);
+    }
+
+    return devices;
+#else
+    return std::vector<LLModel::GPUDevice>();
+#endif
+}
+
+bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string& device)
+{
+#if defined(GGML_USE_KOMPUTE)
+    return ggml_vk_init_device(memoryRequired, device);
+#else
+    return false;
+#endif
+}
+
+bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device)
+{
+#if defined(GGML_USE_KOMPUTE)
+    ggml_vk_device vkDevice;
+    vkDevice.index = device.index;
+    vkDevice.type = device.type;
+    vkDevice.heapSize = device.heapSize;
+    vkDevice.name = device.name;
+    vkDevice.vendor = device.vendor;
+    return ggml_vk_init_device(vkDevice);
+#else
+    return false;
+#endif
+}
+
+bool LLamaModel::initializeGPUDevice(int device)
+{
+#if defined(GGML_USE_KOMPUTE)
+    return ggml_vk_init_device(device);
+#else
+    return false;
+#endif
+}
+
+bool LLamaModel::hasGPUDevice()
+{
+#if defined(GGML_USE_KOMPUTE)
+    return ggml_vk_has_device();
+#else
+    return false;
+#endif
+}
+
+bool LLamaModel::usingGPUDevice()
+{
+#if defined(GGML_USE_KOMPUTE)
+    return ggml_vk_using_vulkan();
+#elif defined(GGML_USE_METAL)
+    return true;
+#endif
+    return false;
+}
+
+#if defined(_WIN32)
+#define DLL_EXPORT __declspec(dllexport)
+#else
+#define DLL_EXPORT __attribute__ ((visibility ("default")))
+#endif
+
+extern "C" {
+DLL_EXPORT bool is_g4a_backend_model_implementation() {
+    return true;
+}
+
+DLL_EXPORT const char *get_model_type() {
+    return modelType_;
+}
+
+DLL_EXPORT const char *get_build_variant() {
+    return GGML_BUILD_VARIANT;
+}
+
+DLL_EXPORT bool magic_match(std::istream& f) {
+    // Check magic
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    if (magic != 0x67676a74) return false;
+    // Check version
+    uint32_t version = 0;
+    f.read(reinterpret_cast<char*>(&version), sizeof(version));
+    if (!(version LLAMA_VERSIONS)) {
+        return false;
+    }
+    llama_file_hparams hparams;
+    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
+    if (!(hparams.n_vocab >= 32000 && hparams.n_vocab <= 32100)) {
+        return false; // not a llama.
+    }
+#ifdef GGML_USE_METAL
+    // Check quant supported on metal
+    // skip fields
+    switch(hparams.ftype) {
+        // currently supported on Metal https://github.com/ggerganov/llama.cpp/blob/ae9663f1887513e152839e91f61c513075a19422/ggml-metal.m#L51-L55
+        case LLAMA_FTYPE_MOSTLY_F16:
+        case LLAMA_FTYPE_MOSTLY_Q2_K:
+        case LLAMA_FTYPE_MOSTLY_Q4_0:
+        case LLAMA_FTYPE_MOSTLY_Q6_K:
+        case LLAMA_FTYPE_MOSTLY_Q4_K_S:
+        case LLAMA_FTYPE_MOSTLY_Q4_K_M:
+            return true;
+        default: // unsupported quant-type for Metal
+            return false;
+    }
+#endif
+    return true;
+}
+
+DLL_EXPORT LLModel *construct() {
+    return new LLamaModel;
+}
+}

gpt4all-backend/llamamodel_impl.h

@@ -0,0 +1,47 @@
+#ifndef LLAMAMODEL_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of llamamodel.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define LLAMAMODEL_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef LLAMAMODEL_H
+#define LLAMAMODEL_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include "llmodel.h"
+
+struct LLamaPrivate;
+class LLamaModel : public LLModel {
+public:
+    LLamaModel();
+    ~LLamaModel();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) override;
+    bool initializeGPUDevice(size_t memoryRequired, const std::string& device) override;
+    bool initializeGPUDevice(const GPUDevice &device) override;
+    bool initializeGPUDevice(int device) override;
+    bool hasGPUDevice() override;
+    bool usingGPUDevice() override;
+
+private:
+    LLamaPrivate *d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::string tokenToString(Token) const override;
+    Token sampleToken(PromptContext& ctx) const override;
+    bool evalTokens(PromptContext& ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // LLAMAMODEL_H

gpt4all-backend/llmodel.cpp

@@ -0,0 +1,181 @@
+#include "llmodel.h"
+#include "dlhandle.h"
+#include "sysinfo.h"
+
+#include <iostream>
+#include <string>
+#include <vector>
+#include <fstream>
+#include <filesystem>
+#include <cassert>
+#include <cstdlib>
+#include <sstream>
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+
+std::string s_implementations_search_path = ".";
+
+static bool has_at_least_minimal_hardware() {
+#if defined(__x86_64__) || defined(_M_X64)
+    #ifndef _MSC_VER
+        return __builtin_cpu_supports("avx");
+    #else
+        int cpuInfo[4];
+        __cpuid(cpuInfo, 1);
+        return cpuInfo[2] & (1 << 28);
+    #endif
+#else
+    return true; // Don't know how to handle non-x86_64
+#endif
+}
+
+static bool requires_avxonly() {
+#if defined(__x86_64__) || defined(_M_X64)
+    #ifndef _MSC_VER
+        return !__builtin_cpu_supports("avx2");
+    #else
+        int cpuInfo[4];
+        __cpuidex(cpuInfo, 7, 0);
+        return !(cpuInfo[1] & (1 << 5));
+    #endif
+#else
+    return false; // Don't know how to handle non-x86_64
+#endif
+}
+
+LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
+    : m_dlhandle(new Dlhandle(std::move(dlhandle_))) {
+    auto get_model_type = m_dlhandle->get<const char *()>("get_model_type");
+    assert(get_model_type);
+    m_modelType = get_model_type();
+    auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
+    assert(get_build_variant);
+    m_buildVariant = get_build_variant();
+    m_magicMatch = m_dlhandle->get<bool(std::ifstream&)>("magic_match");
+    assert(m_magicMatch);
+    m_construct = m_dlhandle->get<LLModel *()>("construct");
+    assert(m_construct);
+}
+
+LLModel::Implementation::Implementation(Implementation &&o)
+    : m_magicMatch(o.m_magicMatch)
+    , m_construct(o.m_construct)
+    , m_modelType(o.m_modelType)
+    , m_buildVariant(o.m_buildVariant)
+    , m_dlhandle(o.m_dlhandle) {
+    o.m_dlhandle = nullptr;
+}
+
+LLModel::Implementation::~Implementation() {
+    if (m_dlhandle) delete m_dlhandle;
+}
+
+bool LLModel::Implementation::isImplementation(const Dlhandle &dl) {
+    return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
+}
+
+const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
+    // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
+    // individual models without the cleanup of the static list interfering
+    static auto* libs = new std::vector<Implementation>([] () {
+        std::vector<Implementation> fres;
+
+        auto search_in_directory = [&](const std::string& paths) {
+            std::stringstream ss(paths);
+            std::string path;
+            // Split the paths string by the delimiter and process each path.
+            while (std::getline(ss, path, ';')) {
+                std::filesystem::path fs_path(path);
+                // Iterate over all libraries
+                for (const auto& f : std::filesystem::directory_iterator(fs_path)) {
+                    const std::filesystem::path& p = f.path();
+                    if (p.extension() != LIB_FILE_EXT) continue;
+                    // Add to list if model implementation
+                    try {
+                        Dlhandle dl(p.string());
+                        if (!Implementation::isImplementation(dl)) {
+                            continue;
+                        }
+                        fres.emplace_back(Implementation(std::move(dl)));
+                    } catch (...) {}
+                }
+            }
+        };
+
+        search_in_directory(s_implementations_search_path);
+
+        return fres;
+    }());
+    // Return static result
+    return *libs;
+}
+
+const LLModel::Implementation* LLModel::Implementation::implementation(std::ifstream& f, const std::string& buildVariant) {
+    for (const auto& i : implementationList()) {
+        f.seekg(0);
+        if (!i.m_magicMatch(f)) continue;
+        if (buildVariant != i.m_buildVariant) continue;
+        return &i;
+    }
+    return nullptr;
+}
+
+LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant) {
+
+    if (!has_at_least_minimal_hardware())
+        return nullptr;
+
+    // Read magic
+    std::ifstream f(modelPath, std::ios::binary);
+    if (!f) return nullptr;
+    // Get correct implementation
+    const Implementation* impl = nullptr;
+
+    #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
+        if (buildVariant == "auto") {
+            size_t total_mem = getSystemTotalRAMInBytes();
+            impl = implementation(f, "metal");
+            if(impl) {
+                LLModel* metalimpl = impl->m_construct();
+                metalimpl->m_implementation = impl;
+                size_t req_mem = metalimpl->requiredMem(modelPath);
+                float req_to_total = (float) req_mem / (float) total_mem;
+                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
+                if (req_to_total >= 0.53) {
+                    delete metalimpl;
+                    impl = nullptr;
+                } else {
+                    return metalimpl;
+                }
+            }
+        }
+    #endif
+
+    if (!impl) {
+        //TODO: Auto-detect CUDA/OpenCL
+        if (buildVariant == "auto") {
+            if (requires_avxonly()) {
+                buildVariant = "avxonly";
+            } else {
+                buildVariant = "default";
+            }
+        }
+        impl = implementation(f, buildVariant);
+        if (!impl) return nullptr;
+    }
+    f.close();
+
+    // Construct and return llmodel implementation
+    auto fres = impl->m_construct();
+    fres->m_implementation = impl;
+    return fres;
+}
+
+void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
+    s_implementations_search_path = path;
+}
+
+const std::string& LLModel::Implementation::implementationsSearchPath() {
+    return s_implementations_search_path;
+}

gpt4all-backend/llmodel.h

@@ -0,0 +1,126 @@
+#ifndef LLMODEL_H
+#define LLMODEL_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <string_view>
+#include <fstream>
+#include <cstdint>
+#include <limits>
+
+#define LLMODEL_MAX_PROMPT_BATCH 128
+
+class Dlhandle;
+class LLModel {
+public:
+    using Token = int32_t;
+    class Implementation {
+    public:
+        Implementation(Dlhandle&&);
+        Implementation(const Implementation&) = delete;
+        Implementation(Implementation&&);
+        ~Implementation();
+
+        std::string_view modelType() const { return m_modelType; }
+        std::string_view buildVariant() const { return m_buildVariant; }
+
+        static bool isImplementation(const Dlhandle&);
+        static const std::vector<Implementation>& implementationList();
+        static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
+        static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
+        static void setImplementationsSearchPath(const std::string& path);
+        static const std::string& implementationsSearchPath();
+
+    private:
+        bool (*m_magicMatch)(std::ifstream& f);
+        LLModel *(*m_construct)();
+
+    private:
+        std::string_view m_modelType;
+        std::string_view m_buildVariant;
+        Dlhandle *m_dlhandle;
+    };
+
+    struct PromptContext {
+        std::vector<float> logits;      // logits of current context
+        std::vector<int32_t> tokens;    // current tokens in the context window
+        int32_t n_past = 0;             // number of tokens in past conversation
+        int32_t n_ctx = 0;              // number of tokens possible in context window
+        int32_t n_predict = 200;
+        int32_t top_k = 40;
+        float   top_p = 0.9f;
+        float   temp = 0.9f;
+        int32_t n_batch = 9;
+        float   repeat_penalty = 1.10f;
+        int32_t repeat_last_n = 64;     // last n tokens to penalize
+        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context
+            // window
+    };
+
+    struct GPUDevice {
+        int index = 0;
+        int type = 0;
+        size_t heapSize = 0;
+        std::string name;
+        std::string vendor;
+    };
+
+    explicit LLModel() {}
+    virtual ~LLModel() {}
+
+    virtual bool supportsEmbedding() const = 0;
+    virtual bool supportsCompletion() const = 0;
+    virtual bool loadModel(const std::string &modelPath) = 0;
+    virtual bool isModelLoaded() const = 0;
+    virtual size_t requiredMem(const std::string &modelPath) = 0;
+    virtual size_t stateSize() const { return 0; }
+    virtual size_t saveState(uint8_t */*dest*/) const { return 0; }
+    virtual size_t restoreState(const uint8_t */*src*/) { return 0; }
+
+    // This method requires the model to return true from supportsCompletion otherwise it will throw
+    // an error
+    virtual void prompt(const std::string &prompt,
+                        std::function<bool(int32_t)> promptCallback,
+                        std::function<bool(int32_t, const std::string&)> responseCallback,
+                        std::function<bool(bool)> recalculateCallback,
+                        PromptContext &ctx);
+
+    virtual std::vector<float> embedding(const std::string &text);
+
+    virtual void setThreadCount(int32_t /*n_threads*/) {}
+    virtual int32_t threadCount() const { return 1; }
+
+    const Implementation& implementation() const {
+        return *m_implementation;
+    }
+
+    virtual std::vector<GPUDevice> availableGPUDevices(size_t /*memoryRequired*/) { return std::vector<GPUDevice>(); }
+    virtual bool initializeGPUDevice(size_t /*memoryRequired*/, const std::string& /*device*/) { return false; }
+    virtual bool initializeGPUDevice(const GPUDevice &/*device*/) { return false; }
+    virtual bool initializeGPUDevice(int /*device*/) { return false; }
+    virtual bool hasGPUDevice() { return false; }
+    virtual bool usingGPUDevice() { return false; }
+    static std::vector<GPUDevice> availableGPUDevices();
+
+protected:
+    // These are pure virtual because subclasses need to implement as the default implementation of
+    // 'prompt' above calls these functions
+    virtual std::vector<Token> tokenize(PromptContext &, const std::string&) const = 0;
+    virtual std::string tokenToString(Token) const = 0;
+    virtual Token sampleToken(PromptContext &ctx) const = 0;
+    virtual bool evalTokens(PromptContext &/*ctx*/, const std::vector<int32_t>& /*tokens*/) const = 0;
+    virtual int32_t contextLength() const = 0;
+    virtual const std::vector<Token>& endTokens() const = 0;
+
+    // This is a helper function called from the default implementation of 'prompt' but it can be
+    // shared by all base classes so it isn't virtual
+    void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate);
+
+    const Implementation *m_implementation = nullptr;
+
+private:
+    friend class LLMImplementation;
+};
+
+#endif // LLMODEL_H

gpt4all-backend/llmodel_c.cpp

@@ -0,0 +1,265 @@
+#include "llmodel_c.h"
+#include "llmodel.h"
+
+#include <cstring>
+#include <cerrno>
+#include <utility>
+
+struct LLModelWrapper {
+    LLModel *llModel = nullptr;
+    LLModel::PromptContext promptContext;
+    ~LLModelWrapper() { delete llModel; }
+};
+
+
+thread_local static std::string last_error_message;
+
+
+llmodel_model llmodel_model_create(const char *model_path) {
+    auto fres = llmodel_model_create2(model_path, "auto", nullptr);
+    if (!fres) {
+        fprintf(stderr, "Invalid model file\n");
+    }
+    return fres;
+}
+
+llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error) {
+    auto wrapper = new LLModelWrapper;
+    int error_code = 0;
+
+    try {
+        wrapper->llModel = LLModel::Implementation::construct(model_path, build_variant);
+    } catch (const std::exception& e) {
+        error_code = EINVAL;
+        last_error_message = e.what();
+    }
+
+    if (!wrapper->llModel) {
+        delete std::exchange(wrapper, nullptr);
+        // Get errno and error message if none
+        if (error_code == 0) {
+            if (errno != 0) {
+                error_code = errno;
+                last_error_message = std::strerror(error_code);
+            } else {
+                error_code = ENOTSUP;
+                last_error_message = "Model format not supported (no matching implementation found)";
+            }
+        }
+        // Set error argument
+        if (error) {
+            error->message = last_error_message.c_str();
+            error->code = error_code;
+        }
+    }
+    return reinterpret_cast<llmodel_model*>(wrapper);
+}
+
+void llmodel_model_destroy(llmodel_model model) {
+    delete reinterpret_cast<LLModelWrapper*>(model);
+}
+
+size_t llmodel_required_mem(llmodel_model model, const char *model_path)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->requiredMem(model_path);
+}
+
+bool llmodel_loadModel(llmodel_model model, const char *model_path)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->loadModel(model_path);
+}
+
+bool llmodel_isModelLoaded(llmodel_model model)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->isModelLoaded();
+}
+
+uint64_t llmodel_get_state_size(llmodel_model model)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->stateSize();
+}
+
+uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->saveState(dest);
+}
+
+uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->restoreState(src);
+}
+
+// Wrapper functions for the C callbacks
+bool prompt_wrapper(int32_t token_id, void *user_data) {
+    llmodel_prompt_callback callback = reinterpret_cast<llmodel_prompt_callback>(user_data);
+    return callback(token_id);
+}
+
+bool response_wrapper(int32_t token_id, const std::string &response, void *user_data) {
+    llmodel_response_callback callback = reinterpret_cast<llmodel_response_callback>(user_data);
+    return callback(token_id, response.c_str());
+}
+
+bool recalculate_wrapper(bool is_recalculating, void *user_data) {
+    llmodel_recalculate_callback callback = reinterpret_cast<llmodel_recalculate_callback>(user_data);
+    return callback(is_recalculating);
+}
+
+void llmodel_prompt(llmodel_model model, const char *prompt,
+                    llmodel_prompt_callback prompt_callback,
+                    llmodel_response_callback response_callback,
+                    llmodel_recalculate_callback recalculate_callback,
+                    llmodel_prompt_context *ctx)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+
+    // Create std::function wrappers that call the C function pointers
+    std::function<bool(int32_t)> prompt_func =
+        std::bind(&prompt_wrapper, std::placeholders::_1, reinterpret_cast<void*>(prompt_callback));
+    std::function<bool(int32_t, const std::string&)> response_func =
+        std::bind(&response_wrapper, std::placeholders::_1, std::placeholders::_2, reinterpret_cast<void*>(response_callback));
+    std::function<bool(bool)> recalc_func =
+        std::bind(&recalculate_wrapper, std::placeholders::_1, reinterpret_cast<void*>(recalculate_callback));
+
+    if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
+        wrapper->promptContext.tokens.resize(ctx->n_past);
+
+    // Copy the C prompt context
+    wrapper->promptContext.n_past = ctx->n_past;
+    wrapper->promptContext.n_ctx = ctx->n_ctx;
+    wrapper->promptContext.n_predict = ctx->n_predict;
+    wrapper->promptContext.top_k = ctx->top_k;
+    wrapper->promptContext.top_p = ctx->top_p;
+    wrapper->promptContext.temp = ctx->temp;
+    wrapper->promptContext.n_batch = ctx->n_batch;
+    wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
+    wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
+    wrapper->promptContext.contextErase = ctx->context_erase;
+
+    // Call the C++ prompt method
+    wrapper->llModel->prompt(prompt, prompt_func, response_func, recalc_func, wrapper->promptContext);
+
+    // Update the C context by giving access to the wrappers raw pointers to std::vector data
+    // which involves no copies
+    ctx->logits = wrapper->promptContext.logits.data();
+    ctx->logits_size = wrapper->promptContext.logits.size();
+    ctx->tokens = wrapper->promptContext.tokens.data();
+    ctx->tokens_size = wrapper->promptContext.tokens.size();
+
+    // Update the rest of the C prompt context
+    ctx->n_past = wrapper->promptContext.n_past;
+    ctx->n_ctx = wrapper->promptContext.n_ctx;
+    ctx->n_predict = wrapper->promptContext.n_predict;
+    ctx->top_k = wrapper->promptContext.top_k;
+    ctx->top_p = wrapper->promptContext.top_p;
+    ctx->temp = wrapper->promptContext.temp;
+    ctx->n_batch = wrapper->promptContext.n_batch;
+    ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
+    ctx->repeat_last_n = wrapper->promptContext.repeat_last_n;
+    ctx->context_erase = wrapper->promptContext.contextErase;
+}
+
+float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size)
+{
+    if (model == nullptr || text == nullptr || !strlen(text)) {
+        *embedding_size = 0;
+        return nullptr;
+    }
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    std::vector<float> embeddingVector = wrapper->llModel->embedding(text);
+    float *embedding = (float *)malloc(embeddingVector.size() * sizeof(float));
+    if (embedding == nullptr) {
+        *embedding_size = 0;
+        return nullptr;
+    }
+    std::copy(embeddingVector.begin(), embeddingVector.end(), embedding);
+    *embedding_size = embeddingVector.size();
+    return embedding;
+}
+
+void llmodel_free_embedding(float *ptr)
+{
+    free(ptr);
+}
+
+void llmodel_setThreadCount(llmodel_model model, int32_t n_threads)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    wrapper->llModel->setThreadCount(n_threads);
+}
+
+int32_t llmodel_threadCount(llmodel_model model)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->threadCount();
+}
+
+void llmodel_set_implementation_search_path(const char *path)
+{
+    LLModel::Implementation::setImplementationsSearchPath(path);
+}
+
+const char *llmodel_get_implementation_search_path()
+{
+    return LLModel::Implementation::implementationsSearchPath().c_str();
+}
+
+struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    std::vector<LLModel::GPUDevice> devices = wrapper->llModel->availableGPUDevices(memoryRequired);
+
+    // Set the num_devices
+    *num_devices = devices.size();
+
+    if (*num_devices == 0) return nullptr;  // Return nullptr if no devices are found
+
+    // Allocate memory for the output array
+    struct llmodel_gpu_device* output = (struct llmodel_gpu_device*) malloc(*num_devices * sizeof(struct llmodel_gpu_device));
+
+    for (int i = 0; i < *num_devices; i++) {
+        output[i].index = devices[i].index;
+        output[i].type = devices[i].type;
+        output[i].heapSize = devices[i].heapSize;
+        output[i].name = strdup(devices[i].name.c_str());  // Convert std::string to char* and allocate memory
+        output[i].vendor = strdup(devices[i].vendor.c_str());  // Convert std::string to char* and allocate memory
+    }
+
+    return output;
+}
+
+bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(memoryRequired, std::string(device));
+}
+
+bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device)
+{
+    LLModel::GPUDevice d;
+    d.index = device->index;
+    d.type = device->type;
+    d.heapSize = device->heapSize;
+    d.name = device->name;
+    d.vendor = device->vendor;
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(d);
+}
+
+bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(device);
+}
+
+bool llmodel_has_gpu_device(llmodel_model model)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->hasGPUDevice();
+}

gpt4all-backend/llmodel_c.h

@@ -0,0 +1,279 @@
+#ifndef LLMODEL_C_H
+#define LLMODEL_C_H
+
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+
+#ifdef __GNUC__
+#define DEPRECATED __attribute__ ((deprecated))
+#elif defined(_MSC_VER)
+#define DEPRECATED __declspec(deprecated)
+#else
+#pragma message("WARNING: You need to implement DEPRECATED for this compiler")
+#define DEPRECATED
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Opaque pointer to the underlying model.
+ */
+typedef void *llmodel_model;
+
+/**
+ * Structure containing any errors that may eventually occur
+ */
+struct llmodel_error {
+    const char *message;  // Human readable error description; Thread-local; guaranteed to survive until next llmodel C API call
+    int code;             // errno; 0 if none
+};
+#ifndef __cplusplus
+typedef struct llmodel_error llmodel_error;
+#endif
+
+/**
+ * llmodel_prompt_context structure for holding the prompt context.
+ * NOTE: The implementation takes care of all the memory handling of the raw logits pointer and the
+ * raw tokens pointer. Attempting to resize them or modify them in any way can lead to undefined
+ * behavior.
+ */
+struct llmodel_prompt_context {
+    float *logits;          // logits of current context
+    size_t logits_size;     // the size of the raw logits vector
+    int32_t *tokens;        // current tokens in the context window
+    size_t tokens_size;     // the size of the raw tokens vector
+    int32_t n_past;         // number of tokens in past conversation
+    int32_t n_ctx;          // number of tokens possible in context window
+    int32_t n_predict;      // number of tokens to predict
+    int32_t top_k;          // top k logits to sample from
+    float top_p;            // nucleus sampling probability threshold
+    float temp;             // temperature to adjust model's output distribution
+    int32_t n_batch;        // number of predictions to generate in parallel
+    float repeat_penalty;   // penalty factor for repeated tokens
+    int32_t repeat_last_n;  // last n tokens to penalize
+    float context_erase;    // percent of context to erase if we exceed the context window
+};
+
+struct llmodel_gpu_device {
+    int index = 0;
+    int type = 0;           // same as VkPhysicalDeviceType
+    size_t heapSize = 0;
+    const char * name;
+    const char * vendor;
+};
+
+#ifndef __cplusplus
+typedef struct llmodel_prompt_context llmodel_prompt_context;
+typedef struct llmodel_gpu_device llmodel_gpu_device;
+#endif
+
+/**
+ * Callback type for prompt processing.
+ * @param token_id The token id of the prompt.
+ * @return a bool indicating whether the model should keep processing.
+ */
+typedef bool (*llmodel_prompt_callback)(int32_t token_id);
+
+/**
+ * Callback type for response.
+ * @param token_id The token id of the response.
+ * @param response The response string. NOTE: a token_id of -1 indicates the string is an error string.
+ * @return a bool indicating whether the model should keep generating.
+ */
+typedef bool (*llmodel_response_callback)(int32_t token_id, const char *response);
+
+/**
+ * Callback type for recalculation of context.
+ * @param whether the model is recalculating the context.
+ * @return a bool indicating whether the model should keep generating.
+ */
+typedef bool (*llmodel_recalculate_callback)(bool is_recalculating);
+
+/**
+ * Create a llmodel instance.
+ * Recognises correct model type from file at model_path
+ * @param model_path A string representing the path to the model file.
+ * @return A pointer to the llmodel_model instance; NULL on error.
+ */
+DEPRECATED llmodel_model llmodel_model_create(const char *model_path);
+
+/**
+ * Create a llmodel instance.
+ * Recognises correct model type from file at model_path
+ * @param model_path A string representing the path to the model file; will only be used to detect model type.
+ * @param build_variant A string representing the implementation to use (auto, default, avxonly, ...),
+ * @param error A pointer to a llmodel_error; will only be set on error.
+ * @return A pointer to the llmodel_model instance; NULL on error.
+ */
+llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error);
+
+/**
+ * Destroy a llmodel instance.
+ * Recognises correct model type using type info
+ * @param model a pointer to a llmodel_model instance.
+ */
+void llmodel_model_destroy(llmodel_model model);
+
+/**
+ * Estimate RAM requirement for a model file
+ * @param model A pointer to the llmodel_model instance.
+ * @param model_path A string representing the path to the model file.
+ * @return size greater than 0 if the model was parsed successfully, 0 if file could not be parsed.
+ */
+size_t llmodel_required_mem(llmodel_model model, const char *model_path);
+
+/**
+ * Load a model from a file.
+ * @param model A pointer to the llmodel_model instance.
+ * @param model_path A string representing the path to the model file.
+ * @return true if the model was loaded successfully, false otherwise.
+ */
+bool llmodel_loadModel(llmodel_model model, const char *model_path);
+
+/**
+ * Check if a model is loaded.
+ * @param model A pointer to the llmodel_model instance.
+ * @return true if the model is loaded, false otherwise.
+ */
+bool llmodel_isModelLoaded(llmodel_model model);
+
+/**
+ * Get the size of the internal state of the model.
+ * NOTE: This state data is specific to the type of model you have created.
+ * @param model A pointer to the llmodel_model instance.
+ * @return the size in bytes of the internal state of the model
+ */
+uint64_t llmodel_get_state_size(llmodel_model model);
+
+/**
+ * Saves the internal state of the model to the specified destination address.
+ * NOTE: This state data is specific to the type of model you have created.
+ * @param model A pointer to the llmodel_model instance.
+ * @param dest A pointer to the destination.
+ * @return the number of bytes copied
+ */
+uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest);
+
+/**
+ * Restores the internal state of the model using data from the specified address.
+ * NOTE: This state data is specific to the type of model you have created.
+ * @param model A pointer to the llmodel_model instance.
+ * @param src A pointer to the src.
+ * @return the number of bytes read
+ */
+uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src);
+
+/**
+ * Generate a response using the model.
+ * @param model A pointer to the llmodel_model instance.
+ * @param prompt A string representing the input prompt.
+ * @param prompt_callback A callback function for handling the processing of prompt.
+ * @param response_callback A callback function for handling the generated response.
+ * @param recalculate_callback A callback function for handling recalculation requests.
+ * @param ctx A pointer to the llmodel_prompt_context structure.
+ */
+void llmodel_prompt(llmodel_model model, const char *prompt,
+                    llmodel_prompt_callback prompt_callback,
+                    llmodel_response_callback response_callback,
+                    llmodel_recalculate_callback recalculate_callback,
+                    llmodel_prompt_context *ctx);
+
+/**
+ * Generate an embedding using the model.
+ * NOTE: If given NULL pointers for the model or text, or an empty text, a NULL pointer will be
+ * returned. Bindings should signal an error when NULL is the return value.
+ * @param model A pointer to the llmodel_model instance.
+ * @param text A string representing the text to generate an embedding for.
+ * @param embedding_size A pointer to a size_t type that will be set by the call indicating the length
+ * of the returned floating point array.
+ * @return A pointer to an array of floating point values passed to the calling method which then will
+ * be responsible for lifetime of this memory.
+ */
+float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size);
+
+/**
+ * Frees the memory allocated by the llmodel_embedding function.
+ * @param ptr A pointer to the embedding as returned from llmodel_embedding.
+ */
+void llmodel_free_embedding(float *ptr);
+
+/**
+ * Set the number of threads to be used by the model.
+ * @param model A pointer to the llmodel_model instance.
+ * @param n_threads The number of threads to be used.
+ */
+void llmodel_setThreadCount(llmodel_model model, int32_t n_threads);
+
+/**
+ * Get the number of threads currently being used by the model.
+ * @param model A pointer to the llmodel_model instance.
+ * @return The number of threads currently being used.
+ */
+int32_t llmodel_threadCount(llmodel_model model);
+
+/**
+ * Set llmodel implementation search path.
+ * Default is "."
+ * @param path The path to the llmodel implementation shared objects. This can be a single path or
+ * a list of paths separated by ';' delimiter.
+ */
+void llmodel_set_implementation_search_path(const char *path);
+
+/**
+ * Get llmodel implementation search path.
+ * @return The current search path; lifetime ends on next set llmodel_set_implementation_search_path() call.
+ */
+const char *llmodel_get_implementation_search_path();
+
+/**
+ * Get a list of available GPU devices given the memory required.
+ * @return A pointer to an array of llmodel_gpu_device's whose number is given by num_devices.
+ */
+struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices);
+
+/**
+ * Initializes a GPU device based on a specified string criterion.
+ *
+ * This function initializes a GPU device based on a string identifier provided. The function
+ * allows initialization based on general device type ("gpu"), vendor name ("amd", "nvidia", "intel"),
+ * or any specific device name.
+ *
+ * @param memoryRequired The amount of memory (in bytes) required by the application or task
+ *                       that will utilize the GPU device.
+ * @param device A string specifying the desired criterion for GPU device selection. It can be:
+ *               - "gpu": To initialize the best available GPU.
+ *               - "amd", "nvidia", or "intel": To initialize the best available GPU from that vendor.
+ *               - A specific GPU device name: To initialize a GPU with that exact name.
+ *
+ * @return True if the GPU device is successfully initialized based on the provided string
+ *         criterion. Returns false if the desired GPU device could not be initialized.
+ */
+bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device);
+
+/**
+ * Initializes a GPU device by specifying a valid gpu device pointer.
+ * @param device A gpu device pointer.
+ * @return True if the GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device);
+
+/**
+ * Initializes a GPU device by its index.
+ * @param device An integer representing the index of the GPU device to be initialized.
+ * @return True if the GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
+
+/**
+ * @return True if a GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_has_gpu_device(llmodel_model model);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // LLMODEL_C_H

gpt4all-backend/llmodel_shared.cpp

@@ -0,0 +1,203 @@
+#include "llmodel.h"
+
+#include <cassert>
+#include <iostream>
+#include <unordered_set>
+
+#ifdef GGML_USE_KOMPUTE
+#include "ggml-vulkan.h"
+#endif
+
+void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
+    size_t i = 0;
+    promptCtx.n_past = 0;
+    while (i < promptCtx.tokens.size()) {
+        size_t batch_end = std::min(i + promptCtx.n_batch, promptCtx.tokens.size());
+        std::vector<int32_t> batch(promptCtx.tokens.begin() + i, promptCtx.tokens.begin() + batch_end);
+        assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
+        if (!evalTokens(promptCtx, batch)) {
+            std::cerr << "LLModel ERROR: Failed to process prompt\n";
+            goto stop_generating;
+        }
+        promptCtx.n_past += batch.size();
+        if (!recalculate(true))
+            goto stop_generating;
+        i = batch_end;
+    }
+    assert(promptCtx.n_past == int32_t(promptCtx.tokens.size()));
+
+stop_generating:
+    recalculate(false);
+}
+
+void LLModel::prompt(const std::string &prompt,
+                     std::function<bool(int32_t)> promptCallback,
+                     std::function<bool(int32_t, const std::string&)> responseCallback,
+                     std::function<bool(bool)> recalculateCallback,
+                     PromptContext &promptCtx)
+{
+    if (!isModelLoaded()) {
+        std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
+        return;
+    }
+
+    if (!supportsCompletion()) {
+        std::string errorMessage = "ERROR: this model does not support text completion or chat!\n";
+        responseCallback(-1, errorMessage);
+        std::cerr << implementation().modelType() << errorMessage;
+        return;
+    }
+
+    // tokenize the prompt
+    std::vector<Token> embd_inp = tokenize(promptCtx, prompt);
+
+    // save the context size
+    promptCtx.n_ctx = contextLength();
+
+    if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
+        responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed.");
+        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
+            " tokens and the context window is " << promptCtx.n_ctx << "!\n";
+        return;
+    }
+
+    promptCtx.n_predict = std::min(promptCtx.n_predict, promptCtx.n_ctx - (int) embd_inp.size());
+    promptCtx.n_past = std::min(promptCtx.n_past, promptCtx.n_ctx);
+    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
+
+    // process the prompt in batches
+    size_t i = 0;
+    while (i < embd_inp.size()) {
+        size_t batch_end = std::min(i + promptCtx.n_batch, embd_inp.size());
+        std::vector<Token> batch(embd_inp.begin() + i, embd_inp.begin() + batch_end);
+
+        // Check if the context has run out...
+        if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
+            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
+            // Erase the first percentage of context from the tokens...
+            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
+            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
+            promptCtx.n_past = promptCtx.tokens.size();
+            recalculateContext(promptCtx, recalculateCallback);
+            assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
+        }
+
+        if (!evalTokens(promptCtx, batch)) {
+            std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
+            return;
+        }
+
+        size_t tokens = batch_end - i;
+        for (size_t t = 0; t < tokens; ++t) {
+            if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
+                promptCtx.tokens.erase(promptCtx.tokens.begin());
+            promptCtx.tokens.push_back(batch.at(t));
+            if (!promptCallback(batch.at(t)))
+                return;
+        }
+        promptCtx.n_past += batch.size();
+        i = batch_end;
+    }
+
+    std::string cachedResponse;
+    std::vector<Token> cachedTokens;
+    std::unordered_set<std::string> reversePrompts
+        = { "### Instruction", "### Prompt", "### Response", "### Human", "### Assistant", "### Context" };
+
+    // predict next tokens
+    for (int i = 0; i < promptCtx.n_predict; i++) {
+
+        // sample next token
+        auto id = sampleToken(promptCtx);
+
+        // Check if the context has run out...
+        if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
+            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
+            // Erase the first percentage of context from the tokens...
+            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
+            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
+            promptCtx.n_past = promptCtx.tokens.size();
+            recalculateContext(promptCtx, recalculateCallback);
+            assert(promptCtx.n_past + 1 <= promptCtx.n_ctx);
+        }
+
+        if (!evalTokens(promptCtx, { id })) {
+            std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
+            return;
+        }
+
+        promptCtx.n_past += 1;
+
+        // display text
+        for (const auto token : endTokens()) {
+            if (id == token) return;
+        }
+
+        const std::string str = tokenToString(id);
+
+        // Check if the provided str is part of our reverse prompts
+        bool foundPartialReversePrompt = false;
+        const std::string completed = cachedResponse + std::string(str);
+        if (reversePrompts.find(completed) != reversePrompts.end())
+            return;
+
+        // Check if it partially matches our reverse prompts and if so, cache
+        for (const auto& s : reversePrompts) {
+            if (s.compare(0, completed.size(), completed) == 0) {
+                foundPartialReversePrompt = true;
+                cachedResponse = completed;
+                break;
+            }
+        }
+
+        // Regardless the token gets added to our cache
+        cachedTokens.push_back(id);
+
+        // Continue if we have found a partial match
+        if (foundPartialReversePrompt)
+            continue;
+
+        // Empty the cache
+        for (auto t : cachedTokens) {
+            if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
+                promptCtx.tokens.erase(promptCtx.tokens.begin());
+            promptCtx.tokens.push_back(t);
+            //TODO: Conversion to std::string can be avoided here...
+            if (!responseCallback(t, std::string(tokenToString(t))))
+                return;
+        }
+        cachedTokens.clear();
+    }
+}
+
+std::vector<float> LLModel::embedding(const std::string &/*text*/)
+{
+    if (!supportsCompletion()) {
+        std::string errorMessage = "ERROR: this model does not support generating embeddings!\n";
+        std::cerr << implementation().modelType() << errorMessage;
+    }
+    return std::vector<float>();
+}
+
+std::vector<LLModel::GPUDevice> LLModel::availableGPUDevices()
+{
+#if defined(GGML_USE_KOMPUTE)
+    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(0);
+
+    std::vector<LLModel::GPUDevice> devices;
+    for(const auto& vkDevice : vkDevices) {
+        LLModel::GPUDevice device;
+        device.index = vkDevice.index;
+        device.type = vkDevice.type;
+        device.heapSize = vkDevice.heapSize;
+        device.name = vkDevice.name;
+        device.vendor = vkDevice.vendor;
+
+        devices.push_back(device);
+    }
+
+    return devices;
+#else
+    return std::vector<LLModel::GPUDevice>();
+#endif
+}

gpt4all-backend/llmodel_shared.h

@@ -0,0 +1,92 @@
+#pragma once
+#include <cstdint>
+#include <cstddef>
+#include <vector>
+#include <ggml.h>
+
+#if defined(GGML_USE_KOMPUTE)
+#include "ggml-vulkan.h"
+struct llm_buffer {
+    uint8_t * addr = NULL;
+    size_t size = 0;
+    ggml_vk_memory memory;
+
+    llm_buffer() = default;
+
+    void resize(size_t size) {
+        free();
+
+        if (!ggml_vk_has_device()) {
+            this->addr = new uint8_t[size];
+            this->size = size;
+        } else {
+            this->memory = ggml_vk_allocate(size);
+            this->addr = (uint8_t*)memory.data;
+            this->size = size;
+        }
+    }
+
+    void free() {
+        if (!memory.primaryMemory) {
+            delete[] addr;
+        } else if (memory.data) {
+            ggml_vk_free_memory(memory);
+        }
+        this->addr = NULL;
+        this->size = 0;
+    }
+
+    ~llm_buffer() {
+        free();
+    }
+
+    // disable copy and move
+    llm_buffer(const llm_buffer&) = delete;
+    llm_buffer(llm_buffer&&) = delete;
+    llm_buffer& operator=(const llm_buffer&) = delete;
+    llm_buffer& operator=(llm_buffer&&) = delete;
+};
+#else
+struct llm_buffer {
+    uint8_t * addr = NULL;
+    size_t size = 0;
+
+    void resize(size_t size) {
+        delete[] addr;
+        addr = new uint8_t[size];
+        this->size = size;
+    }
+
+    ~llm_buffer() {
+        delete[] addr;
+    }
+};
+#endif
+
+struct llm_kv_cache {
+    struct ggml_tensor * k;
+    struct ggml_tensor * v;
+
+    struct ggml_context * ctx = NULL;
+
+    llm_buffer buf;
+
+    int n; // number of tokens currently in the cache
+
+    ~llm_kv_cache() {
+        if (ctx) {
+            ggml_free(ctx);
+        }
+    }
+};
+
+#if LLAMA_DATE >= 230519
+inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads) {
+    struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
+    if (plan.work_size > 0) {
+        buf.resize(plan.work_size);
+        plan.work_data = buf.addr;
+    }
+    ggml_graph_compute(graph, &plan);
+}
+#endif

gpt4all-backend/mpt.cpp

@@ -0,0 +1,893 @@
+#define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#include "mpt_impl.h"
+
+#include "utils.h"
+#include "llmodel_shared.h"
+
+#include <cassert>
+#include <cinttypes>
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+#include <fstream>
+#include <map>
+#include <random>
+#include <string>
+#include <vector>
+#include <iostream>
+#if defined(_WIN32) && defined(_MSC_VER)
+    #define WIN32_LEAN_AND_MEAN
+    #ifndef NOMINMAX
+        #define NOMINMAX
+    #endif
+    #include <windows.h>
+    #include <io.h>
+    #include <stdio.h>
+#else
+    #include <unistd.h>
+#endif
+#include <sstream>
+#include <thread>
+#include <unordered_set>
+#include <regex>
+#include <ggml.h>
+
+
+namespace {
+const char *modelType_ = "MPT";
+}
+
+// default hparams (MPT 7B)
+struct mpt_hparams {
+    int32_t n_vocab      = 50432;
+    int32_t n_ctx        = 2048;
+    int32_t n_embd       = 4096;
+    int32_t n_head       = 32;
+    int32_t n_layer      = 32;
+    float alibi_bias_max = 8;
+    float clip_qkv       = 0;
+    int32_t expand       = 4;
+    int32_t f16          = 1;
+};
+
+struct mpt_layer {
+    // normalization
+    struct ggml_tensor * norm_1_w;
+    struct ggml_tensor * norm_2_w;
+
+    // attention
+    struct ggml_tensor * attn_Wqkv_w;
+    struct ggml_tensor * attn_out_proj_w;
+
+    // ff
+    struct ggml_tensor * ffn_up_proj_w;
+    struct ggml_tensor * ffn_down_proj_w;
+};
+
+struct mpt_model {
+    mpt_hparams hparams;
+
+    // normalization
+    struct ggml_tensor * norm_f_w;
+
+    struct ggml_tensor * wte; // position embedding
+
+    // mpt does weight tying
+
+    std::vector<mpt_layer> layers;
+
+    struct llm_kv_cache kv_self;
+    struct ggml_context * ctx;
+    std::map<std::string, struct ggml_tensor *> tensors;
+
+
+    llm_buffer eval_buf;
+    llm_buffer scr0_buf;
+    llm_buffer scr1_buf;
+
+    ~mpt_model() {
+        if (ctx) {
+            ggml_free(ctx);
+        }
+    }
+};
+
+static bool kv_cache_init(
+        const struct mpt_hparams & hparams,
+             struct llm_kv_cache & cache,
+                         ggml_type   wtype,
+                               int   n_ctx) {
+    const int n_embd  = hparams.n_embd;
+    const int n_layer = hparams.n_layer;
+
+    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
+    const int64_t n_elements = n_embd*n_mem;
+
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+
+    struct ggml_init_params params;
+    params.mem_size   = cache.buf.size;
+    params.mem_buffer = cache.buf.addr;
+    params.no_alloc   = false;
+
+    cache.ctx = ggml_init(params);
+
+    if (!cache.ctx) {
+        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
+        return false;
+    }
+
+    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+
+    return true;
+}
+
+// load the model's weights from a stream. if mem_req ptr is passed the model is
+// only partially parsed to estimate required memory
+bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & model, gpt_vocab & vocab, size_t * mem_req) {
+    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+    if (mem_req != nullptr) {
+        *mem_req = 0;
+    }
+
+    // verify magic
+    {
+        uint32_t magic;
+        fin.read((char *) &magic, sizeof(magic));
+        if (magic != 0x67676d6d) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
+            return false;
+        }
+    }
+
+    // load hparams
+    {
+        auto & hparams = model.hparams;
+
+        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
+        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
+        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
+        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
+        fin.read((char *) &hparams.alibi_bias_max,  sizeof(hparams.alibi_bias_max));
+        fin.read((char *) &hparams.clip_qkv,  sizeof(hparams.clip_qkv));
+        fin.read((char *) &hparams.f16,   sizeof(hparams.f16));
+
+        printf("%s: n_vocab        = %d\n", __func__, hparams.n_vocab);
+        printf("%s: n_ctx          = %d\n", __func__, hparams.n_ctx);
+        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
+        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
+        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
+        printf("%s: alibi_bias_max = %f\n", __func__, hparams.alibi_bias_max);
+        printf("%s: clip_qkv       = %f\n", __func__, hparams.clip_qkv);
+        printf("%s: ftype          = %d\n", __func__, hparams.f16);
+    }
+
+    // load vocab
+    {
+        int32_t n_vocab = model.hparams.n_vocab;
+        fin.read((char *) &n_vocab, sizeof(n_vocab));
+
+        if (n_vocab != model.hparams.n_vocab) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
+                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
+            return false;
+        }
+
+        std::string word;
+        for (int i = 0; i < n_vocab; i++) {
+            uint32_t len;
+            fin.read((char *) &len, sizeof(len));
+            bool special = false;
+            if (len & (1<<31)) {
+                len = len &~ (1<<31);
+                special = true;
+            }
+
+            if (len > 0) {
+                word.resize(len);
+                fin.read((char *) word.data(), len);
+                vocab.token_to_id[word] = i;
+                vocab.id_to_token[i] = word;
+            }
+
+            if(special) {
+                vocab.add_special_token(word);
+            }
+        }
+    }
+
+    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
+    // in order to save memory and also to speed up the computation
+    ggml_type wtype = GGML_TYPE_COUNT;
+    switch (model.hparams.f16) {
+        case 0: wtype = GGML_TYPE_F32;  break;
+        case 1: wtype = GGML_TYPE_F16;  break;
+        case 2: wtype = GGML_TYPE_Q4_0; break;
+        case 3: wtype = GGML_TYPE_Q4_1; break;
+        case 5: wtype = GGML_TYPE_Q4_2; break;
+        default:
+                {
+                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
+                            __func__, fname.c_str(), model.hparams.f16);
+                    return false;
+                }
+    }
+
+    auto & ctx = model.ctx;
+
+    size_t ctx_size = 0;
+
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_ctx   = hparams.n_ctx;
+        const int n_vocab = hparams.n_vocab;
+        const int expand  = hparams.expand;
+
+
+        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_w
+
+        ctx_size += n_embd*n_vocab*ggml_type_sizef(GGML_TYPE_F32); // wte
+
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_1_w
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_2_w
+
+        ctx_size += n_layer*(3*n_embd*n_embd*ggml_type_sizef(wtype)); // attn_Wqkv_w
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // attn_out_proj_w
+
+        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype));  // ffn_up_proj_w
+        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype)); // ffn_down_proj_w
+
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_k
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_v
+
+        // TODO probably less now?
+        ctx_size += (5 + 10*n_layer)*256; // object overhead
+
+        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
+    }
+
+    if (mem_req != nullptr) {
+        *mem_req += ctx_size;
+        const int n_embd  = model.hparams.n_embd;
+        const int n_layer = model.hparams.n_layer;
+
+        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
+        const int64_t n_elements = n_embd*n_mem;
+
+        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+        return false;
+    }
+
+    // create the ggml context
+    {
+        struct ggml_init_params params = {
+            .mem_size   = ctx_size,
+            .mem_buffer = NULL,
+            .no_alloc   = false,
+        };
+
+        model.ctx = ggml_init(params);
+        if (!model.ctx) {
+            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
+            return false;
+        }
+    }
+
+    // prepare memory for the weights
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_vocab = hparams.n_vocab;
+        const int expand  = hparams.expand;
+
+        model.layers.resize(n_layer);
+
+        model.wte    = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab);
+        model.norm_f_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+        // map by name
+        model.tensors["transformer.wte.weight"] = model.wte;
+        model.tensors["transformer.norm_f.weight"] = model.norm_f_w;
+
+        for (int i = 0; i < n_layer; ++i) {
+            auto & layer = model.layers[i];
+
+            layer.norm_1_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.norm_2_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+            layer.attn_Wqkv_w     = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd * 3);
+            layer.attn_out_proj_w = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd);
+            layer.ffn_up_proj_w   = ggml_new_tensor_2d(ctx, wtype,        n_embd, expand*n_embd);
+            layer.ffn_down_proj_w = ggml_new_tensor_2d(ctx, wtype, expand*n_embd, n_embd);
+
+            // map by name
+            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_1.weight"]        = layer.norm_1_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_2.weight"]        = layer.norm_2_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.Wqkv.weight"]     = layer.attn_Wqkv_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.out_proj.weight"] = layer.attn_out_proj_w;
+
+            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.up_proj.weight"]   = layer.ffn_up_proj_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.down_proj.weight"] = layer.ffn_down_proj_w;
+        }
+    }
+
+    // key + value memory
+    {
+        const auto & hparams = model.hparams;
+        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
+            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
+            ggml_free(ctx);
+            return false;
+        }
+
+        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
+        printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
+    }
+
+    // load weights
+    {
+        int n_tensors = 0;
+        size_t total_size = 0;
+
+        printf("%s: ", __func__);
+
+        while (true) {
+            int32_t n_dims;
+            int32_t length;
+            int32_t ttype;
+
+            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
+            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
+
+            if (fin.eof()) {
+                break;
+            }
+
+            int32_t nelements = 1;
+            int32_t ne[2] = { 1, 1 };
+            for (int i = 0; i < n_dims; ++i) {
+                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+                nelements *= ne[i];
+            }
+
+            std::string name(length, 0);
+            fin.read(&name[0], length);
+
+            if (model.tensors.find(name.data()) == model.tensors.end()) {
+                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
+                return false;
+            }
+
+            auto tensor = model.tensors[name.data()];
+            if (ggml_nelements(tensor) != nelements) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
+                return false;
+            }
+
+            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
+                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
+                return false;
+            }
+
+            // for debugging
+            if (0) {
+                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
+            }
+
+            const size_t bpe = ggml_type_size(ggml_type(ttype));
+
+            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
+                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
+                return false;
+            }
+
+            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
+
+            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ttype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
+            total_size += ggml_nbytes(tensor);
+            if (++n_tensors % 8 == 0) {
+                printf(".");
+                fflush(stdout);
+            }
+        }
+
+        printf(" done\n");
+
+        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
+    }
+
+    model.scr0_buf.resize(256u * 1024 * 1024);
+    model.scr1_buf.resize(256u * 1024 * 1024);
+
+    return true;
+}
+
+// load the model's weights from a file path
+bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vocab) {
+
+    auto fin = std::ifstream(fname, std::ios::binary);
+    if (!fin) {
+        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
+        return false;
+    }
+
+    bool loaded = mpt_model_load(fname, fin, model, vocab, nullptr);
+    fin.close();
+    return loaded;
+}
+
+bool mpt_eval(
+        mpt_model & model,
+        const int n_threads,
+        const int n_past,
+        const std::vector<int>           & embd_inp,
+              std::vector<float>         & embd_w,
+              size_t                     & mem_per_token) {
+    const int N = embd_inp.size();
+
+    const auto & hparams = model.hparams;
+
+    const int n_embd  = hparams.n_embd;
+    const int n_layer = hparams.n_layer;
+    const int n_ctx   = hparams.n_ctx;
+    const int n_head  = hparams.n_head;
+    const int n_vocab = hparams.n_vocab;
+
+    const size_t init_buf_size = 1024_MiB;
+    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
+        model.eval_buf.resize(init_buf_size);
+
+    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
+        const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
+        // printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
+
+        // reallocate
+        model.eval_buf.resize(buf_size_new);
+        if (model.eval_buf.addr == nullptr) {
+            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
+            return false;
+        }
+    }
+
+    struct ggml_init_params params = {
+        .mem_size   = model.eval_buf.size,
+        .mem_buffer = model.eval_buf.addr,
+        .no_alloc = false
+    };
+
+    struct ggml_context * ctx0 = ggml_init(params);
+    struct ggml_cgraph gf = {};
+    gf.n_threads = n_threads;
+
+    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
+
+    // wte
+    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
+
+    for (int il = 0; il < n_layer; ++il) {
+        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
+        struct ggml_tensor * inpSA = inpL;
+        struct ggml_tensor * cur = inpSA;
+        // self-attention
+        {
+
+            // norm1
+            cur = ggml_norm(ctx0, cur);
+            cur = ggml_mul(ctx0,
+                    ggml_repeat(ctx0, model.layers[il].norm_1_w, cur),
+                    cur);
+            // compute QKV
+            cur = ggml_mul_mat(ctx0,
+                    model.layers[il].attn_Wqkv_w,
+                    cur);
+
+            // TODO: clip_qkv
+            struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 0*ggml_element_size(cur)*n_embd));
+            struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 1*ggml_element_size(cur)*n_embd));
+            struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 2*ggml_element_size(cur)*n_embd));
+
+            // TODO: qk_ln? (seems to be False in MPT-7B configs)
+            {
+                Vcur = ggml_transpose(ctx0, Vcur);
+
+                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
+                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
+                                        (   n_ctx)*ggml_element_size(model.kv_self.v),
+                                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
+
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
+            }
+            // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
+            struct ggml_tensor * Q =
+                ggml_permute(ctx0,
+                        ggml_reshape_3d(ctx0, Qcur, n_embd/n_head, n_head, N),
+                        0, 2, 1, 3);
+
+            struct ggml_tensor * K =
+                ggml_permute(ctx0,
+                        ggml_reshape_3d(ctx0,
+                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
+                            n_embd/n_head, n_head, n_past + N),
+                        0, 2, 1, 3);
+
+            // K * Q
+            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
+
+            // KQ_scaled = KQ / sqrt(n_embd/n_head)
+            struct ggml_tensor * KQ_scaled =
+                ggml_scale(ctx0,
+                        KQ,
+                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
+                        );
+
+
+            // Alibi
+            struct ggml_tensor * KQ_scaled_biased = ggml_alibi(ctx0, ggml_cont(ctx0, KQ_scaled), n_past, n_head);
+
+            // KQ_masked = mask_past(KQ_scaled)
+            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled_biased, n_past);
+
+            // KQ = soft_max(KQ_masked)
+            struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
+
+            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
+            struct ggml_tensor * V =
+                ggml_view_3d(ctx0, model.kv_self.v,
+                        n_past + N, n_embd/n_head, n_head,
+                        n_ctx*ggml_element_size(model.kv_self.v),
+                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
+                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
+
+            // KQV = transpose(V) * KQ_soft_max
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
+
+            // KQV_merged = KQV.permute(0, 2, 1, 3)
+            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
+
+            // cur = KQV_merged.contiguous().view(n_embd, N)
+            cur = ggml_cpy(ctx0,
+                    KQV_merged,
+                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+
+            // projection (no bias)
+            cur = ggml_mul_mat(ctx0,
+                    model.layers[il].attn_out_proj_w,
+                    cur);
+        }
+
+        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
+        // residual
+        struct ggml_tensor * resSA = ggml_add(ctx0, cur, inpSA);
+        // feed-forward network
+        {
+            cur = resSA;
+            // norm2
+            cur = ggml_norm(ctx0, cur);
+            cur = ggml_mul(ctx0,
+                    ggml_repeat(ctx0, model.layers[il].norm_2_w, cur),
+                    cur);
+            // ffn
+            cur = ggml_mul_mat(ctx0,
+                    model.layers[il].ffn_up_proj_w,
+                    cur);
+            cur = ggml_gelu(ctx0, cur);
+            cur = ggml_mul_mat(ctx0,
+                    model.layers[il].ffn_down_proj_w,
+                    cur);
+
+        }
+
+        // self-attention + FF
+        inpL = ggml_add(ctx0, cur, resSA);
+    }
+    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
+    struct ggml_tensor * out = inpL;
+    // -> logits
+    {
+        out = ggml_norm(ctx0, out);
+        out = ggml_mul(ctx0,
+                    ggml_repeat(ctx0, model.norm_f_w, out),
+                    out);
+        ggml_set_scratch(ctx0, { 0, 0, nullptr, });
+        out = ggml_mul_mat(ctx0, model.wte, out);
+    }
+
+
+    // run the computation
+    ggml_build_forward_expand(&gf, out);
+    ggml_graph_compute       (ctx0, &gf);
+
+
+    // return result for just the last token
+    embd_w.resize(n_vocab);
+    memcpy(embd_w.data(), (float *) ggml_get_data(out) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
+
+    if (mem_per_token == 0) {
+        mem_per_token = ggml_used_mem(ctx0)/N;
+    }
+    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
+
+    ggml_free(ctx0);
+
+    return true;
+}
+
+
+#define MPT_MAX_RNG_STATE 64*1024
+
+size_t mpt_get_state_size(const mpt_model &model)
+{
+    // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state.
+    // for reference, std::mt19937(1337) serializes to 6701 bytes.
+    const size_t s_rng_size        = sizeof(size_t);
+    const size_t s_rng             = MPT_MAX_RNG_STATE;
+    const size_t s_kv_size         = sizeof(size_t);
+    const size_t s_kv_ntok         = sizeof(int);
+    const size_t s_kv              = model.kv_self.buf.size;
+    const size_t s_total = (
+        + s_rng_size
+        + s_rng
+        + s_kv_size
+        + s_kv_ntok
+        + s_kv
+    );
+    fflush(stdout);
+    return s_total;
+}
+
+size_t mpt_copy_state_data(const mpt_model &model, const std::mt19937 &rng, uint8_t *dest)
+{
+    uint8_t * out = dest;
+    fflush(stdout);
+    // copy rng
+    {
+        std::stringstream rng_ss;
+        rng_ss << rng;
+
+        const size_t rng_size = rng_ss.str().size();
+        char rng_buf[MPT_MAX_RNG_STATE];
+
+        memset(&rng_buf[0], 0, MPT_MAX_RNG_STATE);
+        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
+
+        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
+        memcpy(out, &rng_buf[0], MPT_MAX_RNG_STATE); out += MPT_MAX_RNG_STATE;
+    }
+
+    // copy kv cache
+    {
+        const size_t kv_size = model.kv_self.buf.size;
+        const int    kv_ntok = model.kv_self.n;
+
+        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
+        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
+
+        if (kv_size) {
+            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
+        }
+    }
+
+    const size_t written  = out - dest;
+    assert(written == mpt_get_state_size(model));
+    fflush(stdout);
+    return written;
+}
+
+size_t mpt_set_state_data(mpt_model *model, std::mt19937 *rng, const uint8_t *src)
+{
+    const uint8_t * in = src;
+
+    // set rng
+    {
+        size_t rng_size;
+        char   rng_buf[MPT_MAX_RNG_STATE];
+
+        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
+        memcpy(&rng_buf[0], in, MPT_MAX_RNG_STATE); in += MPT_MAX_RNG_STATE;
+
+        std::stringstream rng_ss;
+        rng_ss.str(std::string(&rng_buf[0], rng_size));
+        rng_ss >> *rng;
+
+        assert(rng_ss.fail() == false);
+    }
+
+    // set kv cache
+    {
+        size_t kv_size;
+        int kv_ntok;
+
+        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
+        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
+
+        if (kv_size) {
+            assert(model->kv_self.buf.size == kv_size);
+
+            void * k_data = model->kv_self.k->data; // remember data pointers
+            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
+
+            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
+
+            model->kv_self.k->data = k_data; // restore correct data pointers
+            model->kv_self.v->data = v_data;
+
+        }
+
+        model->kv_self.n = kv_ntok;
+    }
+
+    const size_t nread    = in - src;
+    assert(nread == mpt_get_state_size(*model));
+    fflush(stdout);
+    return nread;
+}
+
+struct MPTPrivate {
+    const std::string modelPath;
+    bool modelLoaded;
+    gpt_vocab vocab;
+    mpt_model *model = nullptr;
+    int64_t n_threads = 0;
+    size_t mem_per_token = 0;
+    std::mt19937 rng;
+    bool has_im_end = false;
+};
+
+MPT::MPT()
+    : d_ptr(new MPTPrivate) {
+    d_ptr->model = new mpt_model;
+    d_ptr->model->ctx = nullptr;
+    d_ptr->modelLoaded = false;
+}
+
+size_t MPT::requiredMem(const std::string &modelPath) {
+    mpt_model dummy_model;
+    gpt_vocab dummy_vocab;
+    size_t mem_req;
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+    mpt_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
+    return mem_req;
+}
+
+bool MPT::loadModel(const std::string &modelPath) {
+    std::mt19937 rng(time(NULL));
+    d_ptr->rng = rng;
+
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+
+    // load the model
+    if (!mpt_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab, nullptr)) {
+        std::cerr << "MPT ERROR: failed to load model from " <<  modelPath;
+        return false;
+    }
+
+    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    d_ptr->modelLoaded = true;
+    d_ptr->has_im_end = d_ptr->vocab.token_to_id.find("<|im_end|>") != d_ptr->vocab.token_to_id.end();
+    fflush(stdout);
+    return true;
+}
+
+void MPT::setThreadCount(int32_t n_threads) {
+    d_ptr->n_threads = n_threads;
+}
+
+int32_t MPT::threadCount() const
+{
+    return d_ptr->n_threads;
+}
+
+MPT::~MPT()
+{
+    delete d_ptr->model;
+}
+
+bool MPT::isModelLoaded() const
+{
+    return d_ptr->modelLoaded;
+}
+
+size_t MPT::stateSize() const
+{
+    return mpt_get_state_size(*d_ptr->model);
+}
+
+size_t MPT::saveState(uint8_t *dest) const
+{
+    return mpt_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
+}
+
+size_t MPT::restoreState(const uint8_t *src)
+{
+    return mpt_set_state_data(d_ptr->model, &d_ptr->rng, src);
+}
+
+std::vector<LLModel::Token> MPT::tokenize(PromptContext &, const std::string &str) const
+{
+    return ::gpt_tokenize(d_ptr->vocab, str);
+}
+
+std::string MPT::tokenToString(Token id) const
+{
+    return d_ptr->vocab.id_to_token[id];
+}
+
+LLModel::Token MPT::sampleToken(PromptContext &promptCtx) const
+{
+    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
+    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
+        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
+        n_prev_toks,
+        promptCtx.logits,
+        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
+        promptCtx.repeat_penalty,
+        d_ptr->rng);
+}
+
+bool MPT::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
+{
+    // determine the required inference memory per token:
+    static bool initialized = false;
+    if (!initialized) {
+        mpt_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
+            d_ptr->mem_per_token);
+        initialized = true;
+    }
+
+    return mpt_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
+}
+
+int32_t MPT::contextLength() const
+{
+    return d_ptr->model->hparams.n_ctx;
+}
+
+const std::vector<LLModel::Token> &MPT::endTokens() const
+{
+    static const std::vector<LLModel::Token> fres = {0, d_ptr->vocab.token_to_id["<|im_end|>"]};
+    return fres;
+}
+
+#if defined(_WIN32)
+#define DLL_EXPORT __declspec(dllexport)
+#else
+#define DLL_EXPORT __attribute__ ((visibility ("default")))
+#endif
+
+extern "C" {
+DLL_EXPORT bool is_g4a_backend_model_implementation() {
+    return true;
+}
+
+DLL_EXPORT const char *get_model_type() {
+    return modelType_;
+}
+
+DLL_EXPORT const char *get_build_variant() {
+    return GGML_BUILD_VARIANT;
+}
+
+DLL_EXPORT bool magic_match(std::istream& f) {
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    return magic == 0x67676d6d;
+}
+
+DLL_EXPORT LLModel *construct() {
+    return new MPT;
+}
+}

gpt4all-backend/mpt_impl.h

@@ -0,0 +1,41 @@
+#ifndef MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of mpt.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef MPT_H
+#define MPT_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include "llmodel.h"
+
+struct MPTPrivate;
+class MPT : public LLModel {
+public:
+    MPT();
+    ~MPT();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    MPTPrivate *d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::string tokenToString(Token) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // MPT_H

gpt4all-backend/replit_impl.h

@@ -0,0 +1,44 @@
+#ifndef REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of replit.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef REPLIT_H
+#define REPLIT_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include "llmodel.h"
+
+#define GGML_QNT_VERSION_FACTOR 1000 // do not change this
+
+struct ReplitPrivate;
+class Replit : public LLModel {
+public:
+    Replit();
+    ~Replit();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string & modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+    bool usingGPUDevice() override;
+
+private:
+    ReplitPrivate *d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::string tokenToString(Token) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // REPLIT_H

gpt4all-backend/scripts/convert_bert_hf_to_ggml.py

@@ -0,0 +1,102 @@
+import sys
+import struct
+import json
+import torch
+import numpy as np
+
+from transformers import AutoModel, AutoTokenizer
+
+if len(sys.argv) < 3:
+    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = sys.argv[1]
+fname_out = sys.argv[1] + "/ggml-model.bin"
+
+with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f:
+    encoder = json.load(f)
+
+with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
+    hparams = json.load(f)
+
+with open(dir_model + "/vocab.txt", "r", encoding="utf-8") as f:
+    vocab = f.readlines()
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+    fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
+
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+model = AutoModel.from_pretrained(dir_model, low_cpu_mem_usage=True)
+print (model)
+
+print(tokenizer.encode('I believe the meaning of life is'))
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+fout = open(fname_out, "wb")
+
+print(hparams)
+
+fout.write(struct.pack("i", 0x62657274)) # magic: ggml in hex
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["max_position_embeddings"]))
+fout.write(struct.pack("i", hparams["hidden_size"]))
+fout.write(struct.pack("i", hparams["intermediate_size"]))
+fout.write(struct.pack("i", hparams["num_attention_heads"]))
+fout.write(struct.pack("i", hparams["num_hidden_layers"]))
+fout.write(struct.pack("i", ftype))
+
+for i in range(hparams["vocab_size"]):
+    text = vocab[i][:-1] # strips newline at the end
+    #print(f"{i}:{text}")
+    data = bytes(text, 'utf-8')
+    fout.write(struct.pack("i", len(data)))
+    fout.write(data)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
+        continue
+    print("Processing variable: " + name + " with shape: ", data.shape)
+
+    n_dims = len(data.shape);
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+            print("  Converting to float16")
+            data = data.astype(np.float16)
+            l_type = 1
+    else:
+        l_type = 0
+
+    # header
+    str = name.encode('utf-8')
+    fout.write(struct.pack("iii", n_dims, len(str), l_type))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str);
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/scripts/convert_falcon_hf_to_ggml.py

@@ -0,0 +1,143 @@
+# Based on: https://github.com/KerfuffleV2/ggml-falcon/blob/feat-improve-falcon-convert-hf/examples/falcon/convert-hf-to-ggml.py
+# Convert Hugging Face fine-tuned bloom-like models to ggml format
+#
+# Usage:
+#
+#   python3 convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]
+#
+# This script is similar to "convert-pt-to-ggml.py"
+#
+
+import io
+import os
+import sys
+import struct
+import json
+import code
+import torch
+import numpy as np
+import gc
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
+
+# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a significant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+if len(sys.argv) < 3:
+    print("INFO: GGML V1 files produced are meant to be finalized through examples/falcon_quantize which will bring them to latest version and precision of choice");
+    print("Usage: python convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]")
+    print("  model_directory: name of the directory and model you convert (it should be a subdirectory)")
+    print("  output-directory: directory where the output file will be written")
+    print("  use-f32:    if present, use float32 instead of float16 (f32 is recommended)")
+    sys.exit(1)
+
+# num_parts = int(sys.argv[1])
+dir_model = sys.argv[1] # name and dir of model
+dir_out = sys.argv[2]   # output directory
+
+# make sure the output directory exists
+os.makedirs(dir_out, exist_ok=True)
+
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+ftype = 1
+if len(sys.argv) > 3:
+    ftype = 0
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+# print(tokenizer)
+config = AutoConfig.from_pretrained(dir_model, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(dir_model, trust_remote_code=True, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)
+hparams = config.to_dict()
+
+n_head = hparams["n_head"]
+n_head_kv = hparams["n_head_kv"] if "n_head_kv" in hparams else 1
+head_dim = hparams["hidden_size"] // n_head
+print("* Loading model from: ", dir_model)
+
+fname_out = dir_out + f"/ggml-model-{dir_model.split('/')[-1]}-{ftype_str[ftype]}.bin"
+fout = open(fname_out, "wb")
+fout.write(struct.pack("i", 0x67676a74)) # magic: ggmf in hex (version 1) - possibly change to ggfc ?
+fout.write(struct.pack("i", 1)) # version
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["hidden_size"]))
+fout.write(struct.pack("i", n_head))
+fout.write(struct.pack("i", n_head_kv))
+fout.write(struct.pack("i", hparams["n_layer"]))
+fout.write(struct.pack("i", 40 if "n_head_kv" in hparams else 7)) # obsolete field that breaks ggml compatibility - todo again remove one day
+fout.write(struct.pack("i", ftype))
+
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+byte_encoder = bytes_to_unicode()
+byte_decoder = {v:k for k, v in byte_encoder.items()}
+
+for i in range(hparams["vocab_size"]):
+    text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+    fout.write(struct.pack("i", len(text)))
+    fout.write(text)
+    fout.write(struct.pack("f", 0.0)) # falcon uses bpe on RefinedWeb - no probability scores used
+
+model = model.state_dict()
+for name in model.keys():
+    src = name
+    # The original query_key_value tensor contains n_head_kv "kv groups",
+    # each consisting of n_head/n_head_kv query weights followed by one key
+    # and one value weight (shared by all query heads in the kv group).
+    # This layout makes it a big pain to work with in GGML.
+    # So we rearrange them here,, so that we have n_head query weights
+    # followed by n_head_kv key weights followed by n_head_kv value weights,
+    # in contiguous fashion.
+
+    if "query_key_value" in src:
+        qkv = model[src].view(
+            n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)
+
+        q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head)
+        k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)
+        v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)
+
+        model[src] = torch.cat((q,k,v)).reshape_as(model[src])
+    data = model[src].squeeze()
+    n_dims = len(data.shape)
+    # default type is fp32
+    ftype_cur = 1 if ftype == 1 and n_dims > 1 else 0
+    data = data.to(dtype = torch.float16 if ftype_cur == 1 else torch.float32).numpy()
+    print(f'  |', name, data.shape, '->', data.dtype)
+    # header
+    str = name.encode('utf-8')
+    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str)
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/scripts/convert_mpt_hf_to_ggml.py

@@ -0,0 +1,145 @@
+# Convert Hugging Face fine-tuned bloom-like models to ggml format
+#
+# Usage:
+#
+#   python3 models/convert-h5-to-ggml.py 
+#
+# This script is similar to "convert-pt-to-ggml.py"
+#
+
+import io
+import os
+import sys
+import struct
+import json
+import code
+import torch
+import numpy as np
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BloomForCausalLM
+
+# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a significant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+if len(sys.argv) < 3:
+    print("Usage: python convert-hf-to-ggml.py model_name dir-output [use-f32]")
+    print("  model_name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
+    print("  dir-output: directory where the output file will be written")
+    print("  use-f32:    if present, use float32 instead of float16")
+    sys.exit(1)
+
+model_name = sys.argv[1]
+dir_out = sys.argv[2]
+
+# make sure the output directory exists
+os.makedirs(dir_out, exist_ok=True)
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+ftype = 1
+if len(sys.argv) > 3:
+    ftype = 0
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
+hparams = config.to_dict()
+print("Loading model: ", model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32, low_cpu_mem_usage=True)
+print("Model loaded: ", model_name)
+
+
+fname_out = dir_out + f"/ggml-model-{model_name.split('/')[-1]}-{ftype_str[ftype]}.bin"
+fout = open(fname_out, "wb")
+vocab = tokenizer.vocab
+
+hparams["multiple_of"] = 1
+fout.write(struct.pack("I", 0x67676d6d)) # magic: ggml in hex
+fout.write(struct.pack("I", model.config.vocab_size))
+fout.write(struct.pack("I", model.config.max_seq_len))
+fout.write(struct.pack("I", model.config.n_layers))
+fout.write(struct.pack("I", model.config.n_heads))
+fout.write(struct.pack("I", model.config.d_model))
+fout.write(struct.pack("f", model.config.attn_config['alibi_bias_max']))
+clip_qkv = model.config.attn_config['clip_qkv']
+fout.write(struct.pack("f",  clip_qkv if clip_qkv is not None else 0))
+fout.write(struct.pack("I", ftype))
+
+# # Is this correct??
+# dot_token = tokenizer.encode(".")[0]
+# write tokens to ggml file 
+dot_token = tokenizer.encode('.')[0]
+fout.write(struct.pack("I", model.config.vocab_size))
+
+for i in range(model.config.vocab_size):
+    text = tokenizer.decode([dot_token, i]).encode('utf-8')
+    # remove the first byte (it's always '.')
+    text = text[1:]
+    enclen = len(text)
+    if i in tokenizer.all_special_ids:
+        print(f"special token: {text}")
+        enclen = enclen | 1<<31
+    fout.write(struct.pack("I", enclen))
+    fout.write(text)
+    
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable: " + name + " with shape: ", data.shape)
+
+    n_dims = len(data.shape);
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0;
+    if ftype != 0:
+        # Keep token embeddings in fp32
+        if name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
+            print("  Converting to float16")
+            data = data.astype(np.float16)
+            ftype_cur = 1
+        else:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+    else:
+        if data.dtype != np.float32:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+
+    # header
+    str = name.encode('utf-8')
+    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str);
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/scripts/convert_replit_hf_to_ggml.py

@@ -0,0 +1,113 @@
+from pathlib import Path
+import sys
+import struct
+import json
+import numpy as np
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import sentencepiece.sentencepiece_model_pb2 as model
+
+if len(sys.argv) < 3:
+    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+
+# output in the same directory as the model
+dir_model = sys.argv[1]
+fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b.bin"
+
+
+with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
+    hparams = json.load(f)
+
+sp_proto = model.ModelProto()
+sp_proto.ParseFromString(open(Path(sys.argv[1]) / "spiece.model", "rb").read())
+
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+    fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".bin"
+
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    dir_model, low_cpu_mem_usage=True, trust_remote_code=True
+)
+# print (model)
+
+# print(tokenizer.encode('I believe the meaning of life is'))
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+fout = open(fname_out, "wb")
+
+print(hparams)
+
+fout.write(struct.pack("i", 0x7265706c))  # magic: repl in hex
+fout.write(struct.pack("i", hparams["vocab_size"]))
+fout.write(struct.pack("i", hparams["max_seq_len"]))
+fout.write(struct.pack("i", hparams["d_model"]))
+fout.write(struct.pack("i", hparams["n_heads"]))
+fout.write(struct.pack("i", hparams["n_layers"]))
+fout.write(struct.pack("i", ftype))
+
+
+# TODO: temporary hack to not deal with implementing the tokenizer
+for piece in sp_proto.pieces:
+    encoded_piece = piece.piece.encode("utf-8")
+    fout.write(struct.pack("i", len(encoded_piece)))
+    fout.write(encoded_piece)
+    fout.write(struct.pack("f", piece.score))
+
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable: " + name + " with shape: ", data.shape)
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0
+    if ftype != 0:
+        if name[-7:] == ".weight" and n_dims == 2:
+            print("  Converting to float16")
+            data = data.astype(np.float16)
+            ftype_cur = 1
+        else:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+    else:
+        if data.dtype != np.float32:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+
+    # header
+    str = name.encode("utf-8")
+    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str)
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/starcoder_impl.h

@@ -0,0 +1,42 @@
+#ifndef STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of starcoder.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef STARCODER_H
+#define STARCODER_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include <memory>
+#include "llmodel.h"
+
+struct StarcoderPrivate;
+class Starcoder : public LLModel {
+public:
+    Starcoder();
+    ~Starcoder();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    std::unique_ptr<StarcoderPrivate> d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    std::string tokenToString(Token) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // STARCODER_H

gpt4all-backend/sysinfo.h

@@ -0,0 +1,61 @@
+#ifndef SYSINFO_H
+#define SYSINFO_H
+
+#include <fstream>
+#include <string>
+#include <sstream>
+#include <iomanip>
+
+#if defined(__linux__)
+#include <unistd.h>
+#elif defined(__APPLE__)
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#elif defined(_WIN32)
+#include <windows.h>
+#endif
+
+static long long getSystemTotalRAMInBytes()
+{
+    long long totalRAM = 0;
+
+#if defined(__linux__)
+    std::ifstream file("/proc/meminfo");
+    std::string line;
+    while (std::getline(file, line)) {
+        if (line.find("MemTotal") != std::string::npos) {
+            std::string memTotalStr = line.substr(line.find(":") + 1);
+            memTotalStr.erase(0, memTotalStr.find_first_not_of(" "));
+            memTotalStr = memTotalStr.substr(0, memTotalStr.find(" "));
+            totalRAM = std::stoll(memTotalStr) * 1024;  // Convert from KB to bytes
+            break;
+        }
+    }
+    file.close();
+#elif defined(__APPLE__)
+    int mib[2] = {CTL_HW, HW_MEMSIZE};
+    size_t length = sizeof(totalRAM);
+    sysctl(mib, 2, &totalRAM, &length, NULL, 0);
+#elif defined(_WIN32)
+    MEMORYSTATUSEX memoryStatus;
+    memoryStatus.dwLength = sizeof(memoryStatus);
+    GlobalMemoryStatusEx(&memoryStatus);
+    totalRAM = memoryStatus.ullTotalPhys;
+#endif
+
+    return totalRAM;
+}
+
+static double getSystemTotalRAMInGB()
+{
+    return static_cast<double>(getSystemTotalRAMInBytes()) / (1024 * 1024 * 1024);
+}
+
+static std::string getSystemTotalRAMInGBString()
+{
+    std::stringstream ss;
+    ss << std::fixed << std::setprecision(2) << getSystemTotalRAMInGB() << " GB";
+    return ss.str();
+}
+
+#endif // SYSINFO_H

gpt4all-backend/utils.cpp

@@ -0,0 +1,328 @@
+#include "utils.h"
+
+#include <fstream>
+#include <regex>
+
+void replace(std::string & str, const std::string & needle, const std::string & replacement) {
+    size_t pos = 0;
+    while ((pos = str.find(needle, pos)) != std::string::npos) {
+        str.replace(pos, needle.length(), replacement);
+        pos += replacement.length();
+    }
+}
+
+std::map<std::string, int32_t> json_parse(const std::string & fname) {
+    std::map<std::string, int32_t> result;
+
+    // read file into string
+    std::string json;
+    {
+        std::ifstream ifs(fname);
+        if (!ifs) {
+            fprintf(stderr, "Failed to open %s\n", fname.c_str());
+            exit(1);
+        }
+
+        json = std::string((std::istreambuf_iterator<char>(ifs)),
+                (std::istreambuf_iterator<char>()));
+    }
+
+    if (json[0] != '{') {
+        return result;
+    }
+
+    // parse json
+    {
+        bool has_key  = false;
+        bool in_token = false;
+
+        std::string str_key = "";
+        std::string str_val = "";
+
+        int n = json.size();
+        for (int i = 1; i < n; ++i) {
+            if (!in_token) {
+                if (json[i] == ' ') continue;
+                if (json[i] == '"') {
+                    in_token = true;
+                    continue;
+                }
+            } else {
+                if (json[i] == '\\' && i+1 < n) {
+                    if (has_key == false) {
+                        str_key += json[i];
+                    } else {
+                        str_val += json[i];
+                    }
+                    ++i;
+                } else if (json[i] == '"') {
+                    if (has_key == false) {
+                        has_key = true;
+                        ++i;
+                        while (json[i] == ' ') ++i;
+                        ++i; // :
+                        while (json[i] == ' ') ++i;
+                        if (json[i] != '\"') {
+                            while (json[i] != ',' && json[i] != '}') {
+                                str_val += json[i++];
+                            }
+                            has_key = false;
+                        } else {
+                            in_token = true;
+                            continue;
+                        }
+                    } else {
+                        has_key = false;
+                    }
+
+                    ::replace(str_key, "\\u0120", " " ); // \u0120 -> space
+                    ::replace(str_key, "\\u010a", "\n"); // \u010a -> new line
+                    ::replace(str_key, "\\\"",    "\""); // \\\"   -> "
+
+                    try {
+                        result[str_key] = std::stoi(str_val);
+                    } catch (...) {
+                        //fprintf(stderr, "%s: ignoring key '%s' with value '%s'\n", fname.c_str(), str_key.c_str(), str_val.c_str());
+
+                    }
+                    str_key = "";
+                    str_val = "";
+                    in_token = false;
+                    continue;
+                }
+                if (has_key == false) {
+                    str_key += json[i];
+                } else {
+                    str_val += json[i];
+                }
+            }
+        }
+    }
+
+    return result;
+}
+
+std::vector<gpt_vocab::id> gpt_tokenize_inner(const gpt_vocab & vocab, const std::string & text) {
+    std::vector<std::string> words;
+
+    // first split the text into words
+    {
+        std::string str = text;
+        std::string pat = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)";
+
+        std::regex re(pat);
+        std::smatch m;
+
+        while (std::regex_search(str, m, re)) {
+            for (auto x : m) {
+                words.push_back(x);
+            }
+            str = m.suffix();
+        }
+    }
+
+    // find the longest tokens that form the words:
+    std::vector<gpt_vocab::id> tokens;
+    for (const auto & word : words) {
+        if (word.size() == 0) continue;
+
+        int i = 0;
+        int n = word.size();
+        while (i < n) {
+            int j = n;
+            while (j > i) {
+                auto it = vocab.token_to_id.find(word.substr(i, j-i));
+                if (it != vocab.token_to_id.end()) {
+                    tokens.push_back(it->second);
+                    i = j;
+                    break;
+                }
+                --j;
+            }
+            if (i == n) {
+                break;
+            }
+            if (j == i) {
+                auto sub = word.substr(i, 1);
+                if (vocab.token_to_id.find(sub) != vocab.token_to_id.end()) {
+                    tokens.push_back(vocab.token_to_id.at(sub));
+                } else {
+                    fprintf(stderr, "%s: unknown token '%s'\n", __func__, sub.data());
+                }
+                ++i;
+            }
+        }
+    }
+
+    return tokens;
+}
+
+std::string regex_escape(const std::string &s) {
+  static const std::regex metacharacters(R"([\.\^\$\-\+\(\)\[\]\{\}\|\?\*])");
+  return std::regex_replace(s, metacharacters, "\\$&");
+}
+
+std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {
+    // Generate the subpattern from the special_tokens vector if it's not empty
+    if (!vocab.special_tokens.empty()) {
+        std::vector<gpt_vocab::id> out;
+        std::vector<std::string> chunks;
+        std::string str = text;
+        std::string special_tokens_subpattern;
+        for (const auto &token : vocab.special_tokens) {
+            if (!special_tokens_subpattern.empty()) {
+                special_tokens_subpattern += "|";
+            }
+            special_tokens_subpattern += regex_escape(token);
+        }
+        std::regex re(special_tokens_subpattern);
+        std::smatch m;
+        while (std::regex_search(str, m, re)) {
+            auto tok = vocab.token_to_id.find(m.str());
+            if (tok != vocab.token_to_id.end()) {
+                auto tokid = tok->second;
+                auto pfxtoks = gpt_tokenize_inner(vocab, m.prefix());
+                out.insert(out.end(), pfxtoks.begin(), pfxtoks.end());
+                out.push_back(tokid);
+                str = m.suffix();
+            }
+        }
+        if (!str.empty()) {
+            auto tokrest = gpt_tokenize_inner(vocab, str);
+            out.insert(out.end(), tokrest.begin(), tokrest.end());
+        }
+        return out;
+    } else {
+        return gpt_tokenize_inner(vocab, text);
+    }
+}
+
+
+bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {
+    printf("%s: loading vocab from '%s'\n", __func__, fname.c_str());
+
+    vocab.token_to_id = ::json_parse(fname);
+
+    for (const auto & kv : vocab.token_to_id) {
+        vocab.id_to_token[kv.second] = kv.first;
+    }
+
+    printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size());
+
+    // print the vocabulary
+    //for (auto kv : vocab.token_to_id) {
+    //    printf("'%s' -> %d\n", kv.first.data(), kv.second);
+    //}
+
+    return true;
+}
+
+gpt_vocab::id gpt_sample_top_k_top_p(
+        const size_t actualVocabSize,
+        const int32_t * last_n_tokens_data,
+        int   last_n_tokens_size,
+        const std::vector<float> logits,
+        int    top_k,
+        double top_p,
+        double temp,
+        float repeat_penalty,
+        std::mt19937 & rng) {
+    int n_logits = actualVocabSize;
+
+    const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_size);
+    const auto * plogits = logits.data();
+
+    if (temp <= 0) {
+        // select the token with the highest logit directly
+        float max_logit = plogits[0];
+        gpt_vocab::id max_id = 0;
+
+        for (int i = 1; i < n_logits; ++i) {
+            if (plogits[i] > max_logit) {
+                max_logit = plogits[i];
+                max_id = i;
+            }
+        }
+        return max_id;
+    }
+    std::vector<std::pair<double, gpt_vocab::id>> logits_id;
+    logits_id.reserve(n_logits);
+
+    {
+        const float scale = 1.0f/temp;
+        for (int i = 0; i < n_logits; ++i) {
+            // repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858)
+            // credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main
+            if (std::find(last_n_tokens.begin(), last_n_tokens.end(), i) != last_n_tokens.end()) {
+                // if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
+                if (plogits[i] < 0.0f) {
+                    logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i));
+                } else {
+                    logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i));
+                }
+            } else {
+                logits_id.push_back(std::make_pair(plogits[i]*scale, i));
+            }
+        }
+    }
+
+    // find the top K tokens
+    std::partial_sort(
+            logits_id.begin(),
+            logits_id.begin() + top_k, logits_id.end(),
+            [](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
+        return a.first > b.first;
+    });
+
+    logits_id.resize(top_k);
+
+    double maxl = -INFINITY;
+    for (const auto & kv : logits_id) {
+        maxl = std::max(maxl, kv.first);
+    }
+
+    // compute probs for the top K tokens
+    std::vector<double> probs;
+    probs.reserve(logits_id.size());
+
+    double sum = 0.0;
+    for (const auto & kv : logits_id) {
+        double p = exp(kv.first - maxl);
+        probs.push_back(p);
+        sum += p;
+    }
+
+    // normalize the probs
+    for (auto & p : probs) {
+        p /= sum;
+    }
+
+    if (top_p < 1.0f) {
+        double cumsum = 0.0f;
+        for (int i = 0; i < top_k; i++) {
+            cumsum += probs[i];
+            if (cumsum >= top_p) {
+                top_k = i + 1;
+                probs.resize(top_k);
+                logits_id.resize(top_k);
+                break;
+            }
+        }
+
+        cumsum = 1.0/cumsum;
+        for (int i = 0; i < (int) probs.size(); i++) {
+            probs[i] *= cumsum;
+        }
+    }
+
+    //printf("\n");
+    //for (int i = 0; i < (int) probs.size(); i++) {
+    //    printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
+    //}
+    //exit(0);
+
+    std::discrete_distribution<> dist(probs.begin(), probs.end());
+    int idx = dist(rng);
+
+    return logits_id[idx].second;
+}

gpt4all-backend/utils.h

@@ -0,0 +1,97 @@
+// Various helper functions and utilities
+
+#pragma once
+
+#include <string>
+#include <map>
+#include <vector>
+#include <random>
+#include <thread>
+
+//
+// General purpose inline functions
+//
+constexpr inline unsigned long long operator ""_MiB(unsigned long long bytes) {
+    return bytes*1024*1024;
+}
+
+//
+// CLI argument parsing
+//
+
+struct gpt_params {
+    int32_t seed      = -1; // RNG seed
+    int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    int32_t n_predict = 200; // new tokens to predict
+
+    // sampling parameters
+    int32_t top_k = 40;
+    float   top_p = 0.9f;
+    float   temp  = 0.9f;
+
+    int32_t n_batch = 8; // batch size for prompt processing
+
+    std::string model = "models/gpt-2-117M/ggml-model.bin"; // model path
+    std::string prompt;
+};
+
+bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
+
+void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
+
+std::string gpt_random_prompt(std::mt19937 & rng);
+
+//
+// Vocab utils
+//
+
+struct gpt_vocab {
+    using id    = int32_t;
+    using token = std::string;
+
+    std::map<token, id> token_to_id;
+    std::map<id, token> id_to_token;
+    std::vector<std::string> special_tokens;
+
+    void add_special_token(const std::string &token) {
+        special_tokens.push_back(token);
+    }
+};
+
+void replace(std::string & str, const std::string & needle, const std::string & replacement);
+
+// poor-man's JSON parsing
+std::map<std::string, int32_t> json_parse(const std::string & fname);
+
+// split text into tokens
+//
+// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53
+//
+// Regex (Python):
+// r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+//
+// Regex (C++):
+// R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"
+//
+std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text);
+
+// load the tokens from encoder.json
+bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab);
+
+// sample next token given probabilities for each embedding
+//
+//   - consider only the top K tokens
+//   - from them, consider only the top tokens with cumulative probability > P
+//
+// TODO: not sure if this implementation is correct
+//
+gpt_vocab::id gpt_sample_top_k_top_p(
+        const size_t actualVocabSize,
+        const int32_t * last_n_tokens_data,
+        int   last_n_tokens_size,
+        const std::vector<float> logits,
+        int    top_k,
+        double top_p,
+        double temp,
+        float repeat_penalty,
+        std::mt19937 & rng);

gpt4all-bindings/README.md

@@ -0,0 +1,3 @@
+# GPT4All Bindings
+This directory will contain language specific bindings on top of the C/C++ model backends.
+We will have one directory per language binding (e.g. Python, Typescript, Golang, etc.).

gpt4all-bindings/cli/README.md

@@ -0,0 +1,44 @@
+# GPT4All Command-Line Interface (CLI)
+
+GPT4All on the command-line.
+
+## Documentation
+<https://docs.gpt4all.io/gpt4all_cli.html>
+
+## Quickstart
+
+The CLI is based on the `gpt4all` Python bindings and the `typer` package.
+
+The following shows one way to get started with the CLI, the documentation has more information.
+Typically, you will want to replace `python` with `python3` on _Unix-like_ systems and `py -3` on
+_Windows_. Also, it's assumed you have all the necessary Python components already installed.
+
+The CLI is a self-contained Python script named [app.py] ([download][app.py-download]). As long as
+its package dependencies are present, you can download and run it from wherever you like.
+
+[app.py]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-bindings/cli/app.py
+[app.py-download]: https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-bindings/cli/app.py
+
+```shell
+# optional but recommended: create and use a virtual environment
+python -m venv gpt4all-cli
+```
+_Windows_ and _Unix-like_ systems differ slightly in how you activate a _virtual environment_:
+- _Unix-like_, typically: `. gpt4all-cli/bin/activate`
+- _Windows_: `gpt4all-cli\Scripts\activate`
+
+Then:
+```shell
+# pip-install the necessary packages; omit '--user' if using a virtual environment
+python -m pip install --user --upgrade gpt4all typer
+# run the CLI
+python app.py repl
+```
+By default, it will automatically download the `groovy` model to `.cache/gpt4all/` in your user
+directory, if necessary.
+
+If you have already saved a model beforehand, specify its path with the `-m`/`--model` argument,
+for example:
+```shell
+python app.py repl --model /home/user/my-gpt4all-models/GPT4All-13B-snoozy.ggmlv3.q4_0.bin
+```

gpt4all-bindings/cli/app.py

@@ -0,0 +1,180 @@
+"""GPT4All CLI
+
+The GPT4All CLI is a self-contained script based on the `gpt4all` and `typer` packages. It offers a
+REPL to communicate with a language model similar to the chat GUI application, but more basic.
+"""
+
+import io
+import pkg_resources  # should be present as a dependency of gpt4all
+import sys
+import typer
+
+from collections import namedtuple
+from typing_extensions import Annotated
+from gpt4all import GPT4All
+
+
+MESSAGES = [
+    {"role": "system", "content": "You are a helpful assistant."},
+    {"role": "user", "content": "Hello there."},
+    {"role": "assistant", "content": "Hi, how can I help you?"},
+]
+
+SPECIAL_COMMANDS = {
+    "/reset": lambda messages: messages.clear(),
+    "/exit": lambda _: sys.exit(),
+    "/clear": lambda _: print("\n" * 100),
+    "/help": lambda _: print("Special commands: /reset, /exit, /help and /clear"),
+}
+
+VersionInfo = namedtuple('VersionInfo', ['major', 'minor', 'micro'])
+VERSION_INFO = VersionInfo(1, 0, 2)
+VERSION = '.'.join(map(str, VERSION_INFO))  # convert to string form, like: '1.2.3'
+
+CLI_START_MESSAGE = f"""
+    
+ ██████  ██████  ████████ ██   ██  █████  ██      ██      
+██       ██   ██    ██    ██   ██ ██   ██ ██      ██      
+██   ███ ██████     ██    ███████ ███████ ██      ██      
+██    ██ ██         ██         ██ ██   ██ ██      ██      
+ ██████  ██         ██         ██ ██   ██ ███████ ███████ 
+                                                          
+
+Welcome to the GPT4All CLI! Version {VERSION}
+Type /help for special commands.
+                                                    
+"""
+
+# create typer app
+app = typer.Typer()
+
+@app.command()
+def repl(
+    model: Annotated[
+        str,
+        typer.Option("--model", "-m", help="Model to use for chatbot"),
+    ] = "ggml-gpt4all-j-v1.3-groovy",
+    n_threads: Annotated[
+        int,
+        typer.Option("--n-threads", "-t", help="Number of threads to use for chatbot"),
+    ] = None,
+):
+    """The CLI read-eval-print loop."""
+    gpt4all_instance = GPT4All(model)
+
+    # if threads are passed, set them
+    if n_threads is not None:
+        num_threads = gpt4all_instance.model.thread_count()
+        print(f"\nAdjusted: {num_threads} →", end="")
+
+        # set number of threads
+        gpt4all_instance.model.set_thread_count(n_threads)
+
+        num_threads = gpt4all_instance.model.thread_count()
+        print(f" {num_threads} threads", end="", flush=True)
+    else:
+        print(f"\nUsing {gpt4all_instance.model.thread_count()} threads", end="")
+
+    print(CLI_START_MESSAGE)
+
+    use_new_loop = False
+    try:
+        version = pkg_resources.Environment()['gpt4all'][0].version
+        version_major = int(version.split('.')[0])
+        if version_major >= 1:
+            use_new_loop = True
+    except:
+        pass  # fall back to old loop
+    if use_new_loop:
+        _new_loop(gpt4all_instance)
+    else:
+        _old_loop(gpt4all_instance)
+
+
+def _old_loop(gpt4all_instance):
+    while True:
+        message = input(" ⇢  ")
+
+        # Check if special command and take action
+        if message in SPECIAL_COMMANDS:
+            SPECIAL_COMMANDS[message](MESSAGES)
+            continue
+
+        # if regular message, append to messages
+        MESSAGES.append({"role": "user", "content": message})
+
+        # execute chat completion and ignore the full response since 
+        # we are outputting it incrementally
+        full_response = gpt4all_instance.chat_completion(
+            MESSAGES,
+            # preferential kwargs for chat ux
+            logits_size=0,
+            tokens_size=0,
+            n_past=0,
+            n_ctx=0,
+            n_predict=200,
+            top_k=40,
+            top_p=0.9,
+            temp=0.9,
+            n_batch=9,
+            repeat_penalty=1.1,
+            repeat_last_n=64,
+            context_erase=0.0,
+            # required kwargs for cli ux (incremental response)
+            verbose=False,
+            streaming=True,
+        )
+        # record assistant's response to messages
+        MESSAGES.append(full_response.get("choices")[0].get("message"))
+        print() # newline before next prompt
+
+
+def _new_loop(gpt4all_instance):
+    with gpt4all_instance.chat_session():
+        while True:
+            message = input(" ⇢  ")
+
+            # Check if special command and take action
+            if message in SPECIAL_COMMANDS:
+                SPECIAL_COMMANDS[message](MESSAGES)
+                continue
+
+            # if regular message, append to messages
+            MESSAGES.append({"role": "user", "content": message})
+
+            # execute chat completion and ignore the full response since 
+            # we are outputting it incrementally
+            response_generator = gpt4all_instance.generate(
+                message,
+                # preferential kwargs for chat ux
+                max_tokens=200,
+                temp=0.9,
+                top_k=40,
+                top_p=0.9,
+                repeat_penalty=1.1,
+                repeat_last_n=64,
+                n_batch=9,
+                # required kwargs for cli ux (incremental response)
+                streaming=True,
+            )
+            response = io.StringIO()
+            for token in response_generator:
+                print(token, end='', flush=True)
+                response.write(token)
+
+            # record assistant's response to messages
+            response_message = {'role': 'assistant', 'content': response.getvalue()}
+            response.close()
+            gpt4all_instance.current_chat_session.append(response_message)
+            MESSAGES.append(response_message)
+            print() # newline before next prompt
+
+
+@app.command()
+def version():
+    """The CLI version command."""
+    print(f"gpt4all-cli v{VERSION}")
+
+
+if __name__ == "__main__":
+    app()

gpt4all-bindings/cli/developer_notes.md

@@ -0,0 +1,25 @@
+# Developing the CLI
+## Documentation
+Documentation can be found in three places:
+- `app.py` docstrings & comments
+- a Readme: `gpt4all-bindings/cli/README.md`
+- the actual CLI documentation: `gpt4all-bindings/python/docs/gpt4all_cli.md`
+
+The _docstrings_ are meant for programmatic use. Since the CLI is primarily geared towards users and
+not to build on top, they're kept terse.
+
+The _Readme_ is mostly meant for users and includes:
+- a link to the _CLI documentation_ (on the [website])
+- a Quickstart section with some guidance on how to get started with a sane setup
+
+The _CLI documentation_ and other documentation are located in the above mentioned `docs/` folder.
+They're in Markdown format and built for the [website]. Of the three, they should be the most
+detailed.
+
+[website]: https://docs.gpt4all.io/gpt4all_cli.html
+
+
+## Versioning
+The version number should now follow the `gpt4all` PyPI package, so compatibility is more clear.
+
+The one place to change it is the `namedtuple` called `VERSION_INFO`.

gpt4all-bindings/csharp/.editorconfig

@@ -0,0 +1,346 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Don't use tabs for indentation.
+[*]
+indent_style = space
+# (Please don't specify an indent_size here; that has too many unintended consequences.)
+
+# Code files
+[*.{cs,csx,vb,vbx}]
+indent_size = 4
+insert_final_newline = true
+charset = utf-8-bom
+
+# XML project files
+[*.{csproj,vbproj,vcxproj,vcxproj.filters,proj,projitems,shproj}]
+indent_size = 4
+
+# XML config files
+[*.{props,targets,ruleset,config,nuspec,resx,vsixmanifest,vsct}]
+indent_size = 2
+
+# JSON files
+[*.json]
+indent_size = 2
+
+# Powershell files
+[*.ps1]
+indent_size = 2
+
+# Shell script files
+[*.sh]
+end_of_line = lf
+indent_size = 2
+insert_final_newline = true
+
+# Dotnet code style settings:
+[*.{cs,vb}]
+
+# IDE0055: Fix formatting
+dotnet_diagnostic.IDE0055.severity = error
+
+# Sort using and Import directives with System.* appearing first
+dotnet_sort_system_directives_first = true
+dotnet_separate_import_directive_groups = false
+
+# Avoid "this." and "Me." if not necessary
+dotnet_style_qualification_for_field = false:suggestion
+dotnet_style_qualification_for_property = false:suggestion
+dotnet_style_qualification_for_method = false:suggestion
+dotnet_style_qualification_for_event = false:suggestion
+
+# Use language keywords instead of framework type names for type references
+dotnet_style_predefined_type_for_locals_parameters_members = true:warning
+dotnet_style_predefined_type_for_member_access = true:warning
+
+# Suggest more modern language features when available
+dotnet_style_object_initializer = true:suggestion
+dotnet_style_collection_initializer = true:suggestion
+dotnet_style_coalesce_expression = true:suggestion
+dotnet_style_null_propagation = true:suggestion
+dotnet_style_explicit_tuple_names = true:suggestion
+
+# Whitespace options
+dotnet_style_allow_multiple_blank_lines_experimental = false
+
+# Private fields are camelCase with '_' prefix
+dotnet_naming_rule.private_members_with_underscore.symbols  = private_fields
+dotnet_naming_rule.private_members_with_underscore.style    = prefix_underscore
+dotnet_naming_rule.private_members_with_underscore.severity = error
+dotnet_naming_symbols.private_fields.applicable_kinds           = field
+dotnet_naming_symbols.private_fields.applicable_accessibilities = private
+dotnet_naming_style.prefix_underscore.capitalization = camel_case
+dotnet_naming_style.prefix_underscore.required_prefix = _
+
+# Non-private static fields are PascalCase
+dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.severity = suggestion
+dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.symbols = non_private_static_fields
+dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.style = non_private_static_field_style
+
+dotnet_naming_symbols.non_private_static_fields.applicable_kinds = field
+dotnet_naming_symbols.non_private_static_fields.applicable_accessibilities = public, protected, internal, protected_internal, private_protected
+dotnet_naming_symbols.non_private_static_fields.required_modifiers = static
+
+dotnet_naming_style.non_private_static_field_style.capitalization = pascal_case
+
+# Non-private readonly fields are PascalCase
+dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.severity = suggestion
+dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.symbols = non_private_readonly_fields
+dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.style = non_private_static_field_style
+
+dotnet_naming_symbols.non_private_readonly_fields.applicable_kinds = field
+dotnet_naming_symbols.non_private_readonly_fields.applicable_accessibilities = public, protected, internal, protected_internal, private_protected
+dotnet_naming_symbols.non_private_readonly_fields.required_modifiers = readonly
+
+dotnet_naming_style.non_private_readonly_field_style.capitalization = pascal_case
+
+# Constants are PascalCase
+dotnet_naming_rule.constants_should_be_pascal_case.severity = suggestion
+dotnet_naming_rule.constants_should_be_pascal_case.symbols = constants
+dotnet_naming_rule.constants_should_be_pascal_case.style = non_private_static_field_style
+
+dotnet_naming_symbols.constants.applicable_kinds = field, local
+dotnet_naming_symbols.constants.required_modifiers = const
+
+dotnet_naming_style.constant_style.capitalization = pascal_case
+
+# Static fields are camelCase and start with s_
+dotnet_naming_rule.static_fields_should_be_camel_case.severity = none
+dotnet_naming_rule.static_fields_should_be_camel_case.symbols = static_fields
+dotnet_naming_rule.static_fields_should_be_camel_case.style = static_field_style
+
+dotnet_naming_symbols.static_fields.applicable_kinds = field
+dotnet_naming_symbols.static_fields.required_modifiers = static
+
+dotnet_naming_style.static_field_style.capitalization = camel_case
+dotnet_naming_style.static_field_style.required_prefix = s_
+
+# Instance fields are camelCase and start with _
+dotnet_naming_rule.instance_fields_should_be_camel_case.severity = none
+dotnet_naming_rule.instance_fields_should_be_camel_case.symbols = instance_fields
+dotnet_naming_rule.instance_fields_should_be_camel_case.style = instance_field_style
+
+dotnet_naming_symbols.instance_fields.applicable_kinds = field
+
+dotnet_naming_style.instance_field_style.capitalization = camel_case
+dotnet_naming_style.instance_field_style.required_prefix = _
+
+# Locals and parameters are camelCase
+dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion
+dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters
+dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style
+
+dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local
+
+dotnet_naming_style.camel_case_style.capitalization = camel_case
+
+# Local functions are PascalCase
+dotnet_naming_rule.local_functions_should_be_pascal_case.severity = suggestion
+dotnet_naming_rule.local_functions_should_be_pascal_case.symbols = local_functions
+dotnet_naming_rule.local_functions_should_be_pascal_case.style = non_private_static_field_style
+
+dotnet_naming_symbols.local_functions.applicable_kinds = local_function
+
+dotnet_naming_style.local_function_style.capitalization = pascal_case
+
+# By default, name items with PascalCase
+dotnet_naming_rule.members_should_be_pascal_case.severity = suggestion
+dotnet_naming_rule.members_should_be_pascal_case.symbols = all_members
+dotnet_naming_rule.members_should_be_pascal_case.style = non_private_static_field_style
+
+dotnet_naming_symbols.all_members.applicable_kinds = *
+
+dotnet_naming_style.pascal_case_style.capitalization = pascal_case
+
+# error RS2008: Enable analyzer release tracking for the analyzer project containing rule '{0}'
+dotnet_diagnostic.RS2008.severity = none
+
+# IDE0073: File header
+dotnet_diagnostic.IDE0073.severity = none
+#file_header_template = Licensed to the .NET Foundation under one or more agreements.\nThe .NET Foundation licenses this file to you under the MIT license.\nSee the LICENSE file in the project root for more information.
+
+# IDE0035: Remove unreachable code
+dotnet_diagnostic.IDE0035.severity = warning
+
+# IDE0036: Order modifiers
+dotnet_diagnostic.IDE0036.severity = warning
+
+# IDE0043: Format string contains invalid placeholder
+dotnet_diagnostic.IDE0043.severity = warning
+
+# IDE0044: Make field readonly
+dotnet_diagnostic.IDE0044.severity = warning
+
+# IDE1006: Naming rule violation
+#dotnet_diagnostic.IDE1006.severity = none
+
+# RS0016: Only enable if API files are present
+dotnet_public_api_analyzer.require_api_files = true
+dotnet_style_operator_placement_when_wrapping = beginning_of_line
+tab_width = 4
+end_of_line = crlf
+dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion
+dotnet_style_prefer_auto_properties = true:silent
+dotnet_style_prefer_simplified_boolean_expressions = true:suggestion
+dotnet_style_prefer_conditional_expression_over_assignment = true:silent
+dotnet_style_prefer_conditional_expression_over_return = true:silent
+dotnet_style_prefer_inferred_tuple_names = true:suggestion
+dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion
+dotnet_style_prefer_compound_assignment = true:suggestion
+dotnet_style_prefer_simplified_interpolation = true:suggestion
+dotnet_style_namespace_match_folder = true:suggestion
+
+# CSharp code style settings:
+[*.cs]
+# Newline settings
+csharp_new_line_before_open_brace = all
+csharp_new_line_before_else = true
+csharp_new_line_before_catch = true
+csharp_new_line_before_finally = true
+csharp_new_line_before_members_in_object_initializers = true
+csharp_new_line_before_members_in_anonymous_types = true
+csharp_new_line_between_query_expression_clauses = true
+
+# Indentation preferences
+csharp_indent_block_contents = true
+csharp_indent_braces = false
+csharp_indent_case_contents = true
+csharp_indent_case_contents_when_block = true
+csharp_indent_switch_labels = true
+csharp_indent_labels = flush_left
+
+# Whitespace options
+csharp_style_allow_embedded_statements_on_same_line_experimental = false
+csharp_style_allow_blank_lines_between_consecutive_braces_experimental = false
+csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = false
+
+# Prefer "var" everywhere
+csharp_style_var_for_built_in_types = true:suggestion
+csharp_style_var_when_type_is_apparent = true:suggestion
+csharp_style_var_elsewhere = true:suggestion
+
+# Prefer method-like constructs to have a block body
+csharp_style_expression_bodied_methods = false:none
+csharp_style_expression_bodied_constructors = false:none
+csharp_style_expression_bodied_operators = false:none
+
+# Prefer property-like constructs to have an expression-body
+csharp_style_expression_bodied_properties = true:none
+csharp_style_expression_bodied_indexers = true:none
+csharp_style_expression_bodied_accessors = true:none
+
+# Suggest more modern language features when available
+csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion
+csharp_style_pattern_matching_over_as_with_null_check = true:suggestion
+csharp_style_inlined_variable_declaration = true:suggestion
+csharp_style_throw_expression = true:suggestion
+csharp_style_conditional_delegate_call = true:suggestion
+
+# Space preferences
+csharp_space_after_cast = false
+csharp_space_after_colon_in_inheritance_clause = true
+csharp_space_after_comma = true
+csharp_space_after_dot = false
+csharp_space_after_keywords_in_control_flow_statements = true
+csharp_space_after_semicolon_in_for_statement = true
+csharp_space_around_binary_operators = before_and_after
+csharp_space_around_declaration_statements = do_not_ignore
+csharp_space_before_colon_in_inheritance_clause = true
+csharp_space_before_comma = false
+csharp_space_before_dot = false
+csharp_space_before_open_square_brackets = false
+csharp_space_before_semicolon_in_for_statement = false
+csharp_space_between_empty_square_brackets = false
+csharp_space_between_method_call_empty_parameter_list_parentheses = false
+csharp_space_between_method_call_name_and_opening_parenthesis = false
+csharp_space_between_method_call_parameter_list_parentheses = false
+csharp_space_between_method_declaration_empty_parameter_list_parentheses = false
+csharp_space_between_method_declaration_name_and_open_parenthesis = false
+csharp_space_between_method_declaration_parameter_list_parentheses = false
+csharp_space_between_parentheses = false
+csharp_space_between_square_brackets = false
+
+# Blocks are allowed
+csharp_prefer_braces = true:silent
+csharp_preserve_single_line_blocks = true
+csharp_preserve_single_line_statements = true
+
+# Target-type new expressio
+csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion
+
+# Currently only enabled for C# due to crash in VB analyzer.  VB can be enabled once
+# https://github.com/dotnet/roslyn/pull/54259 has been published.
+dotnet_style_allow_statement_immediately_after_block_experimental = false
+dotnet_diagnostic.RCS0003.severity=warning
+dotnet_diagnostic.RCS1036.severity=error
+dotnet_diagnostic.IDE0005.severity=warning
+dotnet_diagnostic.IDE0007.severity=error
+csharp_using_directive_placement = outside_namespace:silent
+csharp_prefer_simple_using_statement = true:suggestion
+csharp_style_namespace_declarations = block_scoped:silent
+csharp_style_expression_bodied_lambdas = true:silent
+csharp_style_expression_bodied_local_functions = false:silent
+csharp_style_prefer_null_check_over_type_check = true:suggestion
+dotnet_diagnostic.RCS1075.severity = suggestion
+
+[src/CodeStyle/**.{cs,vb}]
+# warning RS0005: Do not use generic CodeAction.Create to create CodeAction
+dotnet_diagnostic.RS0005.severity = none
+
+[src/{Analyzers,CodeStyle,Features,Workspaces,EditorFeatures,VisualStudio}/**/*.{cs,vb}]
+
+# IDE0011: Add braces
+csharp_prefer_braces = when_multiline:warning
+# NOTE: We need the below severity entry for Add Braces due to https://github.com/dotnet/roslyn/issues/44201
+dotnet_diagnostic.IDE0011.severity = warning
+
+# IDE0040: Add accessibility modifiers
+dotnet_diagnostic.IDE0040.severity = warning
+
+# CONSIDER: Are IDE0051 and IDE0052 too noisy to be warnings for IDE editing scenarios? Should they be made build-only warnings?
+# IDE0051: Remove unused private member
+dotnet_diagnostic.IDE0051.severity = warning
+
+# IDE0052: Remove unread private member
+dotnet_diagnostic.IDE0052.severity = warning
+
+# IDE0059: Unnecessary assignment to a value
+dotnet_diagnostic.IDE0059.severity = warning
+
+# IDE0060: Remove unused parameter
+dotnet_diagnostic.IDE0060.severity = warning
+
+# CA1012: Abstract types should not have public constructors
+dotnet_diagnostic.CA1012.severity = warning
+
+# CA1822: Make member static
+dotnet_diagnostic.CA1822.severity = warning
+
+# Prefer "var" everywhere
+dotnet_diagnostic.IDE0007.severity = warning
+csharp_style_var_for_built_in_types = true:warning
+csharp_style_var_when_type_is_apparent = true:warning
+csharp_style_var_elsewhere = true:warning
+
+# dotnet_style_allow_multiple_blank_lines_experimental
+dotnet_diagnostic.IDE2000.severity = warning
+
+# csharp_style_allow_embedded_statements_on_same_line_experimental
+dotnet_diagnostic.IDE2001.severity = warning
+
+# csharp_style_allow_blank_lines_between_consecutive_braces_experimental
+dotnet_diagnostic.IDE2002.severity = warning
+
+# dotnet_style_allow_statement_immediately_after_block_experimental
+dotnet_diagnostic.IDE2003.severity = warning
+
+# csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental
+dotnet_diagnostic.IDE2004.severity = warning
+
+[src/{VisualStudio}/**/*.{cs,vb}]
+# CA1822: Make member static
+# There is a risk of accidentally breaking an internal API that partners rely on though IVT.
+dotnet_code_quality.CA1822.api_surface = private

gpt4all-bindings/csharp/.gitignore

@@ -0,0 +1,379 @@
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+##
+## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
+
+runtimes
+**/*nuget
+
+*.zip
+include/
+*.exp
+*.lib
+*.dll
+
+# User-specific files
+*.rsuser
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Mono auto generated files
+mono_crash.*
+Tests/**/launchSettings.json
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+[Ww][Ii][Nn]32/
+[Aa][Rr][Mm]/
+[Aa][Rr][Mm]64/
+bld/
+[Bb]in/
+[Oo]bj/
+[Oo]ut/
+[Ll]og/
+[Ll]ogs/
+
+# Visual Studio 2015/2017 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# Visual Studio 2017 auto generated files
+Generated\ Files/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUnit
+*.VisualState.xml
+TestResult.xml
+nunit-*.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# Benchmark Results
+BenchmarkDotNet.Artifacts/
+
+# .NET Core
+project.lock.json
+project.fragment.lock.json
+artifacts/
+
+# ASP.NET Scaffolding
+ScaffoldingReadMe.txt
+
+# StyleCop
+StyleCopReport.xml
+
+# Files built by Visual Studio
+*_i.c
+*_p.c
+*_h.h
+*.ilk
+*.meta
+*.obj
+*.iobj
+*.pch
+*.pdb
+*.ipdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*_wpftmp.csproj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# Visual Studio Trace Files
+*.e2e
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# AxoCover is a Code Coverage Tool
+.axoCover/*
+!.axoCover/settings.json
+
+# Coverlet is a free, cross platform Code Coverage Tool
+coverage*.json
+coverage*.xml
+coverage*.info
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# Note: Comment the next line if you want to checkin your web deploy settings,
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# NuGet Symbol Packages
+*.snupkg
+# The packages folder can be ignored because of Package Restore
+**/[Pp]ackages/*
+# except build/, which is used as an MSBuild target.
+!**/[Pp]ackages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/[Pp]ackages/repositories.config
+# NuGet v3's project.json files produces more ignorable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+*.appx
+*.appxbundle
+*.appxupload
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!?*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.jfm
+*.pfx
+*.publishsettings
+orleans.codegen.cs
+
+# Including strong name files can present a security risk
+# (https://github.com/github/gitignore/pull/2483#issue-259490424)
+#*.snk
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+ServiceFabricBackup/
+*.rptproj.bak
+
+# SQL Server files
+*.mdf
+*.ldf
+*.ndf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+*.rptproj.rsuser
+*- [Bb]ackup.rdl
+*- [Bb]ackup ([0-9]).rdl
+*- [Bb]ackup ([0-9][0-9]).rdl
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+node_modules/
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
+*.vbw
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# CodeRush personal settings
+.cr/personal
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# Cake - Uncomment if you are using it
+# tools/**
+# !tools/packages.config
+
+# Tabs Studio
+*.tss
+
+# Telerik's JustMock configuration file
+*.jmconfig
+
+# BizTalk build output
+*.btp.cs
+*.btm.cs
+*.odx.cs
+*.xsd.cs
+
+# OpenCover UI analysis results
+OpenCover/
+
+# Azure Stream Analytics local run output
+ASALocalRun/
+
+# MSBuild Binary and Structured Log
+*.binlog
+
+# NVidia Nsight GPU debugger configuration file
+*.nvuser
+
+# MFractors (Xamarin productivity tool) working folder
+.mfractor/
+
+# Local History for Visual Studio
+.localhistory/
+
+# BeatPulse healthcheck temp database
+healthchecksdb
+
+# Backup folder for Package Reference Convert tool in Visual Studio 2017
+MigrationBackup/
+
+# Ionide (cross platform F# VS Code tools) working folder
+.ionide/
+
+# Fody - auto-generated XML schema
+FodyWeavers.xsd
+
+# JetBrains Rider
+.idea
+
+# Visual Studio Code
+.vscode

gpt4all-bindings/csharp/Directory.Build.props

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project>
+
+    <PropertyGroup>
+        <Company></Company>
+        <Copyright></Copyright>
+        <NeutralLanguage>en-US</NeutralLanguage>
+        <Version>0.6.3-alpha</Version>
+        <VersionSuffix>$(VersionSuffix)</VersionSuffix>
+        <Version Condition=" '$(VersionSuffix)' != '' ">$(Version)$(VersionSuffix)</Version>
+        <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+        <RepositoryUrl></RepositoryUrl>
+        <RepositoryType>git</RepositoryType>
+        <IncludeSymbols>true</IncludeSymbols>
+        <IncludeSource>true</IncludeSource>
+        <AnalysisLevel>latest-minimum</AnalysisLevel>
+		<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <Using Include="System"/>
+    </ItemGroup>
+
+    <PropertyGroup>
+        <LangVersion>preview</LangVersion>
+        <Features>strict</Features>
+    </PropertyGroup>
+	
+	<ItemGroup>
+        <PackageReference Include="Roslynator.Analyzers" Version="4.2.0">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+        </PackageReference>
+        <PackageReference Include="Roslynator.CodeAnalysis.Analyzers" Version="4.2.0">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+        </PackageReference>
+        <PackageReference Include="Roslynator.Formatting.Analyzers" Version="4.2.0">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+        </PackageReference>
+    </ItemGroup>
+
+</Project>

gpt4all-bindings/csharp/Gpt4All.Samples/Gpt4All.Samples.csproj

@@ -0,0 +1,32 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net7.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <!-- Windows -->
+        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
+        <!-- Linux -->
+        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
+        <!-- MacOS -->
+        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <!-- Windows -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+        <!-- Linux -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+         <!-- MacOS -->
+        <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+        <Content Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.metal" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+    </ItemGroup>
+</Project>

gpt4all-bindings/csharp/Gpt4All.Samples/Program.cs

@@ -0,0 +1,22 @@
+using Gpt4All;
+
+var modelFactory = new Gpt4AllModelFactory();
+if (args.Length < 2)
+{
+    Console.WriteLine($"Usage: Gpt4All.Samples <model-path> <prompt>");
+    return;
+}
+
+var modelPath = args[0];
+var prompt = args[1];
+
+using var model = modelFactory.LoadModel(modelPath);
+
+var result = await model.GetStreamingPredictionAsync(
+    prompt,
+    PredictRequestOptions.Defaults);
+
+await foreach (var token in result.GetPredictionStreamingAsync())
+{
+    Console.Write(token);
+}

gpt4all-bindings/csharp/Gpt4All.Tests/Constants.cs

@@ -0,0 +1,9 @@
+namespace Gpt4All.Tests;
+
+public static class Constants
+{
+    public const string MODELS_BASE_DIR = "../../../models";
+    public const string LLAMA_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-l13b-snoozy.bin";
+    public const string GPTJ_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-j-v1.3-groovy.bin";
+    public const string MPT_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-mpt-7b-chat.bin";
+}

gpt4all-bindings/csharp/Gpt4All.Tests/Gpt4All.Tests.csproj

@@ -0,0 +1,59 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net7.0</TargetFramework>
+        <Nullable>enable</Nullable>
+
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2" />
+        <PackageReference Include="xunit" Version="2.4.2" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+        <PackageReference Include="coverlet.collector" Version="6.0.0">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+    </ItemGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <!-- Windows -->
+        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
+        <!-- Linux -->
+        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
+        <!-- MacOS -->
+        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <!-- Windows -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+        <!-- Linux -->
+        <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+         <!-- MacOS -->
+        <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Update="Roslynator.Analyzers" Version="4.3.0">
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+      </PackageReference>
+      <PackageReference Update="Roslynator.CodeAnalysis.Analyzers" Version="4.3.0">
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+      </PackageReference>
+      <PackageReference Update="Roslynator.Formatting.Analyzers" Version="4.3.0">
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+      </PackageReference>
+    </ItemGroup>
+</Project>

gpt4all-bindings/csharp/Gpt4All.Tests/ModelFactoryTests.cs

@@ -0,0 +1,34 @@
+using Xunit;
+
+namespace Gpt4All.Tests;
+
+public class ModelFactoryTests
+{
+    private readonly Gpt4AllModelFactory _modelFactory;
+
+    public ModelFactoryTests()
+    {
+        _modelFactory = new Gpt4AllModelFactory();
+    }
+
+    [Fact]
+    [Trait(Traits.SkipOnCI, "True")]
+    public void CanLoadLlamaModel()
+    {
+        using var model = _modelFactory.LoadModel(Constants.LLAMA_MODEL_PATH);
+    }
+
+    [Fact]
+    [Trait(Traits.SkipOnCI, "True")]
+    public void CanLoadGptjModel()
+    {
+        using var model = _modelFactory.LoadModel(Constants.GPTJ_MODEL_PATH);
+    }
+
+    [Fact]
+    [Trait(Traits.SkipOnCI, "True")]
+    public void CanLoadMptModel()
+    {
+        using var model = _modelFactory.LoadModel(Constants.MPT_MODEL_PATH);
+    }
+}

gpt4all-bindings/csharp/Gpt4All.Tests/NativeLibraryLoaderTests.cs

@@ -0,0 +1,56 @@
+using System.IO;
+using Gpt4All.LibraryLoader;
+using Xunit;
+
+namespace Gpt4All.Tests;
+
+public class NativeLibraryLoaderTests
+{
+    [Fact]
+    public void NativeLibraryShouldLoad()
+    {
+        var result = NativeLibraryLoader.LoadNativeLibrary(bypassLoading: false);
+        Assert.True(result.IsSuccess);
+    }
+
+    private const string LLModelLib = "libllmodel.{0}";
+
+    [PlatformSpecificFact(Platforms.Windows)]
+    public void NativeLibraryShouldLoad_Windows()
+    {
+        var libraryLoader = new WindowsLibraryLoader();
+
+        var libraryPath = Path.Combine(
+            Environment.CurrentDirectory,
+            string.Format(LLModelLib, "dll"));
+
+        var result = libraryLoader.OpenLibrary(libraryPath);
+        Assert.True(result.IsSuccess);
+    }
+
+    [PlatformSpecificFact(Platforms.Linux)]
+    public void NativeLibraryShouldLoad_Linux()
+    {
+        var libraryLoader = new LinuxLibraryLoader();
+
+        var libraryPath = Path.Combine(
+            Environment.CurrentDirectory,
+            string.Format(LLModelLib, "so"));
+
+        var result = libraryLoader.OpenLibrary(libraryPath);
+        Assert.True(result.IsSuccess);
+    }
+
+    [PlatformSpecificFact(Platforms.MacOS)]
+    public void NativeLibraryShouldLoad_MacOS()
+    {
+        var libraryLoader = new MacOsLibraryLoader();
+
+        var libraryPath = Path.Combine(
+            Environment.CurrentDirectory,
+            string.Format(LLModelLib, "dylib"));
+
+        var result = libraryLoader.OpenLibrary(libraryPath);
+        Assert.True(result.IsSuccess);
+    }
+}

gpt4all-bindings/csharp/Gpt4All.Tests/PlatformSpecificFactAttribute.cs

@@ -0,0 +1,27 @@
+using Xunit;
+
+namespace Gpt4All.Tests;
+
+public static class Platforms
+{
+    public const string Windows = "windows";
+    public const string Linux = "linux";
+    public const string MacOS = "macOS";
+}
+
+/// <summary>
+/// This attribute ensures the Fact is only run on the specified platform.
+/// </summary>
+/// <remarks>
+/// <see cref="OperatingSystem.IsOSPlatform(string)"/> for info about the platform string.
+/// </remarks>
+public class PlatformSpecificFactAttribute : FactAttribute
+{
+    public PlatformSpecificFactAttribute(string platform)
+    {
+        if (!OperatingSystem.IsOSPlatform(platform))
+        {
+            Skip = $"Test only runs on {platform}.";
+        }
+    }
+}

gpt4all-bindings/csharp/Gpt4All.Tests/Traits.cs

@@ -0,0 +1,6 @@
+namespace Gpt4All.Tests;
+
+public static class Traits
+{
+    public const string SkipOnCI = "SKIP_ON_CI";
+}

gpt4all-bindings/csharp/Gpt4All.sln

@@ -0,0 +1,47 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.33516.290
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Gpt4All.Samples", "Gpt4All.Samples\Gpt4All.Samples.csproj", "{59864AE8-E45D-42F7-A7C0-1308EF185F39}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{DA396C11-CEAD-4368-8234-FB12255A30D2}"
+	ProjectSection(SolutionItems) = preProject
+		.gitignore = .gitignore
+		build_linux.sh = build_linux.sh
+		build_win-mingw.ps1 = build_win-mingw.ps1
+		build_win-msvc.ps1 = build_win-msvc.ps1
+		docs\gpt4all_csharp.md = docs\gpt4all_csharp.md
+		README.md = README.md
+	EndProjectSection
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Gpt4All", "Gpt4All\Gpt4All.csproj", "{6015C62B-2008-426B-A334-740D6F1FE38B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gpt4All.Tests", "Gpt4All.Tests\Gpt4All.Tests.csproj", "{33A72341-52C1-4EAE-878B-A98BC77F686A}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6015C62B-2008-426B-A334-740D6F1FE38B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6015C62B-2008-426B-A334-740D6F1FE38B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6015C62B-2008-426B-A334-740D6F1FE38B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6015C62B-2008-426B-A334-740D6F1FE38B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {17632027-F4C2-4903-B88F-310CE3DE386B}
+	EndGlobalSection
+EndGlobal

gpt4all-bindings/csharp/Gpt4All/Bindings/ILLModel.cs

@@ -0,0 +1,31 @@
+namespace Gpt4All.Bindings;
+
+/// <summary>
+/// Represents the interface exposed by the universal wrapper for GPT4All language models built around llmodel C-API.
+/// </summary>
+public interface ILLModel : IDisposable
+{
+    ModelType ModelType { get; }
+
+    ulong GetStateSizeBytes();
+
+    int GetThreadCount();
+
+    void SetThreadCount(int threadCount);
+
+    bool IsLoaded();
+
+    bool Load(string modelPath);
+
+    void Prompt(
+        string text,
+        LLModelPromptContext context,
+        Func<ModelPromptEventArgs, bool>? promptCallback = null,
+        Func<ModelResponseEventArgs, bool>? responseCallback = null,
+        Func<ModelRecalculatingEventArgs, bool>? recalculateCallback = null,
+        CancellationToken cancellationToken = default);
+
+    unsafe ulong RestoreStateData(byte* destination);
+
+    unsafe ulong SaveStateData(byte* source);
+}