chore: reqs.txt

docs: steps to convert
feat: converter scripts from hf
2026-07-17 10:58:08 +00:00 · 2023-07-14 17:50:05 -04:00 · 2023-07-14 17:49:56 -04:00 · 2023-07-14 17:49:46 -04:00
122 changed files with 3615 additions and 8611 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -12,7 +12,6 @@ workflows:
          config-path: .circleci/continue_config.yml
          mapping: |
            gpt4all-bindings/python/.* run-python-workflow true
-            gpt4all-bindings/typescript/.* run-ts-workflow true
            gpt4all-bindings/csharp/.* run-csharp-workflow true
            gpt4all-backend/.* run-chat-workflow true
            gpt4all-chat/.* run-chat-workflow true
--- a/.circleci/continue_config.yml
+++ b/.circleci/continue_config.yml
@@ -2,7 +2,6 @@ version: 2.1
 orbs:
  win: circleci/windows@5.0
  python: circleci/python@1.2
-  node: circleci/node@5.1

 parameters:
  run-default-workflow:
@@ -14,9 +13,6 @@ parameters:
  run-chat-workflow:
    type: boolean
    default: false
-  run-ts-workflow:
-    type: boolean
-    default: false
  run-csharp-workflow:
    type: boolean
    default: false
@@ -41,12 +37,10 @@ jobs:
      - restore_cache:  # this is the new step to restore cache
          keys:
            - linux-qt-cache
-      - run:
+      - run: 
          name: Setup Linux and Dependencies
          command: |
-            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev
      - run:
          name: Installing Qt
          command: |
@@ -94,18 +88,12 @@ jobs:
          key: windows-qt-cache
          paths:
            - C:\Qt
-      - run:
-          name: Install VulkanSDK
-          command: |
-            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
-            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
      - run:
          name: Build
          command: |
            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64"
            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
            $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
-            $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
            $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
@@ -125,7 +113,6 @@ jobs:
              "-DCMAKE_BUILD_TYPE=Release" `
              "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
              "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
-              "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
              "-S ..\gpt4all-chat" `
              "-B ."
            & "C:\Qt\Tools\Ninja\ninja.exe"
@@ -169,29 +156,12 @@ jobs:
              -S ../gpt4all-chat \
              -B .
            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
-  build-ts-docs: 
-    docker: 
-      - image: cimg/base:stable
-    steps:
-      - checkout
-      - node/install:
-          install-yarn: true
-          node-version: "18.16"
-      - run: node --version
-      - node/install-packages:
-          pkg-manager: yarn
-          app-dir: gpt4all-bindings/typescript
-          override-ci-command: yarn install
-      - run:
-          name: build docs ts yo
-          command: |
-            cd gpt4all-bindings/typescript
-            yarn docs:build
+
  build-py-docs:
    docker:
      - image: circleci/python:3.8
    steps:
-      - checkout 
+      - checkout
      - run:
          name: Install dependencies
          command: |
@@ -214,20 +184,15 @@ jobs:
          command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"

  build-py-linux:
-    machine:
-      image: ubuntu-2204:2023.04.2
+    docker:
+      - image: circleci/python:3.8
    steps:
      - checkout
-      - run:
-          name: Set Python Version
-          command: pyenv global 3.11.2
      - run:
          name: Install dependencies
          command: |
-            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
            sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk
+            sudo apt-get install -y cmake build-essential
            pip install setuptools wheel cmake
      - run:
          name: Build C library
@@ -291,15 +256,9 @@ jobs:
      - run:
          name: Add MinGW64 to PATH
          command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
-      - run:
-          name: Install VulkanSDK
-          command: |
-            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
-            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
      - run:
          name: Install dependencies
-          command:
-            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+          command: choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
      - run:
          name: Install Python dependencies
          command: pip install setuptools wheel cmake
@@ -311,8 +270,7 @@ jobs:
            cd gpt4all-backend
            mkdir build
            cd build
-            $env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
-            cmake -G "MinGW Makefiles" .. -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
+            cmake -G "MinGW Makefiles" .. 
            cmake --build . --parallel
      - run:
          name: Build wheel
@@ -364,10 +322,8 @@ jobs:
      - run:
          name: Install dependencies
          command: |
-            wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-            sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list            
            sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk
+            sudo apt-get install -y cmake build-essential
      - run:
          name: Build Libraries
          command: |
@@ -430,11 +386,6 @@ jobs:
      - run:
          name: Install MinGW64
          command: choco install -y mingw --force --no-progress
-      - run:
-          name: Install VulkanSDK
-          command: |
-            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
-            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
      - run:
          name: Install dependencies
          command: |
@@ -445,11 +396,10 @@ jobs:
            $MinGWBin = "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
            $Env:Path += ";$MinGwBin"
            $Env:Path += ";C:\Program Files\CMake\bin"
-            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
            cd gpt4all-backend
            mkdir runtimes/win-x64
            cd runtimes/win-x64
-            cmake -G "MinGW Makefiles" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON ../..
+            cmake -G "MinGW Makefiles" ../..
            cmake --build . --parallel --config Release
            cp "$MinGWBin\libgcc*.dll" .
            cp "$MinGWBin\libstdc++*.dll" .
@@ -472,11 +422,6 @@ jobs:
          command: |
            git submodule sync
            git submodule update --init --recursive
-      - run:
-          name: Install VulkanSDK
-          command: |
-            Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
-            .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
      - run:
          name: Install dependencies
          command: |
@@ -485,11 +430,10 @@ jobs:
          name: Build Libraries
          command: |
            $Env:Path += ";C:\Program Files\CMake\bin"
-            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
            cd gpt4all-backend
            mkdir runtimes/win-x64_msvc
            cd runtimes/win-x64_msvc
-            cmake -G "Visual Studio 17 2022" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -A X64 ../..
+            cmake -G "Visual Studio 17 2022" -A X64 ../..
            cmake --build . --parallel --config Release
            cp bin/Release/*.dll .
      - persist_to_workspace:
@@ -668,160 +612,6 @@ jobs:
      - store_artifacts:
          path: gpt4all-bindings/csharp/Gpt4All/bin/Release

-  build-nodejs-linux: 
-    docker: 
-      - image: cimg/base:stable
-    steps:
-      - checkout
-      - attach_workspace:
-          at: /tmp/gpt4all-backend
-      - node/install:
-          install-yarn: true
-          node-version: "18.16"
-      - run: node --version
-      - node/install-packages:
-          app-dir: gpt4all-bindings/typescript
-          pkg-manager: yarn
-      - run:
-          command: | 
-            cd gpt4all-bindings/typescript
-            yarn prebuildify -t 18.16.0 --napi
-      - run: 
-          command: |
-            mkdir -p gpt4all-backend/prebuilds/linux-x64
-            mkdir -p gpt4all-backend/runtimes/linux-x64
-            cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so gpt4all-backend/runtimes/linux-x64
-            cp gpt4all-bindings/typescript/prebuilds/linux-x64/*.node gpt4all-backend/prebuilds/linux-x64
-      - persist_to_workspace:
-          root: gpt4all-backend
-          paths:
-            - prebuilds/linux-x64/*.node 
-            - runtimes/linux-x64/*-*.so
-  build-nodejs-macos: 
-    macos:
-      xcode: "14.0.0"
-    steps:
-      - checkout
-      - attach_workspace:
-          at: /tmp/gpt4all-backend
-      - node/install:
-          install-yarn: true
-          node-version: "18.16"
-      - run: node --version
-      - node/install-packages:
-          app-dir: gpt4all-bindings/typescript
-          pkg-manager: yarn
-      - run:
-          command: | 
-            cd gpt4all-bindings/typescript
-            yarn prebuildify -t 18.16.0 --napi
-      - run: 
-          name: "Persisting all necessary things to workspace"
-          command: |  
-            mkdir -p gpt4all-backend/prebuilds/darwin-x64
-            mkdir -p gpt4all-backend/runtimes/darwin-x64
-            cp /tmp/gpt4all-backend/runtimes/osx-x64/*-*.* gpt4all-backend/runtimes/darwin-x64
-            cp gpt4all-bindings/typescript/prebuilds/darwin-x64/*.node gpt4all-backend/prebuilds/darwin-x64
-      - persist_to_workspace:
-          root: gpt4all-backend
-          paths:
-            - prebuilds/darwin-x64/*.node
-            - runtimes/darwin-x64/*-*.*
-
-  build-nodejs-windows: 
-    executor:
-      name: win/default
-      size: large
-      shell: powershell.exe -ExecutionPolicy Bypass
-    steps:
-      - checkout
-      - attach_workspace:
-          at: /tmp/gpt4all-backend
-      - run: choco install wget -y
-      - run:
-         command: wget https://nodejs.org/dist/v18.16.0/node-v18.16.0-x86.msi -P C:\Users\circleci\Downloads\
-         shell: cmd.exe
-      - run: MsiExec.exe /i C:\Users\circleci\Downloads\node-v18.16.0-x86.msi /qn
-      - run: 
-          command: |
-              Start-Process powershell -verb runAs -Args "-start GeneralProfile"
-              nvm install 18.16.0
-              nvm use 18.16.0
-      - run: node --version 
-      - run:           
-          command: |
-            npm install -g yarn
-            cd gpt4all-bindings/typescript
-            yarn install
-      - run:
-          command: | 
-            cd gpt4all-bindings/typescript
-            yarn prebuildify -t 18.16.0 --napi 
-      - run: 
-          command: |
-            mkdir -p gpt4all-backend/prebuilds/win32-x64
-            mkdir -p gpt4all-backend/runtimes/win32-x64
-            cp /tmp/gpt4all-backend/runtimes/win-x64_msvc/*-*.dll gpt4all-backend/runtimes/win32-x64
-            cp gpt4all-bindings/typescript/prebuilds/win32-x64/*.node gpt4all-backend/prebuilds/win32-x64
-        
-      - persist_to_workspace:
-          root: gpt4all-backend
-          paths:
-            - prebuilds/win32-x64/*.node
-            - runtimes/win32-x64/*-*.dll
-
-  prepare-npm-pkg:
-    docker: 
-      - image: cimg/base:stable
-    steps:
-      - attach_workspace:
-          at: /tmp/gpt4all-backend
-      - checkout
-      - node/install:
-          install-yarn: true
-          node-version: "18.16"
-      - run: node --version
-      - run: 
-          command: |
-            cd gpt4all-bindings/typescript
-            # excluding llmodel. nodejs bindings dont need llmodel.dll
-            mkdir -p runtimes/win32-x64/native
-            mkdir -p prebuilds/win32-x64/
-            cp /tmp/gpt4all-backend/runtimes/win-x64_msvc/*-*.dll runtimes/win32-x64/native/ 
-            cp /tmp/gpt4all-backend/prebuilds/win32-x64/*.node prebuilds/win32-x64/    
-
-            mkdir -p runtimes/linux-x64/native 
-            mkdir -p prebuilds/linux-x64/
-            cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so runtimes/linux-x64/native/ 
-            cp /tmp/gpt4all-backend/prebuilds/linux-x64/*.node prebuilds/linux-x64/    
-
-            mkdir -p runtimes/darwin-x64/native
-            mkdir -p prebuilds/darwin-x64/
-            cp /tmp/gpt4all-backend/runtimes/darwin-x64/*-*.* runtimes/darwin-x64/native/
-            cp /tmp/gpt4all-backend/prebuilds/darwin-x64/*.node prebuilds/darwin-x64/    
-            
-            # Fallback build if user is not on above prebuilds
-            mv -f binding.ci.gyp binding.gyp
-
-            mkdir gpt4all-backend
-            cd ../../gpt4all-backend
-            mv llmodel.h llmodel.cpp llmodel_c.cpp llmodel_c.h sysinfo.h dlhandle.h ../gpt4all-bindings/typescript/gpt4all-backend/
-            
-      # Test install
-      - node/install-packages:
-          app-dir: gpt4all-bindings/typescript
-          pkg-manager: yarn
-          override-ci-command: yarn install
-      - run: 
-          command: | 
-            cd gpt4all-bindings/typescript
-            yarn run test
-      - run:
-          command: |
-            cd gpt4all-bindings/typescript
-            npm set //registry.npmjs.org/:_authToken=$NPM_TOKEN
-            npm publish --access public --tag alpha
-
 workflows:
  version: 2
  default:
@@ -845,11 +635,6 @@ workflows:
  deploy-docs:
    when: << pipeline.parameters.run-python-workflow >>
    jobs:
-      - build-ts-docs:
-          filters:
-            branches:
-              only:
-                - main
      - build-py-docs:
          filters:
            branches:
@@ -894,14 +679,11 @@ workflows:
        or:
         - << pipeline.parameters.run-python-workflow >>
         - << pipeline.parameters.run-csharp-workflow >>
-         - <<   pipeline.parameters.run-ts-workflow   >>
    jobs:
      - hold:
          type: approval
      - nuget-hold:
          type: approval
-      - npm-hold:
-          type: approval
      - build-bindings-backend-linux:
          filters:
            branches:
@@ -926,61 +708,23 @@ workflows:
              only:
          requires:
            - hold
-
-      # NodeJs Jobs 
-      - prepare-npm-pkg: 
-          filters:
-            branches:
-              only:
-          requires:
-            - npm-hold
-            - build-nodejs-linux
-            - build-nodejs-windows
-            - build-nodejs-macos
-      - build-nodejs-linux:
-          filters:
-            branches:
-              only:
-          requires:
-            - npm-hold
-            - build-bindings-backend-linux
-      - build-nodejs-windows:
-          filters:
-            branches:
-              only:
-          requires:
-            - npm-hold
-            - build-bindings-backend-windows-msvc
-      - build-nodejs-macos:
-          filters:
-            branches:
-              only:
-          requires:
-            - npm-hold
-            - build-bindings-backend-macos
-
-
-      # CSharp Jobs
      - build-csharp-linux:
          filters:
            branches:
              only:
          requires:
-            - nuget-hold
            - build-bindings-backend-linux
      - build-csharp-windows:
          filters:
            branches:
              only:
          requires:
-            - nuget-hold
            - build-bindings-backend-windows
      - build-csharp-macos:
          filters:
            branches:
              only:
          requires:
-            - nuget-hold
            - build-bindings-backend-macos
      - store-and-upload-nupkgs:
          filters:
--- a/.codespellrc
+++ b/.codespellrc
@@ -1,3 +1,3 @@
 [codespell]
-ignore-words-list = blong, belong, afterall, som
+ignore-words-list = blong, belong, afterall
 skip = .git,*.pdf,*.svg,*.lock
--- a/LICENSE_SOM.txt
+++ b/LICENSE_SOM.txt
@@ -1,30 +0,0 @@
-Software for Open Models License (SOM)
-Version 1.0 dated August 30th, 2023
-
-This license governs use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
-
-This license is intended to encourage open release of models created, modified, processed, or otherwise used via the Software under open licensing terms, and should be interpreted in light of that intent.
-
-1. Definitions
-The “Licensor” is the person or entity who is making the Software available under this license. “Software” is the software made available by Licensor under this license.
-A “Model” is the output of a machine learning algorithm, and excludes the Software.
-“Model Source Materials” must include the Model and model weights, and may include any input data, input data descriptions, documentation or training descriptions for the Model.
-“Open Licensing Terms” means: (a) any open source license approved by the Open Source Initiative, or (b) any other terms that make the Model Source Materials publicly available free of charge, and allow recipients to use, modify and distribute the Model Source Materials. Terms described in (b) may include reasonable restrictions such as non-commercial or non-production limitations, or require use in compliance with law.
-
-2. Grant of Rights. Subject to the conditions and limitations in section 3:
-(A) Copyright Grant. Licensor grants you a non-exclusive, worldwide, royalty-free copyright license to copy, modify, and distribute the Software and any modifications of the Software you create under this license. The foregoing license includes without limitation the right to create, modify, and use Models using this Software.
-
-(B) Patent Grant. Licensor grants you a non-exclusive, worldwide, royalty-free license, under any patents owned or controlled by Licensor, to make, have made, use, sell, offer for sale, import, or otherwise exploit the Software.  No license is granted to patent rights that are not embodied in the operation of the Software in the form provided by Licensor.
-
-3. Conditions and Limitations
-(A) Model Licensing and Access. If you use the Software to create, modify, process, or otherwise use any Model, including usage to create inferences with a Model, whether or not you make the Model available to others, you must make that Model Source Materials publicly available under Open Licensing Terms. 
-
-(B) No Re-Licensing. If you redistribute the Software, or modifications to the Software made under the license granted above, you must make it available only under the terms of this license. You may offer additional terms such as warranties, maintenance and support, but You, and not Licensor, are responsible for performing such terms.
-
-(C) No Trademark License. This license does not grant you rights to use the Licensor’s name, logo, or trademarks.
-
-(D) If you assert in writing a claim against any person or entity alleging that the use of the Software infringes any patent, all of your licenses to the Software under Section 2 end automatically as of the date you asserted the claim.
-
-(E) If you distribute any portion of the Software, you must retain all copyright, patent, trademark, and attribution notices that are present in the Software, and you must include a copy of this license.
-
-(F) The Software is licensed “as-is.” You bear the entire risk of using it. Licensor gives You no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws that this license cannot change. To the extent permitted under your local laws, the Licensor disclaims and excludes the implied warranties of merchantability, fitness for a particular purpose and non-infringement. To the extent this disclaimer is unlawful, you, and not Licensor, are responsible for any liability.
--- a/README.md
+++ b/README.md
@@ -1,9 +1,6 @@
 <h1 align="center">GPT4All</h1>
-
 <p align="center">Open-source assistant-style large language models that run locally on your CPU</p>

-<p align="center"><strong>New</strong>: Now with Nomic Vulkan Universal GPU support. <a href="https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan">Learn more</a>.</p>
-
 <p align="center">
 <a href="https://gpt4all.io">GPT4All Website</a>
 </p>
@@ -32,7 +29,7 @@ Run on an M1 macOS Device (not sped up!)
 </p>

 ## GPT4All: An ecosystem of open-source on-edge large language models.
-GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
+GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs.

 Learn more in the [documentation](https://docs.gpt4all.io).

@@ -60,15 +57,12 @@ Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io

 ### Bindings

-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/java">:computer: Official Java Bindings</a>

-### Integrations
-
-* 🗃️ [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)

 ## Contributing
 GPT4All welcomes contributions, involvement, and discussion from the open source community!
--- a/gpt4all-api/.isort.cfg
+++ b/gpt4all-api/.isort.cfg
@@ -1,7 +0,0 @@
-[settings]
-known_third_party=geopy,nltk,np,numpy,pandas,pysbd,fire,torch
-
-line_length=120
-include_trailing_comma=True
-multi_line_output=3
-use_parentheses=True
--- a/gpt4all-api/README.md
+++ b/gpt4all-api/README.md
@@ -4,13 +4,9 @@ for serving inference from GPT4All models. The API matches the OpenAI API spec.

 ## Tutorial

-The following tutorial assumes that you have checked out this repo and cd'd into it.
-
 ### Starting the app

-First change your working directory to `gpt4all/gpt4all-api`.
-
-Now you can build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
+First build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
 ```bash
 DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
 ```
@@ -21,18 +17,6 @@ Then, start the backend with:
 docker compose up --build
 ```

-This will run both the API and locally hosted GPU inference server. If you want to run the API without the GPU inference server, you can run:
-
-```bash
-docker compose up --build gpt4all_api
-```
-
-To run the API with the GPU inference server, you will need to include environment variables (like the `MODEL_ID`). Edit the `.env` file and run
-```bash
-docker compose --env-file .env up --build
-```
-
-
 #### Spinning up your app
 Run `docker compose up` to spin up the backend. Monitor the logs for errors in-case you forgot to set an environment variable above.

--- a/gpt4all-api/docker-compose.gpu.yaml
+++ b/gpt4all-api/docker-compose.gpu.yaml
@@ -1,24 +0,0 @@
-version: "3.8"
-
-services:
-  gpt4all_gpu:
-    image: ghcr.io/huggingface/text-generation-inference:0.9.3
-    container_name: gpt4all_gpu
-    restart: always #restart on error (usually code compilation from save during bad state)
-    environment:
-      - HUGGING_FACE_HUB_TOKEN=token
-      - USE_FLASH_ATTENTION=false
-      - MODEL_ID=''
-      - NUM_SHARD=1
-    command: --model-id $MODEL_ID --num-shard $NUM_SHARD
-    volumes:
-      - ./:/data
-    ports:
-      - "8080:80"
-    shm_size: 1g
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              capabilities: [gpu]
--- a/gpt4all-api/docker-compose.yaml
+++ b/gpt4all-api/docker-compose.yaml
@@ -1,4 +1,4 @@
-version: "3.8"
+version: "3.5"

 services:
  gpt4all_api:
@@ -13,7 +13,6 @@ services:
      - LOGLEVEL=debug
      - PORT=4891
      - model=ggml-mpt-7b-chat.bin
-      - inference_mode=cpu
    volumes:
      - './gpt4all_api/app:/app'
    command: ["/start-reload.sh"]
--- a/gpt4all-api/gpt4all_api/app/api_v1/api.py
+++ b/gpt4all-api/gpt4all_api/app/api_v1/api.py
@@ -1,4 +1,4 @@
-from api_v1.routes import chat, completions, engines, health
+from api_v1.routes import chat, completions, engines
 from fastapi import APIRouter

 router = APIRouter()
@@ -6,4 +6,3 @@ router = APIRouter()
 router.include_router(chat.router)
 router.include_router(completions.router)
 router.include_router(engines.router)
-router.include_router(health.router)
--- a/gpt4all-api/gpt4all_api/app/api_v1/events.py
+++ b/gpt4all-api/gpt4all_api/app/api_v1/events.py
@@ -1,10 +1,8 @@
 import logging
-
-from api_v1.settings import settings
 from fastapi import HTTPException
 from fastapi.responses import JSONResponse
 from starlette.requests import Request
-
+from api_v1.settings import settings
 log = logging.getLogger(__name__)


@@ -21,9 +19,8 @@ async def on_startup(app):
    startup_msg = startup_msg_fmt.format(settings=settings)
    log.info(startup_msg)

-
 def startup_event_handler(app):
    async def start_app() -> None:
        await on_startup(app)

-    return start_app
+    return start_app
--- a/gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py
+++ b/gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py
@@ -1,10 +1,9 @@
-import logging
-import time
-from typing import Dict, List
-
-from api_v1.settings import settings
 from fastapi import APIRouter, Depends, Response, Security, status
 from pydantic import BaseModel, Field
+from typing import List, Dict
+import logging
+import time
+from api_v1.settings import settings

 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -12,11 +11,11 @@ logger.setLevel(logging.DEBUG)
 ### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml


+
 class ChatCompletionMessage(BaseModel):
    role: str
    content: str

-
 class ChatCompletionRequest(BaseModel):
    model: str = Field(..., description='The model to generate a completion from.')
    messages: List[ChatCompletionMessage] = Field(..., description='The model to generate a completion from.')
@@ -27,13 +26,11 @@ class ChatCompletionChoice(BaseModel):
    index: int
    finish_reason: str

-
 class ChatCompletionUsage(BaseModel):
    prompt_tokens: int
    completion_tokens: int
    total_tokens: int

-
 class ChatCompletionResponse(BaseModel):
    id: str
    object: str = 'text_completion'
@@ -45,7 +42,6 @@ class ChatCompletionResponse(BaseModel):

 router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])

-
@router.post("/completions", response_model=ChatCompletionResponse)
 async def chat_completion(request: ChatCompletionRequest):
    '''
@@ -57,5 +53,11 @@ async def chat_completion(request: ChatCompletionRequest):
        created=time.time(),
        model=request.model,
        choices=[{}],
-        usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+        usage={
+            'prompt_tokens': 0,
+            'completion_tokens': 0,
+            'total_tokens': 0
+        }
    )
+
+
--- a/gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py
+++ b/gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py
@@ -1,16 +1,14 @@
 import json
+
+from fastapi import APIRouter, Depends, Response, Security, status
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel, Field
 from typing import List, Dict, Iterable, AsyncIterable
 import logging
-import time
-from typing import Dict, List, Union, Optional
 from uuid import uuid4
-import aiohttp
-import asyncio
 from api_v1.settings import settings
-from fastapi import APIRouter, Depends, Response, Security, status, HTTPException
-from fastapi.responses import StreamingResponse
 from gpt4all import GPT4All
-from pydantic import BaseModel, Field
+import time

 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -18,17 +16,14 @@ logger.setLevel(logging.DEBUG)

 ### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml

-
 class CompletionRequest(BaseModel):
-    model: str = Field(settings.model, description='The model to generate a completion from.')
-    prompt: Union[List[str], str] = Field(..., description='The prompt to begin completing from.')
-    max_tokens: int = Field(None, description='Max tokens to generate')
-    temperature: float = Field(settings.temp, description='Model temperature')
-    top_p: Optional[float] = Field(settings.top_p, description='top_p')
-    top_k: Optional[int] = Field(settings.top_k, description='top_k')
-    n: int = Field(1, description='How many completions to generate for each prompt')
+    model: str = Field(..., description='The model to generate a completion from.')
+    prompt: str = Field(..., description='The prompt to begin completing from.')
+    max_tokens: int = Field(7, description='Max tokens to generate')
+    temperature: float = Field(0, description='Model temperature')
+    top_p: float = Field(1.0, description='top_p')
+    n: int = Field(1, description='')
    stream: bool = Field(False, description='Stream responses')
-    repeat_penalty: float = Field(settings.repeat_penalty, description='Repeat penalty')


 class CompletionChoice(BaseModel):
@@ -63,6 +58,7 @@ class CompletionStreamResponse(BaseModel):

 router = APIRouter(prefix="/completions", tags=["Completion Endpoints"])

+
 def stream_completion(output: Iterable, base_response: CompletionStreamResponse):
    """
    Streams a GPT4All output to the client.
@@ -84,132 +80,49 @@ def stream_completion(output: Iterable, base_response: CompletionStreamResponse)
        ))]
        yield f"data: {json.dumps(dict(chunk))}\n\n"

-async def gpu_infer(payload, header):
-    async with aiohttp.ClientSession() as session:
-        try:
-            async with session.post(
-                settings.hf_inference_server_host, headers=header, data=json.dumps(payload)
-            ) as response:
-                resp = await response.json()
-            return resp
-
-        except aiohttp.ClientError as e:
-            # Handle client-side errors (e.g., connection error, invalid URL)
-            logger.error(f"Client error: {e}")
-        except aiohttp.ServerError as e:
-            # Handle server-side errors (e.g., internal server error)
-            logger.error(f"Server error: {e}")
-        except json.JSONDecodeError as e:
-            # Handle JSON decoding errors
-            logger.error(f"JSON decoding error: {e}")
-        except Exception as e:
-            # Handle other unexpected exceptions
-            logger.error(f"Unexpected error: {e}")

@router.post("/", response_model=CompletionResponse)
 async def completions(request: CompletionRequest):
    '''
    Completes a GPT4All model response.
    '''
-    if settings.inference_mode == "gpu":
-        params = request.dict(exclude={'model', 'prompt', 'max_tokens', 'n'})
-        params["max_new_tokens"] = request.max_tokens
-        params["num_return_sequences"] = request.n

-        header = {"Content-Type": "application/json"}
-        if isinstance(request.prompt, list):
-            tasks = []
-            for prompt in request.prompt:
-                payload = {"parameters": params}
-                payload["inputs"] = prompt
-                task = gpu_infer(payload, header)
-                tasks.append(task)
-            results = await asyncio.gather(*tasks)
+    model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)

-            choices = []
-            for response in results:
-                scores = response["scores"] if "scores" in response else -1.0
-                choices.append(
-                    dict(
-                        CompletionChoice(
-                            text=response["generated_text"], index=0, logprobs=scores, finish_reason='stop'
-                        )
-                    )
-                )
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=choices,
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
-
-        else:
-            payload = {"parameters": params}
-            # If streaming, we need to return a StreamingResponse
-            payload["inputs"] = request.prompt
-
-            resp = await gpu_infer(payload, header)
-
-            output = resp["generated_text"]
-            # this returns all logprobs
-            scores = resp["scores"] if "scores" in resp else -1.0
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(text=output, index=0, logprobs=scores, finish_reason='stop'))],
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
+    output = model.generate(prompt=request.prompt,
+                            n_predict=request.max_tokens,
+                            streaming=request.stream,
+                            top_k=20,
+                            top_p=request.top_p,
+                            temp=request.temperature,
+                            n_batch=1024,
+                            repeat_penalty=1.2,
+                            repeat_last_n=10)

+    # If streaming, we need to return a StreamingResponse
+    if request.stream:
+        base_chunk = CompletionStreamResponse(
+            id=str(uuid4()),
+            created=time.time(),
+            model=request.model,
+            choices=[]
+        )
+        return StreamingResponse((response for response in stream_completion(output, base_chunk)),
+                                 media_type="text/event-stream")
    else:
-
-        if request.model != settings.model:
-            raise HTTPException(status_code=400,
-                                detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
-
-        if isinstance(request.prompt, list):
-            if len(request.prompt) > 1:
-                raise HTTPException(status_code=400, detail="Can only infer one inference per request in CPU mode.")
-            else:
-                request.prompt = request.prompt[0]
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        output = model.generate(prompt=request.prompt,
-                                max_tokens=request.max_tokens,
-                                streaming=request.stream,
-                                top_k=request.top_k,
-                                top_p=request.top_p,
-                                temp=request.temperature,
-                                )
-
-        # If streaming, we need to return a StreamingResponse
-        if request.stream:
-            base_chunk = CompletionStreamResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[]
-            )
-            return StreamingResponse((response for response in stream_completion(output, base_chunk)),
-                                     media_type="text/event-stream")
-        else:
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(
-                    text=output,
-                    index=0,
-                    logprobs=-1,
-                    finish_reason='stop'
-                ))],
-                usage={
-                    'prompt_tokens': 0,  # TODO how to compute this?
-                    'completion_tokens': 0,
-                    'total_tokens': 0
-                }
-            )
+        return CompletionResponse(
+            id=str(uuid4()),
+            created=time.time(),
+            model=request.model,
+            choices=[dict(CompletionChoice(
+                text=output,
+                index=0,
+                logprobs=-1,
+                finish_reason='stop'
+            ))],
+            usage={
+                'prompt_tokens': 0, #TODO how to compute this?
+                'completion_tokens': 0,
+                'total_tokens': 0
+            }
+        )
--- a/gpt4all-api/gpt4all_api/app/api_v1/routes/embeddings.py
+++ b/gpt4all-api/gpt4all_api/app/api_v1/routes/embeddings.py
@@ -1,65 +0,0 @@
-from typing import List, Union
-from fastapi import APIRouter
-from api_v1.settings import settings
-from gpt4all import Embed4All
-from pydantic import BaseModel, Field
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class EmbeddingRequest(BaseModel):
-    model: str = Field(
-        settings.model, description="The model to generate an embedding from."
-    )
-    input: Union[str, List[str], List[int], List[List[int]]] = Field(
-        ..., description="Input text to embed, encoded as a string or array of tokens."
-    )
-
-
-class EmbeddingUsage(BaseModel):
-    prompt_tokens: int = 0
-    total_tokens: int = 0
-
-
-class Embedding(BaseModel):
-    index: int = 0
-    object: str = "embedding"
-    embedding: List[float]
-
-
-class EmbeddingResponse(BaseModel):
-    object: str = "list"
-    model: str
-    data: List[Embedding]
-    usage: EmbeddingUsage
-
-
-router = APIRouter(prefix="/embeddings", tags=["Embedding Endpoints"])
-
-embedder = Embed4All()
-
-
-def get_embedding(data: EmbeddingRequest) -> EmbeddingResponse:
-    """
-    Calculates the embedding for the given input using a specified model.
-
-    Args:
-        data (EmbeddingRequest): An EmbeddingRequest object containing the input data
-        and model name.
-
-    Returns:
-        EmbeddingResponse: An EmbeddingResponse object encapsulating the calculated embedding,
-        usage info, and the model name.
-    """
-    embedding = embedder.embed(data.input)
-    return EmbeddingResponse(
-        data=[Embedding(embedding=embedding)], usage=EmbeddingUsage(), model=data.model
-    )
-
-
-@router.post("/", response_model=EmbeddingResponse)
-def embeddings(data: EmbeddingRequest):
-    """
-    Creates a GPT4All embedding
-    """
-    return get_embedding(data)
--- a/gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py
+++ b/gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py
@@ -1,27 +1,22 @@
-import logging
-from typing import Dict, List
-
-from api_v1.settings import settings
 from fastapi import APIRouter, Depends, Response, Security, status
 from pydantic import BaseModel, Field
+from typing import List, Dict
+import logging
+from api_v1.settings import settings

 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)

 ### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml

-
 class ListEnginesResponse(BaseModel):
    data: List[Dict] = Field(..., description="All available models.")

-
 class EngineResponse(BaseModel):
    data: List[Dict] = Field(..., description="All available models.")

-
 router = APIRouter(prefix="/engines", tags=["Search Endpoints"])

-
@router.get("/", response_model=ListEnginesResponse)
 async def list_engines():
    '''
@@ -34,7 +29,10 @@ async def list_engines():

@router.get("/{engine_id}", response_model=EngineResponse)
 async def retrieve_engine(engine_id: str):
-    ''' '''
+    '''
+
+    '''

    raise NotImplementedError()
    return EngineResponse()
+
--- a/gpt4all-api/gpt4all_api/app/api_v1/routes/health.py
+++ b/gpt4all-api/gpt4all_api/app/api_v1/routes/health.py
@@ -1,7 +1,6 @@
 import logging
 from fastapi import APIRouter
 from fastapi.responses import JSONResponse
-
 log = logging.getLogger(__name__)

 router = APIRouter(prefix="/health", tags=["Health"])
--- a/gpt4all-api/gpt4all_api/app/api_v1/settings.py
+++ b/gpt4all-api/gpt4all_api/app/api_v1/settings.py
@@ -5,15 +5,6 @@ class Settings(BaseSettings):
    app_environment = 'dev'
    model: str = 'ggml-mpt-7b-chat.bin'
    gpt4all_path: str = '/models'
-    inference_mode: str = "cpu"
-    hf_inference_server_host: str = "http://gpt4all_gpu:80/generate"
-    sentry_dns: str = None
-
-    temp: float = 0.18
-    top_p: float = 1.0
-    top_k: int = 50
-    repeat_penalty: float = 1.18
-


 settings = Settings()
--- a/gpt4all-api/gpt4all_api/app/main.py
+++ b/gpt4all-api/gpt4all_api/app/main.py
@@ -1,19 +1,19 @@
-import logging
 import os
-
 import docs
-from api_v1 import events
-from api_v1.api import router as v1_router
-from api_v1.settings import settings
+import logging
 from fastapi import FastAPI, HTTPException, Request
-from fastapi.logger import logger as fastapi_logger
 from starlette.middleware.cors import CORSMiddleware
+from fastapi.logger import logger as fastapi_logger
+from api_v1.settings import settings
+from api_v1.api import router as v1_router
+from api_v1 import events
+import os

 logger = logging.getLogger(__name__)

 app = FastAPI(title='GPT4All API', description=docs.desc)

-# CORS Configuration (in-case you want to deploy)
+#CORS Configuration (in-case you want to deploy)
 app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
@@ -29,41 +29,20 @@ app.include_router(v1_router, prefix='/v1')
 app.add_event_handler('startup', events.startup_event_handler(app))
 app.add_exception_handler(HTTPException, events.on_http_error)

-
@app.on_event("startup")
 async def startup():
    global model
-    if settings.inference_mode == "cpu":
-        logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
-        from gpt4all import GPT4All
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
-    else:
-        # is it possible to do this once the server is up?
-        ## TODO block until HF inference server is up.
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
+    logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
+    from gpt4all import GPT4All
+    model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)

+    logger.info("GPT4All API is ready.")

@app.on_event("shutdown")
 async def shutdown():
    logger.info("Shutting down API")


-if settings.sentry_dns is not None:
-    import sentry_sdk
-
-    def traces_sampler(sampling_context):
-        if 'health' in sampling_context['transaction_context']['name']:
-            return False
-
-    sentry_sdk.init(
-        dsn=settings.sentry_dns, traces_sample_rate=0.1, traces_sampler=traces_sampler, send_default_pii=False
-    )
-
 # This is needed to get logs to show up in the app
 if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
    gunicorn_error_logger = logging.getLogger("gunicorn.error")
@@ -78,7 +57,5 @@ if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
    uvicorn_logger.handlers = gunicorn_error_logger.handlers
 else:
    # https://github.com/tiangolo/fastapi/issues/2019
-    LOG_FORMAT2 = (
-        "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
-    )
-    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)
+    LOG_FORMAT2 = "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
+    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)
--- a/gpt4all-api/gpt4all_api/app/tests/test_endpoints.py
+++ b/gpt4all-api/gpt4all_api/app/tests/test_endpoints.py
@@ -1,25 +1,31 @@
 """
 Use the OpenAI python API to test gpt4all models.
 """
-from typing import List, get_args
-
 import openai
-
 openai.api_base = "http://localhost:4891/v1"

 openai.api_key = "not needed for a local LLM"


 def test_completion():
-    model = "ggml-mpt-7b-chat.bin"
+    model = "gpt4all-j-v1.3-groovy"
    prompt = "Who is Michael Jordan?"
    response = openai.Completion.create(
-        model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
+        model=model,
+        prompt=prompt,
+        max_tokens=50,
+        temperature=0.28,
+        top_p=0.95,
+        n=1,
+        echo=True,
+        stream=False
    )
    assert len(response['choices'][0]['text']) > len(prompt)
+    print(response)
+

 def test_streaming_completion():
-    model = "ggml-mpt-7b-chat.bin"
+    model = "gpt4all-j-v1.3-groovy"
    prompt = "Who is Michael Jordan?"
    tokens = []
    for resp in openai.Completion.create(
@@ -36,24 +42,10 @@ def test_streaming_completion():
    assert (len(tokens) > 0)
    assert (len("".join(tokens)) > len(prompt))

-
-def test_batched_completion():
-    model = "ggml-mpt-7b-chat.bin"
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model, prompt=[prompt] * 3, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-    assert len(response['choices']) == 3
-
-
-def test_embedding():
-    model = "ggml-all-MiniLM-L6-v2-f16.bin"
-    prompt = "Who is Michael Jordan?"
-    response = openai.Embedding.create(model=model, input=prompt)
-    output = response["data"][0]["embedding"]
-    args = get_args(List[float])
-
-    assert response["model"] == model
-    assert isinstance(output, list)
-    assert all(isinstance(x, args) for x in output)
+# def test_chat_completions():
+#     model = "gpt4all-j-v1.3-groovy"
+#     prompt = "Who is Michael Jordan?"
+#     response = openai.ChatCompletion.create(
+#         model=model,
+#         messages=[]
+#     )
--- a/gpt4all-api/gpt4all_api/requirements.txt
+++ b/gpt4all-api/gpt4all_api/requirements.txt
@@ -1,12 +1,10 @@
 aiohttp>=3.6.2
 aiofiles
-pydantic>=1.4.0,<2.0.0
+pydantic>=1.4.0
 requests>=2.24.0
 ujson>=2.0.2
 fastapi>=0.95.0
 Jinja2>=3.0
-gpt4all>=1.0.0
+gpt4all==1.0.1
 pytest
-openai
-black
-isort
+openai
--- a/gpt4all-api/makefile
+++ b/gpt4all-api/makefile
@@ -1,26 +1,22 @@
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 APP_NAME:=gpt4all_api
 PYTHON:=python3.8
-SHELL := /bin/bash

 all: dependencies

 fresh: clean dependencies

 testenv: clean_testenv test_build
-	docker compose -f docker-compose.yaml up --build
-
-testenv_gpu: clean_testenv test_build
-	docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up --build
+	docker compose up --build

 testenv_d: clean_testenv test_build
 	docker compose up --build -d

 test:
-	docker compose exec $(APP_NAME) pytest -svv --disable-warnings -p no:cacheprovider /app/tests
+	docker compose exec gpt4all_api pytest -svv --disable-warnings -p no:cacheprovider /app/tests

 test_build:
-    DOCKER_BUILDKIT=1 docker build -t $(APP_NAME) --progress plain -f $(APP_NAME)/Dockerfile.buildkit .
+    DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .

 clean_testenv:
 	docker compose down -v
@@ -31,7 +27,7 @@ venv:
 	if [ ! -d $(ROOT_DIR)/env ]; then $(PYTHON) -m venv $(ROOT_DIR)/env; fi

 dependencies: venv
-	source $(ROOT_DIR)/env/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
+	source $(ROOT_DIR)/env/bin/activate; yes w | python -m pip install -r $(ROOT_DIR)/gpt4all_api/requirements.txt

 clean: clean_testenv
 	# Remove existing environment
@@ -39,8 +35,3 @@ clean: clean_testenv
 	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;


-black:
-	source $(ROOT_DIR)/env/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
-
-isort:
-	source $(ROOT_DIR)/env/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)
--- a/gpt4all-backend/CMakeLists.txt
+++ b/gpt4all-backend/CMakeLists.txt
@@ -20,7 +20,7 @@ endif()
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")

 set(LLMODEL_VERSION_MAJOR 0)
-set(LLMODEL_VERSION_MINOR 4)
+set(LLMODEL_VERSION_MINOR 3)
 set(LLMODEL_VERSION_PATCH 0)
 set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
 project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
@@ -39,10 +39,6 @@ else()
    message(STATUS "Interprocedural optimization support detected")
 endif()

-if(NOT APPLE)
-  set(LLAMA_KOMPUTE YES)
-endif()
-
 include(llama.cpp.cmake)

 set(BUILD_VARIANTS default avxonly)
@@ -73,6 +69,11 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
    # Include GGML
    set(LLAMA_K_QUANTS YES)
    include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
+    if (NOT LLAMA_METAL)
+        set(LLAMA_K_QUANTS NO)
+        include_ggml(llama.cpp-230511 -230511-${BUILD_VARIANT} ON)
+        include_ggml(llama.cpp-230519 -230519-${BUILD_VARIANT} ON)
+    endif()

    # Function for preparing individual implementations
    function(prepare_target TARGET_NAME BASE_LIB)
@@ -99,33 +100,35 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)

    add_library(replit-mainline-${BUILD_VARIANT} SHARED
    replit.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-    target_compile_definitions(replit-mainline-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
    prepare_target(replit-mainline llama-mainline)

    if (NOT LLAMA_METAL)
-# FIXME: These need to be forward ported to latest ggml
-#        add_library(gptj-${BUILD_VARIANT} SHARED
-#            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-#        prepare_target(gptj ggml-230511)
+        add_library(llamamodel-230519-${BUILD_VARIANT} SHARED
+            llamamodel.cpp llmodel_shared.cpp)
+        target_compile_definitions(llamamodel-230519-${BUILD_VARIANT} PRIVATE
+            LLAMA_VERSIONS===2 LLAMA_DATE=230519)
+        prepare_target(llamamodel-230519 llama-230519)
+        add_library(llamamodel-230511-${BUILD_VARIANT} SHARED
+            llamamodel.cpp llmodel_shared.cpp)
+        target_compile_definitions(llamamodel-230511-${BUILD_VARIANT} PRIVATE
+            LLAMA_VERSIONS=<=1 LLAMA_DATE=230511)
+        prepare_target(llamamodel-230511 llama-230511)
+
+        add_library(gptj-${BUILD_VARIANT} SHARED
+            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        prepare_target(gptj ggml-230511)

        add_library(falcon-${BUILD_VARIANT} SHARED
            falcon.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        target_compile_definitions(falcon-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
        prepare_target(falcon llama-mainline)
-# FIXME: These need to be forward ported to latest ggml
-#        add_library(mpt-${BUILD_VARIANT} SHARED
-#            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-#        prepare_target(mpt ggml-230511)
+
+        add_library(mpt-${BUILD_VARIANT} SHARED
+            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        prepare_target(mpt ggml-230511)

        add_library(bert-${BUILD_VARIANT} SHARED
            bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
        prepare_target(bert llama-mainline)
-
-        add_library(starcoder-${BUILD_VARIANT} SHARED
-            starcoder.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        target_compile_definitions(starcoder-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
-        prepare_target(starcoder llama-mainline)
    endif()
 endforeach()

@@ -134,8 +137,6 @@ add_library(llmodel
    llmodel_c.h llmodel_c.cpp
    dlhandle.h
 )
-target_link_libraries(llmodel PRIVATE ggml-mainline-default)
-target_compile_definitions(llmodel PRIVATE GGML_BUILD_VARIANT="default")
 target_compile_definitions(llmodel PRIVATE LIB_FILE_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}")

 set_target_properties(llmodel PROPERTIES
--- a/gpt4all-backend/bert.cpp
+++ b/gpt4all-backend/bert.cpp
@@ -1,6 +1,5 @@
 #define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
 #include "bert_impl.h"
-#include "llmodel_shared.h"
 #include "ggml.h"

 #include <cassert>
@@ -92,6 +91,22 @@ struct bert_model
 };

 // Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
+struct bert_buffer {
+    uint8_t * data = NULL;
+    size_t size = 0;
+
+    void resize(size_t size) {
+        delete[] data;
+        data = new uint8_t[size];
+        this->size = size;
+    }
+
+    ~bert_buffer() {
+        delete[] data;
+    }
+};
+
+
 struct bert_ctx
 {
    bert_model model;
@@ -100,8 +115,7 @@ struct bert_ctx
    size_t mem_per_token;
    int64_t mem_per_input;
    int32_t max_batch_n;
-    llm_buffer buf_compute;
-    llm_buffer work_buf;
+    bert_buffer buf_compute;
 };

 int32_t bert_n_embd(bert_ctx * ctx)
@@ -314,12 +328,13 @@ void bert_eval(

    struct ggml_init_params params = {
        .mem_size = buf_compute.size,
-        .mem_buffer = buf_compute.addr,
+        .mem_buffer = buf_compute.data,
        .no_alloc = false,
    };

    struct ggml_context *ctx0 = ggml_init(params);
    struct ggml_cgraph gf = {};
+    gf.n_threads = n_threads;

    // Embeddings. word_embeddings + token_type_embeddings + position_embeddings
    struct ggml_tensor *token_layer = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
@@ -451,9 +466,7 @@ void bert_eval(
    ggml_tensor *output = inpL;
    // run the computation
    ggml_build_forward_expand(&gf, output);
-    //ggml_graph_compute_g4a()
-    ggml_graph_compute_g4a(ctx->work_buf, &gf, n_threads);
-    //ggml_graph_compute(ctx0, &gf);
+    ggml_graph_compute(ctx0, &gf);


    // float *dat = ggml_get_data_f32(output);
@@ -620,7 +633,7 @@ struct bert_ctx * bert_load_from_file(const char *fname)
        model_mem_req += n_layer * (n_intermediate * ggml_type_sizef(GGML_TYPE_F32)); // ff_i_b
        model_mem_req += n_layer * (n_embd * ggml_type_sizef(GGML_TYPE_F32)); // ff_o_b

-        model_mem_req += (5 + 16 * n_layer) * ggml_tensor_overhead(); // object overhead
+        model_mem_req += (5 + 16 * n_layer) * 256; // object overhead

 #if defined(DEBUG_BERT)
        printf("%s: ggml ctx size = %6.2f MB\n", __func__, model_mem_req / (1024.0 * 1024.0));
@@ -1050,4 +1063,4 @@ DLL_EXPORT bool magic_match(std::istream& f) {
 DLL_EXPORT LLModel *construct() {
    return new Bert;
 }
-}
+}
--- a/gpt4all-backend/dlhandle.h
+++ b/gpt4all-backend/dlhandle.h
@@ -75,7 +75,7 @@ public:

    Dlhandle() : chandle(nullptr) {}
    Dlhandle(const std::string& fpath) {
-        chandle = LoadLibraryExA(fpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
+        chandle = LoadLibraryA(fpath.c_str());
        if (!chandle) {
            throw Exception("dlopen(\""+fpath+"\"): Error");
        }
--- a/gpt4all-backend/falcon.cpp
+++ b/gpt4all-backend/falcon.cpp
@@ -1,4 +1,3 @@
-#include "ggml.h"
 #define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
 #include "falcon_impl.h"
 #include "llama.h"
@@ -65,7 +64,6 @@ struct falcon_model {
    std::map<std::string, struct ggml_tensor*> tensors;

    llm_buffer eval_buf;
-    llm_buffer work_buf;
    llm_buffer scr0_buf;
    llm_buffer scr1_buf;
 };
@@ -448,7 +446,7 @@ bool falcon_model_load(const std::string & fname, falcon_model & model, gpt_voca
 //   - embd_w:    the predicted logits for the next token
 //
 bool falcon_eval(
-        falcon_model & model,
+        const falcon_model & model,
        const int n_threads,
        const int n_past,
        const std::vector<gpt_vocab::id> & embd_inp,
@@ -475,6 +473,7 @@ bool falcon_eval(

    struct ggml_context * ctx0 = ggml_init(eval_ctx_params);
    struct ggml_cgraph gf = {};
+    gf.n_threads = n_threads;

    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -547,8 +546,8 @@ bool falcon_eval(
                head_dim * (n_head + n_head_kv) * sizeof_wtype);

            // using mode = 2 for neox mode
-            Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, head_dim, 2, n_ctx);
-            Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, head_dim, 2, n_ctx);
+            Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, head_dim, 2);
+            Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, head_dim, 2);

            // store key and value to memory
            {
@@ -679,8 +678,7 @@ bool falcon_eval(

    // run the computation
    ggml_build_forward_expand(&gf, inpL);
-    ggml_graph_compute_g4a(model.work_buf, &gf, n_threads);
-  
+    ggml_graph_compute       (ctx0, &gf);

    //if (n_past%100 == 0) {
    //    ggml_graph_print   (&gf);
--- a/gpt4all-backend/gptj.cpp
+++ b/gpt4all-backend/gptj.cpp
@@ -961,11 +961,6 @@ DLL_EXPORT const char *get_build_variant() {
 DLL_EXPORT bool magic_match(std::istream& f) {
    uint32_t magic = 0;
    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    gptj_hparams hparams;
-    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
-    if (!(hparams.n_vocab >= 50300 && hparams.n_vocab <= 50400)) {
-        return false; // not a gptj.
-    }
    return magic == 0x67676d6c;
 }

--- a/gpt4all-backend/llama.cpp-mainline
+++ b/gpt4all-backend/llama.cpp-mainline
--- a/gpt4all-backend/llama.cpp.cmake
+++ b/gpt4all-backend/llama.cpp.cmake
@@ -1,11 +1,3 @@
-#
-# Copyright (c) 2023 Nomic, Inc. All rights reserved.
-#
-# This software is licensed under the terms of the Software for Open Models License (SOM),
-# version 1.0, as detailed in the LICENSE_SOM.txt file. A copy of this license should accompany
-# this software. Except as expressly granted in the SOM license, all rights are reserved by Nomic, Inc.
-#
-
 cmake_minimum_required(VERSION 3.12) # Don't bump this version for no reason

 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@@ -153,150 +145,6 @@ if (LLAMA_OPENBLAS)
    endif()
 endif()

-if (LLAMA_KOMPUTE)
-    add_compile_definitions(VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
-    find_package(Vulkan COMPONENTS glslc REQUIRED)
-    find_program(glslc_executable NAMES glslc HINTS Vulkan::glslc)
-    if (NOT glslc_executable)
-        message(FATAL_ERROR "glslc not found")
-    endif()
-
-    set(LLAMA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp-mainline)
-
-    function(compile_shader)
-      set(options)
-      set(oneValueArgs)
-      set(multiValueArgs SOURCES)
-      cmake_parse_arguments(compile_shader "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-      foreach(source ${compile_shader_SOURCES})
-        get_filename_component(OP_FILE ${source} NAME)
-        set(spv_file ${CMAKE_CURRENT_BINARY_DIR}/${OP_FILE}.spv)
-        add_custom_command(
-            OUTPUT ${spv_file}
-            DEPENDS ${LLAMA_DIR}/${source} ${LLAMA_DIR}/kompute/common.comp
-            COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
-            COMMENT "Compiling ${source} to ${source}.spv"
-        )
-
-        get_filename_component(RAW_FILE_NAME ${spv_file} NAME)
-        set(FILE_NAME "shader${RAW_FILE_NAME}")
-        string(REPLACE ".comp.spv" ".h" HEADER_FILE ${FILE_NAME})
-        string(TOUPPER ${HEADER_FILE} HEADER_FILE_DEFINE)
-        string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}")
-        set(OUTPUT_HEADER_FILE "${HEADER_FILE}")
-        message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}")
-        if(CMAKE_GENERATOR MATCHES "Visual Studio") 
-            add_custom_command(
-              OUTPUT ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              DEPENDS ${spv_file} xxd
-              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}/xxd"
-            )
-        else()
-            add_custom_command(
-              OUTPUT ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              DEPENDS ${spv_file} xxd
-              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
-            )
-        endif()
-      endforeach()
-    endfunction()
-
-    if (EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
-        message(STATUS "Kompute found")
-        add_subdirectory(${LLAMA_DIR}/kompute)
-
-        # Compile our shaders
-        compile_shader(SOURCES
-          kompute/op_scale.comp
-          kompute/op_add.comp
-          kompute/op_addrow.comp
-          kompute/op_mul.comp
-          kompute/op_mulrow.comp
-          kompute/op_silu.comp
-          kompute/op_relu.comp
-          kompute/op_gelu.comp
-          kompute/op_softmax.comp
-          kompute/op_norm.comp
-          kompute/op_rmsnorm.comp
-          kompute/op_diagmask.comp
-          kompute/op_mul_mat_f16.comp
-          kompute/op_mul_mat_mat_f16.comp
-          kompute/op_mul_mat_q4_0.comp
-          kompute/op_mul_mat_mat_q4_0.comp
-          kompute/op_mul_mat_q4_1.comp
-          kompute/op_getrows_f16.comp
-          kompute/op_getrows_q4_0.comp
-          kompute/op_getrows_q4_1.comp
-          kompute/op_rope.comp
-          kompute/op_cpy_f16_f16.comp
-          kompute/op_cpy_f16_f32.comp
-          kompute/op_cpy_f32_f16.comp
-          kompute/op_cpy_f32_f32.comp
-        )
-
-        # Create a custom target for our generated shaders
-        add_custom_target(generated_shaders DEPENDS
-          shaderop_scale.h
-          shaderop_add.h
-          shaderop_addrow.h
-          shaderop_mul.h
-          shaderop_mulrow.h
-          shaderop_silu.h
-          shaderop_relu.h
-          shaderop_gelu.h
-          shaderop_softmax.h
-          shaderop_norm.h
-          shaderop_rmsnorm.h
-          shaderop_diagmask.h
-          shaderop_mul_mat_f16.h
-          shaderop_mul_mat_mat_f16.h
-          shaderop_mul_mat_q4_0.h
-          shaderop_mul_mat_mat_q4_0.h
-          shaderop_mul_mat_q4_1.h
-          shaderop_getrows_f16.h
-          shaderop_getrows_q4_0.h
-          shaderop_getrows_q4_1.h
-          shaderop_rope.h
-          shaderop_cpy_f16_f16.h
-          shaderop_cpy_f16_f32.h
-          shaderop_cpy_f32_f16.h
-          shaderop_cpy_f32_f32.h
-        )
-
-        # Create a custom command that depends on the generated_shaders
-        add_custom_command(
-            OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp
-            COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp
-            DEPENDS generated_shaders
-            COMMENT "Ensuring shaders are generated before compiling ggml-vulkan.cpp"
-        )
-
-        # Add the stamp to the main sources to ensure dependency tracking
-        set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml-vulkan.cpp ${LLAMA_DIR}/ggml-vulkan.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp)
-        add_compile_definitions(GGML_USE_KOMPUTE)
-        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute)
-        set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CMAKE_BINARY_DIR})
-    else()
-        message(WARNING "Kompute not found")
-    endif()
-endif()
-
 if (LLAMA_ALL_WARNINGS)
    if (NOT MSVC)
        set(c_flags
@@ -448,13 +296,10 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
    add_library(ggml${SUFFIX} OBJECT
                ${DIRECTORY}/ggml.c
                ${DIRECTORY}/ggml.h
-                ${DIRECTORY}/ggml-alloc.c
-                ${DIRECTORY}/ggml-alloc.h
                ${GGML_SOURCES_QUANT_K}
                ${GGML_SOURCES_CUDA}
                ${GGML_METAL_SOURCES}
-                ${GGML_OPENCL_SOURCES}
-                ${GGML_SOURCES_KOMPUTE})
+                ${GGML_OPENCL_SOURCES})

    if (LLAMA_K_QUANTS)
        target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_K_QUANTS)
--- a/gpt4all-backend/llamamodel.cpp
+++ b/gpt4all-backend/llamamodel.cpp
@@ -28,9 +28,6 @@
 #include <llama.h>
 #include <ggml.h>

-#ifdef GGML_USE_KOMPUTE
-#include "ggml-vulkan.h"
-#endif

 namespace {
 const char *modelType_ = "LLaMA";
@@ -158,30 +155,13 @@ bool LLamaModel::loadModel(const std::string &modelPath)
    // currently
    d_ptr->params.n_gpu_layers = 1;
 #endif
-#ifdef GGML_USE_KOMPUTE
-    if (ggml_vk_has_device()) {
-        // vulkan always runs the whole model if n_gpu_layers is not 0, at least
-        // currently
-        d_ptr->params.n_gpu_layers = 1;
-    }
-#endif

    d_ptr->ctx = llama_init_from_file(modelPath.c_str(), d_ptr->params);
    if (!d_ptr->ctx) {
-#ifdef GGML_USE_KOMPUTE
-        // Explicitly free the device so next load it doesn't use it
-        ggml_vk_free_device();
-#endif
        std::cerr << "LLAMA ERROR: failed to load model from " <<  modelPath << std::endl;
        return false;
    }

-#ifdef GGML_USE_KOMPUTE
-    if (ggml_vk_has_device()) {
-        std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
-    }
-#endif
-
    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
    d_ptr->modelLoaded = true;
    fflush(stderr);
@@ -198,7 +178,7 @@ int32_t LLamaModel::threadCount() const {

 LLamaModel::~LLamaModel()
 {
-    if (d_ptr->ctx) {
+    if(d_ptr->ctx) {
        llama_free(d_ptr->ctx);
    }
 }
@@ -272,85 +252,6 @@ const std::vector<LLModel::Token> &LLamaModel::endTokens() const
    return fres;
 }

-#if defined(GGML_USE_KOMPUTE)
-#include "ggml-vulkan.h"
-#endif
-
-std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryRequired)
-{
-#if defined(GGML_USE_KOMPUTE)
-    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(memoryRequired);
-
-    std::vector<LLModel::GPUDevice> devices;
-    for(const auto& vkDevice : vkDevices) {
-        LLModel::GPUDevice device;
-        device.index = vkDevice.index;
-        device.type = vkDevice.type;
-        device.heapSize = vkDevice.heapSize;
-        device.name = vkDevice.name;
-        device.vendor = vkDevice.vendor;
-
-        devices.push_back(device);
-    }
-
-    return devices;
-#else
-    return std::vector<LLModel::GPUDevice>();
-#endif
-}
-
-bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string& device)
-{
-#if defined(GGML_USE_KOMPUTE)
-    return ggml_vk_init_device(memoryRequired, device);
-#else
-    return false;
-#endif
-}
-
-bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device)
-{
-#if defined(GGML_USE_KOMPUTE)
-    ggml_vk_device vkDevice;
-    vkDevice.index = device.index;
-    vkDevice.type = device.type;
-    vkDevice.heapSize = device.heapSize;
-    vkDevice.name = device.name;
-    vkDevice.vendor = device.vendor;
-    return ggml_vk_init_device(vkDevice);
-#else
-    return false;
-#endif
-}
-
-bool LLamaModel::initializeGPUDevice(int device)
-{
-#if defined(GGML_USE_KOMPUTE)
-    return ggml_vk_init_device(device);
-#else
-    return false;
-#endif
-}
-
-bool LLamaModel::hasGPUDevice()
-{
-#if defined(GGML_USE_KOMPUTE)
-    return ggml_vk_has_device();
-#else
-    return false;
-#endif
-}
-
-bool LLamaModel::usingGPUDevice()
-{
-#if defined(GGML_USE_KOMPUTE)
-    return ggml_vk_using_vulkan();
-#elif defined(GGML_USE_METAL)
-    return true;
-#endif
-    return false;
-}
-
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
--- a/gpt4all-backend/llamamodel_impl.h
+++ b/gpt4all-backend/llamamodel_impl.h
@@ -25,12 +25,6 @@ public:
    size_t restoreState(const uint8_t *src) override;
    void setThreadCount(int32_t n_threads) override;
    int32_t threadCount() const override;
-    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) override;
-    bool initializeGPUDevice(size_t memoryRequired, const std::string& device) override;
-    bool initializeGPUDevice(const GPUDevice &device) override;
-    bool initializeGPUDevice(int device) override;
-    bool hasGPUDevice() override;
-    bool usingGPUDevice() override;

 private:
    LLamaPrivate *d_ptr;
--- a/gpt4all-backend/llmodel.cpp
+++ b/gpt4all-backend/llmodel.cpp
@@ -11,7 +11,8 @@
 #include <cstdlib>
 #include <sstream>
 #ifdef _MSC_VER
-#include <intrin.h>
+#include <windows.h>
+#include <processthreadsapi.h>
 #endif

 std::string s_implementations_search_path = ".";
@@ -21,9 +22,7 @@ static bool has_at_least_minimal_hardware() {
    #ifndef _MSC_VER
        return __builtin_cpu_supports("avx");
    #else
-        int cpuInfo[4];
-        __cpuid(cpuInfo, 1);
-        return cpuInfo[2] & (1 << 28);
+        return IsProcessorFeaturePresent(PF_AVX_INSTRUCTIONS_AVAILABLE);
    #endif
 #else
    return true; // Don't know how to handle non-x86_64
@@ -35,9 +34,7 @@ static bool requires_avxonly() {
    #ifndef _MSC_VER
        return !__builtin_cpu_supports("avx2");
    #else
-        int cpuInfo[4];
-        __cpuidex(cpuInfo, 7, 0);
-        return !(cpuInfo[1] & (1 << 5));
+        return !IsProcessorFeaturePresent(PF_AVX2_INSTRUCTIONS_AVAILABLE);
    #endif
 #else
    return false; // Don't know how to handle non-x86_64
--- a/gpt4all-backend/llmodel.h
+++ b/gpt4all-backend/llmodel.h
@@ -58,14 +58,6 @@ public:
            // window
    };

-    struct GPUDevice {
-        int index = 0;
-        int type = 0;
-        size_t heapSize = 0;
-        std::string name;
-        std::string vendor;
-    };
-
    explicit LLModel() {}
    virtual ~LLModel() {}

@@ -95,14 +87,6 @@ public:
        return *m_implementation;
    }

-    virtual std::vector<GPUDevice> availableGPUDevices(size_t /*memoryRequired*/) { return std::vector<GPUDevice>(); }
-    virtual bool initializeGPUDevice(size_t /*memoryRequired*/, const std::string& /*device*/) { return false; }
-    virtual bool initializeGPUDevice(const GPUDevice &/*device*/) { return false; }
-    virtual bool initializeGPUDevice(int /*device*/) { return false; }
-    virtual bool hasGPUDevice() { return false; }
-    virtual bool usingGPUDevice() { return false; }
-    static std::vector<GPUDevice> availableGPUDevices();
-
 protected:
    // These are pure virtual because subclasses need to implement as the default implementation of
    // 'prompt' above calls these functions
--- a/gpt4all-backend/llmodel_c.cpp
+++ b/gpt4all-backend/llmodel_c.cpp
@@ -5,6 +5,7 @@
 #include <cerrno>
 #include <utility>

+
 struct LLModelWrapper {
    LLModel *llModel = nullptr;
    LLModel::PromptContext promptContext;
@@ -167,14 +168,10 @@ void llmodel_prompt(llmodel_model model, const char *prompt,

 float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size)
 {
-    if (model == nullptr || text == nullptr || !strlen(text)) {
-        *embedding_size = 0;
-        return nullptr;
-    }
    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
    std::vector<float> embeddingVector = wrapper->llModel->embedding(text);
    float *embedding = (float *)malloc(embeddingVector.size() * sizeof(float));
-    if (embedding == nullptr) {
+    if(embedding == nullptr) {
        *embedding_size = 0;
        return nullptr;
    }
@@ -209,57 +206,3 @@ const char *llmodel_get_implementation_search_path()
 {
    return LLModel::Implementation::implementationsSearchPath().c_str();
 }
-
-struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices)
-{
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    std::vector<LLModel::GPUDevice> devices = wrapper->llModel->availableGPUDevices(memoryRequired);
-
-    // Set the num_devices
-    *num_devices = devices.size();
-
-    if (*num_devices == 0) return nullptr;  // Return nullptr if no devices are found
-
-    // Allocate memory for the output array
-    struct llmodel_gpu_device* output = (struct llmodel_gpu_device*) malloc(*num_devices * sizeof(struct llmodel_gpu_device));
-
-    for (int i = 0; i < *num_devices; i++) {
-        output[i].index = devices[i].index;
-        output[i].type = devices[i].type;
-        output[i].heapSize = devices[i].heapSize;
-        output[i].name = strdup(devices[i].name.c_str());  // Convert std::string to char* and allocate memory
-        output[i].vendor = strdup(devices[i].vendor.c_str());  // Convert std::string to char* and allocate memory
-    }
-
-    return output;
-}
-
-bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
-{
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->initializeGPUDevice(memoryRequired, std::string(device));
-}
-
-bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device)
-{
-    LLModel::GPUDevice d;
-    d.index = device->index;
-    d.type = device->type;
-    d.heapSize = device->heapSize;
-    d.name = device->name;
-    d.vendor = device->vendor;
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->initializeGPUDevice(d);
-}
-
-bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
-{
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->initializeGPUDevice(device);
-}
-
-bool llmodel_has_gpu_device(llmodel_model model)
-{
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->hasGPUDevice();
-}
--- a/gpt4all-backend/llmodel_c.h
+++ b/gpt4all-backend/llmodel_c.h
@@ -56,18 +56,8 @@ struct llmodel_prompt_context {
    int32_t repeat_last_n;  // last n tokens to penalize
    float context_erase;    // percent of context to erase if we exceed the context window
 };
-
-struct llmodel_gpu_device {
-    int index = 0;
-    int type = 0;           // same as VkPhysicalDeviceType
-    size_t heapSize = 0;
-    const char * name;
-    const char * vendor;
-};
-
 #ifndef __cplusplus
 typedef struct llmodel_prompt_context llmodel_prompt_context;
-typedef struct llmodel_gpu_device llmodel_gpu_device;
 #endif

 /**
@@ -183,8 +173,6 @@ void llmodel_prompt(llmodel_model model, const char *prompt,

 /**
 * Generate an embedding using the model.
- * NOTE: If given NULL pointers for the model or text, or an empty text, a NULL pointer will be
- * returned. Bindings should signal an error when NULL is the return value.
 * @param model A pointer to the llmodel_model instance.
 * @param text A string representing the text to generate an embedding for.
 * @param embedding_size A pointer to a size_t type that will be set by the call indicating the length
@@ -228,50 +216,6 @@ void llmodel_set_implementation_search_path(const char *path);
 */
 const char *llmodel_get_implementation_search_path();

-/**
- * Get a list of available GPU devices given the memory required.
- * @return A pointer to an array of llmodel_gpu_device's whose number is given by num_devices.
- */
-struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices);
-
-/**
- * Initializes a GPU device based on a specified string criterion.
- *
- * This function initializes a GPU device based on a string identifier provided. The function
- * allows initialization based on general device type ("gpu"), vendor name ("amd", "nvidia", "intel"),
- * or any specific device name.
- *
- * @param memoryRequired The amount of memory (in bytes) required by the application or task
- *                       that will utilize the GPU device.
- * @param device A string specifying the desired criterion for GPU device selection. It can be:
- *               - "gpu": To initialize the best available GPU.
- *               - "amd", "nvidia", or "intel": To initialize the best available GPU from that vendor.
- *               - A specific GPU device name: To initialize a GPU with that exact name.
- *
- * @return True if the GPU device is successfully initialized based on the provided string
- *         criterion. Returns false if the desired GPU device could not be initialized.
- */
-bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device);
-
-/**
- * Initializes a GPU device by specifying a valid gpu device pointer.
- * @param device A gpu device pointer.
- * @return True if the GPU device is successfully initialized, false otherwise.
- */
-bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device);
-
-/**
- * Initializes a GPU device by its index.
- * @param device An integer representing the index of the GPU device to be initialized.
- * @return True if the GPU device is successfully initialized, false otherwise.
- */
-bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
-
-/**
- * @return True if a GPU device is successfully initialized, false otherwise.
- */
-bool llmodel_has_gpu_device(llmodel_model model);
-
 #ifdef __cplusplus
 }
 #endif
--- a/gpt4all-backend/llmodel_shared.cpp
+++ b/gpt4all-backend/llmodel_shared.cpp
@@ -4,10 +4,6 @@
 #include <iostream>
 #include <unordered_set>

-#ifdef GGML_USE_KOMPUTE
-#include "ggml-vulkan.h"
-#endif
-
 void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
    size_t i = 0;
    promptCtx.n_past = 0;
@@ -178,26 +174,3 @@ std::vector<float> LLModel::embedding(const std::string &/*text*/)
    }
    return std::vector<float>();
 }
-
-std::vector<LLModel::GPUDevice> LLModel::availableGPUDevices()
-{
-#if defined(GGML_USE_KOMPUTE)
-    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(0);
-
-    std::vector<LLModel::GPUDevice> devices;
-    for(const auto& vkDevice : vkDevices) {
-        LLModel::GPUDevice device;
-        device.index = vkDevice.index;
-        device.type = vkDevice.type;
-        device.heapSize = vkDevice.heapSize;
-        device.name = vkDevice.name;
-        device.vendor = vkDevice.vendor;
-
-        devices.push_back(device);
-    }
-
-    return devices;
-#else
-    return std::vector<LLModel::GPUDevice>();
-#endif
-}
--- a/gpt4all-backend/llmodel_shared.h
+++ b/gpt4all-backend/llmodel_shared.h
@@ -1,52 +1,8 @@
 #pragma once
 #include <cstdint>
 #include <cstddef>
-#include <vector>
 #include <ggml.h>

-#if defined(GGML_USE_KOMPUTE)
-#include "ggml-vulkan.h"
-struct llm_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-    ggml_vk_memory memory;
-
-    llm_buffer() = default;
-
-    void resize(size_t size) {
-        free();
-
-        if (!ggml_vk_has_device()) {
-            this->addr = new uint8_t[size];
-            this->size = size;
-        } else {
-            this->memory = ggml_vk_allocate(size);
-            this->addr = (uint8_t*)memory.data;
-            this->size = size;
-        }
-    }
-
-    void free() {
-        if (!memory.primaryMemory) {
-            delete[] addr;
-        } else if (memory.data) {
-            ggml_vk_free_memory(memory);
-        }
-        this->addr = NULL;
-        this->size = 0;
-    }
-
-    ~llm_buffer() {
-        free();
-    }
-
-    // disable copy and move
-    llm_buffer(const llm_buffer&) = delete;
-    llm_buffer(llm_buffer&&) = delete;
-    llm_buffer& operator=(const llm_buffer&) = delete;
-    llm_buffer& operator=(llm_buffer&&) = delete;
-};
-#else
 struct llm_buffer {
    uint8_t * addr = NULL;
    size_t size = 0;
@@ -61,7 +17,6 @@ struct llm_buffer {
        delete[] addr;
    }
 };
-#endif

 struct llm_kv_cache {
    struct ggml_tensor * k;
@@ -79,14 +34,3 @@ struct llm_kv_cache {
        }
    }
 };
-
-#if LLAMA_DATE >= 230519
-inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads) {
-    struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
-    if (plan.work_size > 0) {
-        buf.resize(plan.work_size);
-        plan.work_data = buf.addr;
-    }
-    ggml_graph_compute(graph, &plan);
-}
-#endif
--- a/gpt4all-backend/replit.cpp
+++ b/gpt4all-backend/replit.cpp
@@ -163,7 +163,7 @@ struct mpt_hparams {
    int32_t n_embd      = 0; //max_seq_len
    int32_t n_head      = 0; // n_heads
    int32_t n_layer     = 0; //n_layers
-    int32_t ftype       = 0;
+    int32_t ftype       = 0; 
 };

 struct replit_layer {
@@ -196,7 +196,6 @@ struct replit_model {

    struct ggml_context * ctx;
    llm_buffer eval_buf;
-    llm_buffer work_buf;
    llm_buffer scr0_buf;
    llm_buffer scr1_buf;
    #ifdef GGML_USE_METAL
@@ -220,7 +219,7 @@ static bool kv_cache_init(
    params.mem_size   = cache.buf.size;
    params.mem_buffer = cache.buf.addr;
    params.no_alloc   = false;
-
+    
    cache.ctx = ggml_init(params);
    if (!cache.ctx) {
        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
@@ -491,7 +490,7 @@ bool replit_model_load(const std::string & fname, std::istream &fin, replit_mode
   model.scr1_buf.resize(256u * 1024 * 1024);

 #ifdef GGML_USE_METAL
-    model.ctx_metal = ggml_metal_init(1);
+    model.ctx_metal = ggml_metal_init();
    void* data_ptr = ggml_get_mem_buffer(model.ctx);
    size_t data_size = ggml_get_mem_size(model.ctx);
    const size_t max_size = ggml_get_max_tensor_size(model.ctx);
@@ -503,7 +502,7 @@ bool replit_model_load(const std::string & fname, std::istream &fin, replit_mode
    }

    GGML_CHECK_BUF(ggml_metal_add_buffer(model.ctx_metal, "data", data_ptr, data_size, max_size));
-    GGML_CHECK_BUF(ggml_metal_add_buffer(model.ctx_metal, "kv", ggml_get_mem_buffer(model.kv_self.ctx),
+    GGML_CHECK_BUF(ggml_metal_add_buffer(model.ctx_metal, "kv", ggml_get_mem_buffer(model.kv_self.ctx), 
                                                                ggml_get_mem_size(model.kv_self.ctx), 0));
    GGML_CHECK_BUF(ggml_metal_add_buffer(model.ctx_metal, "eval", model.eval_buf.addr, model.eval_buf.size, 0));
    GGML_CHECK_BUF(ggml_metal_add_buffer(model.ctx_metal, "scr0", model.scr0_buf.addr, model.scr0_buf.size, 0));
@@ -535,7 +534,7 @@ bool replit_model_load(const std::string & fname, replit_model & model, replit_t
 //   - embd_inp:  the embeddings of the tokens in the context
 //   - embd_w:    the predicted logits for the next token
 //
-bool replit_eval(replit_model & model, const int n_threads, const int n_past,
+bool replit_eval(const replit_model & model, const int n_threads, const int n_past,
                 const std::vector<gpt_vocab::id> & embd_inp, std::vector<float> & embd_w, size_t & mem_per_token) {
    const int N = embd_inp.size();

@@ -553,7 +552,7 @@ bool replit_eval(replit_model & model, const int n_threads, const int n_past,
        .no_alloc = false,
    };
    struct ggml_context * ctx0 = ggml_init(eval_ctx_params);
-    struct ggml_cgraph gf = {};
+    struct ggml_cgraph gf = {.n_threads = n_threads};

    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
    memcpy(embd->data, embd_inp.data(), N * ggml_element_size(embd));
@@ -707,10 +706,10 @@ bool replit_eval(replit_model & model, const int n_threads, const int n_past,
        ggml_metal_get_tensor(model.ctx_metal, model.kv_self.k);
        ggml_metal_get_tensor(model.ctx_metal, model.kv_self.v);

-        ggml_graph_compute_g4a(model.work_buf, &gf, n_threads);
+        ggml_graph_compute(ctx0, &gf);
    }
 #else
-    ggml_graph_compute_g4a(model.work_buf, &gf, n_threads);
+    ggml_graph_compute(ctx0, &gf);
 #endif

    // std::cout << "Qcur" << std::endl;
@@ -975,14 +974,6 @@ const std::vector<LLModel::Token> &Replit::endTokens() const
    return fres;
 }

-bool Replit::usingGPUDevice()
-{
-#if defined(GGML_USE_METAL)
-    return true;
-#endif
-    return false;
-}
-
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
--- a/gpt4all-backend/replit_impl.h
+++ b/gpt4all-backend/replit_impl.h
@@ -27,7 +27,6 @@ public:
    size_t restoreState(const uint8_t *src) override;
    void setThreadCount(int32_t n_threads) override;
    int32_t threadCount() const override;
-    bool usingGPUDevice() override;

 private:
    ReplitPrivate *d_ptr;
--- a/gpt4all-backend/starcoder.cpp
+++ b/gpt4all-backend/starcoder.cpp
--- a/gpt4all-backend/starcoder_impl.h
+++ b/gpt4all-backend/starcoder_impl.h
@@ -1,42 +0,0 @@
-#ifndef STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of starcoder.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define STARCODER_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef STARCODER_H
-#define STARCODER_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include <memory>
-#include "llmodel.h"
-
-struct StarcoderPrivate;
-class Starcoder : public LLModel {
-public:
-    Starcoder();
-    ~Starcoder();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    std::unique_ptr<StarcoderPrivate> d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // STARCODER_H
--- a/gpt4all-bindings/java/README.md
+++ b/gpt4all-bindings/java/README.md
@@ -12,12 +12,12 @@ You can add Java bindings into your Java project by adding the following depende
 <dependency>
    <groupId>com.hexadevlabs</groupId>
    <artifactId>gpt4all-java-binding</artifactId>
-    <version>1.1.5</version>
+    <version>1.1.3</version>
 </dependency>
 ```
 **Gradle**
 ```
-implementation 'com.hexadevlabs:gpt4all-java-binding:1.1.5'
+implementation 'com.hexadevlabs:gpt4all-java-binding:1.1.3'
 ```

 To add the library dependency for another build system see [Maven Central Java bindings](https://central.sonatype.com/artifact/com.hexadevlabs/gpt4all-java-binding/).
@@ -121,6 +121,4 @@ If this is the case you can easily download and install the latest x64 Microsoft
 3. Version **1.1.4**:
   - Java bindings is compatible with gpt4all version 2.4.11
   - Falcon model support included.
-4. Version **1.1.5**:
-   - Add a check for model file readability before loading model.
   
--- a/gpt4all-bindings/java/TODO.md
+++ b/gpt4all-bindings/java/TODO.md
@@ -1,6 +1,2 @@
-## Needed
-1. Integrate with circleci build pipeline like the C# binding.
-
-## These are just ideas
 1. Better Chat completions function.
 2. Chat completion that returns result in OpenAI compatible format.
--- a/gpt4all-bindings/java/pom.xml
+++ b/gpt4all-bindings/java/pom.xml
@@ -6,7 +6,7 @@

    <groupId>com.hexadevlabs</groupId>
    <artifactId>gpt4all-java-binding</artifactId>
-    <version>1.1.5</version>
+    <version>1.1.4</version>
    <packaging>jar</packaging>

    <properties>
--- a/gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java
+++ b/gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java
@@ -184,16 +184,11 @@ public  class LLModel implements AutoCloseable {
            throw new IllegalStateException("Model file does not exist: " + modelPathAbs);
        }

-        // Check if file is Readable
-        if(!Files.isReadable(modelPath)){
-            throw new IllegalStateException("Model file cannot be read: " + modelPathAbs);
-        }
-
        // Create Model Struct. Will load dynamically the correct backend based on model type
        model = library.llmodel_model_create2(modelPathAbs, "auto", error);

        if(model == null) {
-            throw new IllegalStateException("Could not load, gpt4all backend returned error: " + error.message);
+            throw new IllegalStateException("Could not load gpt4all backend :" + error.message);
        }
        library.llmodel_loadModel(model, modelPathAbs);

--- a/gpt4all-bindings/python/README.md
+++ b/gpt4all-bindings/python/README.md
@@ -20,12 +20,12 @@ pip install gpt4all
 1. Setup `llmodel`

 ```
-git clone --recurse-submodules https://github.com/nomic-ai/gpt4all.git
+git clone --recurse-submodules git@github.com:nomic-ai/gpt4all.git
 cd gpt4all/gpt4all-backend/
 mkdir build
 cd build
 cmake ..
-cmake --build . --parallel  # optionally append: --config Release
+cmake --build . --parallel
 ```
 Confirm that `libllmodel.*` exists in `gpt4all-backend/build`.

@@ -47,15 +47,6 @@ output = model.generate("The capital of France is ", max_tokens=3)
 print(output)
 ```

-
-GPU Usage
-```python
-from gpt4all import GPT4All
-model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin", device='gpu') # device='amd', device='intel'
-output = model.generate("The capital of France is ", max_tokens=3)
-print(output)
-```
-
 ## Troubleshooting a Local Build
 - If you're on Windows and have compiled with a MinGW toolchain, you might run into an error like:
  ```
--- a/gpt4all-bindings/python/docs/gpt4all_faq.md
+++ b/gpt4all-bindings/python/docs/gpt4all_faq.md
@@ -2,42 +2,31 @@

 ## What models are supported by the GPT4All ecosystem?

-Currently, there are six different model architectures that are supported:
+Currently, there are five different model architectures that are supported:

 1. GPT-J - Based off of the GPT-J architecture with examples found [here](https://huggingface.co/EleutherAI/gpt-j-6b)
 2. LLaMA - Based off of the LLaMA architecture with examples found [here](https://huggingface.co/models?sort=downloads&search=llama)
 3. MPT - Based off of Mosaic ML's MPT architecture with examples found [here](https://huggingface.co/mosaicml/mpt-7b)
 4. Replit - Based off of Replit Inc.'s Replit architecture with examples found [here](https://huggingface.co/replit/replit-code-v1-3b)
 5. Falcon - Based off of TII's Falcon architecture with examples found [here](https://huggingface.co/tiiuae/falcon-40b)
-6. StarCoder - Based off of BigCode's StarCoder architecture with examples found [here](https://huggingface.co/bigcode/starcoder)

 ## Why so many different architectures? What differentiates them?

-One of the major differences is license. Currently, the LLaMA based models are subject to a non-commercial license, whereas the GPTJ and MPT base
-models allow commercial usage. However, its successor [Llama 2 is commercially licensable](https://ai.meta.com/llama/license/), too. In the early
-advent of the recent explosion of activity in open source local models, the LLaMA models have generally been seen as performing better, but that is
-changing quickly. Every week - even every day! - new models are released with some of the GPTJ and MPT models competitive in performance/quality with
-LLaMA. What's more, there are some very nice architectural innovations with the MPT models that could lead to new performance/quality gains.
+One of the major differences is license. Currently, the LLAMA based models are subject to a non-commercial license, whereas the GPTJ and MPT base models allow commercial usage. In the early advent of the recent explosion of activity in open source local models, the llama models have generally been seen as performing better, but that is changing quickly. Every week - even every day! - new models are released with some of the GPTJ and MPT models competitive in performance/quality with LLAMA. What's more, there are some very nice architectural innovations with the MPT models that could lead to new performance/quality gains.

 ## How does GPT4All make these models available for CPU inference?

-By leveraging the ggml library written by Georgi Gerganov and a growing community of developers. There are currently multiple different versions of
-this library. The original GitHub repo can be found [here](https://github.com/ggerganov/ggml), but the developer of the library has also created a
-LLaMA based version [here](https://github.com/ggerganov/llama.cpp). Currently, this backend is using the latter as a submodule.
+By leveraging the ggml library written by Georgi Gerganov and a growing community of developers. There are currently multiple different versions of this library. The original github repo can be found [here](https://github.com/ggerganov/ggml), but the developer of the library has also created a LLAMA based version [here](https://github.com/ggerganov/llama.cpp). Currently, this backend is using the latter as a submodule.

 ## Does that mean GPT4All is compatible with all llama.cpp models and vice versa?

 Yes!

-The upstream [llama.cpp](https://github.com/ggerganov/llama.cpp) project has introduced several [compatibility breaking] quantization methods recently.
-This is a breaking change that renders all previous models (including the ones that GPT4All uses) inoperative with newer versions of llama.cpp since
-that change.
+The upstream [llama.cpp](https://github.com/ggerganov/llama.cpp) project has introduced several [compatibility breaking](https://github.com/ggerganov/llama.cpp/commit/b9fd7eee57df101d4a3e3eabc9fd6c2cb13c9ca1) quantization methods recently. This is a breaking change that renders all previous models (including the ones that GPT4All uses) inoperative with newer versions of llama.cpp since that change.

 Fortunately, we have engineered a submoduling system allowing us to dynamically load different versions of the underlying library so that
 GPT4All just works.

-[compatibility breaking]: https://github.com/ggerganov/llama.cpp/commit/b9fd7eee57df101d4a3e3eabc9fd6c2cb13c9ca1
-
 ## What are the system requirements?

 Your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) and you need enough RAM to load a model into memory.
@@ -46,55 +35,8 @@ Your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wi

 In newer versions of llama.cpp, there has been some added support for NVIDIA GPU's for inference. We're investigating how to incorporate this into our downloadable installers.

-## Ok, so bottom line... how do I make my model on Hugging Face compatible with GPT4All ecosystem right now?
+## Ok, so bottom line... how do I make my model on huggingface compatible with GPT4All ecosystem right now?

-1. Check to make sure the Hugging Face model is available in one of our three supported architectures
-2. If it is, then you can use the conversion script inside of our pinned llama.cpp submodule for GPTJ and LLaMA based models
+1. Check to make sure the huggingface model is available in one of our three supported architectures
+2. If it is, then you can use the conversion script inside of our pinned llama.cpp submodule for GPTJ and LLAMA based models
 3. Or if your model is an MPT model you can use the conversion script located directly in this backend directory under the scripts subdirectory 
-
-## Language Bindings
-
-#### There's a problem with the download
-
-Some bindings can download a model, if allowed to do so. For example, in Python or TypeScript if `allow_download=True`
-or `allowDownload=true` (default), a model is automatically downloaded into `.cache/gpt4all/` in the user's home folder,
-unless it already exists.
-
-In case of connection issues or errors during the download, you might want to manually verify the model file's MD5
-checksum by comparing it with the one listed in [models.json].
-
-As an alternative to the basic downloader built into the bindings, you can choose to download from the 
-<https://gpt4all.io/> website instead. Scroll down to 'Model Explorer' and pick your preferred model.
-
-[models.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json
-
-#### I need the chat GUI and bindings to behave the same
-
-The chat GUI and bindings are based on the same backend. You can make them behave the same way by following these steps:
-
- First of all, ensure that all parameters in the chat GUI settings match those passed to the generating API, e.g.:
-
-    === "Python"
-        ``` py
-        from gpt4all import GPT4All
-        model = GPT4All(...)
-        model.generate("prompt text", temp=0, ...)  # adjust parameters
-        ```
-    === "TypeScript"
-        ``` ts
-        import { createCompletion, loadModel } from '../src/gpt4all.js'
-        const ll = await loadModel(...);
-        const messages = ...
-        const re = await createCompletion(ll, messages, { temp: 0, ... });  // adjust parameters
-        ```
-
- To make comparing the output easier, set _Temperature_ in both to 0 for now. This will make the output deterministic.
-
- Next you'll have to compare the templates, adjusting them as necessary, based on how you're using the bindings.
-    - Specifically, in Python:
-        - With simple `generate()` calls, the input has to be surrounded with system and prompt templates.
-        - When using a chat session, it depends on whether the bindings are allowed to download [models.json]. If yes,
-          and in the chat GUI the default templates are used, it'll be handled automatically. If no, use
-          `chat_session()` template parameters to customize them.
-
- Once you're done, remember to reset _Temperature_ to its previous value in both chat GUI and your custom code.
--- a/gpt4all-bindings/python/docs/gpt4all_python.md
+++ b/gpt4all-bindings/python/docs/gpt4all_python.md
@@ -2,8 +2,8 @@
 The `GPT4All` python package provides bindings to our C/C++ model backend libraries.
 The source code and local build instructions can be found [here](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python).

-
 ## Quickstart
+
 ```bash
 pip install gpt4all
 ```
@@ -20,16 +20,8 @@ pip install gpt4all
    1. Paris
    ```

-This will:
-
- Instantiate `GPT4All`,  which is the primary public API to your large language model (LLM).
- Automatically download the given model to `~/.cache/gpt4all/` if not already present.
- Through `model.generate(...)` the model starts working on a response. There are various ways to
-  steer that process. Here, `max_tokens` sets an upper limit, i.e. a hard cut-off point to the output.
-
-
 ### Chatting with GPT4All
-Local LLMs can be optimized for chat conversations by reusing previous computational history.
+Local LLMs can be optimized for chat conversions by reusing previous computational history.

 Use the GPT4All `chat_session` context manager to hold chat conversations with the model.

@@ -37,9 +29,9 @@ Use the GPT4All `chat_session` context manager to hold chat conversations with t
    ``` py
    model = GPT4All(model_name='orca-mini-3b.ggmlv3.q4_0.bin')
    with model.chat_session():
-        response1 = model.generate(prompt='hello', temp=0)
-        response2 = model.generate(prompt='write me a short poem', temp=0)
-        response3 = model.generate(prompt='thank you', temp=0)
+        response = model.generate(prompt='hello', top_k=1)
+        response = model.generate(prompt='write me a short poem', top_k=1)
+        response = model.generate(prompt='thank you', top_k=1)
        print(model.current_chat_session)
    ```
 === "Output"
@@ -71,20 +63,19 @@ Use the GPT4All `chat_session` context manager to hold chat conversations with t
       }
    ]
    ```
+When using GPT4All models in the chat_session context:

-When using GPT4All models in the `chat_session` context:
+- The model is given a prompt template which makes it chatty.
+- Internal K/V caches are preserved from previous conversation history speeding up inference.

- Consecutive chat exchanges are taken into account and not discarded until the session ends; as long as the model has capacity.
- Internal K/V caches are preserved from previous conversation history, speeding up inference.
- The model is given a system and prompt template which make it chatty. Depending on `allow_download=True` (default),
-  it will obtain the latest version of [models.json] from the repository, which contains specifically tailored templates
-  for models. Conversely, if it is not allowed to download, it falls back to default templates instead.

-[models.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json
+### Generation Parameters
+
+::: gpt4all.gpt4all.GPT4All.generate


 ### Streaming Generations
-To interact with GPT4All responses as the model generates, use the `streaming=True` flag during generation.
+To interact with GPT4All responses as the model generates, use the `streaming = True` flag during generation.

 === "GPT4All Streaming Example"
    ``` py
@@ -100,338 +91,22 @@ To interact with GPT4All responses as the model generates, use the `streaming=Tr
    [' Paris', ' is', ' a', ' city', ' that', ' has', ' been', ' a', ' major', ' cultural', ' and', ' economic', ' center', ' for', ' over', ' ', '2', ',', '0', '0']
    ```

+#### Streaming and Chat Sessions
+When streaming tokens in a chat session, you must manually handle collection and updating of the chat history.

-### The Generate Method API
-::: gpt4all.gpt4all.GPT4All.generate
-
-
-## Examples & Explanations
-### Influencing Generation
-The three most influential parameters in generation are _Temperature_ (`temp`), _Top-p_ (`top_p`) and _Top-K_ (`top_k`).
-In a nutshell, during the process of selecting the next token, not just one or a few are considered, but every single
-token in the vocabulary is given a probability. The parameters can change the field of candidate tokens.
-
- **Temperature** makes the process either more or less random. A _Temperature_ above 1 increasingly "levels the playing
-  field", while at a _Temperature_ between 0 and 1 the likelihood of the best token candidates grows even more. A
-  _Temperature_ of 0 results in selecting the best token, making the output deterministic. A _Temperature_ of 1
-  represents a neutral setting with regard to randomness in the process.
-
- _Top-p_ and _Top-K_ both narrow the field:
-    - **Top-K** limits candidate tokens to a fixed number after sorting by probability. Setting it higher than the
-      vocabulary size deactivates this limit.
-    - **Top-p** selects tokens based on their total probabilities. For example, a value of 0.8 means "include the best
-      tokens, whose accumulated probabilities reach or just surpass 80%". Setting _Top-p_ to 1, which is 100%,
-      effectively disables it.
-
-The recommendation is to keep at least one of _Top-K_ and _Top-p_ active. Other parameters can also influence
-generation; be sure to review all their descriptions.
-
-
-### Specifying the Model Folder
-The model folder can be set with the `model_path` parameter when creating a `GPT4All` instance. The example below is
-is the same as if it weren't provided; that is, `~/.cache/gpt4all/` is the default folder.
-
-=== "GPT4All Model Folder Example"
-    ``` py
-    from pathlib import Path
-    from gpt4all import GPT4All
-    model = GPT4All(model_name='orca-mini-3b.ggmlv3.q4_0.bin',
-                    model_path=(Path.home() / '.cache' / 'gpt4all'),
-                    allow_download=False)
-    response = model.generate('my favorite 3 fruits are:', temp=0)
-    print(response)
-    ```
-=== "Output"
-    ```
-    My favorite three fruits are apples, bananas and oranges.
-    ```
-
-If you want to point it at the chat GUI's default folder, it should be:
-=== "macOS"
-    ``` py
-    from pathlib import Path
-    from gpt4all import GPT4All
-
-    model_name = 'orca-mini-3b.ggmlv3.q4_0.bin'
-    model_path = Path.home() / 'Library' / 'Application Support' / 'nomic.ai' / 'GPT4All'
-    model = GPT4All(model_name, model_path)
-    ```
-=== "Windows"
-    ``` py
-    from pathlib import Path
-    from gpt4all import GPT4All
-    import os
-    model_name = 'orca-mini-3b.ggmlv3.q4_0.bin'
-    model_path = Path(os.environ['LOCALAPPDATA']) / 'nomic.ai' / 'GPT4All'
-    model = GPT4All(model_name, model_path)
-    ```
-=== "Linux"
-    ``` py
-    from pathlib import Path
-    from gpt4all import GPT4All
-
-    model_name = 'orca-mini-3b.ggmlv3.q4_0.bin'
-    model_path = Path.home() / '.local' / 'share' / 'nomic.ai' / 'GPT4All'
-    model = GPT4All(model_name, model_path)
-    ```
-
-Alternatively, you could also change the module's default model directory:
-
-``` py
-from pathlib import Path
-import gpt4all.gpt4all
-gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = Path.home() / 'my' / 'models-directory'
+```python
 from gpt4all import GPT4All
-model = GPT4All('orca-mini-3b.ggmlv3.q4_0.bin')
-...
+model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin")
+
+with model.chat_session():
+    tokens = list(model.generate(prompt='hello', top_k=1, streaming=True))
+    model.current_chat_session.append({'role': 'assistant', 'content': ''.join(tokens)})
+
+    tokens = list(model.generate(prompt='write me a poem about dogs', top_k=1, streaming=True))
+    model.current_chat_session.append({'role': 'assistant', 'content': ''.join(tokens)})
+
+    print(model.current_chat_session)
 ```

-
-### Managing Templates
-Session templates can be customized when starting a `chat_session` context:
-
-=== "GPT4All Custom Session Templates Example"
-    ``` py
-    from gpt4all import GPT4All
-    model = GPT4All('ggml-Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin')
-    system_template = 'A chat between a curious user and an artificial intelligence assistant.'
-    # many models use triple hash '###' for keywords, Vicunas are simpler:
-    prompt_template = 'USER: {0}\nASSISTANT: '
-    with model.chat_session(system_template, prompt_template):
-        response1 = model.generate('why is the grass green?')
-        print(response1)
-        print()
-        response2 = model.generate('why is the sky blue?')
-        print(response2)
-    ```
-=== "Possible Output"
-    ```
-    The color of grass can be attributed to its chlorophyll content, which allows it
-    to absorb light energy from sunlight through photosynthesis. Chlorophyll absorbs
-    blue and red wavelengths of light while reflecting other colors such as yellow
-    and green. This is why the leaves appear green to our eyes.
-
-    The color of the sky appears blue due to a phenomenon called Rayleigh scattering,
-    which occurs when sunlight enters Earth's atmosphere and interacts with air
-    molecules such as nitrogen and oxygen. Blue light has shorter wavelength than
-    other colors in the visible spectrum, so it is scattered more easily by these
-    particles, making the sky appear blue to our eyes.
-    ```
-
-To do the same outside a session, the input has to be formatted manually. For example:
-
-=== "GPT4All Templates Outside a Session Example"
-    ``` py
-    model = GPT4All('ggml-Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin')
-    system_template = 'A chat between a curious user and an artificial intelligence assistant.'
-    prompt_template = 'USER: {0}\nASSISTANT: '
-    prompts = ['name 3 colors', 'now name 3 fruits', 'what were the 3 colors in your earlier response?']
-    first_input = system_template + prompt_template.format(prompts[0])
-    response = model.generate(first_input, temp=0)
-    print(response)
-    for prompt in prompts[1:]:
-        response = model.generate(prompt_template.format(prompt), temp=0)
-        print(response)
-    ```
-=== "Output"
-    ```
-    1) Red
-    2) Blue
-    3) Green
-
-    1. Apple
-    2. Banana
-    3. Orange
-
-    The colors in my previous response are blue, green and red.
-    ```
-
-Ultimately, the method `GPT4All._format_chat_prompt_template()` is responsible for formatting templates. It can be
-customized in a subclass. As an example:
-
-=== "Custom Subclass"
-    ``` py
-    from itertools import cycle
-    from gpt4all import GPT4All
-
-    class RotatingTemplateGPT4All(GPT4All):
-        def __init__(self, *args, **kwargs):
-            super().__init__(*args, **kwargs)
-            self._templates = [
-                "Respond like a pirate.",
-                "Respond like a politician.",
-                "Respond like a philosopher.",
-                "Respond like a Klingon.",
-            ]
-            self._cycling_templates = cycle(self._templates)
-
-        def _format_chat_prompt_template(
-            self,
-            messages: list,
-            default_prompt_header: str = "",
-            default_prompt_footer: str = "",
-        ) -> str:
-            full_prompt = default_prompt_header + "\n\n" if default_prompt_header != "" else ""
-            for message in messages:
-                if message["role"] == "user":
-                    user_message = f"USER: {message['content']} {next(self._cycling_templates)}\n"
-                    full_prompt += user_message
-                if message["role"] == "assistant":
-                    assistant_message = f"ASSISTANT: {message['content']}\n"
-                    full_prompt += assistant_message
-            full_prompt += "\n\n" + default_prompt_footer if default_prompt_footer != "" else ""
-            print(full_prompt)
-            return full_prompt
-    ```
-=== "GPT4All Custom Subclass Example"
-    ``` py
-    model = RotatingTemplateGPT4All('ggml-Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin')
-    with model.chat_session():  # starting a session is optional in this example
-        response1 = model.generate("hi, who are you?")
-        print(response1)
-        print()
-        response2 = model.generate("what can you tell me about snakes?")
-        print(response2)
-        print()
-        response3 = model.generate("what's your opinion on Chess?")
-        print(response3)
-        print()
-        response4 = model.generate("tell me about ancient Rome.")
-        print(response4)
-    ```
-=== "Possible Output"
-    ```
-    USER: hi, who are you? Respond like a pirate.
-
-    Pirate: Ahoy there mateys! I be Cap'n Jack Sparrow of the Black Pearl.
-
-    USER: what can you tell me about snakes? Respond like a politician.
-
-    Politician: Snakes have been making headlines lately due to their ability to
-    slither into tight spaces and evade capture, much like myself during my last
-    election campaign. However, I believe that with proper education and
-    understanding of these creatures, we can work together towards creating a
-    safer environment for both humans and snakes alike.
-
-    USER: what's your opinion on Chess? Respond like a philosopher.
-
-    Philosopher: The game of chess is often used as an analogy to illustrate the
-    complexities of life and decision-making processes. However, I believe that it
-    can also be seen as a reflection of our own consciousness and subconscious mind.
-    Just as each piece on the board has its unique role to play in shaping the
-    outcome of the game, we too have different roles to fulfill in creating our own
-    personal narrative.
-
-    USER: tell me about ancient Rome. Respond like a Klingon.
-
-    Klingon: Ancient Rome was once a great empire that ruled over much of Europe and
-    the Mediterranean region. However, just as the Empire fell due to internal strife
-    and external threats, so too did my own house come crashing down when I failed to
-    protect our homeworld from invading forces.
-    ```
-
-
-### Introspection
-A less apparent feature is the capacity to log the final prompt that gets sent to the model. It relies on
-[Python's logging facilities][py-logging] implemented in the `pyllmodel` module at the `INFO` level. You can activate it
-for example with a `basicConfig`, which displays it on the standard error stream. It's worth mentioning that Python's
-logging infrastructure offers [many more customization options][py-logging-cookbook].
-
-[py-logging]: https://docs.python.org/3/howto/logging.html
-[py-logging-cookbook]: https://docs.python.org/3/howto/logging-cookbook.html
-
-=== "GPT4All Prompt Logging Example"
-    ``` py
-    import logging
-    from gpt4all import GPT4All
-    logging.basicConfig(level=logging.INFO)
-    model = GPT4All('nous-hermes-13b.ggmlv3.q4_0.bin')
-    with model.chat_session('You are a geography expert.\nBe terse.',
-                            '### Instruction:\n{0}\n### Response:\n'):
-        response = model.generate('who are you?', temp=0)
-        print(response)
-        response = model.generate('what are your favorite 3 mountains?', temp=0)
-        print(response)
-    ```
-=== "Output"
-    ```
-    INFO:gpt4all.pyllmodel:LLModel.prompt_model -- prompt:
-    You are a geography expert.
-    Be terse.
-
-    ### Instruction:
-    who are you?
-    ### Response:
-
-    ===/LLModel.prompt_model -- prompt/===
-    I am an AI-powered chatbot designed to assist users with their queries related to geographical information.
-    INFO:gpt4all.pyllmodel:LLModel.prompt_model -- prompt:
-    ### Instruction:
-    what are your favorite 3 mountains?
-    ### Response:
-
-    ===/LLModel.prompt_model -- prompt/===
-    1) Mount Everest - Located in the Himalayas, it is the highest mountain on Earth and a significant challenge for mountaineers.
-    2) Kangchenjunga - This mountain is located in the Himalayas and is the third-highest peak in the world after Mount Everest and K2.
-    3) Lhotse - Located in the Himalayas, it is the fourth highest mountain on Earth and offers a challenging climb for experienced mountaineers.
-    ```
-
-
-### Without Online Connectivity
-To prevent GPT4All from accessing online resources, instantiate it with `allow_download=False`. This will disable both
-downloading missing models and [models.json], which contains information about them. As a result, predefined templates
-are used instead of model-specific system and prompt templates:
-
-=== "GPT4All Default Templates Example"
-    ``` py
-    from gpt4all import GPT4All
-    model = GPT4All('ggml-mpt-7b-chat.bin', allow_download=False)
-    # when downloads are disabled, it will use the default templates:
-    print("default system template:", repr(model.config['systemPrompt']))
-    print("default prompt template:", repr(model.config['promptTemplate']))
-    print()
-    # even when inside a session:
-    with model.chat_session():
-        assert model.current_chat_session[0]['role'] == 'system'
-        print("session system template:", repr(model.current_chat_session[0]['content']))
-        print("session prompt template:", repr(model._current_prompt_template))
-    ```
-=== "Output"
-    ```
-    default system template: ''
-    default prompt template: '### Human: \n{0}\n### Assistant:\n'
-
-    session system template: ''
-    session prompt template: '### Human: \n{0}\n### Assistant:\n'
-    ```
-
-
-### Interrupting Generation
-The simplest way to stop generation is to set a fixed upper limit with the `max_tokens` parameter.
-
-If you know exactly when a model should stop responding, you can add a custom callback, like so:
-
-=== "GPT4All Custom Stop Callback"
-    ``` py
-    from gpt4all import GPT4All
-    model = GPT4All('orca-mini-3b.ggmlv3.q4_0.bin')
-
-    def stop_on_token_callback(token_id, token_string):
-        # one sentence is enough:
-        if '.' in token_string:
-            return False
-        else:
-            return True
-
-    response = model.generate('Blue Whales are the biggest animal to ever inhabit the Earth.',
-                              temp=0, callback=stop_on_token_callback)
-    print(response)
-    ```
-=== "Output"
-    ```
-     They can grow up to 100 feet (30 meters) long and weigh as much as 20 tons (18 metric tons).
-    ```
-
-
-## API Documentation
+### API documentation
 ::: gpt4all.gpt4all.GPT4All
--- a/gpt4all-bindings/python/docs/gpt4all_typescript.md
+++ b/gpt4all-bindings/python/docs/gpt4all_typescript.md
@@ -1,731 +0,0 @@
-# GPT4All Node.js API
-
-```sh
-yarn add gpt4all@alpha
-
-npm install gpt4all@alpha
-
-pnpm install gpt4all@alpha
-```
-
-The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
-
-*   New bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
-*   The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
-*   Everything should work out the box.
-*   See [API Reference](#api-reference)
-
-### Chat Completion (alpha)
-
-```js
-import { createCompletion, loadModel } from '../src/gpt4all.js'
-
-const model = await loadModel('ggml-vicuna-7b-1.1-q4_2', { verbose: true });
-
-const response = await createCompletion(model, [
-    { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
-    { role : 'user', content: 'What is 1 + 1?'  } 
-]);
-
-```
-
-### Embedding (alpha)
-
-```js
-import { createEmbedding, loadModel } from '../src/gpt4all.js'
-
-const model = await loadModel('ggml-all-MiniLM-L6-v2-f16', { verbose: true });
-
-const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional");
-```
-
-### Build Instructions
-
-*   binding.gyp is compile config
-*   Tested on Ubuntu. Everything seems to work fine
-*   Tested on Windows. Everything works fine.
-*   Sparse testing on mac os.
-*   MingW works as well to build the gpt4all-backend. **HOWEVER**, this package works only with MSVC built dlls.
-
-### Requirements
-
-*   git
-*   [node.js >= 18.0.0](https://nodejs.org/en)
-*   [yarn](https://yarnpkg.com/)
-*   [node-gyp](https://github.com/nodejs/node-gyp)
-    *   all of its requirements.
-*   (unix) gcc version 12
-*   (win) msvc version 143
-    *   Can be obtained with visual studio 2022 build tools
-*   python 3
-
-### Build (from source)
-
-```sh
-git clone https://github.com/nomic-ai/gpt4all.git
-cd gpt4all-bindings/typescript
-```
-
-*   The below shell commands assume the current working directory is `typescript`.
-
-*   To Build and Rebuild:
-
-```sh
-yarn
-```
-
-*   llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
-
-```sh
-git submodule update --init --depth 1 --recursive
-```
-
-**AS OF NEW BACKEND** to build the backend,
-
-```sh
-yarn build:backend
-```
-
-This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native The only current way to use them is to put them in the current working directory of your application. That is, **WHEREVER YOU RUN YOUR NODE APPLICATION**
-
-*   llama-xxxx.dll is required.
-*   According to whatever model you are using, you'll need to select the proper model loader.
-    *   For example, if you running an Mosaic MPT model, you will need to select the mpt-(buildvariant).(dynamiclibrary)
-
-### Test
-
-```sh
-yarn test
-```
-
-### Source Overview
-
-#### src/
-
-*   Extra functions to help aid devex
-*   Typings for the native node addon
-*   the javascript interface
-
-#### test/
-
-*   simple unit testings for some functions exported.
-*   more advanced ai testing is not handled
-
-#### spec/
-
-*   Average look and feel of the api
-*   Should work assuming a model and libraries are installed locally in working directory
-
-#### index.cc
-
-*   The bridge between nodejs and c. Where the bindings are.
-
-#### prompt.cc
-
-*   Handling prompting and inference of models in a threadsafe, asynchronous way.
-
-### Known Issues
-
-*   why your model may be spewing bull 💩
-    *   The downloaded model is broken (just reinstall or download from official site)
-    *   That's it so far
-
-### Roadmap
-
-This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
-
-*   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
-*   \[ ] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
-*   \[x] proper unit testing (integrate with circle ci)
-*   \[x] publish to npm under alpha tag `gpt4all@alpha`
-*   \[x] have more people test on other platforms (mac tester needed)
-*   \[x] switch to new pluggable backend
-*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
-    *   Should include prebuilds to avoid painful node-gyp errors
-*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
-
-### API Reference
-
-<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
-
-##### Table of Contents
-
-*   [ModelType](#modeltype)
-*   [ModelFile](#modelfile)
-    *   [gptj](#gptj)
-    *   [llama](#llama)
-    *   [mpt](#mpt)
-    *   [replit](#replit)
-*   [type](#type)
-*   [LLModel](#llmodel)
-    *   [constructor](#constructor)
-        *   [Parameters](#parameters)
-    *   [type](#type-1)
-    *   [name](#name)
-    *   [stateSize](#statesize)
-    *   [threadCount](#threadcount)
-    *   [setThreadCount](#setthreadcount)
-        *   [Parameters](#parameters-1)
-    *   [raw\_prompt](#raw_prompt)
-        *   [Parameters](#parameters-2)
-    *   [embed](#embed)
-        *   [Parameters](#parameters-3)
-    *   [isModelLoaded](#ismodelloaded)
-    *   [setLibraryPath](#setlibrarypath)
-        *   [Parameters](#parameters-4)
-    *   [getLibraryPath](#getlibrarypath)
-*   [loadModel](#loadmodel)
-    *   [Parameters](#parameters-5)
-*   [createCompletion](#createcompletion)
-    *   [Parameters](#parameters-6)
-*   [createEmbedding](#createembedding)
-    *   [Parameters](#parameters-7)
-*   [CompletionOptions](#completionoptions)
-    *   [verbose](#verbose)
-    *   [systemPromptTemplate](#systemprompttemplate)
-    *   [promptTemplate](#prompttemplate)
-    *   [promptHeader](#promptheader)
-    *   [promptFooter](#promptfooter)
-*   [PromptMessage](#promptmessage)
-    *   [role](#role)
-    *   [content](#content)
-*   [prompt\_tokens](#prompt_tokens)
-*   [completion\_tokens](#completion_tokens)
-*   [total\_tokens](#total_tokens)
-*   [CompletionReturn](#completionreturn)
-    *   [model](#model)
-    *   [usage](#usage)
-    *   [choices](#choices)
-*   [CompletionChoice](#completionchoice)
-    *   [message](#message)
-*   [LLModelPromptContext](#llmodelpromptcontext)
-    *   [logitsSize](#logitssize)
-    *   [tokensSize](#tokenssize)
-    *   [nPast](#npast)
-    *   [nCtx](#nctx)
-    *   [nPredict](#npredict)
-    *   [topK](#topk)
-    *   [topP](#topp)
-    *   [temp](#temp)
-    *   [nBatch](#nbatch)
-    *   [repeatPenalty](#repeatpenalty)
-    *   [repeatLastN](#repeatlastn)
-    *   [contextErase](#contexterase)
-*   [createTokenStream](#createtokenstream)
-    *   [Parameters](#parameters-8)
-*   [DEFAULT\_DIRECTORY](#default_directory)
-*   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
-*   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
-*   [DEFAULT\_PROMT\_CONTEXT](#default_promt_context)
-*   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
-*   [downloadModel](#downloadmodel)
-    *   [Parameters](#parameters-9)
-    *   [Examples](#examples)
-*   [DownloadModelOptions](#downloadmodeloptions)
-    *   [modelPath](#modelpath)
-    *   [verbose](#verbose-1)
-    *   [url](#url)
-    *   [md5sum](#md5sum)
-*   [DownloadController](#downloadcontroller)
-    *   [cancel](#cancel)
-    *   [promise](#promise)
-
-#### ModelType
-
-Type of the model
-
-Type: (`"gptj"` | `"llama"` | `"mpt"` | `"replit"`)
-
-#### ModelFile
-
-Full list of models available
-@deprecated These model names are outdated and this type will not be maintained, please use a string literal instead
-
-##### gptj
-
-List of GPT-J Models
-
-Type: (`"ggml-gpt4all-j-v1.3-groovy.bin"` | `"ggml-gpt4all-j-v1.2-jazzy.bin"` | `"ggml-gpt4all-j-v1.1-breezy.bin"` | `"ggml-gpt4all-j.bin"`)
-
-##### llama
-
-List Llama Models
-
-Type: (`"ggml-gpt4all-l13b-snoozy.bin"` | `"ggml-vicuna-7b-1.1-q4_2.bin"` | `"ggml-vicuna-13b-1.1-q4_2.bin"` | `"ggml-wizardLM-7B.q4_2.bin"` | `"ggml-stable-vicuna-13B.q4_2.bin"` | `"ggml-nous-gpt4-vicuna-13b.bin"` | `"ggml-v3-13b-hermes-q5_1.bin"`)
-
-##### mpt
-
-List of MPT Models
-
-Type: (`"ggml-mpt-7b-base.bin"` | `"ggml-mpt-7b-chat.bin"` | `"ggml-mpt-7b-instruct.bin"`)
-
-##### replit
-
-List of Replit Models
-
-Type: `"ggml-replit-code-v1-3b.bin"`
-
-#### type
-
-Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
-
-Type: [ModelType](#modeltype)
-
-#### LLModel
-
-LLModel class representing a language model.
-This is a base class that provides common functionality for different types of language models.
-
-##### constructor
-
-Initialize a new LLModel.
-
-###### Parameters
-
-*   `path` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Absolute path to the model file.
-
-<!---->
-
-*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model file does not exist.
-
-##### type
-
-either 'gpt', mpt', or 'llama' or undefined
-
-Returns **([ModelType](#modeltype) | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
-
-##### name
-
-The name of the model.
-
-Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-
-##### stateSize
-
-Get the size of the internal state of the model.
-NOTE: This state data is specific to the type of model you have created.
-
-Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the size in bytes of the internal state of the model
-
-##### threadCount
-
-Get the number of threads used for model inference.
-The default is the number of physical cores your computer has.
-
-Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The number of threads used for model inference.
-
-##### setThreadCount
-
-Set the number of threads used for model inference.
-
-###### Parameters
-
-*   `newNumber` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The new number of threads.
-
-Returns **void**&#x20;
-
-##### raw\_prompt
-
-Prompt the model with a given input and optional parameters.
-This is the raw output from model.
-Use the prompt function exported for a value
-
-###### Parameters
-
-*   `q` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
-*   `params` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
-*   `callback` **function (res: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): void**&#x20;
-
-Returns **void** The result of the model prompt.
-
-##### embed
-
-Embed text with the model. Keep in mind that
-not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
-Use the prompt function exported for a value
-
-###### Parameters
-
-*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-*   `q`  The prompt input.
-*   `params`  Optional parameters for the prompt context.
-
-Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The result of the model prompt.
-
-##### isModelLoaded
-
-Whether the model is loaded or not.
-
-Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)**&#x20;
-
-##### setLibraryPath
-
-Where to search for the pluggable backend libraries
-
-###### Parameters
-
-*   `s` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-
-Returns **void**&#x20;
-
-##### getLibraryPath
-
-Where to get the pluggable backend libraries
-
-Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-
-#### loadModel
-
-Loads a machine learning model with the specified name. The defacto way to create a model.
-By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
-
-##### Parameters
-
-*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The name of the model to load.
-*   `options` **(LoadModelOptions | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))?** (Optional) Additional options for loading the model.
-
-Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<(InferenceModel | EmbeddingModel)>** A promise that resolves to an instance of the loaded LLModel.
-
-#### createCompletion
-
-The nodejs equivalent to python binding's chat\_completion
-
-##### Parameters
-
-*   `model` **InferenceModel** The language model object.
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
-*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
-
-Returns **[CompletionReturn](#completionreturn)** The completion result.
-
-#### createEmbedding
-
-The nodejs moral equivalent to python binding's Embed4All().embed()
-meow
-
-##### Parameters
-
-*   `model` **EmbeddingModel** The language model object.
-*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** text to embed
-
-Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The completion result.
-
-#### CompletionOptions
-
-**Extends Partial\<LLModelPromptContext>**
-
-The options for creating the completion.
-
-##### verbose
-
-Indicates if verbose logging is enabled.
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### systemPromptTemplate
-
-Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
-Note that if this is not defined, system messages will not be included in the prompt.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### promptTemplate
-
-Template for user messages, with %1 being replaced by the message.
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### promptHeader
-
-The initial instruction for the model, on top of the prompt
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### promptFooter
-
-The last instruction for the model, appended to the end of the prompt.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### PromptMessage
-
-A message in the conversation, identical to OpenAI's chat message.
-
-##### role
-
-The role of the message.
-
-Type: (`"system"` | `"assistant"` | `"user"`)
-
-##### content
-
-The message content.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### prompt\_tokens
-
-The number of tokens used in the prompt.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### completion\_tokens
-
-The number of tokens used in the completion.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### total\_tokens
-
-The total number of tokens used.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### CompletionReturn
-
-The result of the completion, similar to OpenAI's format.
-
-##### model
-
-The model used for the completion.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### usage
-
-Token usage report.
-
-Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
-
-##### choices
-
-The generated completions.
-
-Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[CompletionChoice](#completionchoice)>
-
-#### CompletionChoice
-
-A completion choice, similar to OpenAI's format.
-
-##### message
-
-Response message
-
-Type: [PromptMessage](#promptmessage)
-
-#### LLModelPromptContext
-
-Model inference arguments for generating completions.
-
-##### logitsSize
-
-The size of the raw logits vector.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### tokensSize
-
-The size of the raw tokens vector.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nPast
-
-The number of tokens in the past conversation.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nCtx
-
-The number of tokens possible in the context window.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nPredict
-
-The number of tokens to predict.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### topK
-
-The top-k logits to sample from.
-Top-K sampling selects the next token only from the top K most likely tokens predicted by the model.
-It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit
-the diversity of the output. A higher value for top-K (eg., 100) will consider more tokens and lead
-to more diverse text, while a lower value (eg., 10) will focus on the most probable tokens and generate
-more conservative text. 30 - 60 is a good range for most tasks.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### topP
-
-The nucleus sampling probability threshold.
-Top-P limits the selection of the next token to a subset of tokens with a cumulative probability
-above a threshold P. This method, also known as nucleus sampling, finds a balance between diversity
-and quality by considering both token probabilities and the number of tokens available for sampling.
-When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
-On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
-The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
-for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### temp
-
-The temperature to adjust the model's output distribution.
-Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
-(eg., 1.2) increase randomness, resulting in more imaginative and diverse text. Lower temperatures (eg., 0.5)
-make the output more focused, predictable, and conservative. When the temperature is set to 0, the output
-becomes completely deterministic, always selecting the most probable next token and producing identical results
-each time. A safe range would be around 0.6 - 0.85, but you are free to search what value fits best for you.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nBatch
-
-The number of predictions to generate in parallel.
-By splitting the prompt every N tokens, prompt-batch-size reduces RAM usage during processing. However,
-this can increase the processing time as a trade-off. If the N value is set too low (e.g., 10), long prompts
-with 500+ tokens will be most affected, requiring numerous processing runs to complete the prompt processing.
-To ensure optimal performance, setting the prompt-batch-size to 2048 allows processing of all tokens in a single run.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### repeatPenalty
-
-The penalty factor for repeated tokens.
-Repeat-penalty can help penalize tokens based on how frequently they occur in the text, including the input prompt.
-A token that has already appeared five times is penalized more heavily than a token that has appeared only one time.
-A value of 1 means that there is no penalty and values larger than 1 discourage repeated tokens.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### repeatLastN
-
-The number of last tokens to penalize.
-The repeat-penalty-tokens N option controls the number of tokens in the history to consider for penalizing repetition.
-A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only
-consider recent tokens.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### contextErase
-
-The percentage of context to erase if the context window is exceeded.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### createTokenStream
-
-TODO: Help wanted to implement this
-
-##### Parameters
-
-*   `llmodel` **[LLModel](#llmodel)**&#x20;
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>**&#x20;
-*   `options` **[CompletionOptions](#completionoptions)**&#x20;
-
-Returns **function (ll: [LLModel](#llmodel)): AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>**&#x20;
-
-#### DEFAULT\_DIRECTORY
-
-From python api:
-models will be stored in (homedir)/.cache/gpt4all/\`
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### DEFAULT\_LIBRARIES\_DIRECTORY
-
-From python api:
-The default path for dynamic libraries to be stored.
-You may separate paths by a semicolon to search in multiple areas.
-This searches DEFAULT\_DIRECTORY/libraries, cwd/libraries, and finally cwd.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### DEFAULT\_MODEL\_CONFIG
-
-Default model configuration.
-
-Type: ModelConfig
-
-#### DEFAULT\_PROMT\_CONTEXT
-
-Default prompt context.
-
-Type: [LLModelPromptContext](#llmodelpromptcontext)
-
-#### DEFAULT\_MODEL\_LIST\_URL
-
-Default model list url.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### downloadModel
-
-Initiates the download of a model file.
-By default this downloads without waiting. use the controller returned to alter this behavior.
-
-##### Parameters
-
-*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The model to be downloaded.
-*   `options` **DownloadOptions** to pass into the downloader. Default is { location: (cwd), verbose: false }.
-
-##### Examples
-
-```javascript
-const download = downloadModel('ggml-gpt4all-j-v1.3-groovy.bin')
-download.promise.then(() => console.log('Downloaded!'))
-```
-
-*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model already exists in the specified location.
-*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model cannot be found at the specified url.
-
-Returns **[DownloadController](#downloadcontroller)** object that allows controlling the download process.
-
-#### DownloadModelOptions
-
-Options for the model download process.
-
-##### modelPath
-
-location to download the model.
-Default is process.cwd(), or the current working directory
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### verbose
-
-Debug mode -- check how long it took to download in seconds
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### url
-
-Remote download url. Defaults to `https://gpt4all.io/models/<modelName>`
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### md5sum
-
-MD5 sum of the model file. If this is provided, the downloaded file will be checked against this sum.
-If the sums do not match, an error will be thrown and the file will be deleted.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### DownloadController
-
-Model download controller.
-
-##### cancel
-
-Cancel the request to download if this is called.
-
-Type: function (): void
-
-##### promise
-
-A promise resolving to the downloaded models config once the download is done
-
-Type: [Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<ModelConfig>
--- a/gpt4all-bindings/python/gpt4all/init.py
+++ b/gpt4all-bindings/python/gpt4all/init.py
@@ -1,2 +1,2 @@
-from .gpt4all import Embed4All, GPT4All  # noqa
+from .gpt4all import GPT4All, Embed4All  # noqa
 from .pyllmodel import LLModel  # noqa
--- a/gpt4all-bindings/python/gpt4all/gpt4all.py
+++ b/gpt4all-bindings/python/gpt4all/gpt4all.py
@@ -5,7 +5,7 @@ import os
 import time
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Union
+from typing import Dict, Iterable, List, Union, Optional

 import requests
 from tqdm import tqdm
@@ -15,20 +15,10 @@ from . import pyllmodel
 # TODO: move to config
 DEFAULT_MODEL_DIRECTORY = os.path.join(str(Path.home()), ".cache", "gpt4all").replace("\\", "\\\\")

-DEFAULT_MODEL_CONFIG = {
-    "systemPrompt": "",
-    "promptTemplate": "### Human: \n{0}\n### Assistant:\n",
-}
-
-ConfigType = Dict[str, str]
-MessageType = Dict[str, str]
-
-
 class Embed4All:
    """
    Python class that handles embeddings for GPT4All.
    """
-
    def __init__(
        self,
        n_threads: Optional[int] = None,
@@ -41,7 +31,10 @@ class Embed4All:
        """
        self.gpt4all = GPT4All(model_name='ggml-all-MiniLM-L6-v2-f16.bin', n_threads=n_threads)

-    def embed(self, text: str) -> List[float]:
+    def embed(
+        self,
+        text: str
+    ) -> list[float]:
        """
        Generate an embedding.

@@ -53,7 +46,6 @@ class Embed4All:
        """
        return self.gpt4all.model.generate_embedding(text)

-
 class GPT4All:
    """
    Python class that handles instantiation, downloading, generation and chat with GPT4All models.
@@ -66,7 +58,6 @@ class GPT4All:
        model_type: Optional[str] = None,
        allow_download: bool = True,
        n_threads: Optional[int] = None,
-        device: Optional[str] = "cpu",
    ):
        """
        Constructor
@@ -79,33 +70,21 @@ class GPT4All:
                descriptive identifier for user. Default is None.
            allow_download: Allow API to download models from gpt4all.io. Default is True.
            n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
-            device: The processing unit on which the GPT4All model will run. It can be set to:
-                - "cpu": Model will run on the central processing unit.
-                - "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
-                - "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
-                Alternatively, a specific GPU name can also be provided, and the model will run on the GPU that matches the name if it's available.
-                Default is "cpu".
-
-                Note: If a selected GPU device does not have sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the model.
        """
        self.model_type = model_type
        self.model = pyllmodel.LLModel()
        # Retrieve model and download if allowed
-        self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download)
-        if device is not None:
-            if device != "cpu":
-                self.model.init_gpu(model_path=self.config["path"], device=device)
-        self.model.load_model(self.config["path"])
+        model_dest = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download)
+        self.model.load_model(model_dest)
        # Set n_threads
        if n_threads is not None:
            self.model.set_thread_count(n_threads)

-        self._is_chat_session_activated: bool = False
-        self.current_chat_session: List[MessageType] = empty_chat_session()
-        self._current_prompt_template: str = "{0}"
+        self._is_chat_session_activated = False
+        self.current_chat_session = []

    @staticmethod
-    def list_models() -> List[ConfigType]:
+    def list_models() -> Dict:
        """
        Fetch model list from https://gpt4all.io/models/models.json.

@@ -116,11 +95,8 @@ class GPT4All:

    @staticmethod
    def retrieve_model(
-        model_name: str,
-        model_path: Optional[str] = None,
-        allow_download: bool = True,
-        verbose: bool = True,
-    ) -> ConfigType:
+        model_name: str, model_path: Optional[str] = None, allow_download: bool = True, verbose: bool = True
+    ) -> str:
        """
        Find model file, and if it doesn't exist, download the model.

@@ -132,25 +108,11 @@ class GPT4All:
            verbose: If True (default), print debug messages.

        Returns:
-            Model config.
+            Model file destination.
        """

        model_filename = append_bin_suffix_if_missing(model_name)

-        # get the config for the model
-        config: ConfigType = DEFAULT_MODEL_CONFIG
-        if allow_download:
-            available_models = GPT4All.list_models()
-
-            for m in available_models:
-                if model_filename == m["filename"]:
-                    config.update(m)
-                    config["systemPrompt"] = config["systemPrompt"].strip()
-                    config["promptTemplate"] = config["promptTemplate"].replace(
-                        "%1", "{0}", 1
-                    )  # change to Python-style formatting
-                    break
-
        # Validate download directory
        if model_path is None:
            try:
@@ -169,28 +131,31 @@ class GPT4All:

        model_dest = os.path.join(model_path, model_filename).replace("\\", "\\\\")
        if os.path.exists(model_dest):
-            config.pop("url", None)
-            config["path"] = model_dest
            if verbose:
                print("Found model file at ", model_dest)
+            return model_dest

        # If model file does not exist, download
        elif allow_download:
-            url = config.pop("url", None)
+            # Make sure valid model filename before attempting download
+            available_models = GPT4All.list_models()

-            config["path"] = GPT4All.download_model(model_filename, model_path, verbose=verbose, url=url)
+            selected_model = None
+            for m in available_models:
+                if model_filename == m['filename']:
+                    selected_model = m
+                    break
+
+            if selected_model is None:
+                raise ValueError(f"Model filename not in model list: {model_filename}")
+            url = selected_model.pop('url', None)
+
+            return GPT4All.download_model(model_filename, model_path, verbose=verbose, url=url)
        else:
            raise ValueError("Failed to retrieve model")

-        return config
-
    @staticmethod
-    def download_model(
-        model_filename: str,
-        model_path: str,
-        verbose: bool = True,
-        url: Optional[str] = None,
-    ) -> str:
+    def download_model(model_filename: str, model_path: str, verbose: bool = True, url: Optional[str] = None) -> str:
        """
        Download model from https://gpt4all.io.

@@ -226,7 +191,7 @@ class GPT4All:
            except Exception:
                if os.path.exists(download_path):
                    if verbose:
-                        print("Cleaning up the interrupted download...")
+                        print('Cleaning up the interrupted download...')
                    os.remove(download_path)
                raise

@@ -247,14 +212,13 @@ class GPT4All:
        max_tokens: int = 200,
        temp: float = 0.7,
        top_k: int = 40,
-        top_p: float = 0.4,
+        top_p: float = 0.1,
        repeat_penalty: float = 1.18,
        repeat_last_n: int = 64,
        n_batch: int = 8,
        n_predict: Optional[int] = None,
        streaming: bool = False,
-        callback: pyllmodel.ResponseCallbackType = pyllmodel.empty_response_callback,
-    ) -> Union[str, Iterable[str]]:
+    ) -> Union[str, Iterable]:
        """
        Generate outputs from any GPT4All model.

@@ -269,14 +233,12 @@ class GPT4All:
            n_batch: Number of prompt tokens processed in parallel. Larger values decrease latency but increase resource requirements.
            n_predict: Equivalent to max_tokens, exists for backwards compatibility.
            streaming: If True, this method will instead return a generator that yields tokens as the model generates them.
-            callback: A function with arguments token_id:int and response:str, which receives the tokens from the model as they are generated and stops the generation by returning False.

        Returns:
            Either the entire completion or a generator that yields the completion token by token.
        """
-
-        # Preparing the model request
-        generate_kwargs: Dict[str, Any] = dict(
+        generate_kwargs = dict(
+            prompt=prompt,
            temp=temp,
            top_k=top_k,
            top_p=top_p,
@@ -287,92 +249,42 @@ class GPT4All:
        )

        if self._is_chat_session_activated:
-            # check if there is only one message, i.e. system prompt:
-            generate_kwargs["reset_context"] = len(self.current_chat_session) == 1
            self.current_chat_session.append({"role": "user", "content": prompt})
-
-            prompt = self._format_chat_prompt_template(
-                messages=self.current_chat_session[-1:],
-                default_prompt_header=self.current_chat_session[0]["content"]
-                if generate_kwargs["reset_context"]
-                else "",
-            )
+            generate_kwargs['prompt'] = self._format_chat_prompt_template(messages=self.current_chat_session[-1:])
+            generate_kwargs['reset_context'] = len(self.current_chat_session) == 1
        else:
-            generate_kwargs["reset_context"] = True
+            generate_kwargs['reset_context'] = True

-        # Prepare the callback, process the model response
-        output_collector: List[MessageType]
-        output_collector = [
-            {"content": ""}
-        ]  # placeholder for the self.current_chat_session if chat session is not activated
+        if streaming:
+            return self.model.prompt_model_streaming(**generate_kwargs)
+
+        output = self.model.prompt_model(**generate_kwargs)

        if self._is_chat_session_activated:
-            self.current_chat_session.append({"role": "assistant", "content": ""})
-            output_collector = self.current_chat_session
+            self.current_chat_session.append({"role": "assistant", "content": output})

-        def _callback_wrapper(
-            callback: pyllmodel.ResponseCallbackType,
-            output_collector: List[MessageType],
-        ) -> pyllmodel.ResponseCallbackType:
-
-            def _callback(token_id: int, response: str) -> bool:
-                nonlocal callback, output_collector
-
-                output_collector[-1]["content"] += response
-
-                return callback(token_id, response)
-
-            return _callback
-
-        # Send the request to the model
-        if streaming:
-            return self.model.prompt_model_streaming(
-                prompt=prompt,
-                callback=_callback_wrapper(callback, output_collector),
-                **generate_kwargs,
-            )
-
-        self.model.prompt_model(
-            prompt=prompt,
-            callback=_callback_wrapper(callback, output_collector),
-            **generate_kwargs,
-        )
-
-        return output_collector[-1]["content"]
+        return output

    @contextmanager
-    def chat_session(
-        self,
-        system_prompt: str = "",
-        prompt_template: str = "",
-    ):
-        """
+    def chat_session(self):
+        '''
        Context manager to hold an inference optimized chat session with a GPT4All model.
-
-        Args:
-            system_prompt: An initial instruction for the model.
-            prompt_template: Template for the prompts with {0} being replaced by the user message.
-        """
+        '''
        # Code to acquire resource, e.g.:
        self._is_chat_session_activated = True
-        self.current_chat_session = empty_chat_session(system_prompt or self.config["systemPrompt"])
-        self._current_prompt_template = prompt_template or self.config["promptTemplate"]
+        self.current_chat_session = []
        try:
            yield self
        finally:
            # Code to release resource, e.g.:
            self._is_chat_session_activated = False
-            self.current_chat_session = empty_chat_session()
-            self._current_prompt_template = "{0}"
+            self.current_chat_session = []

    def _format_chat_prompt_template(
-        self,
-        messages: List[MessageType],
-        default_prompt_header: str = "",
-        default_prompt_footer: str = "",
+        self, messages: List[Dict], default_prompt_header=True, default_prompt_footer=True
    ) -> str:
        """
-        Helper method for building a prompt from list of messages using the self._current_prompt_template as a template for each message.
+        Helper method for building a prompt using template from list of messages.

        Args:
            messages:  List of dictionaries. Each dictionary should have a "role" key
@@ -384,44 +296,19 @@ class GPT4All:
        Returns:
            Formatted prompt.
        """
-
-        if isinstance(default_prompt_header, bool):
-            import warnings
-
-            warnings.warn(
-                "Using True/False for the 'default_prompt_header' is deprecated. Use a string instead.",
-                DeprecationWarning,
-            )
-            default_prompt_header = ""
-
-        if isinstance(default_prompt_footer, bool):
-            import warnings
-
-            warnings.warn(
-                "Using True/False for the 'default_prompt_footer' is deprecated. Use a string instead.",
-                DeprecationWarning,
-            )
-            default_prompt_footer = ""
-
-        full_prompt = default_prompt_header + "\n\n" if default_prompt_header != "" else ""
+        full_prompt = ""

        for message in messages:
            if message["role"] == "user":
-                user_message = self._current_prompt_template.format(message["content"])
+                user_message = "### Human: \n" + message["content"] + "\n### Assistant:\n"
                full_prompt += user_message
            if message["role"] == "assistant":
-                assistant_message = message["content"] + "\n"
+                assistant_message = message["content"] + '\n'
                full_prompt += assistant_message

-        full_prompt += "\n\n" + default_prompt_footer if default_prompt_footer != "" else ""
-
        return full_prompt


-def empty_chat_session(system_prompt: str = "") -> List[MessageType]:
-    return [{"role": "system", "content": system_prompt}]
-
-
 def append_bin_suffix_if_missing(model_name):
    if not model_name.endswith(".bin"):
        model_name += ".bin"
--- a/gpt4all-bindings/python/gpt4all/pyllmodel.py
+++ b/gpt4all-bindings/python/gpt4all/pyllmodel.py
@@ -1,17 +1,26 @@
 import ctypes
-import logging
 import os
 import platform
-from queue import Queue
+import queue
 import re
 import subprocess
 import sys
 import threading
-from typing import Callable, Iterable, List
+from typing import Iterable

 import pkg_resources

-logger: logging.Logger = logging.getLogger(__name__)
+
+class DualStreamProcessor:
+    def __init__(self, stream=None):
+        self.stream = stream
+        self.output = ""
+
+    def write(self, text):
+        if self.stream is not None:
+            self.stream.write(text)
+            self.stream.flush()
+        self.output += text


 # TODO: provide a config file to make this more robust
@@ -34,9 +43,9 @@ def load_llmodel_library():

    c_lib_ext = get_c_shared_lib_extension()

-    llmodel_file = "libllmodel" + "." + c_lib_ext
+    llmodel_file = "libllmodel" + '.' + c_lib_ext

-    llmodel_dir = str(pkg_resources.resource_filename("gpt4all", os.path.join(LLMODEL_PATH, llmodel_file))).replace(
+    llmodel_dir = str(pkg_resources.resource_filename('gpt4all', os.path.join(LLMODEL_PATH, llmodel_file))).replace(
        "\\", "\\\\"
    )

@@ -70,14 +79,6 @@ class LLModelPromptContext(ctypes.Structure):
        ("context_erase", ctypes.c_float),
    ]

-class LLModelGPUDevice(ctypes.Structure):
-    _fields_ = [
-        ("index", ctypes.c_int32),
-        ("type", ctypes.c_int32),
-        ("heapSize", ctypes.c_size_t),
-        ("name", ctypes.c_char_p),
-        ("vendor", ctypes.c_char_p),
-    ]

 # Define C function signatures using ctypes
 llmodel.llmodel_model_create.argtypes = [ctypes.c_char_p]
@@ -119,7 +120,9 @@ llmodel.llmodel_embedding.argtypes = [

 llmodel.llmodel_embedding.restype = ctypes.POINTER(ctypes.c_float)

-llmodel.llmodel_free_embedding.argtypes = [ctypes.POINTER(ctypes.c_float)]
+llmodel.llmodel_free_embedding.argtypes = [
+    ctypes.POINTER(ctypes.c_float)
+]
 llmodel.llmodel_free_embedding.restype = None

 llmodel.llmodel_setThreadCount.argtypes = [ctypes.c_void_p, ctypes.c_int32]
@@ -131,29 +134,7 @@ llmodel.llmodel_set_implementation_search_path.restype = None
 llmodel.llmodel_threadCount.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_threadCount.restype = ctypes.c_int32

-llmodel.llmodel_set_implementation_search_path(MODEL_LIB_PATH.encode("utf-8"))
-
-llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
-llmodel.llmodel_available_gpu_devices.restype = ctypes.POINTER(LLModelGPUDevice)
-
-llmodel.llmodel_gpu_init_gpu_device_by_string.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_char_p]
-llmodel.llmodel_gpu_init_gpu_device_by_string.restype = ctypes.c_bool
-
-llmodel.llmodel_gpu_init_gpu_device_by_struct.argtypes = [ctypes.c_void_p, ctypes.POINTER(LLModelGPUDevice)]
-llmodel.llmodel_gpu_init_gpu_device_by_struct.restype = ctypes.c_bool
-
-llmodel.llmodel_gpu_init_gpu_device_by_int.argtypes = [ctypes.c_void_p, ctypes.c_int32]
-llmodel.llmodel_gpu_init_gpu_device_by_int.restype = ctypes.c_bool
-
-llmodel.llmodel_has_gpu_device.argtypes = [ctypes.c_void_p]
-llmodel.llmodel_has_gpu_device.restype = ctypes.c_bool
-
-ResponseCallbackType = Callable[[int, str], bool]
-RawResponseCallbackType = Callable[[int, bytes], bool]
-
-
-def empty_response_callback(token_id: int, response: str) -> bool:
-    return True
+llmodel.llmodel_set_implementation_search_path(MODEL_LIB_PATH.encode('utf-8'))


 class LLModel:
@@ -175,9 +156,6 @@ class LLModel:
        self.context = None
        self.llmodel_lib = llmodel

-        self.buffer = bytearray()
-        self.buff_expecting_cont_bytes: int = 0
-
    def __del__(self):
        if self.model is not None:
            self.llmodel_lib.llmodel_model_destroy(self.model)
@@ -191,60 +169,6 @@ class LLModel:
        else:
            raise ValueError("Unable to instantiate model")

-    def list_gpu(self, model_path: str) -> list:
-        """
-        Lists available GPU devices that satisfy the model's memory requirements.
-
-        Parameters
-        ----------
-        model_path : str
-            Path to the model.
-
-        Returns
-        -------
-        list
-            A list of LLModelGPUDevice structures representing available GPU devices.
-        """
-        if self.model is not None:
-            model_path_enc = model_path.encode("utf-8")
-            mem_required = llmodel.llmodel_required_mem(self.model, model_path_enc)
-        else:
-            mem_required = self.memory_needed(model_path)
-        num_devices = ctypes.c_int32(0)
-        devices_ptr = self.llmodel_lib.llmodel_available_gpu_devices(self.model, mem_required, ctypes.byref(num_devices))
-        if not devices_ptr:
-            raise ValueError("Unable to retrieve available GPU devices")
-        devices = [devices_ptr[i] for i in range(num_devices.value)]
-        return devices
-
-    def init_gpu(self, model_path: str, device: str):
-        if self.model is not None:
-            model_path_enc = model_path.encode("utf-8")
-            mem_required = llmodel.llmodel_required_mem(self.model, model_path_enc)
-        else:
-            mem_required = self.memory_needed(model_path)
-        device_enc = device.encode("utf-8")
-        success = self.llmodel_lib.llmodel_gpu_init_gpu_device_by_string(self.model, mem_required, device_enc)
-        if not success:
-            # Retrieve all GPUs without considering memory requirements.
-            num_devices = ctypes.c_int32(0)
-            all_devices_ptr = self.llmodel_lib.llmodel_available_gpu_devices(self.model, 0, ctypes.byref(num_devices))
-            if not all_devices_ptr:
-                raise ValueError("Unable to retrieve list of all GPU devices")
-            all_gpus = [all_devices_ptr[i].name.decode('utf-8') for i in range(num_devices.value)]
-
-            # Retrieve GPUs that meet the memory requirements using list_gpu
-            available_gpus = [device.name.decode('utf-8') for device in self.list_gpu(model_path)]
-
-            # Identify GPUs that are unavailable due to insufficient memory or features
-            unavailable_gpus = set(all_gpus) - set(available_gpus)
-
-            # Formulate the error message
-            error_msg = "Unable to initialize model on GPU: '{}'.".format(device)
-            error_msg += "\nAvailable GPUs: {}.".format(available_gpus)
-            error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}.".format(unavailable_gpus)
-            raise ValueError(error_msg)
-
    def load_model(self, model_path: str) -> bool:
        """
        Load model from a file.
@@ -323,10 +247,10 @@ class LLModel:
        self.context.repeat_last_n = repeat_last_n
        self.context.context_erase = context_erase

-    def generate_embedding(self, text: str) -> List[float]:
-        if not text:
-            raise ValueError("Text must not be None or empty")
-
+    def generate_embedding(
+        self,
+        text: str
+    ) -> list[float]:
        embedding_size = ctypes.c_size_t()
        c_text = ctypes.c_char_p(text.encode('utf-8'))
        embedding_ptr = llmodel.llmodel_embedding(self.model, c_text, ctypes.byref(embedding_size))
@@ -337,7 +261,6 @@ class LLModel:
    def prompt_model(
        self,
        prompt: str,
-        callback: ResponseCallbackType,
        n_predict: int = 4096,
        top_k: int = 40,
        top_p: float = 0.9,
@@ -347,7 +270,8 @@ class LLModel:
        repeat_last_n: int = 10,
        context_erase: float = 0.75,
        reset_context: bool = False,
-    ):
+        streaming=False,
+    ) -> str:
        """
        Generate response from model from a prompt.

@@ -355,27 +279,26 @@ class LLModel:
        ----------
        prompt: str
            Question, task, or conversation for model to respond to
-        callback(token_id:int, response:str): bool
-            The model sends response tokens to callback
+        streaming: bool
+            Stream response to stdout

        Returns
        -------
-        None
+        Model response str
        """

-        self.buffer.clear()
-        self.buff_expecting_cont_bytes = 0
-
-        logger.info(
-            "LLModel.prompt_model -- prompt:\n"
-            + "%s\n"
-            + "===/LLModel.prompt_model -- prompt/===",
-            prompt,
-        )
-
-        prompt_bytes = prompt.encode("utf-8")
+        prompt_bytes = prompt.encode('utf-8')
        prompt_ptr = ctypes.c_char_p(prompt_bytes)

+        old_stdout = sys.stdout
+
+        stream_processor = DualStreamProcessor()
+
+        if streaming:
+            stream_processor.stream = sys.stdout
+
+        sys.stdout = stream_processor
+
        self._set_context(
            n_predict=n_predict,
            top_k=top_k,
@@ -392,43 +315,70 @@ class LLModel:
            self.model,
            prompt_ptr,
            PromptCallback(self._prompt_callback),
-            ResponseCallback(self._callback_decoder(callback)),
+            ResponseCallback(self._response_callback),
            RecalculateCallback(self._recalculate_callback),
            self.context,
        )

+        # Revert to old stdout
+        sys.stdout = old_stdout
+        # Force new line
+        return stream_processor.output

    def prompt_model_streaming(
-        self, prompt: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
-    ) -> Iterable[str]:
+        self,
+        prompt: str,
+        n_predict: int = 4096,
+        top_k: int = 40,
+        top_p: float = 0.9,
+        temp: float = 0.1,
+        n_batch: int = 8,
+        repeat_penalty: float = 1.2,
+        repeat_last_n: int = 10,
+        context_erase: float = 0.75,
+        reset_context: bool = False,
+    ) -> Iterable:
        # Symbol to terminate from generator
        TERMINATING_SYMBOL = object()

-        output_queue: Queue = Queue()
+        output_queue = queue.Queue()
+
+        prompt_bytes = prompt.encode('utf-8')
+        prompt_ptr = ctypes.c_char_p(prompt_bytes)
+
+        self._set_context(
+            n_predict=n_predict,
+            top_k=top_k,
+            top_p=top_p,
+            temp=temp,
+            n_batch=n_batch,
+            repeat_penalty=repeat_penalty,
+            repeat_last_n=repeat_last_n,
+            context_erase=context_erase,
+            reset_context=reset_context,
+        )

        # Put response tokens into an output queue
-        def _generator_callback_wrapper(callback: ResponseCallbackType) -> ResponseCallbackType:
-            def _generator_callback(token_id: int, response: str):
-                nonlocal callback
+        def _generator_response_callback(token_id, response):
+            output_queue.put(response.decode('utf-8', 'replace'))
+            return True

-                if callback(token_id, response):
-                    output_queue.put(response)
-                    return True
-
-                return False
-
-            return _generator_callback
-
-        def run_llmodel_prompt(prompt: str, callback: ResponseCallbackType, **kwargs):
-            self.prompt_model(prompt, callback, **kwargs)
+        def run_llmodel_prompt(model, prompt, prompt_callback, response_callback, recalculate_callback, context):
+            llmodel.llmodel_prompt(model, prompt, prompt_callback, response_callback, recalculate_callback, context)
            output_queue.put(TERMINATING_SYMBOL)

        # Kick off llmodel_prompt in separate thread so we can return generator
        # immediately
        thread = threading.Thread(
            target=run_llmodel_prompt,
-            args=(prompt, _generator_callback_wrapper(callback)),
-            kwargs=kwargs,
+            args=(
+                self.model,
+                prompt_ptr,
+                PromptCallback(self._prompt_callback),
+                ResponseCallback(_generator_response_callback),
+                RecalculateCallback(self._recalculate_callback),
+                self.context,
+            ),
        )
        thread.start()

@@ -439,53 +389,18 @@ class LLModel:
                break
            yield response

-    def _callback_decoder(self, callback: ResponseCallbackType) -> RawResponseCallbackType:
-        def _raw_callback(token_id: int, response: bytes) -> bool:
-            nonlocal self, callback
-
-            decoded = []
-
-            for byte in response:
-                
-                bits = "{:08b}".format(byte)
-                (high_ones, _, _) = bits.partition('0')
-
-                if len(high_ones) == 1: 
-                    # continuation byte
-                    self.buffer.append(byte)
-                    self.buff_expecting_cont_bytes -= 1
-
-                else: 
-                    # beginning of a byte sequence
-                    if len(self.buffer) > 0:
-                        decoded.append(self.buffer.decode('utf-8', 'replace'))
-
-                        self.buffer.clear()
-
-                    self.buffer.append(byte)
-                    self.buff_expecting_cont_bytes = max(0, len(high_ones) - 1)
-
-                if self.buff_expecting_cont_bytes <= 0: 
-                    # received the whole sequence or an out of place continuation byte
-                    decoded.append(self.buffer.decode('utf-8', 'replace'))
-
-                    self.buffer.clear()
-                    self.buff_expecting_cont_bytes = 0
-                    
-            if len(decoded) == 0 and self.buff_expecting_cont_bytes > 0:
-                # wait for more continuation bytes
-                return True
-            
-            return callback(token_id, ''.join(decoded))     
-
-        return _raw_callback
-
    # Empty prompt callback
    @staticmethod
-    def _prompt_callback(token_id: int) -> bool:
+    def _prompt_callback(token_id):
+        return True
+
+    # Empty response callback method that just prints response to be collected
+    @staticmethod
+    def _response_callback(token_id, response):
+        sys.stdout.write(response.decode('utf-8', 'replace'))
        return True

    # Empty recalculate callback
    @staticmethod
-    def _recalculate_callback(is_recalculating: bool) -> bool:
+    def _recalculate_callback(is_recalculating):
        return is_recalculating
--- a/gpt4all-bindings/python/gpt4all/tests/test_embed_timings.py
+++ b/gpt4all-bindings/python/gpt4all/tests/test_embed_timings.py
@@ -1,9 +1,8 @@
 import sys
-import time
 from io import StringIO

-from gpt4all import Embed4All, GPT4All
-
+from gpt4all import GPT4All, Embed4All
+import time

 def time_embedding(i, embedder):
    text = 'foo bar ' * i
@@ -13,7 +12,6 @@ def time_embedding(i, embedder):
    elapsed_time = end_time - start_time
    print(f"Time report: {2 * i / elapsed_time} tokens/second with {2 * i} tokens taking {elapsed_time} seconds")

-
 if __name__ == "__main__":
    embedder = Embed4All(n_threads=8)
    for i in [2**n for n in range(6, 14)]:
--- a/gpt4all-bindings/python/gpt4all/tests/test_gpt4all.py
+++ b/gpt4all-bindings/python/gpt4all/tests/test_gpt4all.py
@@ -1,11 +1,8 @@
 import sys
 from io import StringIO
-from pathlib import Path

 from gpt4all import GPT4All, Embed4All
 import time
-import pytest
-

 def test_inference():
    model = GPT4All(model_name='orca-mini-3b.ggmlv3.q4_0.bin')
@@ -103,7 +100,6 @@ def test_inference_mpt():
    assert isinstance(output, str)
    assert len(output) > 0

-
 def test_embedding():
    text = 'The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox'
    embedder = Embed4All()
@@ -111,22 +107,3 @@ def test_embedding():
    #for i, value in enumerate(output):
        #print(f'Value at index {i}: {value}')
    assert len(output) == 384
-
-
-def test_empty_embedding():
-    text = ''
-    embedder = Embed4All()
-    with pytest.raises(ValueError):
-        output = embedder.embed(text)
-
-def test_download_model(tmp_path: Path):
-    import gpt4all.gpt4all
-    old_default_dir = gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY
-    gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = tmp_path  # temporary pytest directory to ensure a download happens
-    try:
-        model = GPT4All(model_name='ggml-all-MiniLM-L6-v2-f16.bin')
-        model_path = tmp_path / model.config['filename']
-        assert model_path.absolute() == Path(model.config['path']).absolute()
-        assert model_path.stat().st_size == int(model.config['filesize'])
-    finally:
-        gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = old_default_dir
--- a/gpt4all-bindings/python/mkdocs.yml
+++ b/gpt4all-bindings/python/mkdocs.yml
@@ -9,12 +9,11 @@ use_directory_urls: false

 nav:
    - 'index.md'
-    - 'GPT4All Chat Client': 'gpt4all_chat.md'
    - 'Bindings':
      - 'GPT4All in Python':
        - 'Generation': 'gpt4all_python.md'
        - 'Embedding': 'gpt4all_python_embedding.md'
-      - 'GPT4ALL in NodeJs': 'gpt4all_typescript.md'
+      - 'GPT4All Chat Client': 'gpt4all_chat.md'
      - 'gpt4all_cli.md'
 #    - 'Tutorials':
 #      - 'gpt4all_modal.md'
--- a/gpt4all-bindings/python/setup.py
+++ b/gpt4all-bindings/python/setup.py
@@ -61,10 +61,10 @@ copy_prebuilt_C_lib(SRC_CLIB_DIRECtORY,

 setup(
    name=package_name,
-    version="1.0.12",
+    version="1.0.6",
    description="Python bindings for GPT4All",
-    author="Nomic and the Open Source Community",
-    author_email="support@nomic.ai",
+    author="Richard Guo",
+    author_email="richard@nomic.ai",
    url="https://pypi.org/project/gpt4all/",
    classifiers = [
        "Programming Language :: Python :: 3",
--- a/gpt4all-bindings/typescript/.gitignore
+++ b/gpt4all-bindings/typescript/.gitignore
@@ -1,10 +1,3 @@
 node_modules/
 build/
 prebuilds/
-.yarn/*
-!.yarn/patches
-!.yarn/plugins
-!.yarn/releases
-!.yarn/sdks
-!.yarn/versions
-runtimes/
--- a/gpt4all-bindings/typescript/README.md
+++ b/gpt4all-bindings/typescript/README.md
@@ -1,731 +1,98 @@
-# GPT4All Node.js API
-
-```sh
-yarn add gpt4all@alpha
-
-npm install gpt4all@alpha
-
-pnpm install gpt4all@alpha
-```
-
+### Javascript Bindings
 The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.

-*   New bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
-*   The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
-*   Everything should work out the box.
-*   See [API Reference](#api-reference)
-
-### Chat Completion
+- created by [jacoobes](https://github.com/jacoobes) and [nomic ai](https://home.nomic.ai) :D, for all to use.

+### Code (alpha)
 ```js
-import { createCompletion, loadModel } from '../src/gpt4all.js'
+import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY } from '../src/gpt4all.js'

-const model = await loadModel('ggml-vicuna-7b-1.1-q4_2', { verbose: true });
+const ll = new LLModel({
+    model_name: 'ggml-vicuna-7b-1.1-q4_2.bin',
+    model_path: './', 
+    library_path: DEFAULT_LIBRARIES_DIRECTORY
+});

-const response = await createCompletion(model, [
+const response = await createCompletion(ll, [
    { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
    { role : 'user', content: 'What is 1 + 1?'  } 
 ]);

 ```
-
-### Embedding
-
-```js
-import { createEmbedding, loadModel } from '../src/gpt4all.js'
-
-const model = await loadModel('ggml-all-MiniLM-L6-v2-f16', { verbose: true });
-
-const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional");
-```
-
+### API 
+- The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
+- [docs](./docs/api.md)
 ### Build Instructions

-*   binding.gyp is compile config
-*   Tested on Ubuntu. Everything seems to work fine
-*   Tested on Windows. Everything works fine.
-*   Sparse testing on mac os.
-*   MingW works as well to build the gpt4all-backend. **HOWEVER**, this package works only with MSVC built dlls.
+- As of 05/21/2023, Tested on windows (MSVC). (somehow got it to work on MSVC 🤯)
+    - binding.gyp is compile config
+- Tested on Ubuntu. Everything seems to work fine
+- MingW works as well to build the gpt4all-backend. HOWEVER, this package works only with MSVC built dlls.

 ### Requirements
+- git
+- [node.js >= 18.0.0](https://nodejs.org/en)
+- [yarn](https://yarnpkg.com/)
+- [node-gyp](https://github.com/nodejs/node-gyp)
+    - all of its requirements.

-*   git
-*   [node.js >= 18.0.0](https://nodejs.org/en)
-*   [yarn](https://yarnpkg.com/)
-*   [node-gyp](https://github.com/nodejs/node-gyp)
-    *   all of its requirements.
-*   (unix) gcc version 12
-*   (win) msvc version 143
-    *   Can be obtained with visual studio 2022 build tools
-*   python 3
-
-### Build (from source)
-
+### Build
 ```sh
 git clone https://github.com/nomic-ai/gpt4all.git
 cd gpt4all-bindings/typescript
 ```
+- The below shell commands assume the current working directory is `typescript`.

-*   The below shell commands assume the current working directory is `typescript`.
-
-*   To Build and Rebuild:
-
-```sh
-yarn
-```
-
-*   llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
-
-```sh
+- To Build and Rebuild: 
+ ```sh
+ yarn
+ ```
+ - llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
+ ```sh
 git submodule update --init --depth 1 --recursive
-```
-
+ ```
 **AS OF NEW BACKEND** to build the backend,
-
 ```sh
 yarn build:backend
 ```
-
 This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native The only current way to use them is to put them in the current working directory of your application. That is, **WHEREVER YOU RUN YOUR NODE APPLICATION**
-
-*   llama-xxxx.dll is required.
-*   According to whatever model you are using, you'll need to select the proper model loader.
-    *   For example, if you running an Mosaic MPT model, you will need to select the mpt-(buildvariant).(dynamiclibrary)
+- llama-xxxx.dll is required.
+- According to whatever model you are using, you'll need to select the proper model loader.
+    - For example, if you running an Mosaic MPT model, you will need to select the mpt-(buildvariant).(dynamiclibrary)

 ### Test
-
 ```sh
 yarn test
 ```
-
 ### Source Overview

 #### src/
-
-*   Extra functions to help aid devex
-*   Typings for the native node addon
-*   the javascript interface
+- Extra functions to help aid devex
+- Typings for the native node addon
+- the javascript interface

 #### test/
-
-*   simple unit testings for some functions exported.
-*   more advanced ai testing is not handled
+- simple unit testings for some functions exported.
+- more advanced ai testing is not handled 

 #### spec/
-
-*   Average look and feel of the api
-*   Should work assuming a model and libraries are installed locally in working directory
+- Average look and feel of the api
+- Should work assuming a model and libraries are installed locally in working directory

 #### index.cc
-
-*   The bridge between nodejs and c. Where the bindings are.
-
-#### prompt.cc
-
-*   Handling prompting and inference of models in a threadsafe, asynchronous way.
-
-### Known Issues
-
-*   why your model may be spewing bull 💩
-    *   The downloaded model is broken (just reinstall or download from official site)
-    *   That's it so far
+- The bridge between nodejs and c. Where the bindings are.
+#### prompt.cc 
+- Handling prompting and inference of models in a threadsafe, asynchronous way.
+#### docs/
+- Autogenerated documentation using the script `yarn docs:build`

 ### Roadmap
-
 This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:

-*   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
-*   \[ ] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
-*   \[x] proper unit testing (integrate with circle ci)
-*   \[x] publish to npm under alpha tag `gpt4all@alpha`
-*   \[x] have more people test on other platforms (mac tester needed)
-*   \[x] switch to new pluggable backend
-*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
-    *   Should include prebuilds to avoid painful node-gyp errors
-*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
+- [x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
+- [ ] createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)
+- [ ] proper unit testing (integrate with circle ci)
+- [ ] publish to npm under alpha tag `gpt4all@alpha`
+- [ ] have more people test on other platforms (mac tester needed)
+- [x] switch to new pluggable backend

-### API Reference
-
-<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
-
-##### Table of Contents
-
-*   [ModelType](#modeltype)
-*   [ModelFile](#modelfile)
-    *   [gptj](#gptj)
-    *   [llama](#llama)
-    *   [mpt](#mpt)
-    *   [replit](#replit)
-*   [type](#type)
-*   [LLModel](#llmodel)
-    *   [constructor](#constructor)
-        *   [Parameters](#parameters)
-    *   [type](#type-1)
-    *   [name](#name)
-    *   [stateSize](#statesize)
-    *   [threadCount](#threadcount)
-    *   [setThreadCount](#setthreadcount)
-        *   [Parameters](#parameters-1)
-    *   [raw\_prompt](#raw_prompt)
-        *   [Parameters](#parameters-2)
-    *   [embed](#embed)
-        *   [Parameters](#parameters-3)
-    *   [isModelLoaded](#ismodelloaded)
-    *   [setLibraryPath](#setlibrarypath)
-        *   [Parameters](#parameters-4)
-    *   [getLibraryPath](#getlibrarypath)
-*   [loadModel](#loadmodel)
-    *   [Parameters](#parameters-5)
-*   [createCompletion](#createcompletion)
-    *   [Parameters](#parameters-6)
-*   [createEmbedding](#createembedding)
-    *   [Parameters](#parameters-7)
-*   [CompletionOptions](#completionoptions)
-    *   [verbose](#verbose)
-    *   [systemPromptTemplate](#systemprompttemplate)
-    *   [promptTemplate](#prompttemplate)
-    *   [promptHeader](#promptheader)
-    *   [promptFooter](#promptfooter)
-*   [PromptMessage](#promptmessage)
-    *   [role](#role)
-    *   [content](#content)
-*   [prompt\_tokens](#prompt_tokens)
-*   [completion\_tokens](#completion_tokens)
-*   [total\_tokens](#total_tokens)
-*   [CompletionReturn](#completionreturn)
-    *   [model](#model)
-    *   [usage](#usage)
-    *   [choices](#choices)
-*   [CompletionChoice](#completionchoice)
-    *   [message](#message)
-*   [LLModelPromptContext](#llmodelpromptcontext)
-    *   [logitsSize](#logitssize)
-    *   [tokensSize](#tokenssize)
-    *   [nPast](#npast)
-    *   [nCtx](#nctx)
-    *   [nPredict](#npredict)
-    *   [topK](#topk)
-    *   [topP](#topp)
-    *   [temp](#temp)
-    *   [nBatch](#nbatch)
-    *   [repeatPenalty](#repeatpenalty)
-    *   [repeatLastN](#repeatlastn)
-    *   [contextErase](#contexterase)
-*   [createTokenStream](#createtokenstream)
-    *   [Parameters](#parameters-8)
-*   [DEFAULT\_DIRECTORY](#default_directory)
-*   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
-*   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
-*   [DEFAULT\_PROMT\_CONTEXT](#default_promt_context)
-*   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
-*   [downloadModel](#downloadmodel)
-    *   [Parameters](#parameters-9)
-    *   [Examples](#examples)
-*   [DownloadModelOptions](#downloadmodeloptions)
-    *   [modelPath](#modelpath)
-    *   [verbose](#verbose-1)
-    *   [url](#url)
-    *   [md5sum](#md5sum)
-*   [DownloadController](#downloadcontroller)
-    *   [cancel](#cancel)
-    *   [promise](#promise)
-
-#### ModelType
-
-Type of the model
-
-Type: (`"gptj"` | `"llama"` | `"mpt"` | `"replit"`)
-
-#### ModelFile
-
-Full list of models available
-@deprecated These model names are outdated and this type will not be maintained, please use a string literal instead
-
-##### gptj
-
-List of GPT-J Models
-
-Type: (`"ggml-gpt4all-j-v1.3-groovy.bin"` | `"ggml-gpt4all-j-v1.2-jazzy.bin"` | `"ggml-gpt4all-j-v1.1-breezy.bin"` | `"ggml-gpt4all-j.bin"`)
-
-##### llama
-
-List Llama Models
-
-Type: (`"ggml-gpt4all-l13b-snoozy.bin"` | `"ggml-vicuna-7b-1.1-q4_2.bin"` | `"ggml-vicuna-13b-1.1-q4_2.bin"` | `"ggml-wizardLM-7B.q4_2.bin"` | `"ggml-stable-vicuna-13B.q4_2.bin"` | `"ggml-nous-gpt4-vicuna-13b.bin"` | `"ggml-v3-13b-hermes-q5_1.bin"`)
-
-##### mpt
-
-List of MPT Models
-
-Type: (`"ggml-mpt-7b-base.bin"` | `"ggml-mpt-7b-chat.bin"` | `"ggml-mpt-7b-instruct.bin"`)
-
-##### replit
-
-List of Replit Models
-
-Type: `"ggml-replit-code-v1-3b.bin"`
-
-#### type
-
-Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
-
-Type: [ModelType](#modeltype)
-
-#### LLModel
-
-LLModel class representing a language model.
-This is a base class that provides common functionality for different types of language models.
-
-##### constructor
-
-Initialize a new LLModel.
-
-###### Parameters
-
-*   `path` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Absolute path to the model file.
-
-<!---->
-
-*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model file does not exist.
-
-##### type
-
-either 'gpt', mpt', or 'llama' or undefined
-
-Returns **([ModelType](#modeltype) | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
-
-##### name
-
-The name of the model.
-
-Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-
-##### stateSize
-
-Get the size of the internal state of the model.
-NOTE: This state data is specific to the type of model you have created.
-
-Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the size in bytes of the internal state of the model
-
-##### threadCount
-
-Get the number of threads used for model inference.
-The default is the number of physical cores your computer has.
-
-Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The number of threads used for model inference.
-
-##### setThreadCount
-
-Set the number of threads used for model inference.
-
-###### Parameters
-
-*   `newNumber` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The new number of threads.
-
-Returns **void**&#x20;
-
-##### raw\_prompt
-
-Prompt the model with a given input and optional parameters.
-This is the raw output from model.
-Use the prompt function exported for a value
-
-###### Parameters
-
-*   `q` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
-*   `params` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
-*   `callback` **function (res: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): void**&#x20;
-
-Returns **void** The result of the model prompt.
-
-##### embed
-
-Embed text with the model. Keep in mind that
-not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
-Use the prompt function exported for a value
-
-###### Parameters
-
-*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-*   `q`  The prompt input.
-*   `params`  Optional parameters for the prompt context.
-
-Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The result of the model prompt.
-
-##### isModelLoaded
-
-Whether the model is loaded or not.
-
-Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)**&#x20;
-
-##### setLibraryPath
-
-Where to search for the pluggable backend libraries
-
-###### Parameters
-
-*   `s` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-
-Returns **void**&#x20;
-
-##### getLibraryPath
-
-Where to get the pluggable backend libraries
-
-Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-
-#### loadModel
-
-Loads a machine learning model with the specified name. The defacto way to create a model.
-By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
-
-##### Parameters
-
-*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The name of the model to load.
-*   `options` **(LoadModelOptions | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))?** (Optional) Additional options for loading the model.
-
-Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<(InferenceModel | EmbeddingModel)>** A promise that resolves to an instance of the loaded LLModel.
-
-#### createCompletion
-
-The nodejs equivalent to python binding's chat\_completion
-
-##### Parameters
-
-*   `model` **InferenceModel** The language model object.
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
-*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
-
-Returns **[CompletionReturn](#completionreturn)** The completion result.
-
-#### createEmbedding
-
-The nodejs moral equivalent to python binding's Embed4All().embed()
-meow
-
-##### Parameters
-
-*   `model` **EmbeddingModel** The language model object.
-*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** text to embed
-
-Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The completion result.
-
-#### CompletionOptions
-
-**Extends Partial\<LLModelPromptContext>**
-
-The options for creating the completion.
-
-##### verbose
-
-Indicates if verbose logging is enabled.
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### systemPromptTemplate
-
-Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
-Note that if this is not defined, system messages will not be included in the prompt.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### promptTemplate
-
-Template for user messages, with %1 being replaced by the message.
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### promptHeader
-
-The initial instruction for the model, on top of the prompt
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### promptFooter
-
-The last instruction for the model, appended to the end of the prompt.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### PromptMessage
-
-A message in the conversation, identical to OpenAI's chat message.
-
-##### role
-
-The role of the message.
-
-Type: (`"system"` | `"assistant"` | `"user"`)
-
-##### content
-
-The message content.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### prompt\_tokens
-
-The number of tokens used in the prompt.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### completion\_tokens
-
-The number of tokens used in the completion.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### total\_tokens
-
-The total number of tokens used.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### CompletionReturn
-
-The result of the completion, similar to OpenAI's format.
-
-##### model
-
-The model used for the completion.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### usage
-
-Token usage report.
-
-Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
-
-##### choices
-
-The generated completions.
-
-Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[CompletionChoice](#completionchoice)>
-
-#### CompletionChoice
-
-A completion choice, similar to OpenAI's format.
-
-##### message
-
-Response message
-
-Type: [PromptMessage](#promptmessage)
-
-#### LLModelPromptContext
-
-Model inference arguments for generating completions.
-
-##### logitsSize
-
-The size of the raw logits vector.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### tokensSize
-
-The size of the raw tokens vector.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nPast
-
-The number of tokens in the past conversation.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nCtx
-
-The number of tokens possible in the context window.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nPredict
-
-The number of tokens to predict.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### topK
-
-The top-k logits to sample from.
-Top-K sampling selects the next token only from the top K most likely tokens predicted by the model.
-It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit
-the diversity of the output. A higher value for top-K (eg., 100) will consider more tokens and lead
-to more diverse text, while a lower value (eg., 10) will focus on the most probable tokens and generate
-more conservative text. 30 - 60 is a good range for most tasks.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### topP
-
-The nucleus sampling probability threshold.
-Top-P limits the selection of the next token to a subset of tokens with a cumulative probability
-above a threshold P. This method, also known as nucleus sampling, finds a balance between diversity
-and quality by considering both token probabilities and the number of tokens available for sampling.
-When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
-On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
-The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
-for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### temp
-
-The temperature to adjust the model's output distribution.
-Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
-(eg., 1.2) increase randomness, resulting in more imaginative and diverse text. Lower temperatures (eg., 0.5)
-make the output more focused, predictable, and conservative. When the temperature is set to 0, the output
-becomes completely deterministic, always selecting the most probable next token and producing identical results
-each time. A safe range would be around 0.6 - 0.85, but you are free to search what value fits best for you.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nBatch
-
-The number of predictions to generate in parallel.
-By splitting the prompt every N tokens, prompt-batch-size reduces RAM usage during processing. However,
-this can increase the processing time as a trade-off. If the N value is set too low (e.g., 10), long prompts
-with 500+ tokens will be most affected, requiring numerous processing runs to complete the prompt processing.
-To ensure optimal performance, setting the prompt-batch-size to 2048 allows processing of all tokens in a single run.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### repeatPenalty
-
-The penalty factor for repeated tokens.
-Repeat-penalty can help penalize tokens based on how frequently they occur in the text, including the input prompt.
-A token that has already appeared five times is penalized more heavily than a token that has appeared only one time.
-A value of 1 means that there is no penalty and values larger than 1 discourage repeated tokens.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### repeatLastN
-
-The number of last tokens to penalize.
-The repeat-penalty-tokens N option controls the number of tokens in the history to consider for penalizing repetition.
-A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only
-consider recent tokens.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### contextErase
-
-The percentage of context to erase if the context window is exceeded.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### createTokenStream
-
-TODO: Help wanted to implement this
-
-##### Parameters
-
-*   `llmodel` **[LLModel](#llmodel)**&#x20;
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>**&#x20;
-*   `options` **[CompletionOptions](#completionoptions)**&#x20;
-
-Returns **function (ll: [LLModel](#llmodel)): AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>**&#x20;
-
-#### DEFAULT\_DIRECTORY
-
-From python api:
-models will be stored in (homedir)/.cache/gpt4all/\`
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### DEFAULT\_LIBRARIES\_DIRECTORY
-
-From python api:
-The default path for dynamic libraries to be stored.
-You may separate paths by a semicolon to search in multiple areas.
-This searches DEFAULT\_DIRECTORY/libraries, cwd/libraries, and finally cwd.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### DEFAULT\_MODEL\_CONFIG
-
-Default model configuration.
-
-Type: ModelConfig
-
-#### DEFAULT\_PROMT\_CONTEXT
-
-Default prompt context.
-
-Type: [LLModelPromptContext](#llmodelpromptcontext)
-
-#### DEFAULT\_MODEL\_LIST\_URL
-
-Default model list url.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### downloadModel
-
-Initiates the download of a model file.
-By default this downloads without waiting. use the controller returned to alter this behavior.
-
-##### Parameters
-
-*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The model to be downloaded.
-*   `options` **DownloadOptions** to pass into the downloader. Default is { location: (cwd), verbose: false }.
-
-##### Examples
-
-```javascript
-const download = downloadModel('ggml-gpt4all-j-v1.3-groovy.bin')
-download.promise.then(() => console.log('Downloaded!'))
-```
-
-*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model already exists in the specified location.
-*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model cannot be found at the specified url.
-
-Returns **[DownloadController](#downloadcontroller)** object that allows controlling the download process.
-
-#### DownloadModelOptions
-
-Options for the model download process.
-
-##### modelPath
-
-location to download the model.
-Default is process.cwd(), or the current working directory
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### verbose
-
-Debug mode -- check how long it took to download in seconds
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### url
-
-Remote download url. Defaults to `https://gpt4all.io/models/<modelName>`
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### md5sum
-
-MD5 sum of the model file. If this is provided, the downloaded file will be checked against this sum.
-If the sums do not match, an error will be thrown and the file will be deleted.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### DownloadController
-
-Model download controller.
-
-##### cancel
-
-Cancel the request to download if this is called.
-
-Type: function (): void
-
-##### promise
-
-A promise resolving to the downloaded models config once the download is done
-
-Type: [Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<ModelConfig>
--- a/gpt4all-bindings/typescript/binding.ci.gyp
+++ b/gpt4all-bindings/typescript/binding.ci.gyp
@@ -1,62 +0,0 @@
-{
-  "targets": [
-    {
-      "target_name": "gpt4all", # gpt4all-ts will cause compile error
-      "include_dirs": [
-        "<!@(node -p \"require('node-addon-api').include\")",
-        "gpt4all-backend",
-      ],
-      "sources": [ 
-        # PREVIOUS VERSION: had to required the sources, but with newest changes do not need to
-        #"../../gpt4all-backend/llama.cpp/examples/common.cpp",
-        #"../../gpt4all-backend/llama.cpp/ggml.c",
-        #"../../gpt4all-backend/llama.cpp/llama.cpp",
-        # "../../gpt4all-backend/utils.cpp", 
-        "gpt4all-backend/llmodel_c.cpp",
-        "gpt4all-backend/llmodel.cpp",
-        "prompt.cc",
-        "index.cc",
-       ],
-      "conditions": [
-        ['OS=="mac"', {
-            'xcode_settings': {
-                'GCC_ENABLE_CPP_EXCEPTIONS': 'YES'
-            },
-            'defines': [
-                'LIB_FILE_EXT=".dylib"',
-                'NAPI_CPP_EXCEPTIONS',
-            ],
-            'cflags_cc': [
-                "-fexceptions"
-            ]
-        }],
-        ['OS=="win"', {
-            'defines': [
-                'LIB_FILE_EXT=".dll"',
-                'NAPI_CPP_EXCEPTIONS',
-            ],
-            "msvs_settings": {
-                "VCCLCompilerTool": {
-                    "AdditionalOptions": [
-                        "/std:c++20",
-                        "/EHsc",
-                  ], 
-                },
-            },
-        }],
-        ['OS=="linux"', {
-            'defines': [
-                'LIB_FILE_EXT=".so"',
-                'NAPI_CPP_EXCEPTIONS',
-            ],
-            'cflags_cc!': [
-                '-fno-rtti',
-            ],
-            'cflags_cc': [
-                '-std=c++2a',
-                '-fexceptions'
-            ]
-        }]
-      ]
-    }]
-}
--- a/gpt4all-bindings/typescript/binding.gyp
+++ b/gpt4all-bindings/typescript/binding.gyp
@@ -2,6 +2,7 @@
  "targets": [
    {
      "target_name": "gpt4all", # gpt4all-ts will cause compile error
+      "cflags_cc!": [ "-fno-exceptions"],
      "include_dirs": [
        "<!@(node -p \"require('node-addon-api').include\")",
        "../../gpt4all-backend",
@@ -19,15 +20,9 @@
       ],
      "conditions": [
        ['OS=="mac"', {
-            'xcode_settings': {
-                'GCC_ENABLE_CPP_EXCEPTIONS': 'YES'
-            },
            'defines': [
                'LIB_FILE_EXT=".dylib"',
                'NAPI_CPP_EXCEPTIONS',
-            ],
-            'cflags_cc': [
-                "-fexceptions"
            ]
        }],
        ['OS=="win"', {
@@ -53,8 +48,7 @@
                '-fno-rtti',
            ],
            'cflags_cc': [
-                '-std=c++2a',
-                '-fexceptions'
+                '-std=c++20'
            ]
        }]
      ]
--- a/gpt4all-bindings/typescript/docs/api.md
+++ b/gpt4all-bindings/typescript/docs/api.md
@@ -0,0 +1,623 @@
+<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
+
+### Table of Contents
+
+*   [download][1]
+    *   [Parameters][2]
+    *   [Examples][3]
+*   [DownloadOptions][4]
+    *   [location][5]
+    *   [debug][6]
+    *   [url][7]
+*   [DownloadController][8]
+    *   [cancel][9]
+    *   [promise][10]
+*   [ModelType][11]
+*   [ModelFile][12]
+    *   [gptj][13]
+    *   [llama][14]
+    *   [mpt][15]
+*   [type][16]
+*   [LLModel][17]
+    *   [constructor][18]
+        *   [Parameters][19]
+    *   [type][20]
+    *   [name][21]
+    *   [stateSize][22]
+    *   [threadCount][23]
+    *   [setThreadCount][24]
+        *   [Parameters][25]
+    *   [raw\_prompt][26]
+        *   [Parameters][27]
+    *   [isModelLoaded][28]
+    *   [setLibraryPath][29]
+        *   [Parameters][30]
+    *   [getLibraryPath][31]
+*   [createCompletion][32]
+    *   [Parameters][33]
+    *   [Examples][34]
+*   [CompletionOptions][35]
+    *   [verbose][36]
+    *   [hasDefaultHeader][37]
+    *   [hasDefaultFooter][38]
+*   [PromptMessage][39]
+    *   [role][40]
+    *   [content][41]
+*   [prompt\_tokens][42]
+*   [completion\_tokens][43]
+*   [total\_tokens][44]
+*   [CompletionReturn][45]
+    *   [model][46]
+    *   [usage][47]
+    *   [choices][48]
+*   [CompletionChoice][49]
+    *   [message][50]
+*   [LLModelPromptContext][51]
+    *   [logits\_size][52]
+    *   [tokens\_size][53]
+    *   [n\_past][54]
+    *   [n\_ctx][55]
+    *   [n\_predict][56]
+    *   [top\_k][57]
+    *   [top\_p][58]
+    *   [temp][59]
+    *   [n\_batch][60]
+    *   [repeat\_penalty][61]
+    *   [repeat\_last\_n][62]
+    *   [context\_erase][63]
+*   [createTokenStream][64]
+    *   [Parameters][65]
+*   [DEFAULT\_DIRECTORY][66]
+*   [DEFAULT\_LIBRARIES\_DIRECTORY][67]
+
+## download
+
+Initiates the download of a model file of a specific model type.
+By default this downloads without waiting. use the controller returned to alter this behavior.
+
+### Parameters
+
+*   `model` **[ModelFile][12]** The model file to be downloaded.
+*   `options` **[DownloadOptions][4]** to pass into the downloader. Default is { location: (cwd), debug: false }.
+
+### Examples
+
+```javascript
+const controller = download('ggml-gpt4all-j-v1.3-groovy.bin')
+controller.promise().then(() => console.log('Downloaded!'))
+```
+
+*   Throws **[Error][68]** If the model already exists in the specified location.
+*   Throws **[Error][68]** If the model cannot be found at the specified url.
+
+Returns **[DownloadController][8]** object that allows controlling the download process.
+
+## DownloadOptions
+
+Options for the model download process.
+
+### location
+
+location to download the model.
+Default is process.cwd(), or the current working directory
+
+Type: [string][69]
+
+### debug
+
+Debug mode -- check how long it took to download in seconds
+
+Type: [boolean][70]
+
+### url
+
+Remote download url. Defaults to `https://gpt4all.io/models`
+
+Type: [string][69]
+
+## DownloadController
+
+Model download controller.
+
+### cancel
+
+Cancel the request to download from gpt4all website if this is called.
+
+Type: function (): void
+
+### promise
+
+Convert the downloader into a promise, allowing people to await and manage its lifetime
+
+Type: function (): [Promise][71]\<void>
+
+## ModelType
+
+Type of the model
+
+Type: (`"gptj"` | `"llama"` | `"mpt"`)
+
+## ModelFile
+
+Full list of models available
+
+### gptj
+
+List of GPT-J Models
+
+Type: (`"ggml-gpt4all-j-v1.3-groovy.bin"` | `"ggml-gpt4all-j-v1.2-jazzy.bin"` | `"ggml-gpt4all-j-v1.1-breezy.bin"` | `"ggml-gpt4all-j.bin"`)
+
+### llama
+
+List Llama Models
+
+Type: (`"ggml-gpt4all-l13b-snoozy.bin"` | `"ggml-vicuna-7b-1.1-q4_2.bin"` | `"ggml-vicuna-13b-1.1-q4_2.bin"` | `"ggml-wizardLM-7B.q4_2.bin"` | `"ggml-stable-vicuna-13B.q4_2.bin"` | `"ggml-nous-gpt4-vicuna-13b.bin"`)
+
+### mpt
+
+List of MPT Models
+
+Type: (`"ggml-mpt-7b-base.bin"` | `"ggml-mpt-7b-chat.bin"` | `"ggml-mpt-7b-instruct.bin"`)
+
+## type
+
+Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
+
+Type: [ModelType][11]
+
+## LLModel
+
+LLModel class representing a language model.
+This is a base class that provides common functionality for different types of language models.
+
+### constructor
+
+Initialize a new LLModel.
+
+#### Parameters
+
+*   `path` **[string][69]** Absolute path to the model file.
+
+<!---->
+
+*   Throws **[Error][68]** If the model file does not exist.
+
+### type
+
+either 'gpt', mpt', or 'llama' or undefined
+
+Returns **([ModelType][11] | [undefined][72])**&#x20;
+
+### name
+
+The name of the model.
+
+Returns **[ModelFile][12]**&#x20;
+
+### stateSize
+
+Get the size of the internal state of the model.
+NOTE: This state data is specific to the type of model you have created.
+
+Returns **[number][73]** the size in bytes of the internal state of the model
+
+### threadCount
+
+Get the number of threads used for model inference.
+The default is the number of physical cores your computer has.
+
+Returns **[number][73]** The number of threads used for model inference.
+
+### setThreadCount
+
+Set the number of threads used for model inference.
+
+#### Parameters
+
+*   `newNumber` **[number][73]** The new number of threads.
+
+Returns **void**&#x20;
+
+### raw\_prompt
+
+Prompt the model with a given input and optional parameters.
+This is the raw output from std out.
+Use the prompt function exported for a value
+
+#### Parameters
+
+*   `q` **[string][69]** The prompt input.
+*   `params` **Partial<[LLModelPromptContext][51]>?** Optional parameters for the prompt context.
+
+Returns **any** The result of the model prompt.
+
+### isModelLoaded
+
+Whether the model is loaded or not.
+
+Returns **[boolean][70]**&#x20;
+
+### setLibraryPath
+
+Where to search for the pluggable backend libraries
+
+#### Parameters
+
+*   `s` **[string][69]**&#x20;
+
+Returns **void**&#x20;
+
+### getLibraryPath
+
+Where to get the pluggable backend libraries
+
+Returns **[string][69]**&#x20;
+
+## createCompletion
+
+The nodejs equivalent to python binding's chat\_completion
+
+### Parameters
+
+*   `llmodel` **[LLModel][17]** The language model object.
+*   `messages` **[Array][74]<[PromptMessage][39]>** The array of messages for the conversation.
+*   `options` **[CompletionOptions][35]** The options for creating the completion.
+
+### Examples
+
+```javascript
+const llmodel = new LLModel(model)
+const messages = [ 
+{ role: 'system', message: 'You are a weather forecaster.' },
+{ role: 'user', message: 'should i go out today?' } ]
+const completion = await createCompletion(llmodel, messages, {
+ verbose: true,
+ temp: 0.9,
+})
+console.log(completion.choices[0].message.content)
+// No, it's going to be cold and rainy.
+```
+
+Returns **[CompletionReturn][45]** The completion result.
+
+## CompletionOptions
+
+**Extends Partial\<LLModelPromptContext>**
+
+The options for creating the completion.
+
+### verbose
+
+Indicates if verbose logging is enabled.
+
+Type: [boolean][70]
+
+### hasDefaultHeader
+
+Indicates if the default header is included in the prompt.
+
+Type: [boolean][70]
+
+### hasDefaultFooter
+
+Indicates if the default footer is included in the prompt.
+
+Type: [boolean][70]
+
+## PromptMessage
+
+A message in the conversation, identical to OpenAI's chat message.
+
+### role
+
+The role of the message.
+
+Type: (`"system"` | `"assistant"` | `"user"`)
+
+### content
+
+The message content.
+
+Type: [string][69]
+
+## prompt\_tokens
+
+The number of tokens used in the prompt.
+
+Type: [number][73]
+
+## completion\_tokens
+
+The number of tokens used in the completion.
+
+Type: [number][73]
+
+## total\_tokens
+
+The total number of tokens used.
+
+Type: [number][73]
+
+## CompletionReturn
+
+The result of the completion, similar to OpenAI's format.
+
+### model
+
+The model name.
+
+Type: [ModelFile][12]
+
+### usage
+
+Token usage report.
+
+Type: {prompt\_tokens: [number][73], completion\_tokens: [number][73], total\_tokens: [number][73]}
+
+### choices
+
+The generated completions.
+
+Type: [Array][74]<[CompletionChoice][49]>
+
+## CompletionChoice
+
+A completion choice, similar to OpenAI's format.
+
+### message
+
+Response message
+
+Type: [PromptMessage][39]
+
+## LLModelPromptContext
+
+Model inference arguments for generating completions.
+
+### logits\_size
+
+The size of the raw logits vector.
+
+Type: [number][73]
+
+### tokens\_size
+
+The size of the raw tokens vector.
+
+Type: [number][73]
+
+### n\_past
+
+The number of tokens in the past conversation.
+
+Type: [number][73]
+
+### n\_ctx
+
+The number of tokens possible in the context window.
+
+Type: [number][73]
+
+### n\_predict
+
+The number of tokens to predict.
+
+Type: [number][73]
+
+### top\_k
+
+The top-k logits to sample from.
+
+Type: [number][73]
+
+### top\_p
+
+The nucleus sampling probability threshold.
+
+Type: [number][73]
+
+### temp
+
+The temperature to adjust the model's output distribution.
+
+Type: [number][73]
+
+### n\_batch
+
+The number of predictions to generate in parallel.
+
+Type: [number][73]
+
+### repeat\_penalty
+
+The penalty factor for repeated tokens.
+
+Type: [number][73]
+
+### repeat\_last\_n
+
+The number of last tokens to penalize.
+
+Type: [number][73]
+
+### context\_erase
+
+The percentage of context to erase if the context window is exceeded.
+
+Type: [number][73]
+
+## createTokenStream
+
+TODO: Help wanted to implement this
+
+### Parameters
+
+*   `llmodel` **[LLModel][17]**&#x20;
+*   `messages` **[Array][74]<[PromptMessage][39]>**&#x20;
+*   `options` **[CompletionOptions][35]**&#x20;
+
+Returns **function (ll: [LLModel][17]): AsyncGenerator<[string][69]>**&#x20;
+
+## DEFAULT\_DIRECTORY
+
+From python api:
+models will be stored in (homedir)/.cache/gpt4all/\`
+
+Type: [string][69]
+
+## DEFAULT\_LIBRARIES\_DIRECTORY
+
+From python api:
+The default path for dynamic libraries to be stored.
+You may separate paths by a semicolon to search in multiple areas.
+This searches DEFAULT\_DIRECTORY/libraries, cwd/libraries, and finally cwd.
+
+Type: [string][69]
+
+[1]: #download
+
+[2]: #parameters
+
+[3]: #examples
+
+[4]: #downloadoptions
+
+[5]: #location
+
+[6]: #debug
+
+[7]: #url
+
+[8]: #downloadcontroller
+
+[9]: #cancel
+
+[10]: #promise
+
+[11]: #modeltype
+
+[12]: #modelfile
+
+[13]: #gptj
+
+[14]: #llama
+
+[15]: #mpt
+
+[16]: #type
+
+[17]: #llmodel
+
+[18]: #constructor
+
+[19]: #parameters-1
+
+[20]: #type-1
+
+[21]: #name
+
+[22]: #statesize
+
+[23]: #threadcount
+
+[24]: #setthreadcount
+
+[25]: #parameters-2
+
+[26]: #raw_prompt
+
+[27]: #parameters-3
+
+[28]: #ismodelloaded
+
+[29]: #setlibrarypath
+
+[30]: #parameters-4
+
+[31]: #getlibrarypath
+
+[32]: #createcompletion
+
+[33]: #parameters-5
+
+[34]: #examples-1
+
+[35]: #completionoptions
+
+[36]: #verbose
+
+[37]: #hasdefaultheader
+
+[38]: #hasdefaultfooter
+
+[39]: #promptmessage
+
+[40]: #role
+
+[41]: #content
+
+[42]: #prompt_tokens
+
+[43]: #completion_tokens
+
+[44]: #total_tokens
+
+[45]: #completionreturn
+
+[46]: #model
+
+[47]: #usage
+
+[48]: #choices
+
+[49]: #completionchoice
+
+[50]: #message
+
+[51]: #llmodelpromptcontext
+
+[52]: #logits_size
+
+[53]: #tokens_size
+
+[54]: #n_past
+
+[55]: #n_ctx
+
+[56]: #n_predict
+
+[57]: #top_k
+
+[58]: #top_p
+
+[59]: #temp
+
+[60]: #n_batch
+
+[61]: #repeat_penalty
+
+[62]: #repeat_last_n
+
+[63]: #context_erase
+
+[64]: #createtokenstream
+
+[65]: #parameters-6
+
+[66]: #default_directory
+
+[67]: #default_libraries_directory
+
+[68]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error
+
+[69]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String
+
+[70]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean
+
+[71]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise
+
+[72]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined
+
+[73]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number
+
+[74]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array
--- a/gpt4all-bindings/typescript/index.cc
+++ b/gpt4all-bindings/typescript/index.cc
@@ -10,7 +10,6 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
       InstanceMethod("stateSize", &NodeModelWrapper::StateSize),
       InstanceMethod("raw_prompt", &NodeModelWrapper::Prompt),
       InstanceMethod("setThreadCount", &NodeModelWrapper::SetThreadCount),
-       InstanceMethod("embed", &NodeModelWrapper::GenerateEmbedding),
       InstanceMethod("threadCount", &NodeModelWrapper::ThreadCount),
       InstanceMethod("getLibraryPath", &NodeModelWrapper::GetLibraryPath),
    });
@@ -58,19 +57,15 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
        }
    }
    llmodel_set_implementation_search_path(library_path.c_str());
-    llmodel_error e = {
-        .message="looks good to me",
-        .code=0,
-    };
-    inference_ = std::make_shared<llmodel_model>(llmodel_model_create2(full_weight_path.c_str(), "auto", &e));
-    if(e.code != 0) {
-       Napi::Error::New(env, e.message).ThrowAsJavaScriptException(); 
+    llmodel_error* e = nullptr;
+    inference_ = std::make_shared<llmodel_model>(llmodel_model_create2(full_weight_path.c_str(), "auto", e));
+    if(e != nullptr) {
+       Napi::Error::New(env, e->message).ThrowAsJavaScriptException(); 
       return;
    }
    if(GetInference() == nullptr) {
       std::cerr << "Tried searching libraries in \"" << library_path << "\"" <<  std::endl;
       std::cerr << "Tried searching for model weight in \"" << full_weight_path << "\"" << std::endl;
-       std::cerr << "Do you have runtime libraries installed?" << std::endl;
       Napi::Error::New(env, "Had an issue creating llmodel object, inference is null").ThrowAsJavaScriptException(); 
       return;
    }
@@ -95,33 +90,6 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
    return Napi::Number::New(info.Env(), static_cast<int64_t>(llmodel_get_state_size(GetInference())));
  }
  
-  Napi::Value NodeModelWrapper::GenerateEmbedding(const Napi::CallbackInfo& info) {
-    auto env = info.Env();
-    std::string text = info[0].As<Napi::String>().Utf8Value();
-    size_t embedding_size = 0;
-    float* arr = llmodel_embedding(GetInference(), text.c_str(), &embedding_size);
-    if(arr == nullptr) {
-        Napi::Error::New(
-            env, 
-            "Cannot embed. native embedder returned 'nullptr'"
-        ).ThrowAsJavaScriptException(); 
-        return env.Undefined();
-    }
-
-    if(embedding_size == 0 && text.size() != 0 ) {
-        std::cout << "Warning: embedding length 0 but input text length > 0" << std::endl;
-    }
-    Napi::Float32Array js_array = Napi::Float32Array::New(env, embedding_size);
-    
-    for (size_t i = 0; i < embedding_size; ++i) {
-        float element = *(arr + i);
-        js_array[i] = element;
-    }
-
-    llmodel_free_embedding(arr);
-
-    return js_array;
-  }

 /**
 * Generate a response using the model.
@@ -188,12 +156,12 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
            promptContext.context_erase = inputObject.Get("context_erase").As<Napi::Number>().FloatValue();
    }
    //copy to protect llmodel resources when splitting to new thread
-    llmodel_prompt_context copiedPrompt = promptContext;

+    llmodel_prompt_context copiedPrompt = promptContext;
    std::string copiedQuestion = question;
    PromptWorkContext pc = {
        copiedQuestion,
-        std::ref(inference_),
+        inference_.load(),
        copiedPrompt,
    };
    auto threadSafeContext = new TsfnContext(env, pc);
@@ -233,7 +201,7 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
  }

  llmodel_model NodeModelWrapper::GetInference() {
-    return *inference_;
+    return *inference_.load();
  }

 //Exports Bindings
--- a/gpt4all-bindings/typescript/index.h
+++ b/gpt4all-bindings/typescript/index.h
@@ -23,7 +23,6 @@ public:
  void SetThreadCount(const Napi::CallbackInfo& info);
  Napi::Value getName(const Napi::CallbackInfo& info);
  Napi::Value ThreadCount(const Napi::CallbackInfo& info);
-  Napi::Value GenerateEmbedding(const Napi::CallbackInfo& info);
  /*
   * The path that is used to search for the dynamic libraries
   */
@@ -37,7 +36,7 @@ private:
  /**
   * The underlying inference that interfaces with the C interface
   */
-  std::shared_ptr<llmodel_model> inference_;
+  std::atomic<std::shared_ptr<llmodel_model>> inference_;

  std::string type;
  // corresponds to LLModel::name() in typescript
--- a/gpt4all-bindings/typescript/package.json
+++ b/gpt4all-bindings/typescript/package.json
@@ -1,29 +1,17 @@
 {
  "name": "gpt4all",
-  "version": "2.2.0",
-  "packageManager": "yarn@3.6.1",
+  "version": "2.0.0",
+  "packageManager": "yarn@3.5.1",
  "main": "src/gpt4all.js",
  "repository": "nomic-ai/gpt4all",
  "scripts": {
-    "install": "node-gyp-build",
-    "test": "jest",
+    "test": "node ./test/index.mjs",
    "build:backend": "node scripts/build.js",
-    "build": "node-gyp-build",
-    "predocs:build": "node scripts/docs.js",
-    "docs:build": "documentation readme ./src/gpt4all.d.ts --parse-extension js d.ts --format md --section \"API Reference\" --readme-file ../python/docs/gpt4all_typescript.md",
-    "postdocs:build": "documentation readme ./src/gpt4all.d.ts --parse-extension js d.ts --format md --section \"API Reference\" --readme-file README.md"
+    "install": "node-gyp-build",
+    "prebuild": "node scripts/prebuild.js",
+    "docs:build": "documentation build ./src/gpt4all.d.ts --parse-extension d.ts --format md --output docs/api.md"
  },
-  "files": [
-    "src/**/*",
-    "runtimes/**/*",
-    "binding.gyp",
-    "prebuilds/**/*",
-    "*.h",
-    "*.cc",
-    "gpt4all-backend/**/*"
-  ],
  "dependencies": {
-    "md5-file": "^5.0.0",
    "mkdirp": "^3.0.1",
    "node-addon-api": "^6.1.0",
    "node-gyp-build": "^4.6.0"
@@ -31,21 +19,14 @@
  "devDependencies": {
    "@types/node": "^20.1.5",
    "documentation": "^14.0.2",
-    "jest": "^29.5.0",
    "prebuildify": "^5.0.1",
    "prettier": "^2.8.8"
  },
-  "optionalDependencies": {
-    "node-gyp": "9.x.x"
-  },
  "engines": {
    "node": ">= 18.x.x"
  },
  "prettier": {
    "endOfLine": "lf",
    "tabWidth": 4
-  },
-  "jest": {
-    "verbose": true
  }
 }
--- a/gpt4all-bindings/typescript/prompt.cc
+++ b/gpt4all-bindings/typescript/prompt.cc
@@ -4,12 +4,11 @@
 TsfnContext::TsfnContext(Napi::Env env, const PromptWorkContext& pc) 
    : deferred_(Napi::Promise::Deferred::New(env)), pc(pc) {
 }
-namespace {
-    static std::string *res;
-}

+std::mutex mtx;
+static thread_local std::string res;
 bool response_callback(int32_t token_id, const char *response) {
-   *res += response;
+   res+=response;
   return token_id != -1;
 }
 bool recalculate_callback (bool isrecalculating) {
@@ -22,12 +21,10 @@ bool prompt_callback (int32_t tid) {
 // The thread entry point. This takes as its arguments the specific
 // threadsafe-function context created inside the main thread.
 void threadEntry(TsfnContext* context) {
-  static std::mutex mtx;
  std::lock_guard<std::mutex> lock(mtx);
-  res = &context->pc.res;
  // Perform a call into JavaScript.
  napi_status status =
-    context->tsfn.BlockingCall(&context->pc,
+    context->tsfn.NonBlockingCall(&context->pc,
    [](Napi::Env env, Napi::Function jsCallback, PromptWorkContext* pc) {
        llmodel_prompt(
            *pc->inference_,
@@ -37,6 +34,8 @@ void threadEntry(TsfnContext* context) {
            &recalculate_callback,
            &pc->prompt_params
        );
+        jsCallback.Call({ Napi::String::New(env, res)} );
+        res.clear();
  });

  if (status != napi_ok) {
@@ -44,6 +43,7 @@ void threadEntry(TsfnContext* context) {
        "ThreadEntry",
        "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
  }
+
  // Release the thread-safe function. This decrements the internal thread
  // count, and will perform finalization since the count will reach 0.
  context->tsfn.Release();
@@ -52,9 +52,11 @@ void threadEntry(TsfnContext* context) {
 void FinalizerCallback(Napi::Env env,
                       void* finalizeData,
                       TsfnContext* context) {
-  // Resolve the Promise previously returned to JS 
-    context->deferred_.Resolve(Napi::String::New(env, context->pc.res));
-    // Wait for the thread to finish executing before proceeding.
-    context->nativeThread.join();
-    delete context;
+  // Join the thread
+  context->nativeThread.join();
+  // Resolve the Promise previously returned to JS via the CreateTSFN method.
+  context->deferred_.Resolve(Napi::Boolean::New(env, true));
+  delete context;
 }
+
+
--- a/gpt4all-bindings/typescript/prompt.h
+++ b/gpt4all-bindings/typescript/prompt.h
@@ -10,10 +10,8 @@
 #include <memory>
 struct PromptWorkContext {
    std::string question;
-    std::shared_ptr<llmodel_model>& inference_;
+    std::shared_ptr<llmodel_model> inference_;
    llmodel_prompt_context prompt_params;
-    std::string res;
-
 };

 struct TsfnContext {
@@ -31,12 +29,12 @@ public:

 // The thread entry point. This takes as its arguments the specific
 // threadsafe-function context created inside the main thread.
-void threadEntry(TsfnContext*);
+void threadEntry(TsfnContext* context);

 // The thread-safe function finalizer callback. This callback executes
 // at destruction of thread-safe function, taking as arguments the finalizer
 // data and threadsafe-function context.
-void FinalizerCallback(Napi::Env, void* finalizeData, TsfnContext*);
+void FinalizerCallback(Napi::Env env, void* finalizeData, TsfnContext* context);

 bool response_callback(int32_t token_id, const char *response);
 bool recalculate_callback (bool isrecalculating);
--- a/gpt4all-bindings/typescript/scripts/build.js
+++ b/gpt4all-bindings/typescript/scripts/build.js
@@ -2,6 +2,7 @@ const { spawn } = require("node:child_process");
 const { resolve } = require("path");
 const args = process.argv.slice(2);
 const platform = process.platform;
+
 //windows 64bit or 32
 if (platform === "win32") {
    const path = "scripts/build_msvc.bat";
@@ -9,9 +10,8 @@ if (platform === "win32") {
    process.on("data", (s) => console.log(s.toString()));
 } else if (platform === "linux" || platform === "darwin") {
    const path = "scripts/build_unix.sh";
-    spawn(`sh `, [path, args], {
-        shell: true,
+    const bash = spawn(`sh`, [path, ...args]);
+    bash.stdout.on("data", (s) => console.log(s.toString()), {
        stdio: "inherit",
    });
-    process.on("data", (s) => console.log(s.toString()));
 }
--- a/gpt4all-bindings/typescript/scripts/build_msvc.bat
+++ b/gpt4all-bindings/typescript/scripts/build_msvc.bat
@@ -1,33 +0,0 @@
-@echo off
-
-set "BUILD_TYPE=Release"
-set "BUILD_DIR=.\build\win-x64-msvc"
-set "LIBS_DIR=.\runtimes\win32-x64"
-
-REM Cleanup env
-rmdir /s /q %BUILD_DIR%
-
-REM Create directories
-mkdir %BUILD_DIR%
-mkdir %LIBS_DIR%
-
-REM Build
-cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=%BUILD_TYPE% -S ..\..\gpt4all-backend -B %BUILD_DIR% -A x64
-
-:BUILD
-REM Build the project
-cmake --build "%BUILD_DIR%" --parallel --config %BUILD_TYPE%
-
-REM Check the exit code of the build command
-if %errorlevel% neq 0 (
-    echo Build failed. Retrying...
-    goto BUILD
-)
-
-mkdir runtimes\win32-x64
-
-REM Copy the DLLs to the desired location
-del /F /A /Q %LIBS_DIR%
-xcopy /Y "%BUILD_DIR%\bin\%BUILD_TYPE%\*.dll" runtimes\win32-x64\native\
-
-echo Batch script execution completed.
--- a/gpt4all-bindings/typescript/scripts/build_unix.sh
+++ b/gpt4all-bindings/typescript/scripts/build_unix.sh
@@ -24,9 +24,7 @@ mkdir -p "$NATIVE_DIR" "$BUILD_DIR"

 cmake -S ../../gpt4all-backend -B "$BUILD_DIR" &&
 cmake --build "$BUILD_DIR" -j --config Release && {
-  cp "$BUILD_DIR"/libbert*.$LIB_EXT   "$NATIVE_DIR"/
-  cp "$BUILD_DIR"/libfalcon*.$LIB_EXT "$NATIVE_DIR"/
-  cp "$BUILD_DIR"/libreplit*.$LIB_EXT "$NATIVE_DIR"/
+  cp "$BUILD_DIR"/libllmodel.$LIB_EXT "$NATIVE_DIR"/
  cp "$BUILD_DIR"/libgptj*.$LIB_EXT   "$NATIVE_DIR"/
  cp "$BUILD_DIR"/libllama*.$LIB_EXT  "$NATIVE_DIR"/
  cp "$BUILD_DIR"/libmpt*.$LIB_EXT    "$NATIVE_DIR"/
--- a/gpt4all-bindings/typescript/scripts/docs.js
+++ b/gpt4all-bindings/typescript/scripts/docs.js
@@ -1,8 +0,0 @@
-//Maybe some command line piping would work better, but can't think of platform independent command line tool
-
-const fs = require('fs');
-
-const newPath = '../python/docs/gpt4all_typescript.md';
-const filepath = 'README.md';
-const data = fs.readFileSync(filepath);
-fs.writeFileSync(newPath, data);
--- a/gpt4all-bindings/typescript/scripts/prebuild.js
+++ b/gpt4all-bindings/typescript/scripts/prebuild.js
@@ -6,7 +6,6 @@ async function createPrebuilds(combinations) {
            platform,
            arch,
            napi: true,
-            targets: ["18.16.0"]
        };
        try {
            await createPrebuild(opts);
@@ -34,24 +33,17 @@ function createPrebuild(opts) {
    });
 }

-let prebuildConfigs;
-if(process.platform === 'win32') {
-   prebuildConfigs = [
-    { platform: "win32", arch: "x64" }
-   ];
-} else if(process.platform === 'linux') {
-   //Unsure if darwin works, need mac tester!
-   prebuildConfigs = [
+const prebuildConfigs = [
+    { platform: "win32", arch: "x64" },
+    { platform: "win32", arch: "arm64" },
+    // { platform: 'win32', arch: 'armv7' },
+    { platform: "darwin", arch: "x64" },
+    { platform: "darwin", arch: "arm64" },
+    // { platform: 'darwin', arch: 'armv7' },
    { platform: "linux", arch: "x64" },
-    //{ platform: "linux", arch: "arm64" },
-    //{ platform: "linux", arch: "armv7" },
-   ]
-} else if(process.platform === 'darwin') {
-    prebuildConfigs = [
-       { platform: "darwin", arch: "x64" },
-       { platform: "darwin", arch: "arm64" },
-    ]
-}
+    { platform: "linux", arch: "arm64" },
+    { platform: "linux", arch: "armv7" },
+];

 createPrebuilds(prebuildConfigs)
    .then(() => console.log("All builds succeeded"))
--- a/gpt4all-bindings/typescript/spec/chat.mjs
+++ b/gpt4all-bindings/typescript/spec/chat.mjs
@@ -1,66 +0,0 @@
-import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY, loadModel } from '../src/gpt4all.js'
-
-const model = await loadModel(
-    'orca-mini-3b.ggmlv3.q4_0.bin',
-    { verbose: true }
-);
-const ll = model.llm;
-
-try {
-   class Extended extends LLModel {
-   }
-
-} catch(e) {
-    console.log("Extending from native class gone wrong " + e)
-}
-
-console.log("state size " + ll.stateSize())
-
-console.log("thread count " + ll.threadCount());
-ll.setThreadCount(5);
-
-console.log("thread count " + ll.threadCount());
-ll.setThreadCount(4);
-console.log("thread count " + ll.threadCount());
-console.log("name " + ll.name());
-console.log("type: " + ll.type());
-console.log("Default directory for models", DEFAULT_DIRECTORY);
-console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY);
-
-const completion1 = await createCompletion(model, [ 
-    { role : 'system', content: 'You are an advanced mathematician.'  },
-    { role : 'user', content: 'What is 1 + 1?'  }, 
-], { verbose: true })
-console.log(completion1.choices[0].message)
-
-const completion2 = await createCompletion(model, [
-    { role : 'system', content: 'You are an advanced mathematician.'  },
-    { role : 'user', content: 'What is two plus two?'  }, 
-], {  verbose: true })
-
-console.log(completion2.choices[0].message)
-
-// At the moment, from testing this code, concurrent model prompting is not possible. 
-// Behavior: The last prompt gets answered, but the rest are cancelled
-// my experience with threading is not the best, so if anyone who is good is willing to give this a shot,
-// maybe this is possible
-// INFO: threading with llama.cpp is not the best maybe not even possible, so this will be left here as reference
-
-//const responses = await Promise.all([
-//    createCompletion(ll, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//    ], { verbose: true }),
-//    createCompletion(ll, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//    ], { verbose: true }),
-//
-//createCompletion(ll, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//], { verbose: true })
-//
-//])
-//console.log(responses.map(s => s.choices[0].message))
-
--- a/gpt4all-bindings/typescript/spec/embed.mjs
+++ b/gpt4all-bindings/typescript/spec/embed.mjs
@@ -1,8 +0,0 @@
-import {  loadModel, createEmbedding } from '../src/gpt4all.js'
-
-const embedder = await loadModel("ggml-all-MiniLM-L6-v2-f16.bin", { verbose: true })
-
-console.log(
-    createEmbedding(embedder, "Accept your current situation")
-)
-
--- a/gpt4all-bindings/typescript/spec/index.mjs
+++ b/gpt4all-bindings/typescript/spec/index.mjs
@@ -0,0 +1,46 @@
+import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY } from '../src/gpt4all.js'
+
+const ll = new LLModel({
+    model_name: 'ggml-vicuna-7b-1.1-q4_2.bin',
+    model_path: './', 
+    library_path: DEFAULT_LIBRARIES_DIRECTORY
+});
+
+try {
+   class Extended extends LLModel {
+   }
+
+} catch(e) {
+    console.log("Extending from native class gone wrong " + e)
+}
+
+console.log("state size " + ll.stateSize())
+
+console.log("thread count " + ll.threadCount());
+ll.setThreadCount(5);
+console.log("thread count " + ll.threadCount());
+ll.setThreadCount(4);
+console.log("thread count " + ll.threadCount());
+console.log("name " + ll.name());
+console.log("type: " + ll.type());
+console.log("Default directory for models", DEFAULT_DIRECTORY);
+console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY);
+
+console.log(await createCompletion(
+    ll,
+    [
+        { role : 'system', content: 'You are a girl who likes playing league of legends.'  },
+        { role : 'user', content: 'What is the best top laner to play right now?'  }, 
+    ],
+    { verbose: false}
+));
+
+
+console.log(await createCompletion(
+    ll,
+    [
+        { role : 'user', content: 'What is the best bottom laner to play right now?'  }, 
+    ],
+))
+
+
--- a/gpt4all-bindings/typescript/src/config.js
+++ b/gpt4all-bindings/typescript/src/config.js
@@ -16,26 +16,7 @@ const librarySearchPaths = [

 const DEFAULT_LIBRARIES_DIRECTORY = librarySearchPaths.join(";");

-const DEFAULT_MODEL_CONFIG = {
-    systemPrompt: "",
-    promptTemplate: "### Human: \n%1\n### Assistant:\n",
-}
-
-const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models.json";
-
-const DEFAULT_PROMPT_CONTEXT = {
-    temp: 0.7,
-    topK: 40,
-    topP: 0.4,
-    repeatPenalty: 1.18,
-    repeatLastN: 64,
-    nBatch: 8,
-}
-
 module.exports = {
    DEFAULT_DIRECTORY,
    DEFAULT_LIBRARIES_DIRECTORY,
-    DEFAULT_MODEL_CONFIG,
-    DEFAULT_MODEL_LIST_URL,
-    DEFAULT_PROMPT_CONTEXT,
 };
--- a/gpt4all-bindings/typescript/src/gpt4all.d.ts
+++ b/gpt4all-bindings/typescript/src/gpt4all.d.ts
@@ -1,13 +1,13 @@
 /// <reference types="node" />
 declare module "gpt4all";

+export * from "./util.d.ts";
+
 /** Type of the model */
 type ModelType = "gptj" | "llama" | "mpt" | "replit";

-// NOTE: "deprecated" tag in below comment breaks the doc generator https://github.com/documentationjs/documentation/issues/1596
 /**
 * Full list of models available
- * @deprecated These model names are outdated and this type will not be maintained, please use a string literal instead
 */
 interface ModelFile {
    /** List of GPT-J Models */
@@ -40,37 +40,10 @@ interface LLModelOptions {
     * Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
     */
    type?: ModelType;
-    model_name: string;
+    model_name: ModelFile[ModelType];
    model_path: string;
    library_path?: string;
 }
-
-interface ModelConfig {
-    systemPrompt: string;
-    promptTemplate: string;
-    path: string;
-    url?: string;
-}
-
-declare class InferenceModel {
-    constructor(llm: LLModel, config: ModelConfig);
-    llm: LLModel;
-    config: ModelConfig;
-
-    generate(
-        prompt: string,
-        options?: Partial<LLModelPromptContext>
-    ): Promise<string>;
-}
-
-declare class EmbeddingModel {
-    constructor(llm: LLModel, config: ModelConfig);
-    llm: LLModel;
-    config: ModelConfig;
-
-    embed(text: string): Float32Array;
-}
-
 /**
 * LLModel class representing a language model.
 * This is a base class that provides common functionality for different types of language models.
@@ -88,7 +61,7 @@ declare class LLModel {
    type(): ModelType | undefined;

    /** The name of the model. */
-    name(): string;
+    name(): ModelFile;

    /**
     * Get the size of the internal state of the model.
@@ -112,27 +85,14 @@ declare class LLModel {

    /**
     * Prompt the model with a given input and optional parameters.
-     * This is the raw output from model.
+     * This is the raw output from std out.
     * Use the prompt function exported for a value
     * @param q The prompt input.
     * @param params Optional parameters for the prompt context.
     * @returns The result of the model prompt.
     */
-    raw_prompt(
-        q: string,
-        params: Partial<LLModelPromptContext>,
-        callback: (res: string) => void
-    ): void; // TODO work on return type
+    raw_prompt(q: string, params: Partial<LLModelPromptContext>, callback: (res: string) => void): void; // TODO work on return type

-    /**
-     * Embed text with the model. Keep in mind that
-     * not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
-     * Use the prompt function exported for a value
-     * @param q The prompt input.
-     * @param params Optional parameters for the prompt context.
-     * @returns The result of the model prompt.
-     */
-    embed(text: string): Float32Array;
    /**
     * Whether the model is loaded or not.
     */
@@ -151,67 +111,39 @@ declare class LLModel {
 interface LoadModelOptions {
    modelPath?: string;
    librariesPath?: string;
-    modelConfigFile?: string;
    allowDownload?: boolean;
    verbose?: boolean;
 }

-interface InferenceModelOptions extends LoadModelOptions {
-    type?: "inference";
-}
-
-interface EmbeddingModelOptions extends LoadModelOptions {
-    type: "embedding";
-}
-
-/**
- * Loads a machine learning model with the specified name. The defacto way to create a model.
- * By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
- *
- * @param {string} modelName - The name of the model to load.
- * @param {LoadModelOptions|undefined} [options] - (Optional) Additional options for loading the model.
- * @returns {Promise<InferenceModel | EmbeddingModel>} A promise that resolves to an instance of the loaded LLModel.
- */
 declare function loadModel(
    modelName: string,
-    options?: InferenceModelOptions
-): Promise<InferenceModel>;
-
-declare function loadModel(
-    modelName: string,
-    options?: EmbeddingModelOptions
-): Promise<EmbeddingModel>;
-
-declare function loadModel(
-    modelName: string,
-    options?: EmbeddingOptions | InferenceOptions
-): Promise<InferenceModel | EmbeddingModel>;
+    options?: LoadModelOptions
+): Promise<LLModel>;

 /**
 * The nodejs equivalent to python binding's chat_completion
- * @param {InferenceModel} model - The language model object.
+ * @param {LLModel} llmodel - The language model object.
 * @param {PromptMessage[]} messages - The array of messages for the conversation.
 * @param {CompletionOptions} options - The options for creating the completion.
 * @returns {CompletionReturn} The completion result.
+ * @example
+ * const llmodel = new LLModel(model)
+ * const messages = [
+ * { role: 'system', message: 'You are a weather forecaster.' },
+ * { role: 'user', message: 'should i go out today?' } ]
+ * const completion = await createCompletion(llmodel, messages, {
+ *  verbose: true,
+ *  temp: 0.9,
+ * })
+ * console.log(completion.choices[0].message.content)
+ * // No, it's going to be cold and rainy.
 */
 declare function createCompletion(
-    model: InferenceModel,
+    llmodel: LLModel,
    messages: PromptMessage[],
    options?: CompletionOptions
 ): Promise<CompletionReturn>;

-/**
- * The nodejs moral equivalent to python binding's Embed4All().embed()
- * meow
- * @param {EmbeddingModel} model - The language model object.
- * @param {string} text - text to embed
- * @returns {Float32Array} The completion result.
- */
-declare function createEmbedding(
-    model: EmbeddingModel,
-    text: string
-): Float32Array;
-
 /**
 * The options for creating the completion.
 */
@@ -223,25 +155,16 @@ interface CompletionOptions extends Partial<LLModelPromptContext> {
    verbose?: boolean;

    /**
-     * Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
-     * Note that if this is not defined, system messages will not be included in the prompt.
+     * Indicates if the default header is included in the prompt.
+     * @default true
     */
-    systemPromptTemplate?: string;
+    hasDefaultHeader?: boolean;

    /**
-     * Template for user messages, with %1 being replaced by the message.
+     * Indicates if the default footer is included in the prompt.
+     * @default true
     */
-    promptTemplate?: boolean;
-
-    /**
-     * The initial instruction for the model, on top of the prompt
-     */
-    promptHeader?: string;
-
-    /**
-     * The last instruction for the model, appended to the end of the prompt.
-     */
-    promptFooter?: string;
+    hasDefaultFooter?: boolean;
 }

 /**
@@ -259,8 +182,10 @@ interface PromptMessage {
 * The result of the completion, similar to OpenAI's format.
 */
 interface CompletionReturn {
-    /** The model used for the completion. */
-    model: string;
+    /** The model name.
+     * @type {ModelFile}
+     */
+    model: ModelFile[ModelType];

    /** Token usage report. */
    usage: {
@@ -291,85 +216,58 @@ interface CompletionChoice {
 */
 interface LLModelPromptContext {
    /** The size of the raw logits vector. */
-    logitsSize: number;
+    logits_size: number;

    /** The size of the raw tokens vector. */
-    tokensSize: number;
+    tokens_size: number;

    /** The number of tokens in the past conversation. */
-    nPast: number;
+    n_past: number;

    /** The number of tokens possible in the context window.
     * @default 1024
     */
-    nCtx: number;
+    n_ctx: number;

    /** The number of tokens to predict.
     * @default 128
     * */
-    nPredict: number;
+    n_predict: number;

    /** The top-k logits to sample from.
-     * Top-K sampling selects the next token only from the top K most likely tokens predicted by the model.
-     * It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit
-     * the diversity of the output. A higher value for top-K (eg., 100) will consider more tokens and lead
-     * to more diverse text, while a lower value (eg., 10) will focus on the most probable tokens and generate
-     * more conservative text. 30 - 60 is a good range for most tasks.
     * @default 40
     * */
-    topK: number;
+    top_k: number;

    /** The nucleus sampling probability threshold.
-     * Top-P limits the selection of the next token to a subset of tokens with a cumulative probability 
-     * above a threshold P. This method, also known as nucleus sampling, finds a balance between diversity
-     * and quality by considering both token probabilities and the number of tokens available for sampling.
-     * When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
-     * On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
-     * The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
-     * for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
-     * @default 0.4
+     * @default 0.9
     * */
-    topP: number;
+    top_p: number;

    /** The temperature to adjust the model's output distribution.
-     * Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
-     * (eg., 1.2) increase randomness, resulting in more imaginative and diverse text. Lower temperatures (eg., 0.5)
-     * make the output more focused, predictable, and conservative. When the temperature is set to 0, the output
-     * becomes completely deterministic, always selecting the most probable next token and producing identical results
-     * each time. A safe range would be around 0.6 - 0.85, but you are free to search what value fits best for you.
-     * @default 0.7
+     * @default 0.72
     * */
    temp: number;

    /** The number of predictions to generate in parallel.
-     * By splitting the prompt every N tokens, prompt-batch-size reduces RAM usage during processing. However,
-     * this can increase the processing time as a trade-off. If the N value is set too low (e.g., 10), long prompts
-     * with 500+ tokens will be most affected, requiring numerous processing runs to complete the prompt processing.
-     * To ensure optimal performance, setting the prompt-batch-size to 2048 allows processing of all tokens in a single run.
     * @default 8
     * */
-    nBatch: number;
+    n_batch: number;

    /** The penalty factor for repeated tokens.
-     * Repeat-penalty can help penalize tokens based on how frequently they occur in the text, including the input prompt.
-     * A token that has already appeared five times is penalized more heavily than a token that has appeared only one time.
-     * A value of 1 means that there is no penalty and values larger than 1 discourage repeated tokens.
-     * @default 1.18
+     * @default 1
     * */
-    repeatPenalty: number;
+    repeat_penalty: number;

    /** The number of last tokens to penalize.
-     * The repeat-penalty-tokens N option controls the number of tokens in the history to consider for penalizing repetition.
-     * A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only
-     * consider recent tokens.
-     * @default 64
+     * @default 10
     * */
-    repeatLastN: number;
+    repeat_last_n: number;

    /** The percentage of context to erase if the context window is exceeded.
     * @default 0.5
     * */
-    contextErase: number;
+    context_erase: number;
 }

 /**
@@ -392,104 +290,13 @@ declare const DEFAULT_DIRECTORY: string;
 * This searches DEFAULT_DIRECTORY/libraries, cwd/libraries, and finally cwd.
 */
 declare const DEFAULT_LIBRARIES_DIRECTORY: string;
-
-/**
- * Default model configuration.
- */
-declare const DEFAULT_MODEL_CONFIG: ModelConfig;
-
-/**
- * Default prompt context.
- */
-declare const DEFAULT_PROMT_CONTEXT: LLModelPromptContext;
-
-/**
- * Default model list url.
- */
-declare const DEFAULT_MODEL_LIST_URL: string;
-
-/**
- * Initiates the download of a model file.
- * By default this downloads without waiting. use the controller returned to alter this behavior.
- * @param {string} modelName - The model to be downloaded.
- * @param {DownloadOptions} options - to pass into the downloader. Default is { location: (cwd), verbose: false }.
- * @returns {DownloadController} object that allows controlling the download process.
- *
- * @throws {Error} If the model already exists in the specified location.
- * @throws {Error} If the model cannot be found at the specified url.
- *
- * @example
- * const download = downloadModel('ggml-gpt4all-j-v1.3-groovy.bin')
- * download.promise.then(() => console.log('Downloaded!'))
- */
-declare function downloadModel(
-    modelName: string,
-    options?: DownloadModelOptions
-): DownloadController;
-
-/**
- * Options for the model download process.
- */
-interface DownloadModelOptions {
-    /**
-     * location to download the model.
-     * Default is process.cwd(), or the current working directory
-     */
-    modelPath?: string;
-
-    /**
-     * Debug mode -- check how long it took to download in seconds
-     * @default false
-     */
-    verbose?: boolean;
-
-    /**
-     * Remote download url. Defaults to `https://gpt4all.io/models/<modelName>`
-     * @default https://gpt4all.io/models/<modelName>
-     */
-    url?: string;
-    /**
-     * MD5 sum of the model file. If this is provided, the downloaded file will be checked against this sum.
-     * If the sums do not match, an error will be thrown and the file will be deleted.
-     */
-    md5sum?: string;
+interface PromptMessage {
+    role: "system" | "assistant" | "user";
+    content: string;
 }
-
-interface ListModelsOptions {
-    url?: string;
-    file?: string;
-}
-
-declare function listModels(options?: ListModelsOptions): Promise<ModelConfig[]>;
-
-interface RetrieveModelOptions {
-    allowDownload?: boolean;
-    verbose?: boolean;
-    modelPath?: string;
-    modelConfigFile?: string;
-}
-
-declare function retrieveModel(
-    modelName: string,
-    options?: RetrieveModelOptions
-): Promise<ModelConfig>;
-
-/**
- * Model download controller.
- */
-interface DownloadController {
-    /** Cancel the request to download if this is called. */
-    cancel: () => void;
-    /** A promise resolving to the downloaded models config once the download is done */
-    promise: Promise<ModelConfig>;
-}
-
 export {
    ModelType,
    ModelFile,
-    ModelConfig,
-    InferenceModel,
-    EmbeddingModel,
    LLModel,
    LLModelPromptContext,
    PromptMessage,
@@ -497,17 +304,7 @@ export {
    LoadModelOptions,
    loadModel,
    createCompletion,
-    createEmbedding,
    createTokenStream,
    DEFAULT_DIRECTORY,
    DEFAULT_LIBRARIES_DIRECTORY,
-    DEFAULT_MODEL_CONFIG,
-    DEFAULT_PROMT_CONTEXT,
-    DEFAULT_MODEL_LIST_URL,
-    downloadModel,
-    retrieveModel,
-    listModels,
-    DownloadController,
-    RetrieveModelOptions,
-    DownloadModelOptions,
 };
--- a/gpt4all-bindings/typescript/src/gpt4all.js
+++ b/gpt4all-bindings/typescript/src/gpt4all.js
@@ -10,36 +10,19 @@ const {
    downloadModel,
    appendBinSuffixIfMissing,
 } = require("./util.js");
-const {
-    DEFAULT_DIRECTORY,
-    DEFAULT_LIBRARIES_DIRECTORY,
-    DEFAULT_PROMPT_CONTEXT,
-    DEFAULT_MODEL_CONFIG,
-    DEFAULT_MODEL_LIST_URL,
-} = require("./config.js");
-const { InferenceModel, EmbeddingModel } = require("./models.js");
+const config = require("./config.js");

-/**
- * Loads a machine learning model with the specified name. The defacto way to create a model.
- * By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
- *
- * @param {string} modelName - The name of the model to load.
- * @param {LoadModelOptions|undefined} [options] - (Optional) Additional options for loading the model.
- * @returns {Promise<InferenceModel | EmbeddingModel>} A promise that resolves to an instance of the loaded LLModel.
- */
 async function loadModel(modelName, options = {}) {
    const loadOptions = {
-        modelPath: DEFAULT_DIRECTORY,
-        librariesPath: DEFAULT_LIBRARIES_DIRECTORY,
-        type: "inference",
+        modelPath: config.DEFAULT_DIRECTORY,
+        librariesPath: config.DEFAULT_LIBRARIES_DIRECTORY,
        allowDownload: true,
        verbose: true,
        ...options,
    };

-    const modelConfig = await retrieveModel(modelName, {
+    await retrieveModel(modelName, {
        modelPath: loadOptions.modelPath,
-        modelConfigFile: loadOptions.modelConfigFile,
        allowDownload: loadOptions.allowDownload,
        verbose: loadOptions.verbose,
    });
@@ -54,9 +37,7 @@ async function loadModel(modelName, options = {}) {
            break;
        }
    }
-    if (!libPath) {
-        throw Error("Could not find a valid path from " + libSearchPaths);
-    }
+
    const llmOptions = {
        model_name: appendBinSuffixIfMissing(modelName),
        model_path: loadOptions.modelPath,
@@ -64,183 +45,94 @@ async function loadModel(modelName, options = {}) {
    };

    if (loadOptions.verbose) {
-        console.debug("Creating LLModel with options:", llmOptions);
+        console.log("Creating LLModel with options:", llmOptions);
    }
    const llmodel = new LLModel(llmOptions);

-    if (loadOptions.type === "embedding") {
-        return new EmbeddingModel(llmodel, modelConfig);
-    } else if (loadOptions.type === "inference") {
-        return new InferenceModel(llmodel, modelConfig);
-    } else {
-        throw Error("Invalid model type: " + loadOptions.type);
-    }
+    return llmodel;
 }

-/**
- * Formats a list of messages into a single prompt string.
- */
-function formatChatPrompt(
-    messages,
-    {
-        systemPromptTemplate,
-        defaultSystemPrompt,
-        promptTemplate,
-        promptFooter,
-        promptHeader,
-    }
-) {
-    const systemMessages = messages
-        .filter((message) => message.role === "system")
-        .map((message) => message.content);
-
+function createPrompt(messages, hasDefaultHeader, hasDefaultFooter) {
    let fullPrompt = "";

-    if (promptHeader) {
-        fullPrompt += promptHeader + "\n\n";
-    }
-
-    if (systemPromptTemplate) {
-        // if user specified a template for the system prompt, put all system messages in the template
-        let systemPrompt = "";
-
-        if (systemMessages.length > 0) {
-            systemPrompt += systemMessages.join("\n");
+    for (const message of messages) {
+        if (message.role === "system") {
+            const systemMessage = message.content + "\n";
+            fullPrompt += systemMessage;
        }
-
-        if (systemPrompt) {
-            fullPrompt +=
-                systemPromptTemplate.replace("%1", systemPrompt) + "\n";
-        }
-    } else if (defaultSystemPrompt) {
-        // otherwise, use the system prompt from the model config and ignore system messages
-        fullPrompt += defaultSystemPrompt + "\n\n";
    }
-
-    if (systemMessages.length > 0 && !systemPromptTemplate) {
-        console.warn(
-            "System messages were provided, but no systemPromptTemplate was specified. System messages will be ignored."
-        );
+    if (hasDefaultHeader) {
+        fullPrompt += `### Instruction: 
+        The prompt below is a question to answer, a task to complete, or a conversation 
+        to respond to; decide which and write an appropriate response.
+        \n### Prompt: 
+        `;
    }
-
    for (const message of messages) {
        if (message.role === "user") {
-            const userMessage = promptTemplate.replace(
-                "%1",
-                message["content"]
-            );
-            fullPrompt += userMessage;
+            const user_message = "\n" + message["content"];
+            fullPrompt += user_message;
        }
        if (message["role"] == "assistant") {
-            const assistantMessage = message["content"] + "\n";
-            fullPrompt += assistantMessage;
+            const assistant_message = "\nResponse: " + message["content"];
+            fullPrompt += assistant_message;
        }
    }
-
-    if (promptFooter) {
-        fullPrompt += "\n\n" + promptFooter;
+    if (hasDefaultFooter) {
+        fullPrompt += "\n### Response:";
    }

    return fullPrompt;
 }

-function createEmbedding(model, text) {
-    return model.embed(text);
-}
-
-const defaultCompletionOptions = {
-    verbose: false,
-    ...DEFAULT_PROMPT_CONTEXT,
-};
-
 async function createCompletion(
-    model,
+    llmodel,
    messages,
-    options = defaultCompletionOptions
+    options = {
+        hasDefaultHeader: true,
+        hasDefaultFooter: false,
+        verbose: true,
+    }
 ) {
-    if (options.hasDefaultHeader !== undefined) {
-        console.warn(
-            "hasDefaultHeader (bool) is deprecated and has no effect, use promptHeader (string) instead"
-        );
+    //creating the keys to insert into promptMaker.
+    const fullPrompt = createPrompt(
+        messages,
+        options.hasDefaultHeader ?? true,
+        options.hasDefaultFooter
+    );
+    if (options.verbose) {
+        console.log("Sent: " + fullPrompt);
    }
-
-    if (options.hasDefaultFooter !== undefined) {
-        console.warn(
-            "hasDefaultFooter (bool) is deprecated and has no effect, use promptFooter (string) instead"
-        );
-    }
-
-    const optionsWithDefaults = {
-        ...defaultCompletionOptions,
-        ...options,
-    };
-
-    const {
-        verbose,
-        systemPromptTemplate,
-        promptTemplate,
-        promptHeader,
-        promptFooter,
-        ...promptContext
-    } = optionsWithDefaults;
-
-    const prompt = formatChatPrompt(messages, {
-        systemPromptTemplate,
-        defaultSystemPrompt: model.config.systemPrompt,
-        promptTemplate: promptTemplate || model.config.promptTemplate || "%1",
-        promptHeader: promptHeader || "",
-        promptFooter: promptFooter || "",
-        // These were the default header/footer prompts used for non-chat single turn completions.
-        // both seem to be working well still with some models, so keeping them here for reference.
-        // promptHeader: '### Instruction: The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.',
-        // promptFooter: '### Response:',
+    const promisifiedRawPrompt = new Promise((resolve, rej) => {
+        llmodel.raw_prompt(fullPrompt, options, (s) => {
+            resolve(s);
+        });
    });
-
-    if (verbose) {
-        console.debug("Sending Prompt:\n" + prompt);
-    }
-
-    const response = await model.generate(prompt, promptContext);
-
-    if (verbose) {
-        console.debug("Received Response:\n" + response);
-    }
-
-    return {
-        llmodel: model.llm.name(),
-        usage: {
-            prompt_tokens: prompt.length,
-            completion_tokens: response.length, //TODO
-            total_tokens: prompt.length + response.length, //TODO
-        },
-        choices: [
-            {
-                message: {
-                    role: "assistant",
-                    content: response,
-                },
+    return promisifiedRawPrompt.then((response) => {
+        return {
+            llmodel: llmodel.name(),
+            usage: {
+                prompt_tokens: fullPrompt.length,
+                completion_tokens: response.length, //TODO
+                total_tokens: fullPrompt.length + response.length, //TODO
            },
-        ],
-    };
-}
-
-function createTokenStream() {
-    throw Error("This API has not been completed yet!");
+            choices: [
+                {
+                    message: {
+                        role: "assistant",
+                        content: response,
+                    },
+                },
+            ],
+        };
+    });
 }

 module.exports = {
-    DEFAULT_LIBRARIES_DIRECTORY,
-    DEFAULT_DIRECTORY,
-    DEFAULT_PROMPT_CONTEXT,
-    DEFAULT_MODEL_CONFIG,
-    DEFAULT_MODEL_LIST_URL,
+    ...config,
    LLModel,
-    InferenceModel,
-    EmbeddingModel,
    createCompletion,
-    createEmbedding,
    downloadModel,
    retrieveModel,
    loadModel,
-    createTokenStream,
 };
--- a/gpt4all-bindings/typescript/src/models.js
+++ b/gpt4all-bindings/typescript/src/models.js
@@ -1,38 +0,0 @@
-const { normalizePromptContext, warnOnSnakeCaseKeys } = require('./util');
-
-class InferenceModel {
-    llm;
-    config;
-
-    constructor(llmodel, config) {
-        this.llm = llmodel;
-        this.config = config;
-    }
-
-    async generate(prompt, promptContext) {
-        warnOnSnakeCaseKeys(promptContext);
-        const normalizedPromptContext = normalizePromptContext(promptContext);
-        const result = this.llm.raw_prompt(prompt, normalizedPromptContext, () => {});
-        return result;
-    }
-}
-
-class EmbeddingModel {
-    llm;
-    config;
-
-    constructor(llmodel, config) {
-        this.llm = llmodel;
-        this.config = config;
-    }
-
-    embed(text) {
-        return this.llm.embed(text)
-    }
-}
-
-
-module.exports = {
-    InferenceModel,
-    EmbeddingModel,
-};
--- a/gpt4all-bindings/typescript/src/util.d.ts
+++ b/gpt4all-bindings/typescript/src/util.d.ts
@@ -0,0 +1,69 @@
+/// <reference types="node" />
+declare module "gpt4all";
+
+/**
+ * Initiates the download of a model file of a specific model type.
+ * By default this downloads without waiting. use the controller returned to alter this behavior.
+ * @param {ModelFile} model - The model file to be downloaded.
+ * @param {DownloadOptions} options - to pass into the downloader. Default is { location: (cwd), debug: false }.
+ * @returns {DownloadController} object that allows controlling the download process.
+ *
+ * @throws {Error} If the model already exists in the specified location.
+ * @throws {Error} If the model cannot be found at the specified url.
+ *
+ * @example
+ * const controller = download('ggml-gpt4all-j-v1.3-groovy.bin')
+ * controller.promise().then(() => console.log('Downloaded!'))
+ */
+declare function downloadModel(
+    modelName: string,
+    options?: DownloadModelOptions
+): DownloadController;
+
+/**
+ * Options for the model download process.
+ */
+export interface DownloadModelOptions {
+    /**
+     * location to download the model.
+     * Default is process.cwd(), or the current working directory
+     */
+    modelPath?: string;
+
+    /**
+     * Debug mode -- check how long it took to download in seconds
+     * @default false
+     */
+    debug?: boolean;
+
+    /**
+     * Remote download url. Defaults to `https://gpt4all.io/models`
+     * @default https://gpt4all.io/models
+     */
+    url?: string;
+}
+
+declare function listModels(): Promise<Record<string, string>[]>;
+
+interface RetrieveModelOptions {
+    allowDownload?: boolean;
+    verbose?: boolean;
+    modelPath?: string;
+}
+
+declare async function retrieveModel(
+    model: string,
+    options?: RetrieveModelOptions
+): Promise<string>;
+
+/**
+ * Model download controller.
+ */
+interface DownloadController {
+    /** Cancel the request to download from gpt4all website if this is called. */
+    cancel: () => void;
+    /** Convert the downloader into a promise, allowing people to await and manage its lifetime */
+    promise: () => Promise<void>;
+}
+
+export { downloadModel, DownloadModelOptions, DownloadController, listModels, retrieveModel, RetrieveModelOptions };
--- a/gpt4all-bindings/typescript/src/util.js
+++ b/gpt4all-bindings/typescript/src/util.js
@@ -1,45 +1,13 @@
-const { createWriteStream, existsSync, statSync } = require("node:fs");
-const fsp = require("node:fs/promises");
+const { createWriteStream, existsSync } = require("fs");
 const { performance } = require("node:perf_hooks");
 const path = require("node:path");
-const { mkdirp } = require("mkdirp");
-const md5File = require("md5-file");
-const {
-    DEFAULT_DIRECTORY,
-    DEFAULT_MODEL_CONFIG,
-    DEFAULT_MODEL_LIST_URL,
-} = require("./config.js");
+const {mkdirp} = require("mkdirp");
+const { DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY } = require("./config.js");

-async function listModels(
-    options = {
-        url: DEFAULT_MODEL_LIST_URL,
-    }
-) {
-    if (!options || (!options.url && !options.file)) {
-        throw new Error(
-            `No model list source specified. Please specify either a url or a file.`
-        );
-    }
-
-    if (options.file) {
-        if (!existsSync(options.file)) {
-            throw new Error(`Model list file ${options.file} does not exist.`);
-        }
-
-        const fileContents = await fsp.readFile(options.file, "utf-8");
-        const modelList = JSON.parse(fileContents);
-        return modelList;
-    } else if (options.url) {
-        const res = await fetch(options.url);
-
-        if (!res.ok) {
-            throw Error(
-                `Failed to retrieve model list from ${url} - ${res.status} ${res.statusText}`
-            );
-        }
-        const modelList = await res.json();
-        return modelList;
-    }
+async function listModels() {
+    const res = await fetch("https://gpt4all.io/models/models.json");
+    const modelList = await res.json();
+    return modelList;
 }

 function appendBinSuffixIfMissing(name) {
@@ -63,160 +31,76 @@ function readChunks(reader) {
    };
 }

-/**
- * Prints a warning if any keys in the prompt context are snake_case.
- */
-function warnOnSnakeCaseKeys(promptContext) {
-    const snakeCaseKeys = Object.keys(promptContext).filter((key) =>
-        key.includes("_")
-    );
-
-    if (snakeCaseKeys.length > 0) {
-        console.warn(
-            "Prompt context keys should be camelCase. Support for snake_case might be removed in the future. Found keys: " +
-                snakeCaseKeys.join(", ")
-        );
-    }
-}
-
-/**
- * Converts all keys in the prompt context to snake_case
- * For duplicate definitions, the value of the last occurrence will be used.
- */
-function normalizePromptContext(promptContext) {
-    const normalizedPromptContext = {};
-
-    for (const key in promptContext) {
-        if (promptContext.hasOwnProperty(key)) {
-            const snakeKey = key.replace(
-                /[A-Z]/g,
-                (match) => `_${match.toLowerCase()}`
-            );
-            normalizedPromptContext[snakeKey] = promptContext[key];
-        }
-    }
-
-    return normalizedPromptContext;
-}
-
-function downloadModel(modelName, options = {}) {
+function downloadModel(
+    modelName,
+    options = {}
+) {
    const downloadOptions = {
        modelPath: DEFAULT_DIRECTORY,
-        verbose: false,
+        debug: false,
+        url: "https://gpt4all.io/models",
        ...options,
    };
-    
+
    const modelFileName = appendBinSuffixIfMissing(modelName);
-    const partialModelPath = path.join(
-        downloadOptions.modelPath,
-        modelName + ".part"
-    );
-    const finalModelPath = path.join(downloadOptions.modelPath, modelFileName);
-    const modelUrl =
-        downloadOptions.url ?? `https://gpt4all.io/models/${modelFileName}`;
+    const fullModelPath = path.join(downloadOptions.modelPath, modelFileName);
+    const modelUrl = `${downloadOptions.url}/${modelFileName}`

-    mkdirp.sync(downloadOptions.modelPath)
-
-    if (existsSync(finalModelPath)) {
-        throw Error(`Model already exists at ${finalModelPath}`);
-    }
-    
-    if (downloadOptions.verbose) {
-        console.log(`Downloading ${modelName} from ${modelUrl}`);
-    }
-
-    const headers = {
-        "Accept-Ranges": "arraybuffer",
-        "Response-Type": "arraybuffer",
-    };
-
-    const writeStreamOpts = {};
-
-    if (existsSync(partialModelPath)) {
-        console.log("Partial model exists, resuming download...");
-        const startRange = statSync(partialModelPath).size;
-        headers["Range"] = `bytes=${startRange}-`;
-        writeStreamOpts.flags = "a";
+    if (existsSync(fullModelPath)) {
+        throw Error(`Model already exists at ${fullModelPath}`);
    }

    const abortController = new AbortController();
    const signal = abortController.signal;

-    const finalizeDownload = async () => {
-        if (options.md5sum) {
-            const fileHash = await md5File(partialModelPath);
-            if (fileHash !== options.md5sum) {
-                await fsp.unlink(partialModelPath);
-                const message = `Model "${modelName}" failed verification: Hashes mismatch. Expected ${options.md5sum}, got ${fileHash}`;
-                throw Error(message);
-            }
-            if (options.verbose) {
-                console.log(`MD5 hash verified: ${fileHash}`);
-            }
-        }
-
-        await fsp.rename(partialModelPath, finalModelPath);
-    };
-
-    // a promise that executes and writes to a stream. Resolves to the path the model was downloaded to when done writing.
-    const downloadPromise = new Promise((resolve, reject) => {
-        let timestampStart;
-
-        if (options.verbose) {
-            console.log(`Downloading @ ${partialModelPath} ...`);
-            timestampStart = performance.now();
-        }
-
-        const writeStream = createWriteStream(
-            partialModelPath,
-            writeStreamOpts
-        );
-
-        writeStream.on("error", (e) => {
-            writeStream.close();
-            reject(e);
-        });
-
-        writeStream.on("finish", () => {
-            if (options.verbose) {
-                const elapsed = performance.now() - timestampStart;
-                console.log(`Finished. Download took ${elapsed.toFixed(2)} ms`);
-            }
-
-            finalizeDownload()
-                .then(() => {
-                    resolve(finalModelPath);
-                })
-                .catch(reject);
-        });
-
+    //wrapper function to get the readable stream from request
+    // const baseUrl = options.url ?? "https://gpt4all.io/models";
+    const fetchModel = () =>
        fetch(modelUrl, {
            signal,
-            headers,
-        })
-            .then((res) => {
-                if (!res.ok) {
-                    const message = `Failed to download model from ${modelUrl} - ${res.status} ${res.statusText}`;
-                    reject(Error(message));
+        }).then((res) => {
+            if (!res.ok) {
+                throw Error(`Failed to download model from ${modelUrl} - ${res.statusText}`);
+            }
+            return res.body.getReader();
+        });
+
+    //a promise that executes and writes to a stream. Resolves when done writing.
+    const res = new Promise((resolve, reject) => {
+        fetchModel()
+            //Resolves an array of a reader and writestream.
+            .then((reader) => [reader, createWriteStream(fullModelPath)])
+            .then(async ([readable, wstream]) => {
+                console.log("Downloading @ ", fullModelPath);
+                let perf;
+                if (options.debug) {
+                    perf = performance.now();
                }
-                return res.body.getReader();
-            })
-            .then(async (reader) => {
-                for await (const chunk of readChunks(reader)) {
-                    writeStream.write(chunk);
+                for await (const chunk of readChunks(readable)) {
+                    wstream.write(chunk);
                }
-                writeStream.end();
+                if (options.debug) {
+                    console.log(
+                        "Time taken: ",
+                        (performance.now() - perf).toFixed(2),
+                        " ms"
+                    );
+                }
+                resolve(fullModelPath);
            })
            .catch(reject);
    });

    return {
        cancel: () => abortController.abort(),
-        promise: downloadPromise,
+        promise: () => res,
    };
-}
+};

-async function retrieveModel(modelName, options = {}) {
+async function retrieveModel (
+    modelName,
+    options = {}
+) {
    const retrieveOptions = {
        modelPath: DEFAULT_DIRECTORY,
        allowDownload: true,
@@ -230,68 +114,43 @@ async function retrieveModel(modelName, options = {}) {
    const fullModelPath = path.join(retrieveOptions.modelPath, modelFileName);
    const modelExists = existsSync(fullModelPath);

-    let config = { ...DEFAULT_MODEL_CONFIG };
+    if (modelExists) {
+        return fullModelPath;
+    }

-    const availableModels = await listModels({
-        file: retrieveOptions.modelConfigFile,
-        url:
-            retrieveOptions.allowDownload &&
-            "https://gpt4all.io/models/models.json",
+    if (!retrieveOptions.allowDownload) {
+        throw Error(`Model does not exist at ${fullModelPath}`);
+    }
+
+    const availableModels = await listModels();
+    const foundModel = availableModels.find((model) => model.filename === modelFileName);
+
+    if (!foundModel) {
+        throw Error(`Model "${modelName}" is not available.`);
+    }
+
+    if (retrieveOptions.verbose) {
+        console.log(`Downloading ${modelName}...`);
+    }
+
+    const downloadController = downloadModel(modelName, {
+        modelPath: retrieveOptions.modelPath,
+        debug: retrieveOptions.verbose,
    });

-    const loadedModelConfig = availableModels.find(
-        (model) => model.filename === modelFileName
-    );
+    const downloadPath = await downloadController.promise();

-    if (loadedModelConfig) {
-        config = {
-            ...config,
-            ...loadedModelConfig,
-        };
-    } else {
-        // if there's no local modelConfigFile specified, and allowDownload is false, the default model config will be used.
-        // warning the user here because the model may not work as expected.
-        console.warn(
-            `Failed to load model config for ${modelName}. Using defaults.`
-        );
+    if (retrieveOptions.verbose) {
+        console.log(`Model downloaded to ${downloadPath}`);
    }

-    config.systemPrompt = config.systemPrompt.trim();
+    return downloadPath

-    if (modelExists) {
-        config.path = fullModelPath;
-
-        if (retrieveOptions.verbose) {
-            console.log(`Found ${modelName} at ${fullModelPath}`);
-        }
-    } else if (retrieveOptions.allowDownload) {
-
-        const downloadController = downloadModel(modelName, {
-            modelPath: retrieveOptions.modelPath,
-            verbose: retrieveOptions.verbose,
-            filesize: config.filesize,
-            url: config.url,
-            md5sum: config.md5sum,
-        });
-
-        const downloadPath = await downloadController.promise;
-        config.path = downloadPath;
-
-        if (retrieveOptions.verbose) {
-            console.log(`Model downloaded to ${downloadPath}`);
-        }
-    } else {
-        throw Error("Failed to retrieve model.");
-    }
-
-    return config;
 }

+
 module.exports = {
    appendBinSuffixIfMissing,
    downloadModel,
    retrieveModel,
-    listModels,
-    normalizePromptContext,
-    warnOnSnakeCaseKeys,
-};
+};
--- a/gpt4all-bindings/typescript/test/gpt4all.test.js
+++ b/gpt4all-bindings/typescript/test/gpt4all.test.js
@@ -1,246 +0,0 @@
-const path = require("node:path");
-const os = require("node:os");
-const fsp = require("node:fs/promises");
-const { existsSync } = require('node:fs');
-const { LLModel } = require("node-gyp-build")(path.resolve(__dirname, ".."));
-const {
-    listModels,
-    downloadModel,
-    appendBinSuffixIfMissing,
-    normalizePromptContext,
-} = require("../src/util.js");
-const {
-    DEFAULT_DIRECTORY,
-    DEFAULT_LIBRARIES_DIRECTORY,
-    DEFAULT_MODEL_LIST_URL,
-} = require("../src/config.js");
-const {
-    loadModel,
-    createPrompt,
-    createCompletion,
-} = require("../src/gpt4all.js");
-const { mock } = require("node:test");
-const { mkdirp } = require("mkdirp");
-
-describe("config", () => {
-    test("default paths constants are available and correct", () => {
-        expect(DEFAULT_DIRECTORY).toBe(
-            path.resolve(os.homedir(), ".cache/gpt4all")
-        );
-        const paths = [
-            path.join(DEFAULT_DIRECTORY, "libraries"),
-            path.resolve("./libraries"),
-            path.resolve(
-                __dirname,
-                "..",
-                `runtimes/${process.platform}-${process.arch}/native`
-            ),
-            process.cwd(),
-        ];
-        expect(typeof DEFAULT_LIBRARIES_DIRECTORY).toBe("string");
-        expect(DEFAULT_LIBRARIES_DIRECTORY).toBe(paths.join(";"));
-    });
-});
-
-describe("listModels", () => {
-    const fakeModels = require("./models.json");
-    const fakeModel = fakeModels[0];
-    const mockResponse = JSON.stringify([fakeModel]);
-
-    let mockFetch, originalFetch;
-
-    beforeAll(() => {
-        // Mock the fetch function for all tests
-        mockFetch = jest.fn().mockResolvedValue({
-            ok: true,
-            json: () => JSON.parse(mockResponse),
-        });
-        originalFetch = global.fetch;
-        global.fetch = mockFetch;
-    });
-
-    afterEach(() => {
-        // Reset the fetch counter after each test
-        mockFetch.mockClear();
-    });
-    afterAll(() => {
-        // Restore fetch
-        global.fetch = originalFetch;
-    });
-
-    it("should load the model list from remote when called without args", async () => {
-        const models = await listModels();
-        expect(fetch).toHaveBeenCalledTimes(1);
-        expect(fetch).toHaveBeenCalledWith(DEFAULT_MODEL_LIST_URL);
-        expect(models[0]).toEqual(fakeModel);
-    });
-
-    it("should load the model list from a local file, if specified", async () => {
-        const models = await listModels({
-            file: path.resolve(__dirname, "models.json"),
-        });
-        expect(fetch).toHaveBeenCalledTimes(0);
-        expect(models[0]).toEqual(fakeModel);
-    });
-    
-    it("should throw an error if neither url nor file is specified", async () => {
-        await expect(listModels(null)).rejects.toThrow(
-            "No model list source specified. Please specify either a url or a file."
-        );
-    });
-});
-
-describe("appendBinSuffixIfMissing", () => {
-    it("should make sure the suffix is there", () => {
-        expect(appendBinSuffixIfMissing("filename")).toBe("filename.bin");
-        expect(appendBinSuffixIfMissing("filename.bin")).toBe("filename.bin");
-    });
-});
-
-describe("downloadModel", () => {
-    let mockAbortController, mockFetch;
-    const fakeModelName = "fake-model";
-
-    const createMockFetch = () => {
-        const mockData = new Uint8Array([1, 2, 3, 4]);
-        const mockResponse = new ReadableStream({
-            start(controller) {
-                controller.enqueue(mockData);
-                controller.close();
-            },
-        });
-        const mockFetchImplementation = jest.fn(() =>
-            Promise.resolve({
-                ok: true,
-                body: mockResponse,
-            })
-        );
-        return mockFetchImplementation;
-    };
-
-    beforeEach(async () => {
-        // Mocking the AbortController constructor
-        mockAbortController = jest.fn();
-        global.AbortController = mockAbortController;
-        mockAbortController.mockReturnValue({
-            signal: "signal",
-            abort: jest.fn(),
-        });
-        mockFetch = createMockFetch();
-        jest.spyOn(global, "fetch").mockImplementation(mockFetch);
-
-    });
-
-    afterEach(async () => {
-        // Clean up mocks
-        mockAbortController.mockReset();
-        mockFetch.mockClear();
-        global.fetch.mockRestore();
-        
-        const rootDefaultPath = path.resolve(DEFAULT_DIRECTORY),
-              partialPath = path.resolve(rootDefaultPath, fakeModelName+'.part'),
-              fullPath = path.resolve(rootDefaultPath, fakeModelName+'.bin')  
-
-        //if tests fail, remove the created files
-        // acts as cleanup if tests fail
-        //
-        if(existsSync(fullPath)) {
-            await fsp.rm(fullPath)
-        }
-        if(existsSync(partialPath)) {
-            await fsp.rm(partialPath)
-        }
-
-    });
-
-    test("should successfully download a model file", async () => {
-        const downloadController = downloadModel(fakeModelName);
-        const modelFilePath = await downloadController.promise;
-        expect(modelFilePath).toBe(path.resolve(DEFAULT_DIRECTORY, `${fakeModelName}.bin`));
-
-        expect(global.fetch).toHaveBeenCalledTimes(1);
-        expect(global.fetch).toHaveBeenCalledWith(
-            "https://gpt4all.io/models/fake-model.bin",
-            {
-                signal: "signal",
-                headers: {
-                    "Accept-Ranges": "arraybuffer",
-                    "Response-Type": "arraybuffer",
-                },
-            }
-        );
-
-        // final model file should be present
-        await expect(fsp.access(modelFilePath)).resolves.not.toThrow();
-
-        // remove the testing model file
-        await fsp.unlink(modelFilePath);
-    });
-
-    test("should error and cleanup if md5sum is not matching", async () => {
-        const downloadController = downloadModel(fakeModelName, {
-            md5sum: "wrong-md5sum",
-        });
-        // the promise should reject with a mismatch
-        await expect(downloadController.promise).rejects.toThrow(
-            `Model "fake-model" failed verification: Hashes mismatch. Expected wrong-md5sum, got 08d6c05a21512a79a1dfeb9d2a8f262f`
-        );
-        // fetch should have been called
-        expect(global.fetch).toHaveBeenCalledTimes(1);
-        // the file should be missing
-        await expect(
-            fsp.access(path.resolve(DEFAULT_DIRECTORY, `${fakeModelName}.bin`))
-        ).rejects.toThrow();
-        // partial file should also be missing
-        await expect(
-            fsp.access(path.resolve(DEFAULT_DIRECTORY, `${fakeModelName}.part`))
-        ).rejects.toThrow();
-    });
-
-    // TODO
-    // test("should be able to cancel and resume a download", async () => {
-    // });
-});
-
-describe("normalizePromptContext", () => {
-    it("should convert a dict with camelCased keys to snake_case", () => {
-        const camelCased = {
-            topK: 20,
-            repeatLastN: 10,
-        };
-
-        const expectedSnakeCased = {
-            top_k: 20,
-            repeat_last_n: 10,
-        };
-
-        const result = normalizePromptContext(camelCased);
-        expect(result).toEqual(expectedSnakeCased);
-    });
-
-    it("should convert a mixed case dict to snake_case, last value taking precedence", () => {
-        const mixedCased = {
-            topK: 20,
-            top_k: 10,
-            repeatLastN: 10,
-        };
-
-        const expectedSnakeCased = {
-            top_k: 10,
-            repeat_last_n: 10,
-        };
-
-        const result = normalizePromptContext(mixedCased);
-        expect(result).toEqual(expectedSnakeCased);
-    });
-
-    it("should not modify already snake cased dict", () => {
-        const snakeCased = {
-            top_k: 10,
-            repeast_last_n: 10,
-        };
-
-        const result = normalizePromptContext(snakeCased);
-        expect(result).toEqual(snakeCased);
-    });
-});
--- a/gpt4all-bindings/typescript/test/index.mjs
+++ b/gpt4all-bindings/typescript/test/index.mjs
@@ -0,0 +1,8 @@
+import * as assert from 'node:assert'
+import { download } from '../src/gpt4all.js'
+
+
+assert.rejects(async () => download('poo.bin').promise());
+
+
+console.log('OK')
--- a/gpt4all-bindings/typescript/test/models.json
+++ b/gpt4all-bindings/typescript/test/models.json
@@ -1,10 +0,0 @@
-[
-  {
-    "order": "a",
-    "md5sum": "08d6c05a21512a79a1dfeb9d2a8f262f",
-    "name": "Not a real model",
-    "filename": "fake-model.bin",
-    "filesize": "4",
-    "systemPrompt": " "
-  }
-]
--- a/gpt4all-bindings/typescript/yarn.lock
+++ b/gpt4all-bindings/typescript/yarn.lock
--- a/gpt4all-chat/CMakeLists.txt
+++ b/gpt4all-chat/CMakeLists.txt
@@ -18,7 +18,7 @@ endif()

 set(APP_VERSION_MAJOR 2)
 set(APP_VERSION_MINOR 4)
-set(APP_VERSION_PATCH 20)
+set(APP_VERSION_PATCH 14)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")

 # Include the binary directory for the generated header file
@@ -180,10 +180,18 @@ install(TARGETS llmodel DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})

 # We should probably iterate through the list of the cmake for backend, but these need to be installed
 # to the this component's dir for the finicky qt installer to work
-#install(TARGETS gptj-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-#install(TARGETS gptj-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS gptj-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS gptj-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS llama-230511-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS llama-230511-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS llama-230519-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS llama-230519-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llama-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llama-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS llamamodel-230511-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS llamamodel-230511-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS llamamodel-230519-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS llamamodel-230519-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llamamodel-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llamamodel-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 if(APPLE)
@@ -191,8 +199,8 @@ install(TARGETS llamamodel-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_
 endif()
 install(TARGETS falcon-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS falcon-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-#install(TARGETS mpt-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-#install(TARGETS mpt-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS mpt-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
+install(TARGETS mpt-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS replit-mainline-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS replit-mainline-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 if(APPLE)
@@ -200,8 +208,6 @@ install(TARGETS replit-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME
 endif()
 install(TARGETS bert-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS bert-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS starcoder-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS starcoder-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})

 set(CPACK_GENERATOR "IFW")
 set(CPACK_VERBATIM_VARIABLES YES)
@@ -259,11 +265,7 @@ set(CPACK_IFW_PACKAGE_WIZARD_SHOW_PAGE_LIST OFF)
 include(InstallRequiredSystemLibraries)
 include(CPack)
 include(CPackIFW)
-if(GPT4ALL_OFFLINE_INSTALLER)
-  cpack_add_component(${COMPONENT_NAME_MAIN})
-else()
-  cpack_add_component(${COMPONENT_NAME_MAIN} DOWNLOADED)
-endif()
+cpack_add_component(${COMPONENT_NAME_MAIN} DOWNLOADED)
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} ESSENTIAL FORCED_INSTALLATION)
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} VERSION ${APP_VERSION})
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} LICENSES "MIT LICENSE" ${CPACK_RESOURCE_FILE_LICENSE})
@@ -273,7 +275,7 @@ cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} REPLACES "gpt4all-chat") #W
 if (GPT4ALL_LOCALHOST)
    cpack_ifw_add_repository("GPT4AllRepository" URL "http://localhost/repository")
 elseif(GPT4ALL_OFFLINE_INSTALLER)
-  # noop
+    cpack_ifw_add_repository("GPT4AllRepository" URL "file://${CMAKE_BINARY_DIR}/packages")
 else()
  if(${CMAKE_SYSTEM_NAME} MATCHES Linux)
      cpack_ifw_add_repository("GPT4AllRepository" URL "https://gpt4all.io/installer_repos/linux/repository")
--- a/gpt4all-chat/README.md
+++ b/gpt4all-chat/README.md
@@ -40,7 +40,6 @@ One click installers for macOS, Linux, and Windows at https://gpt4all.io
 * Syntax highlighting support for programming languages, etc.
 * REST API with a built-in webserver in the chat gui itself with a headless operation mode as well
 * Advanced settings for changing temperature, topk, etc. (DONE)
-* * Improve the accessibility of the installer for screen reader users
 * YOUR IDEA HERE

 ## Building and running
--- a/gpt4all-chat/chat.cpp
+++ b/gpt4all-chat/chat.cpp
@@ -56,7 +56,6 @@ void Chat::connectLLM()
    connect(m_llmodel, &ChatLLM::recalcChanged, this, &Chat::handleRecalculating, Qt::QueuedConnection);
    connect(m_llmodel, &ChatLLM::generatedNameChanged, this, &Chat::generatedNameChanged, Qt::QueuedConnection);
    connect(m_llmodel, &ChatLLM::reportSpeed, this, &Chat::handleTokenSpeedChanged, Qt::QueuedConnection);
-    connect(m_llmodel, &ChatLLM::reportDevice, this, &Chat::handleDeviceChanged, Qt::QueuedConnection);
    connect(m_llmodel, &ChatLLM::databaseResultsChanged, this, &Chat::handleDatabaseResultsChanged, Qt::QueuedConnection);
    connect(m_llmodel, &ChatLLM::modelInfoChanged, this, &Chat::handleModelInfoChanged, Qt::QueuedConnection);

@@ -346,12 +345,6 @@ void Chat::handleTokenSpeedChanged(const QString &tokenSpeed)
    emit tokenSpeedChanged();
 }

-void Chat::handleDeviceChanged(const QString &device)
-{
-    m_device = device;
-    emit deviceChanged();
-}
-
 void Chat::handleDatabaseResultsChanged(const QList<ResultInfo> &results)
 {
    m_databaseResults = results;
--- a/gpt4all-chat/chat.h
+++ b/gpt4all-chat/chat.h
@@ -25,7 +25,6 @@ class Chat : public QObject
    Q_PROPERTY(QList<QString> collectionList READ collectionList NOTIFY collectionListChanged)
    Q_PROPERTY(QString modelLoadingError READ modelLoadingError NOTIFY modelLoadingErrorChanged)
    Q_PROPERTY(QString tokenSpeed READ tokenSpeed NOTIFY tokenSpeedChanged);
-    Q_PROPERTY(QString device READ device NOTIFY deviceChanged);
    QML_ELEMENT
    QML_UNCREATABLE("Only creatable from c++!")

@@ -89,7 +88,6 @@ public:
    QString modelLoadingError() const { return m_modelLoadingError; }

    QString tokenSpeed() const { return m_tokenSpeed; }
-    QString device() const { return m_device; }

 public Q_SLOTS:
    void serverNewPromptResponsePair(const QString &prompt);
@@ -117,7 +115,6 @@ Q_SIGNALS:
    void isServerChanged();
    void collectionListChanged(const QList<QString> &collectionList);
    void tokenSpeedChanged();
-    void deviceChanged();

 private Q_SLOTS:
    void handleResponseChanged(const QString &response);
@@ -128,7 +125,6 @@ private Q_SLOTS:
    void handleRecalculating();
    void handleModelLoadingError(const QString &error);
    void handleTokenSpeedChanged(const QString &tokenSpeed);
-    void handleDeviceChanged(const QString &device);
    void handleDatabaseResultsChanged(const QList<ResultInfo> &results);
    void handleModelInfoChanged(const ModelInfo &modelInfo);
    void handleModelInstalled();
@@ -141,7 +137,6 @@ private:
    ModelInfo m_modelInfo;
    QString m_modelLoadingError;
    QString m_tokenSpeed;
-    QString m_device;
    QString m_response;
    QList<QString> m_collections;
    ChatModel *m_chatModel;
--- a/gpt4all-chat/chatllm.cpp
+++ b/gpt4all-chat/chatllm.cpp
@@ -15,7 +15,6 @@
 #define LLAMA_INTERNAL_STATE_VERSION 0
 #define FALCON_INTERNAL_STATE_VERSION 0
 #define BERT_INTERNAL_STATE_VERSION 0
-#define STARCODER_INTERNAL_STATE_VERSION 0

 class LLModelStore {
 public:
@@ -81,7 +80,6 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
    connect(parent, &Chat::idChanged, this, &ChatLLM::handleChatIdChanged);
    connect(&m_llmThread, &QThread::started, this, &ChatLLM::handleThreadStarted);
    connect(MySettings::globalInstance(), &MySettings::forceMetalChanged, this, &ChatLLM::handleForceMetalChanged);
-    connect(MySettings::globalInstance(), &MySettings::deviceChanged, this, &ChatLLM::handleDeviceChanged);

    // The following are blocking operations and will block the llm thread
    connect(this, &ChatLLM::requestRetrieveFromDB, LocalDocs::globalInstance()->database(), &Database::retrieveFromDB,
@@ -125,16 +123,6 @@ void ChatLLM::handleForceMetalChanged(bool forceMetal)
 #endif
 }

-void ChatLLM::handleDeviceChanged()
-{
-    if (isModelLoaded() && m_shouldBeLoaded) {
-        m_reloadingToChangeVariant = true;
-        unloadModel();
-        reloadModel();
-        m_reloadingToChangeVariant = false;
-    }
-}
-
 bool ChatLLM::loadDefaultModel()
 {
    ModelInfo defaultModel = ModelList::globalInstance()->defaultModelInfo();
@@ -261,52 +249,16 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 #endif

            if (m_llModelInfo.model) {
-                // Update the settings that a model is being loaded and update the device list
                MySettings::globalInstance()->setAttemptModelLoad(filePath);
-
-                // Pick the best match for the device
-                QString actualDevice = m_llModelInfo.model->implementation().buildVariant() == "metal" ? "Metal" : "CPU";
-                const QString requestedDevice = MySettings::globalInstance()->device();
-                if (requestedDevice != "CPU") {
-                    const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString());
-                    std::vector<LLModel::GPUDevice> availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
-                    if (!availableDevices.empty() && requestedDevice == "Auto" && availableDevices.front().type == 2 /*a discrete gpu*/) {
-                        m_llModelInfo.model->initializeGPUDevice(availableDevices.front());
-                        actualDevice = QString::fromStdString(availableDevices.front().name);
-                    } else {
-                        for (LLModel::GPUDevice &d : availableDevices) {
-                            if (QString::fromStdString(d.name) == requestedDevice) {
-                                m_llModelInfo.model->initializeGPUDevice(d);
-                                actualDevice = QString::fromStdString(d.name);
-                                break;
-                            }
-                        }
-                    }
-                }
-
-                // Report which device we're actually using
-                emit reportDevice(actualDevice);
-
                bool success = m_llModelInfo.model->loadModel(filePath.toStdString());
-                if (!success && actualDevice != "CPU") {
-                    emit reportDevice("CPU");
-                    success = m_llModelInfo.model->loadModel(filePath.toStdString());
-                }
-
                MySettings::globalInstance()->setAttemptModelLoad(QString());
                if (!success) {
-                    delete m_llModelInfo.model;
-                    m_llModelInfo.model = nullptr;
+                    delete std::exchange(m_llModelInfo.model, nullptr);
                    if (!m_isServer)
                        LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
                    m_llModelInfo = LLModelInfo();
                    emit modelLoadingError(QString("Could not load model due to invalid model file for %1").arg(modelInfo.filename()));
                } else {
-                    // We might have had to fallback to CPU after load if the model is not possible to accelerate
-                    // for instance if the quantization method is not supported on Vulkan yet
-                    if (actualDevice != "CPU" && !m_llModelInfo.model->usingGPUDevice())
-                        emit reportDevice("CPU");
-
                    switch (m_llModelInfo.model->implementation().modelType()[0]) {
                    case 'L': m_llModelType = LLModelType::LLAMA_; break;
                    case 'G': m_llModelType = LLModelType::GPTJ_; break;
@@ -314,11 +266,9 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                    case 'R': m_llModelType = LLModelType::REPLIT_; break;
                    case 'F': m_llModelType = LLModelType::FALCON_; break;
                    case 'B': m_llModelType = LLModelType::BERT_; break;
-                    case 'S': m_llModelType = LLModelType::STARCODER_; break;
                    default:
                        {
-                            delete m_llModelInfo.model;
-                            m_llModelInfo.model = nullptr;
+                            delete std::exchange(m_llModelInfo.model, nullptr);
                            if (!m_isServer)
                                LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
                            m_llModelInfo = LLModelInfo();
@@ -449,7 +399,7 @@ bool ChatLLM::handlePrompt(int32_t token)
 #endif
    ++m_promptTokens;
    ++m_promptResponseTokens;
-    m_timer->start();
+    m_timer->inc();
    return !m_stopGenerating;
 }

@@ -723,7 +673,6 @@ bool ChatLLM::serialize(QDataStream &stream, int version)
        case LLAMA_: stream << LLAMA_INTERNAL_STATE_VERSION; break;
        case FALCON_: stream << FALCON_INTERNAL_STATE_VERSION; break;
        case BERT_: stream << BERT_INTERNAL_STATE_VERSION; break;
-        case STARCODER_: stream << STARCODER_INTERNAL_STATE_VERSION; break;
        default: Q_UNREACHABLE();
        }
    }
--- a/gpt4all-chat/chatllm.h
+++ b/gpt4all-chat/chatllm.h
@@ -16,8 +16,7 @@ enum LLModelType {
    CHATGPT_,
    REPLIT_,
    FALCON_,
-    BERT_,
-    STARCODER_
+    BERT_
 };

 struct LLModelInfo {
@@ -111,7 +110,6 @@ public Q_SLOTS:
    void handleShouldBeLoadedChanged();
    void handleThreadStarted();
    void handleForceMetalChanged(bool forceMetal);
-    void handleDeviceChanged();
    void processSystemPrompt();

 Q_SIGNALS:
@@ -129,7 +127,6 @@ Q_SIGNALS:
    void shouldBeLoadedChanged();
    void requestRetrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
    void reportSpeed(const QString &speed);
-    void reportDevice(const QString &device);
    void databaseResultsChanged(const QList<ResultInfo>&);
    void modelInfoChanged(const ModelInfo &modelInfo);

--- a/gpt4all-chat/cmake/deploy-qt-mac.cmake.in
+++ b/gpt4all-chat/cmake/deploy-qt-mac.cmake.in
@@ -8,7 +8,6 @@ file(GLOB MYLLAMALIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_
 file(GLOB MYREPLITLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libreplit*)
 file(GLOB MYFALCONLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libfalcon*)
 file(GLOB MYBERTLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libbert*)
-file(GLOB MYSTARCODERLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libstarcoder*)
 file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllmodel.*)
 file(COPY ${MYGPTJLIBS}
     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
@@ -22,10 +21,6 @@ file(COPY ${MYFALCONLLIBS}
     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYBERTLLIBS}
     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYSTARCODERLLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYLLAMALIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLMODELLIBS}
     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns"
--- a/gpt4all-chat/main.qml
+++ b/gpt4all-chat/main.qml
@@ -313,7 +313,6 @@ Window {
                Label {
                    anchors.verticalCenter: parent.verticalCenter
                    text: qsTr("Loading model...")
-                    font.pixelSize: theme.fontSizeLarge
                    color: theme.textAccent
                }
            }
@@ -472,21 +471,18 @@ Window {
        id: copyMessage
        anchors.centerIn: parent
        text: qsTr("Conversation copied to clipboard.")
-        font.pixelSize: theme.fontSizeLarge
    }

    PopupDialog {
        id: copyCodeMessage
        anchors.centerIn: parent
        text: qsTr("Code copied to clipboard.")
-        font.pixelSize: theme.fontSizeLarge
    }

    PopupDialog {
        id: healthCheckFailed
        anchors.centerIn: parent
        text: qsTr("Connection to datalake failed.")
-        font.pixelSize: theme.fontSizeLarge
    }

    PopupDialog {
@@ -495,7 +491,6 @@ Window {
        shouldTimeOut: false
        shouldShowBusy: true
        text: qsTr("Recalculating context.")
-        font.pixelSize: theme.fontSizeLarge

        Connections {
            target: currentChat
@@ -514,7 +509,6 @@ Window {
        shouldTimeOut: false
        shouldShowBusy: true
        text: qsTr("Saving chats.")
-        font.pixelSize: theme.fontSizeLarge
    }

    MyToolButton {
@@ -620,7 +614,6 @@ Window {
                   If you can't start it manually, then I'm afraid you'll have to<br>
                   reinstall.")
            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
            Accessible.role: Accessible.Dialog
            Accessible.name: text
            Accessible.description: qsTr("Dialog indicating an error")
@@ -682,7 +675,7 @@ Window {
            anchors.top: parent.top
            anchors.bottom: !currentChat.isServer ? textInputView.top : parent.bottom
            anchors.bottomMargin: !currentChat.isServer ? 30 : 0
-            ScrollBar.vertical.policy: ScrollBar.AlwaysOff
+            ScrollBar.vertical.policy: ScrollBar.AlwaysOn

            Rectangle {
                anchors.fill: parent
@@ -692,7 +685,6 @@ Window {
                    id: warningLabel
                    text: qsTr("You must install a model to continue. Models are available via the download dialog or you can install them manually by downloading from <a href=\"https://gpt4all.io\">the GPT4All website</a> (look for the Models Explorer) and placing them in the model folder. The model folder can be found in the settings dialog under the application tab.")
                    color: theme.textColor
-                    font.pixelSize: theme.fontSizeLarge
                    width: 600
                    linkColor: theme.linkColor
                    wrapMode: Text.WordWrap
@@ -737,7 +729,6 @@ Window {
                    visible: ModelList.installedModels.count !== 0
                    anchors.fill: parent
                    model: chatModel
-                    ScrollBar.vertical: ScrollBar { policy: ScrollBar.AlwaysOn }

                    Accessible.role: Accessible.List
                    Accessible.name: qsTr("List of prompt/response pairs")
@@ -1006,15 +997,13 @@ Window {
        }

        Text {
-            id: device
+            id: speed
            anchors.bottom: textInputView.top
            anchors.bottomMargin: 20
            anchors.right: parent.right
            anchors.rightMargin: 30
            color: theme.mutedTextColor
-            visible: currentChat.tokenSpeed !== ""
-            text: qsTr("Speed: ") + currentChat.tokenSpeed + "<br>" + qsTr("Device: ") + currentChat.device
-            font.pixelSize: theme.fontSizeLarge
+            text: currentChat.tokenSpeed
        }

        RectangularGlow {
--- a/gpt4all-chat/metadata/models.json
+++ b/gpt4all-chat/metadata/models.json
@@ -44,6 +44,19 @@
    "url": "https://huggingface.co/TheBloke/Nous-Hermes-13B-GGML/resolve/main/nous-hermes-13b.ggmlv3.q4_0.bin",
    "promptTemplate": "### Instruction:\n%1\n### Response:\n"
  },
+  {
+    "order": "e",
+    "md5sum": "81a09a0ddf89690372fc296ff7f625af",
+    "name": "Groovy",
+    "filename": "ggml-gpt4all-j-v1.3-groovy.bin",
+    "filesize": "3785248281",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "GPT-J",
+    "systemPrompt": " ",
+    "description": "<strong>Creative model can be used for commercial purposes</strong><br><ul><li>Fast responses<li>Creative responses</li><li>Instruction based</li><li>Trained by Nomic AI<li>Licensed for commercial use</ul>"
+  },
  {
    "order": "f",
    "md5sum": "11d9f060ca24575a2c303bdc39952486",
@@ -59,6 +72,21 @@
    "description": "<strong>Very good overall model</strong><br><ul><li>Instruction based<li>Based on the same dataset as Groovy<li>Slower than Groovy, with higher quality responses<li>Trained by Nomic AI<li>Cannot be used commercially</ul>",
    "url": "https://huggingface.co/TheBloke/GPT4All-13B-snoozy-GGML/resolve/main/GPT4All-13B-snoozy.ggmlv3.q4_0.bin"
  },
+  {
+    "order": "g",
+    "md5sum": "756249d3d6abe23bde3b1ae272628640",
+    "name": "MPT Chat",
+    "filename": "ggml-mpt-7b-chat.bin",
+    "filesize": "4854401050",
+    "requires": "2.4.1",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "MPT",
+    "description": "<strong>Best overall smaller model</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|><|im_start|>assistant\n",
+    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
+  },
  {
    "order": "h",
    "md5sum": "e64e74375ce9d36a3d0af3db1523fd0a",
@@ -107,6 +135,99 @@
    "promptTemplate": "### User:\n%1\n### Response:\n",
    "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
  },
+  {
+    "order": "k",
+    "md5sum": "29119f8fa11712704c6b22ac5ab792ea",
+    "name": "Vicuna",
+    "filename": "ggml-vicuna-7b-1.1-q4_2.bin",
+    "filesize": "4212859520",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_2",
+    "type": "LLaMA",
+    "systemPrompt": " ",
+    "description": "<strong>Good small model - trained by teams from UC Berkeley, CMU, Stanford, MBZUAI, and UC San Diego</strong><br><ul><li>Instruction based<li>Cannot be used commercially</ul>"
+  },
+  {
+    "order": "l",
+    "md5sum": "95999b7b0699e2070af63bf5d34101a8",
+    "name": "Vicuna (large)",
+    "filename": "ggml-vicuna-13b-1.1-q4_2.bin",
+    "filesize": "8136770688",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_2",
+    "type": "LLaMA",
+    "systemPrompt": " ",
+    "description": "<strong>Good larger model - trained by teams from UC Berkeley, CMU, Stanford, MBZUAI, and UC San Diego</strong><br><ul><li>Instruction based<li>Cannot be used commercially</ul>"
+  },
+  {
+    "order": "m",
+    "md5sum": "99e6d129745a3f1fb1121abed747b05a",
+    "name": "Wizard",
+    "filename": "ggml-wizardLM-7B.q4_2.bin",
+    "filesize": "4212864640",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_2",
+    "type": "LLaMA",
+    "systemPrompt": " ",
+    "description": "<strong>Good small model - trained by by Microsoft and Peking University</strong><br><ul><li>Instruction based<li>Cannot be used commercially</ul>"
+  },
+  {
+    "order": "n",
+    "md5sum": "6cb4ee297537c9133bddab9692879de0",
+    "name": "Stable Vicuna",
+    "filename": "ggml-stable-vicuna-13B.q4_2.bin",
+    "filesize": "8136777088",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_2",
+    "type": "LLaMA",
+    "description": "<strong>Trained with RLHF by Stability AI</strong><br><ul><li>Instruction based<li>Cannot be used commercially</ul>",
+    "systemPrompt": "## Assistant: I am StableVicuna, a large language model created by CarperAI. I am here to chat!\n\n"
+  },
+  {
+    "order": "o",
+    "md5sum": "1cfa4958f489f0a0d1ffdf6b37322809",
+    "name": "MPT Instruct",
+    "filename": "ggml-mpt-7b-instruct.bin",
+    "filesize": "4854401028",
+    "requires": "2.4.1",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "MPT",
+    "systemPrompt": " ",
+    "description": "<strong>Mosaic's instruction model</strong><br><ul><li>Instruction based<li>Trained by Mosaic ML<li>Licensed for commercial use</ul>"
+  },
+  {
+    "order": "p",
+    "md5sum": "120c32a51d020066288df045ef5d52b9",
+    "name": "MPT Base",
+    "filename": "ggml-mpt-7b-base.bin",
+    "filesize": "4854401028",
+    "requires": "2.4.1",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "MPT",
+    "systemPrompt": " ",
+    "description": "<strong>Trained for text completion with no assistant finetuning</strong><br><ul><li>Completion based<li>Trained by Mosaic ML<li>Licensed for commercial use</ul>"
+  },
+  {
+    "order": "q",
+    "md5sum": "d5eafd5b0bd0d615cfd5fd763f642dfe",
+    "name": "Nous Vicuna",
+    "filename": "ggml-nous-gpt4-vicuna-13b.bin",
+    "filesize": "8136777088",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_0",
+    "type": "LLaMA",
+    "systemPrompt": " ",
+    "description": "<strong>Trained on ~180,000 instructions</strong><br><ul><li>Instruction based<li>Trained by Nous Research<li>Cannot be used commercially</ul>"
+  },
  {
    "order": "r",
    "md5sum": "489d21fd48840dcb31e5f92f453f3a20",
@@ -135,7 +256,6 @@
    "quant": "f16",
    "type": "Replit",
    "systemPrompt": " ",
-    "promptTemplate": "%1",
    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>Licensed for commercial use</ul>",
    "url": "https://huggingface.co/nomic-ai/ggml-replit-code-v1-3b/resolve/main/ggml-replit-code-v1-3b.bin"
  },
@@ -153,53 +273,5 @@
    "type": "Bert",
    "systemPrompt": " ",
    "description": "<strong>Sbert</strong><br><ul><li>For embeddings"
-  },
-  {
-    "order": "u",
-    "md5sum": "379ee1bab9a7a9c27c2314daa097528e",
-    "disableGUI": "true",
-    "name": "Starcoder (Small)",
-    "filename": "starcoderbase-3b-ggml.bin",
-    "filesize": "7503121552",
-    "requires": "2.4.14",
-    "ramrequired": "8",
-    "parameters": "3 billion",
-    "quant": "f16",
-    "type": "Starcoder",
-    "systemPrompt": " ",
-    "promptTemplate": "%1",
-    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based</ul>"
-  },
-  {
-    "order": "w",
-    "md5sum": "f981ab8fbd1ebbe4932ddd667c108ba7",
-    "disableGUI": "true",
-    "name": "Starcoder",
-    "filename": "starcoderbase-7b-ggml.bin",
-    "filesize": "17860448016",
-    "requires": "2.4.14",
-    "ramrequired": "16",
-    "parameters": "7 billion",
-    "quant": "f16",
-    "type": "Starcoder",
-    "systemPrompt": " ",
-    "promptTemplate": "%1",
-    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based</ul>"
-  },
-  {
-    "order": "w",
-    "md5sum": "c7ebc61eec1779bddae1f2bcbf2007cc",
-    "name": "Llama-2-7B Chat",
-    "filename": "llama-2-7b-chat.ggmlv3.q4_0.bin",
-    "filesize": "3791725184",
-    "requires": "2.4.14",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "LLaMA2",
-    "description": "<strong>New LLaMA2 model from Meta AI.</strong><br><ul><li>Fine-tuned for dialogue.<li>static model trained on an offline dataset<li>RLHF dataset<li>Licensed for commercial use</ul>",
-    "url": "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/resolve/main/llama-2-7b-chat.ggmlv3.q4_0.bin",
-    "promptTemplate": "[INST] %1 [/INST] ",
-    "systemPrompt": "[INST]<<SYS>>You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.<</SYS>>[/INST] "
  }
 ]
--- a/gpt4all-chat/metadata/release.json
+++ b/gpt4all-chat/metadata/release.json
@@ -450,85 +450,6 @@
 * Aaron Miller (Nomic AI)
 * Adam Treat (Nomic AI)
 * Community (beta testers, bug reporters)
-"
-  },
-  {
-    "version": "2.4.14",
-    "notes":
-"
-* Add starcoder model support
-* Add ability to switch between light mode/dark mode
-* Increase the size of fonts in monospace code blocks a bit
-",
-    "contributors":
-"
-* Lakshay Kansal (Nomic AI)
-* Adam Treat (Nomic AI)
-"
-  },
-  {
-    "version": "2.4.15",
-    "notes":
-"
-* Add Vulkan GPU backend which allows inference on AMD, Intel and NVIDIA GPUs
-* Add ability to switch font sizes
-* Various bug fixes
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-* Nils Sauer (Nomic AI)
-* Lakshay Kansal (Nomic AI)
-"
-  },
-  {
-    "version": "2.4.16",
-    "notes":
-"
-* Bugfix for properly falling back to CPU when GPU can't be used
-* Report the actual device we're using
-* Fix context bugs for GPU accelerated models
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-"
-  },
-  {
-    "version": "2.4.17",
-    "notes":
-"
-* Bugfix for properly falling back to CPU when GPU is out of memory
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-"
-  },
-  {
-    "version": "2.4.18",
-    "notes":
-"
-* Bugfix for devices to show up in the settings combobox on application start and not just on model load
-* Send information on requested device and actual device on model load to help assess which model/gpu/os combos are working
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-"
-  },
-  {
-    "version": "2.4.19",
-    "notes":
-"
-* Fix a crasher on systems with corrupted vulkan drivers or corrupted vulkan dlls
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
 "
  }
 ]
--- a/gpt4all-chat/modellist.h
+++ b/gpt4all-chat/modellist.h
@@ -108,7 +108,7 @@ private:
    QString m_name;
    QString m_filename;
    double  m_temperature         = 0.7;
-    double  m_topP                = 0.4;
+    double  m_topP                = 0.1;
    int     m_topK                = 40;
    int     m_maxLength           = 4096;
    int     m_promptBatchSize     = 128;
--- a/gpt4all-chat/mysettings.cpp
+++ b/gpt4all-chat/mysettings.cpp
@@ -1,6 +1,5 @@
 #include "mysettings.h"
 #include "modellist.h"
-#include "../gpt4all-backend/llmodel.h"

 #include <QDir>
 #include <QFile>
@@ -17,14 +16,11 @@ static QString  default_userDefaultModel    = "Application default";
 static bool     default_forceMetal          = false;
 static QString  default_lastVersionStarted  = "";
 static int      default_localDocsChunkSize  = 256;
-static QString  default_chatTheme           = "Dark";
-static QString  default_fontSize            = "Small";
 static int      default_localDocsRetrievalSize  = 3;
 static bool     default_localDocsShowReferences = true;
 static QString  default_networkAttribution      = "";
 static bool     default_networkIsActive         = false;
 static bool     default_networkUsageStatsActive = false;
-static QString  default_device              = "Auto";

 static QString defaultLocalModelsPath()
 {
@@ -64,24 +60,6 @@ MySettings::MySettings()
    : QObject{nullptr}
 {
    QSettings::setDefaultFormat(QSettings::IniFormat);
-
-    std::vector<LLModel::GPUDevice> devices = LLModel::availableGPUDevices();
-    QVector<QString> deviceList{ "Auto" };
-    for (LLModel::GPUDevice &d : devices)
-        deviceList << QString::fromStdString(d.name);
-    deviceList << "CPU";
-    setDeviceList(deviceList);
-}
-
-Q_INVOKABLE QVector<QString> MySettings::deviceList() const
-{
-    return m_deviceList;
-}
-
-void MySettings::setDeviceList(const QVector<QString> &deviceList)
-{
-    m_deviceList = deviceList;
-    emit deviceListChanged();
 }

 void MySettings::restoreModelDefaults(const ModelInfo &model)
@@ -99,9 +77,6 @@ void MySettings::restoreModelDefaults(const ModelInfo &model)

 void MySettings::restoreApplicationDefaults()
 {
-    setChatTheme(default_chatTheme);
-    setFontSize(default_fontSize);
-    setDevice(default_device);
    setThreadCount(default_threadCount);
    setSaveChats(default_saveChats);
    setSaveChatGPTChats(default_saveChatGPTChats);
@@ -499,60 +474,6 @@ void MySettings::setUserDefaultModel(const QString &u)
    emit userDefaultModelChanged();
 }

-QString MySettings::chatTheme() const
-{
-    QSettings setting;
-    setting.sync();
-    return setting.value("chatTheme", default_chatTheme).toString();
-}
-
-void MySettings::setChatTheme(const QString &u)
-{
-    if (chatTheme() == u)
-        return;
-
-    QSettings setting;
-    setting.setValue("chatTheme", u);
-    setting.sync();
-    emit chatThemeChanged();
-}
-
-QString MySettings::fontSize() const
-{
-    QSettings setting;
-    setting.sync();
-    return setting.value("fontSize", default_fontSize).toString();
-}
-
-void MySettings::setFontSize(const QString &u)
-{
-    if (fontSize() == u)
-        return;
-
-    QSettings setting;
-    setting.setValue("fontSize", u);
-    setting.sync();
-    emit fontSizeChanged();
-}
-
-QString MySettings::device() const
-{
-    QSettings setting;
-    setting.sync();
-    return setting.value("device", default_device).toString();
-}
-
-void MySettings::setDevice(const QString &u)
-{
-    if (device() == u)
-        return;
-
-    QSettings setting;
-    setting.setValue("device", u);
-    setting.sync();
-    emit deviceChanged();
-}
-
 bool MySettings::forceMetal() const
 {
    return m_forceMetal;
--- a/gpt4all-chat/mysettings.h
+++ b/gpt4all-chat/mysettings.h
@@ -15,8 +15,6 @@ class MySettings : public QObject
    Q_PROPERTY(bool serverChat READ serverChat WRITE setServerChat NOTIFY serverChatChanged)
    Q_PROPERTY(QString modelPath READ modelPath WRITE setModelPath NOTIFY modelPathChanged)
    Q_PROPERTY(QString userDefaultModel READ userDefaultModel WRITE setUserDefaultModel NOTIFY userDefaultModelChanged)
-    Q_PROPERTY(QString chatTheme READ chatTheme WRITE setChatTheme NOTIFY chatThemeChanged)
-    Q_PROPERTY(QString fontSize READ fontSize WRITE setFontSize NOTIFY fontSizeChanged)
    Q_PROPERTY(bool forceMetal READ forceMetal WRITE setForceMetal NOTIFY forceMetalChanged)
    Q_PROPERTY(QString lastVersionStarted READ lastVersionStarted WRITE setLastVersionStarted NOTIFY lastVersionStartedChanged)
    Q_PROPERTY(int localDocsChunkSize READ localDocsChunkSize WRITE setLocalDocsChunkSize NOTIFY localDocsChunkSizeChanged)
@@ -25,8 +23,6 @@ class MySettings : public QObject
    Q_PROPERTY(QString networkAttribution READ networkAttribution WRITE setNetworkAttribution NOTIFY networkAttributionChanged)
    Q_PROPERTY(bool networkIsActive READ networkIsActive WRITE setNetworkIsActive NOTIFY networkIsActiveChanged)
    Q_PROPERTY(bool networkUsageStatsActive READ networkUsageStatsActive WRITE setNetworkUsageStatsActive NOTIFY networkUsageStatsActiveChanged)
-    Q_PROPERTY(QString device READ device WRITE setDevice NOTIFY deviceChanged)
-    Q_PROPERTY(QVector<QString> deviceList READ deviceList NOTIFY deviceListChanged)

 public:
    static MySettings *globalInstance();
@@ -74,14 +70,8 @@ public:
    void setModelPath(const QString &p);
    QString userDefaultModel() const;
    void setUserDefaultModel(const QString &u);
-    QString chatTheme() const;
-    void setChatTheme(const QString &u);
-    QString fontSize() const;
-    void setFontSize(const QString &u);
    bool forceMetal() const;
    void setForceMetal(bool b);
-    QString device() const;
-    void setDevice(const QString &u);

    // Release/Download settings
    QString lastVersionStarted() const;
@@ -106,9 +96,6 @@ public:
    QString attemptModelLoad() const;
    void setAttemptModelLoad(const QString &modelFile);

-    QVector<QString> deviceList() const;
-    void setDeviceList(const QVector<QString> &deviceList);
-
 Q_SIGNALS:
    void nameChanged(const ModelInfo &model);
    void filenameChanged(const ModelInfo &model);
@@ -127,8 +114,6 @@ Q_SIGNALS:
    void serverChatChanged();
    void modelPathChanged();
    void userDefaultModelChanged();
-    void chatThemeChanged();
-    void fontSizeChanged();
    void forceMetalChanged(bool);
    void lastVersionStartedChanged();
    void localDocsChunkSizeChanged();
@@ -138,12 +123,9 @@ Q_SIGNALS:
    void networkIsActiveChanged();
    void networkUsageStatsActiveChanged();
    void attemptModelLoadChanged();
-    void deviceChanged();
-    void deviceListChanged();

 private:
    bool m_forceMetal;
-    QVector<QString> m_deviceList;

 private:
    explicit MySettings();
--- a/gpt4all-chat/network.cpp
+++ b/gpt4all-chat/network.cpp
@@ -393,8 +393,6 @@ void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &valu
    properties.insert("name", QCoreApplication::applicationName() + " v"
        + QCoreApplication::applicationVersion());
    properties.insert("model", ChatListModel::globalInstance()->currentChat()->modelInfo().filename());
-    properties.insert("requestedDevice", MySettings::globalInstance()->device());
-    properties.insert("actualDevice", ChatListModel::globalInstance()->currentChat()->device());

    // Some additional startup information
    if (ev == "startup") {
--- a/gpt4all-chat/qml/AboutDialog.qml
+++ b/gpt4all-chat/qml/AboutDialog.qml
@@ -39,7 +39,6 @@ MyDialog {
                anchors.leftMargin: 30
                anchors.verticalCenter: img.verticalCenter
                text: qsTr("About GPT4All")
-                font.pixelSize: theme.fontSizeLarger
                color: theme.textColor
            }
        }
@@ -62,7 +61,6 @@ MyDialog {
                    + qsTr("### Contributors\n")
                    + Download.releaseInfo.contributors
                color: theme.textColor
-                font.pixelSize: theme.fontSizeLarge
                focus: false
                readOnly: true
                Accessible.role: Accessible.Paragraph
@@ -81,7 +79,6 @@ MyDialog {
            textFormat: Text.StyledText
            wrapMode: Text.WordWrap
            text: qsTr("Check out our discord channel <a href=\"https://discord.gg/4M2QFmTt2k\">https://discord.gg/4M2QFmTt2k</a>")
-            font.pixelSize: theme.fontSizeLarge
            onLinkActivated: { Qt.openUrlExternally("https://discord.gg/4M2QFmTt2k") }
            color: theme.textColor
            linkColor: theme.linkColor
@@ -96,7 +93,6 @@ MyDialog {
            textFormat: Text.StyledText
            wrapMode: Text.WordWrap
            text: qsTr("Thank you to <a href=\"https://home.nomic.ai\">Nomic AI</a> and the community for contributing so much great data, code, ideas, and energy to the growing open source AI ecosystem!")
-            font.pixelSize: theme.fontSizeLarge
            onLinkActivated: { Qt.openUrlExternally("https://home.nomic.ai") }
            color: theme.textColor
            linkColor: theme.linkColor
--- a/gpt4all-chat/qml/ApplicationSettings.qml
+++ b/gpt4all-chat/qml/ApplicationSettings.qml
@@ -18,125 +18,17 @@ MySettingsTab {
        columns: 3
        rowSpacing: 10
        columnSpacing: 10
-        Label {
-            id: themeLabel
-            text: qsTr("Theme:")
-            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
-            Layout.row: 1
-            Layout.column: 0
-        }
-        MyComboBox {
-            id: themeBox
-            Layout.row: 1
-            Layout.column: 1
-            Layout.columnSpan: 1
-            Layout.minimumWidth: 50
-            Layout.fillWidth: false
-            model: ["Dark", "Light"]
-            Accessible.role: Accessible.ComboBox
-            Accessible.name: qsTr("ComboBox for displaying/picking the color theme")
-            Accessible.description: qsTr("Use this for picking the color theme for the chat client to use")
-            function updateModel() {
-                themeBox.currentIndex = themeBox.indexOfValue(MySettings.chatTheme);
-            }
-            Component.onCompleted: {
-                themeBox.updateModel()
-            }
-            Connections {
-                target: MySettings
-                function onChatThemeChanged() {
-                    themeBox.updateModel()
-                }
-            }
-            onActivated: {
-                MySettings.chatTheme = themeBox.currentText
-            }
-        }
-        Label {
-            id: fontLabel
-            text: qsTr("Font Size:")
-            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
-            Layout.row: 2
-            Layout.column: 0
-        }
-        MyComboBox {
-            id: fontBox
-            Layout.row: 2
-            Layout.column: 1
-            Layout.columnSpan: 1
-            Layout.minimumWidth: 100
-            Layout.fillWidth: false
-            model: ["Small", "Medium", "Large"]
-            Accessible.role: Accessible.ComboBox
-            Accessible.name: qsTr("ComboBox for displaying/picking the font size")
-            Accessible.description: qsTr("Use this for picking the font size of the chat client")
-            function updateModel() {
-                fontBox.currentIndex = fontBox.indexOfValue(MySettings.fontSize);
-            }
-            Component.onCompleted: {
-                fontBox.updateModel()
-            }
-            Connections {
-                target: MySettings
-                function onFontSizeChanged() {
-                    fontBox.updateModel()
-                }
-            }
-            onActivated: {
-                MySettings.fontSize = fontBox.currentText
-            }
-        }
-        Label {
-            id: deviceLabel
-            text: qsTr("Device:")
-            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
-            Layout.row: 3
-            Layout.column: 0
-        }
-        MyComboBox {
-            id: deviceBox
-            Layout.row: 3
-            Layout.column: 1
-            Layout.columnSpan: 1
-            Layout.minimumWidth: 350
-            Layout.fillWidth: false
-            model: MySettings.deviceList
-            Accessible.role: Accessible.ComboBox
-            Accessible.name: qsTr("ComboBox for displaying/picking the device")
-            Accessible.description: qsTr("Use this for picking the device of the chat client")
-            function updateModel() {
-                deviceBox.currentIndex = deviceBox.indexOfValue(MySettings.device);
-            }
-            Component.onCompleted: {
-                deviceBox.updateModel()
-            }
-            Connections {
-                target: MySettings
-                function onDeviceChanged() {
-                    deviceBox.updateModel()
-                }
-                function onDeviceListChanged() {
-                    deviceBox.updateModel()
-                }
-            }
-            onActivated: {
-                MySettings.device = deviceBox.currentText
-            }
-        }
+
        Label {
            id: defaultModelLabel
            text: qsTr("Default model:")
            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
-            Layout.row: 4
+            Layout.row: 1
            Layout.column: 0
        }
        MyComboBox {
            id: comboBox
-            Layout.row: 4
+            Layout.row: 1
            Layout.column: 1
            Layout.columnSpan: 2
            Layout.minimumWidth: 350
@@ -165,16 +57,14 @@ MySettingsTab {
            id: modelPathLabel
            text: qsTr("Download path:")
            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
-            Layout.row: 5
+            Layout.row: 2
            Layout.column: 0
        }
        MyDirectoryField {
            id: modelPathDisplayField
            text: MySettings.modelPath
-            font.pixelSize: theme.fontSizeLarge
            implicitWidth: 300
-            Layout.row: 5
+            Layout.row: 2
            Layout.column: 1
            Layout.fillWidth: true
            ToolTip.text: qsTr("Path where model files will be downloaded to")
@@ -191,7 +81,7 @@ MySettingsTab {
            }
        }
        MyButton {
-            Layout.row: 5
+            Layout.row: 2
            Layout.column: 2
            text: qsTr("Browse")
            Accessible.description: qsTr("Opens a folder picker dialog to choose where to save model files")
@@ -205,17 +95,15 @@ MySettingsTab {
            id: nThreadsLabel
            text: qsTr("CPU Threads:")
            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
-            Layout.row: 6
+            Layout.row: 3
            Layout.column: 0
        }
        MyTextField {
            text: MySettings.threadCount
            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
            ToolTip.text: qsTr("Amount of processing threads to use bounded by 1 and number of logical processors")
            ToolTip.visible: hovered
-            Layout.row: 6
+            Layout.row: 3
            Layout.column: 1
            validator: IntValidator {
                bottom: 1
@@ -237,13 +125,12 @@ MySettingsTab {
            id: saveChatsLabel
            text: qsTr("Save chats to disk:")
            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
-            Layout.row: 7
+            Layout.row: 4
            Layout.column: 0
        }
        MyCheckBox {
            id: saveChatsBox
-            Layout.row: 7
+            Layout.row: 4
            Layout.column: 1
            checked: MySettings.saveChats
            onClicked: {
@@ -257,13 +144,12 @@ MySettingsTab {
            id: saveChatGPTChatsLabel
            text: qsTr("Save ChatGPT chats to disk:")
            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
-            Layout.row: 8
+            Layout.row: 5
            Layout.column: 0
        }
        MyCheckBox {
            id: saveChatGPTChatsBox
-            Layout.row: 8
+            Layout.row: 5
            Layout.column: 1
            checked: MySettings.saveChatGPTChats
            onClicked: {
@@ -274,13 +160,12 @@ MySettingsTab {
            id: serverChatLabel
            text: qsTr("Enable API server:")
            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
-            Layout.row: 9
+            Layout.row: 6
            Layout.column: 0
        }
        MyCheckBox {
            id: serverChatBox
-            Layout.row: 9
+            Layout.row: 6
            Layout.column: 1
            checked: MySettings.serverChat
            onClicked: {
@@ -290,7 +175,7 @@ MySettingsTab {
            ToolTip.visible: hovered
        }
        Rectangle {
-            Layout.row: 10
+            Layout.row: 7
            Layout.column: 0
            Layout.columnSpan: 3
            Layout.fillWidth: true
@@ -314,7 +199,6 @@ MySettingsTab {
            id: gpuOverrideLabel
            text: qsTr("Force Metal (macOS+arm):")
            color: theme.textColor
-            font.pixelSize: theme.fontSizeLarge
            Layout.row: 1
            Layout.column: 0
        }
--- a/gpt4all-chat/qml/ChatDrawer.qml
+++ b/gpt4all-chat/qml/ChatDrawer.qml
@@ -63,15 +63,15 @@ Drawer {
            anchors.top: newChat.bottom
            anchors.bottom: checkForUpdatesButton.top
            anchors.bottomMargin: 10
-            ScrollBar.vertical.policy: ScrollBar.AlwaysOff
+            ScrollBar.vertical.policy: ScrollBar.AlwaysOn
            clip: true

            ListView {
                id: conversationList
                anchors.fill: parent
                anchors.rightMargin: 10
+
                model: ChatListModel
-                ScrollBar.vertical: ScrollBar { policy: ScrollBar.AlwaysOn }

                delegate: Rectangle {
                    id: chatRectangle
@@ -255,7 +255,6 @@ Drawer {
            anchors.bottom: downloadButton.top
            anchors.bottomMargin: 10
            text: qsTr("Updates")
-            font.pixelSize: theme.fontSizeLarge
            Accessible.description: qsTr("Use this to launch an external application that will check for updates to the installer")
            onClicked: {
                if (!LLM.checkForUpdates())
@@ -288,4 +287,4 @@ Drawer {
            }
        }
    }
-}
+}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Zach Nussbaum	b8658c489a	chore: reqs.txt	2023-07-14 17:50:05 -04:00
Zach Nussbaum	1b3b32c630	docs: steps to convert	2023-07-14 17:49:56 -04:00
Zach Nussbaum	c7f0cf0cd2	feat: converter scripts from hf	2023-07-14 17:49:46 -04:00