chore: requirement

.circleci/config.yml

@@ -1,20 +1,194 @@
 version: 2.1
-setup: true
 orbs:
-  path-filtering: circleci/path-filtering@0.0.1
+  win: circleci/windows@5.0
+  python: circleci/python@1.2
+
+jobs:
+  build-py-docs:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          # TODO: eventually this will be cleaned up so we aren't building
+          # new dependencies each time unnecessarily.
+          # This will be introduced once we setup branch and path filtering
+          command: |
+            sudo apt-get update
+            sudo apt-get -y install python3 python3-pip
+            sudo pip3 install awscli --upgrade
+            sudo pip3 install mkdocs mkdocs-material mkautodoc 'mkdocstrings[python]'
+      - run:
+          name: Make Documentation
+          command: |
+            cd gpt4all-bindings/python/
+            mkdocs build
+      - run:
+          name: Deploy Documentation
+          command: |
+            cd gpt4all-bindings/python/
+            aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat
+      - run:
+          name: Invalidate docs.gpt4all.io cloudfront
+          command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"
+
+            
+
+  build-py-linux:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y cmake build-essential
+            pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            cmake ..
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          command: |
+            cd gpt4all-bindings/python/
+            python setup.py bdist_wheel --plat-name=manylinux1_x86_64
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  build-py-macos:
+    macos:
+      xcode: "14.2.0"
+    resource_class: macos.m1.large.gen1
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            brew install cmake
+            pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            cmake .. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          command: |
+            cd gpt4all-bindings/python
+            python setup.py bdist_wheel --plat-name=macosx_10_9_universal2
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  build-py-windows:
+    executor:
+      name: win/default
+    steps:
+      - checkout
+      - run:
+          name: Install MinGW64
+          command: choco install -y mingw --force --no-progress
+      - run:
+          name: Add MinGW64 to PATH
+          command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
+      - run:
+          name: Install dependencies
+          command: choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+      - run:
+          name: Install Python dependencies
+          command: pip install setuptools wheel cmake
+      - run:
+          name: Build C library
+          command: |
+            git submodule init
+            git submodule update
+            cd gpt4all-backend
+            mkdir build
+            cd build
+            cmake -G "MinGW Makefiles" .. 
+            cmake --build . --parallel
+      - run:
+          name: Build wheel
+          # TODO: As part of this task, we need to move mingw64 binaries into package.
+          # This is terrible and needs a more robust solution eventually.
+          command: |
+            cd gpt4all-bindings/python
+            cd gpt4all
+            mkdir llmodel_DO_NOT_MODIFY
+            mkdir llmodel_DO_NOT_MODIFY/build/
+            cp 'C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
+            cd ..
+            python setup.py bdist_wheel --plat-name=win_amd64
+      - persist_to_workspace:
+          root: gpt4all-bindings/python/dist
+          paths:
+            - "*.whl"
+
+  store-and-upload-wheels:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - setup_remote_docker
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y cmake build-essential
+            pip install setuptools wheel twine
+      - run:
+          name: Upload Python package
+          command: |
+            twine upload /tmp/workspace/*.whl --username __token__ --password $PYPI_CRED
+      - store_artifacts:
+          path: /tmp/workspace
 
 workflows:
-  version: 2.1
-  generate-config:
+  version: 2
+  deploy-docs:
     jobs:
-      - path-filtering/filter:
-          base-revision: main
-          config-path: .circleci/continue_config.yml
-          mapping: |
-            .circleci/.* run-all-workflows true
-            gpt4all-backend/.* run-all-workflows true
-            gpt4all-bindings/python/.* run-python-workflow true
-            gpt4all-bindings/typescript/.* run-ts-workflow true
-            gpt4all-bindings/csharp/.* run-csharp-workflow true
-            gpt4all-chat/.* run-chat-workflow true
-            .* run-default-workflow true
+      - build-py-docs:
+          filters:
+            branches:
+              only:
+                - main
+  # build-py-deploy:
+  #   jobs:
+  #     - build-py-linux:
+  #         filters:
+  #           branches:
+  #             only:
+  #     - build-py-macos:
+  #         filters:
+  #           branches:
+  #             only:
+  #     - build-py-windows:
+  #         filters:
+  #           branches:
+  #             only:
+  #     - store-and-upload-wheels:
+  #         filters:
+  #           branches:
+  #             only:
+  #         requires:
+  #           - build-py-windows
+  #           - build-py-linux
+  #           - build-py-macos

.codespellrc

@@ -1,3 +1,4 @@
 [codespell]
-ignore-words-list = blong, afterall, som, assistent, crasher
-skip = .git,*.pdf,*.svg,*.lock
+skip = .git,*.pdf,*.svg
+#
+# ignore-words-list =

.github/ISSUE_TEMPLATE/bindings-bug.md

@@ -1,35 +0,0 @@
----
-name: "\U0001F6E0 Bindings Bug Report"
-about: A bug report for the GPT4All Bindings
-labels: ["bindings", "bug-unconfirmed"]
----
-
-<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
-
-### Bug Report
-
-<!-- A clear and concise description of what the bug is. -->
-
-### Example Code
-
-<!-- Please provide a minimal code example that can be used to experience this issue. Delete this section if it does not apply. -->
-
-### Steps to Reproduce
-
-<!-- List the steps that should be taken to experience this issue. -->
-
-1.
-2.
-3.
-
-### Expected Behavior
-
-<!-- In a few words, what did you expect to happen? -->
-
-### Your Environment
-
-- Bindings version (e.g. "Version" from `pip show gpt4all`):
-- Operating System:
-- Chat model used (if applicable):
-
-<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,70 @@
+name: "\U0001F41B Bug Report"
+description: Submit a bug report to help us improve GPT4All
+labels: ["02 Bug Report"]
+body:
+  - type: markdown
+    attributes:
+      value: >
+        Thank you for taking the time to file a bug report. Before creating a new
+        issue, please make sure to take a few moments to check the issue tracker
+        for existing issues about the bug.
+
+  - type: textarea
+    id: system-info
+    attributes:
+      label: System Info
+      description: Please share your system info with us.
+      placeholder: GPT4All version, platform, python version, etc...
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: information-scripts-examples
+    attributes:
+      label: Information
+      description: "The problem arises when using:"
+      options:
+        - label: "The official example notebooks/scripts"
+        - label: "My own modified scripts"
+
+  - type: checkboxes
+    id: related-components
+    attributes:
+      label: Related Components
+      description: "Select the components related to the issue (if applicable):"
+      options:
+        - label: "backend"
+        - label: "bindings"
+        - label: "python-bindings"
+        - label: "chat-ui"
+        - label: "models"
+        - label: "circleci"
+        - label: "docker"
+        - label: "api"
+
+  - type: textarea
+    id: reproduction
+    validations:
+      required: true
+    attributes:
+      label: Reproduction
+      description: |
+        Please provide a [code sample](https://stackoverflow.com/help/minimal-reproducible-example) that reproduces the problem you ran into. It can be a Colab link or just a code snippet.
+        If you have code snippets, error messages, stack traces please provide them here as well.
+        Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting
+        Avoid screenshots when possible, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code.
+
+      placeholder: |
+        Steps to reproduce the behavior:
+
+          1.
+          2.
+          3.
+
+  - type: textarea
+    id: expected-behavior
+    validations:
+      required: true
+    attributes:
+      label: Expected behavior
+      description: "A clear and concise description of what you would expect to happen."

.github/ISSUE_TEMPLATE/chat-bug.md

@@ -1,31 +0,0 @@
----
-name: "\U0001F4AC GPT4All Bug Report"
-about: A bug report for GPT4All Chat
-labels: ["chat", "bug-unconfirmed"]
----
-
-<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
-
-### Bug Report
-
-<!-- A clear and concise description of what the bug is. -->
-
-### Steps to Reproduce
-
-<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. -->
-
-1.
-2.
-3.
-
-### Expected Behavior
-
-<!-- In a few words, what did you expect to happen? -->
-
-### Your Environment
-
-- GPT4All version:
-- Operating System:
-- Chat model used (if applicable):
-
-<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -1 +1,2 @@
-version: 2.1
+blank_issues_enabled: false
+version: 2.1

.github/ISSUE_TEMPLATE/documentation.md

@@ -1,9 +0,0 @@
----
-name: "\U0001F4C4 Documentation"
-about: An issue related to the GPT4All documentation
-labels: ["documentation"]
----
-
-### Documentation
-
-<!-- Please describe the issue with the documentation as clearly as possible. -->

.github/ISSUE_TEMPLATE/documentation.yml

@@ -0,0 +1,19 @@
+name: Documentation
+description: Report an issue related to the GPT4All documentation.
+title: "DOC: <Please write a comprehensive title after the 'DOC: ' prefix>"
+labels: [03 - Documentation]
+
+body:
+- type: textarea
+  attributes: 
+    label: "Issue with current documentation:"
+    description: >
+      Please make sure to leave a reference to the document/code you're
+      referring to.
+
+- type: textarea
+  attributes:
+    label: "Idea or request for content:"
+    description: >
+      Please describe as clearly as possible what topics you think are missing
+      from the current documentation.

.github/ISSUE_TEMPLATE/feature-request.md

@@ -1,10 +0,0 @@
----
-name: "\U0001F680 Feature Request"
-about: Submit a proposal/request for a new GPT4All feature
-title: "[Feature] Feature request title..."
-labels: ["enhancement"]
----
-
-### Feature Request
-
-<!-- A clear and concise description of the feature proposal. -->

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -0,0 +1,30 @@
+name: "\U0001F680 Feature Request"
+description: Submit a proposal/request for a new GPT4All feature
+labels: ["02 Feature Request"]
+body:
+  - type: textarea
+    id: feature-request
+    validations:
+      required: true
+    attributes:
+      label: Feature request
+      description: |
+        A clear and concise description of the feature proposal. Please provide links to any relevant GitHub repos, papers, or other resources if relevant.
+
+  - type: textarea
+    id: motivation
+    validations:
+      required: true
+    attributes:
+      label: Motivation
+      description: |
+        Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too.
+
+  - type: textarea
+    id: contribution
+    validations:
+      required: true
+    attributes:
+      label: Your contribution
+      description: |
+        Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/nomic-ai/gpt4all/blob/main/CONTRIBUTING.md)

.github/ISSUE_TEMPLATE/other-bug.md

@@ -1,32 +0,0 @@
----
-name: "\U0001F41B Other Bug Report"
-about: A bug in another component of GPT4All
-labels: ["bug-unconfirmed"]
----
-
-<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
-
-### Bug Report
-
-<!-- A clear and concise description of what the bug is. -->
-
-### Steps to Reproduce
-
-<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. If this bug involves original code, please provide a minimal version that can reproduce the issue. -->
-
-1.
-2.
-3.
-
-### Expected Behavior
-
-<!-- In a few words, what did you expect to happen? -->
-
-### Your Environment
-
-- GPT4All version (if applicable):
-- Operating System:
-- Chat model used (if applicable):
-
-<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->
-

.github/ISSUE_TEMPLATE/other.yml

@@ -0,0 +1,18 @@
+name: Other Issue
+description: Raise an issue that wouldn't be covered by the other templates.
+title: "Issue: <Please write a comprehensive title after the 'Issue: ' prefix>"
+labels: [04 - Other]
+
+body:
+  - type: textarea
+    attributes:
+      label: "Issue you'd like to raise."
+      description: >
+        Please describe the issue you'd like to raise as clearly as possible.
+        Make sure to include any relevant links or references.
+
+  - type: textarea
+    attributes:
+      label: "Suggestion:"
+      description: >
+        Please outline a suggestion to improve the issue here.

.gitignore

@@ -1,6 +1,3 @@
-*.arrow
-squad_*
-*sbert_embedded*
 *.pkl
 ckpts*
 .deepspeed_env
@@ -181,9 +178,3 @@ CMakeLists.txt.user
 gpt4all-chat/models/*
 build_*
 build-*
-
-# IntelliJ
-.idea/
-
-# LLM models
-*.gguf

.gitmodules

@@ -1,4 +1,9 @@
+[submodule "llama.cpp-230519"]
+	path = gpt4all-backend/llama.cpp-230519
+	url = https://github.com/ggerganov/llama.cpp.git
+[submodule "llama.cpp-230511"]
+	path = gpt4all-backend/llama.cpp-230511
+	url = https://github.com/manyoso/llama.cpp.git
 [submodule "llama.cpp-mainline"]
 	path = gpt4all-backend/llama.cpp-mainline
-	url = https://github.com/nomic-ai/llama.cpp.git
-	branch = master
+	url = https://github.com/ggerganov/llama.cpp.git

LICENSE_SOM.txt

@@ -1,30 +0,0 @@
-Software for Open Models License (SOM)
-Version 1.0 dated August 30th, 2023
-
-This license governs use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
-
-This license is intended to encourage open release of models created, modified, processed, or otherwise used via the Software under open licensing terms, and should be interpreted in light of that intent.
-
-1. Definitions
-The “Licensor” is the person or entity who is making the Software available under this license. “Software” is the software made available by Licensor under this license.
-A “Model” is the output of a machine learning algorithm, and excludes the Software.
-“Model Source Materials” must include the Model and model weights, and may include any input data, input data descriptions, documentation or training descriptions for the Model.
-“Open Licensing Terms” means: (a) any open source license approved by the Open Source Initiative, or (b) any other terms that make the Model Source Materials publicly available free of charge, and allow recipients to use, modify and distribute the Model Source Materials. Terms described in (b) may include reasonable restrictions such as non-commercial or non-production limitations, or require use in compliance with law.
-
-2. Grant of Rights. Subject to the conditions and limitations in section 3:
-(A) Copyright Grant. Licensor grants you a non-exclusive, worldwide, royalty-free copyright license to copy, modify, and distribute the Software and any modifications of the Software you create under this license. The foregoing license includes without limitation the right to create, modify, and use Models using this Software.
-
-(B) Patent Grant. Licensor grants you a non-exclusive, worldwide, royalty-free license, under any patents owned or controlled by Licensor, to make, have made, use, sell, offer for sale, import, or otherwise exploit the Software.  No license is granted to patent rights that are not embodied in the operation of the Software in the form provided by Licensor.
-
-3. Conditions and Limitations
-(A) Model Licensing and Access. If you use the Software to create, modify, process, or otherwise use any Model, including usage to create inferences with a Model, whether or not you make the Model available to others, you must make that Model Source Materials publicly available under Open Licensing Terms. 
-
-(B) No Re-Licensing. If you redistribute the Software, or modifications to the Software made under the license granted above, you must make it available only under the terms of this license. You may offer additional terms such as warranties, maintenance and support, but You, and not Licensor, are responsible for performing such terms.
-
-(C) No Trademark License. This license does not grant you rights to use the Licensor’s name, logo, or trademarks.
-
-(D) If you assert in writing a claim against any person or entity alleging that the use of the Software infringes any patent, all of your licenses to the Software under Section 2 end automatically as of the date you asserted the claim.
-
-(E) If you distribute any portion of the Software, you must retain all copyright, patent, trademark, and attribution notices that are present in the Software, and you must include a copy of this license.
-
-(F) The Software is licensed “as-is.” You bear the entire risk of using it. Licensor gives You no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws that this license cannot change. To the extent permitted under your local laws, the Licensor disclaims and excludes the implied warranties of merchantability, fitness for a particular purpose and non-infringement. To the extent this disclaimer is unlawful, you, and not Licensor, are responsible for any liability.

README.md

@@ -1,16 +1,17 @@
 <h1 align="center">GPT4All</h1>
-
-<p align="center">Open-source large language models that run locally on your CPU and nearly any GPU</p>
-
+<p align="center">Open-source assistant-style large language models that run locally on your CPU</p>
 
 <p align="center">
-Join the <a href="https://discord.gg/tyc74KNVK3?event=1227642051294658621">GPT4All 2024 Roadmap Townhall</a> on April 18, 2024 at 12pm EST
+<a href="https://gpt4all.io">GPT4All Website</a>
 </p>
 
 <p align="center">
-<a href="https://gpt4all.io">GPT4All Website and Models</a> • <a href="https://docs.gpt4all.io">GPT4All Documentation</a> • <a href="https://discord.gg/mGZE39AS3e">Discord</a>
+<a href="https://docs.gpt4all.io">GPT4All Documentation</a>
 </p>
 
+<p align="center">
+<a href="https://discord.gg/mGZE39AS3e">Discord</a>
+</p>
 
 <p align="center">
 <a href="https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html">🦜️🔗 Official Langchain Backend</a> 
@@ -20,33 +21,21 @@ Join the <a href="https://discord.gg/tyc74KNVK3?event=1227642051294658621">GPT4A
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
 </p>
 
-<p align="center">
- <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg?&logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNSIgaGVpZ2h0PSI0IiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxwYXRoIGQ9Ik00LjQzIDEuODgyYTEuNDQgMS40NCAwIDAgMS0uMDk4LjQyNmMtLjA1LjEyMy0uMTE1LjIzLS4xOTIuMzIyLS4wNzUuMDktLjE2LjE2NS0uMjU1LjIyNmExLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxMmMtLjA5OS4wMTItLjE5Mi4wMTQtLjI3OS4wMDZsLTEuNTkzLS4xNHYtLjQwNmgxLjY1OGMuMDkuMDAxLjE3LS4xNjkuMjQ2LS4xOTFhLjYwMy42MDMgMCAwIDAgLjItLjEwNi41MjkuNTI5IDAgMCAwIC4xMzgtLjE3LjY1NC42NTQgMCAwIDAgLjA2NS0uMjRsLjAyOC0uMzJhLjkzLjkzIDAgMCAwLS4wMzYtLjI0OS41NjcuNTY3IDAgMCAwLS4xMDMtLjIuNTAyLjUwMiAwIDAgMC0uMTY4LS4xMzguNjA4LjYwOCAwIDAgMC0uMjQtLjA2N0wyLjQzNy43MjkgMS42MjUuNjcxYS4zMjIuMzIyIDAgMCAwLS4yMzIuMDU4LjM3NS4zNzUgMCAwIDAtLjExNi4yMzJsLS4xMTYgMS40NS0uMDU4LjY5Ny0uMDU4Ljc1NEwuNzA1IDRsLS4zNTctLjA3OUwuNjAyLjkwNkMuNjE3LjcyNi42NjMuNTc0LjczOS40NTRhLjk1OC45NTggMCAwIDEgLjI3NC0uMjg1Ljk3MS45NzEgMCAwIDEgLjMzNy0uMTRjLjExOS0uMDI2LjIyNy0uMDM0LjMyNS0uMDI2TDMuMjMyLjE2Yy4xNTkuMDE0LjMzNi4wMy40NTkuMDgyYTEuMTczIDEuMTczIDAgMCAxIC41NDUuNDQ3Yy4wNi4wOTQuMTA5LjE5Mi4xNDQuMjkzYTEuMzkyIDEuMzkyIDAgMCAxIC4wNzguNThsLS4wMjkuMzJaIiBmaWxsPSIjRjI3NzdBIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+Cjwvc3ZnPgo=" alt="phorm.ai"></a>
-</p>
-
 <p align="center">
   <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
 </p>
 <p align="center">
-Run on an M1 macOS Device (not sped up!)
+Run on an M1 Mac (not sped up!)
 </p>
 
 ## GPT4All: An ecosystem of open-source on-edge large language models.
-
-GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and any GPU. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
+GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs.
 
 Learn more in the [documentation](https://docs.gpt4all.io).
 
-A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models.
+The goal is simple - be the best instruction tuned assistant-style language model that any person or enterprise can freely use, distribute and build on.
 
-### What's New ([Issue Tracker](https://github.com/orgs/nomic-ai/projects/2))
-- **October 19th, 2023**: GGUF Support Launches with Support for:
-    - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
-    - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4\_0 and Q4\_1 quantizations in GGUF.
-    - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
-- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on AMD, Intel, Samsung, Qualcomm and NVIDIA GPUs.
-- **August 15th, 2023**: GPT4All API launches allowing inference of local LLMs from docker containers.
-- **July 2023**: Stable support for LocalDocs, a GPT4All Plugin that allows you to privately and locally chat with your data.
+A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models. 
 
 
 ### Chat Client
@@ -54,12 +43,20 @@ Run any GPT4All model natively on your home desktop with the auto-updating deskt
 
 Direct Installer Links:
 
-* [macOS](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
+* [Mac/OSX](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
 
 * [Windows](https://gpt4all.io/installers/gpt4all-installer-win64.exe)
 
 * [Ubuntu](https://gpt4all.io/installers/gpt4all-installer-linux.run)
 
+If you have older hardware that only supports avx and not avx2 you can use these.
+
+* [Mac/OSX - avx-only](https://gpt4all.io/installers/gpt4all-installer-darwin-avx-only.dmg)
+
+* [Windows - avx-only](https://gpt4all.io/installers/gpt4all-installer-win64-avx-only.exe)
+
+* [Ubuntu - avx-only](https://gpt4all.io/installers/gpt4all-installer-linux-avx-only.run)
+
 Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io/)
 
 ### Chat Client building and running
@@ -68,15 +65,11 @@ Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io
 
 ### Bindings
 
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/java">:computer: Official Java Bindings</a>
 
-### Integrations
-
-* 🗃️ [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
 
 ## Contributing
 GPT4All welcomes contributions, involvement, and discussion from the open source community!

gpt4all-api/.gitignore

@@ -1,112 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-app/__pycache__/
-gpt4all_api/__pycache__/
-gpt4all_api/app/api_v1/__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# VS Code
-.vscode/
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-
-*.lock
-*.cache

gpt4all-api/.isort.cfg

@@ -1,7 +0,0 @@
-[settings]
-known_third_party=geopy,nltk,np,numpy,pandas,pysbd,fire,torch
-
-line_length=120
-include_trailing_comma=True
-multi_line_output=3
-use_parentheses=True

gpt4all-api/LICENSE

@@ -1,13 +0,0 @@
-Copyright 2023 Nomic, Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

gpt4all-api/README.md

@@ -1,90 +1,2 @@
-# GPT4All REST API
-
-NOTICE: We are considering to deprecate this API as it has become challenging to maintain and test. If you have any interest in maintaining this or would like to takeover and adopt or discuss the future of this API please speak up in the discord channel.
-
-This directory contains the source code to run and build docker images that run a FastAPI app
-for serving inference from GPT4All models. The API matches the OpenAI API spec.
-
-## Tutorial
-
-The following tutorial assumes that you have checked out this repo and cd'd into it.
-
-### Starting the app
-
-First change your working directory to `gpt4all/gpt4all-api`.
-
-Now you can build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
-```bash
-DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
-```
-
-Then, start the backend with:
-
-```bash
-docker compose up --build
-```
-
-This will run both the API and locally hosted GPU inference server. If you want to run the API without the GPU inference server, you can run:
-
-```bash
-docker compose up --build gpt4all_api
-```
-
-To run the API with the GPU inference server, you will need to include environment variables (like the `MODEL_ID`). Edit the `.env` file and run
-```bash
-docker compose --env-file .env up --build
-```
-
-
-#### Spinning up your app
-Run `docker compose up` to spin up the backend. Monitor the logs for errors in-case you forgot to set an environment variable above.
-
-
-#### Development
-Run
-
-```bash
-docker compose up --build
-```
-and edit files in the `app` directory. The api will hot-reload on changes.
-
-You can run the unit tests with
-
-```bash
-make test
-```
-
-#### Viewing API documentation
-
-Once the FastAPI ap is started you can access its documentation and test the search endpoint by going to:
-```
-localhost:80/docs
-```
-
-This documentation should match the OpenAI OpenAPI spec located at https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-#### Running inference
-```python
-import openai
-openai.api_base = "http://localhost:4891/v1"
-
-openai.api_key = "not needed for a local LLM"
-
-
-def test_completion():
-    model = "gpt4all-j-v1.3-groovy"
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model,
-        prompt=prompt,
-        max_tokens=50,
-        temperature=0.28,
-        top_p=0.95,
-        n=1,
-        echo=True,
-        stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-    print(response)
-```
+# GPT4All API
+This directory will contain code to build out a RESTful API for GPT4All models. Exact details TBD, but as an MVP, user should be able to send requests to list, download, and generate text with different models.

gpt4all-api/docker-compose.gpu.yaml

@@ -1,24 +0,0 @@
-version: "3.8"
-
-services:
-  gpt4all_gpu:
-    image: ghcr.io/huggingface/text-generation-inference:0.9.3
-    container_name: gpt4all_gpu
-    restart: always #restart on error (usually code compilation from save during bad state)
-    environment:
-      - HUGGING_FACE_HUB_TOKEN=token
-      - USE_FLASH_ATTENTION=false
-      - MODEL_ID=''
-      - NUM_SHARD=1
-    command: --model-id $MODEL_ID --num-shard $NUM_SHARD
-    volumes:
-      - ./:/data
-    ports:
-      - "8080:80"
-    shm_size: 1g
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              capabilities: [gpu]

gpt4all-api/docker-compose.yaml

@@ -1,22 +0,0 @@
-version: "3.8"
-
-services:
-  gpt4all_api:
-    image: gpt4all_api
-    container_name: gpt4all_api
-    restart: always #restart on error (usually code compilation from save during bad state)
-    ports:
-      - "4891:4891"
-    env_file:
-      - .env
-    environment:
-      - APP_ENVIRONMENT=dev
-      - WEB_CONCURRENCY=2
-      - LOGLEVEL=debug
-      - PORT=4891
-      - model=${MODEL_BIN} # using variable from .env file
-      - inference_mode=cpu
-    volumes:
-      - './gpt4all_api/app:/app'
-      - './gpt4all_api/models:/models' # models are mounted in the container
-    command: ["/start-reload.sh"]

gpt4all-api/gpt4all_api/Dockerfile.buildkit

@@ -1,17 +0,0 @@
-# syntax=docker/dockerfile:1.0.0-experimental
-FROM tiangolo/uvicorn-gunicorn:python3.11
-
-# Put first so anytime this file changes other cached layers are invalidated.
-COPY gpt4all_api/requirements.txt /requirements.txt
-
-RUN pip install --upgrade pip
-
-# Run various pip install commands with ssh keys from host machine.
-RUN --mount=type=ssh pip install -r /requirements.txt && \
-  rm -Rf /root/.cache && rm -Rf /tmp/pip-install*
-
-# Finally, copy app and client.
-COPY gpt4all_api/app /app
-
-RUN mkdir -p /models
-

gpt4all-api/gpt4all_api/README.md

@@ -1 +0,0 @@
-# FastAPI app for serving GPT4All models

gpt4all-api/gpt4all_api/app/api_v1/api.py

@@ -1,9 +0,0 @@
-from api_v1.routes import chat, completions, engines, health
-from fastapi import APIRouter
-
-router = APIRouter()
-
-router.include_router(chat.router)
-router.include_router(completions.router)
-router.include_router(engines.router)
-router.include_router(health.router)

gpt4all-api/gpt4all_api/app/api_v1/events.py

@@ -1,29 +0,0 @@
-import logging
-
-from api_v1.settings import settings
-from fastapi import HTTPException
-from fastapi.responses import JSONResponse
-from starlette.requests import Request
-
-log = logging.getLogger(__name__)
-
-
-startup_msg_fmt = """
- Starting up GPT4All API
-"""
-
-
-async def on_http_error(request: Request, exc: HTTPException):
-    return JSONResponse({'detail': exc.detail}, status_code=exc.status_code)
-
-
-async def on_startup(app):
-    startup_msg = startup_msg_fmt.format(settings=settings)
-    log.info(startup_msg)
-
-
-def startup_event_handler(app):
-    async def start_app() -> None:
-        await on_startup(app)
-
-    return start_app

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -1,103 +0,0 @@
-import logging
-import time
-from typing import List
-from uuid import uuid4
-from fastapi import APIRouter, HTTPException
-from gpt4all import GPT4All
-from pydantic import BaseModel, Field
-from api_v1.settings import settings
-from fastapi.responses import StreamingResponse
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-class ChatCompletionMessage(BaseModel):
-    role: str
-    content: str
-
-class ChatCompletionRequest(BaseModel):
-    model: str = Field(settings.model, description='The model to generate a completion from.')
-    messages: List[ChatCompletionMessage] = Field(..., description='Messages for the chat completion.')
-    temperature: float = Field(settings.temp, description='Model temperature')
-
-class ChatCompletionChoice(BaseModel):
-    message: ChatCompletionMessage
-    index: int
-    logprobs: float
-    finish_reason: str
-
-class ChatCompletionUsage(BaseModel):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-class ChatCompletionResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[ChatCompletionChoice]
-    usage: ChatCompletionUsage
-
-router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
-
-@router.post("/completions", response_model=ChatCompletionResponse)
-async def chat_completion(request: ChatCompletionRequest):
-    '''
-    Completes a GPT4All model response based on the last message in the chat.
-    '''
-    # GPU is not implemented yet
-    if settings.inference_mode == "gpu":
-        raise HTTPException(status_code=400,
-              detail=f"Not implemented yet: Can only infer in CPU mode.")
-
-    # we only support the configured model
-    if request.model != settings.model:
-        raise HTTPException(status_code=400,
-              detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
-
-    # run only of we have a message
-    if request.messages:
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        # format system message and conversation history correctly
-        formatted_messages = ""
-        for message in request.messages:
-            formatted_messages += f"<|im_start|>{message.role}\n{message.content}<|im_end|>\n"
-
-        # the LLM will complete the response of the assistant
-        formatted_messages += "<|im_start|>assistant\n"
-        response = model.generate(
-            prompt=formatted_messages,
-            temp=request.temperature
-            )
-
-        # the LLM may continue to hallucinate the conversation, but we want only the first response
-        # so, cut off everything after first <|im_end|>
-        index = response.find("<|im_end|>")
-        response_content = response[:index].strip()
-    else:
-        response_content = "No messages received."
-
-    # Create a chat message for the response
-    response_message = ChatCompletionMessage(role="assistant", content=response_content)
-
-    # Create a choice object with the response message
-    response_choice = ChatCompletionChoice(
-        message=response_message,
-        index=0,
-        logprobs=-1.0,  # Placeholder value
-        finish_reason="length"  # Placeholder value
-    )
-
-    # Create the response object
-    chat_response = ChatCompletionResponse(
-        id=str(uuid4()),
-        created=int(time.time()),
-        model=request.model,
-        choices=[response_choice],
-        usage=ChatCompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0),  # Placeholder values
-    )
-
-    return chat_response

gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py

@@ -1,215 +0,0 @@
-import json
-from typing import List, Dict, Iterable, AsyncIterable
-import logging
-import time
-from typing import Dict, List, Union, Optional
-from uuid import uuid4
-import aiohttp
-import asyncio
-from api_v1.settings import settings
-from fastapi import APIRouter, Depends, Response, Security, status, HTTPException
-from fastapi.responses import StreamingResponse
-from gpt4all import GPT4All
-from pydantic import BaseModel, Field
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class CompletionRequest(BaseModel):
-    model: str = Field(settings.model, description='The model to generate a completion from.')
-    prompt: Union[List[str], str] = Field(..., description='The prompt to begin completing from.')
-    max_tokens: int = Field(None, description='Max tokens to generate')
-    temperature: float = Field(settings.temp, description='Model temperature')
-    top_p: Optional[float] = Field(settings.top_p, description='top_p')
-    top_k: Optional[int] = Field(settings.top_k, description='top_k')
-    n: int = Field(1, description='How many completions to generate for each prompt')
-    stream: bool = Field(False, description='Stream responses')
-    repeat_penalty: float = Field(settings.repeat_penalty, description='Repeat penalty')
-
-
-class CompletionChoice(BaseModel):
-    text: str
-    index: int
-    logprobs: float
-    finish_reason: str
-
-
-class CompletionUsage(BaseModel):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-
-class CompletionResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[CompletionChoice]
-    usage: CompletionUsage
-
-
-class CompletionStreamResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[CompletionChoice]
-
-
-router = APIRouter(prefix="/completions", tags=["Completion Endpoints"])
-
-def stream_completion(output: Iterable, base_response: CompletionStreamResponse):
-    """
-    Streams a GPT4All output to the client.
-
-    Args:
-        output: The output of GPT4All.generate(), which is an iterable of tokens.
-        base_response: The base response object, which is cloned and modified for each token.
-
-    Returns:
-        A Generator of CompletionStreamResponse objects, which are serialized to JSON Event Stream format.
-    """
-    for token in output:
-        chunk = base_response.copy()
-        chunk.choices = [dict(CompletionChoice(
-            text=token,
-            index=0,
-            logprobs=-1,
-            finish_reason=''
-        ))]
-        yield f"data: {json.dumps(dict(chunk))}\n\n"
-
-async def gpu_infer(payload, header):
-    async with aiohttp.ClientSession() as session:
-        try:
-            async with session.post(
-                settings.hf_inference_server_host, headers=header, data=json.dumps(payload)
-            ) as response:
-                resp = await response.json()
-            return resp
-
-        except aiohttp.ClientError as e:
-            # Handle client-side errors (e.g., connection error, invalid URL)
-            logger.error(f"Client error: {e}")
-        except aiohttp.ServerError as e:
-            # Handle server-side errors (e.g., internal server error)
-            logger.error(f"Server error: {e}")
-        except json.JSONDecodeError as e:
-            # Handle JSON decoding errors
-            logger.error(f"JSON decoding error: {e}")
-        except Exception as e:
-            # Handle other unexpected exceptions
-            logger.error(f"Unexpected error: {e}")
-
-@router.post("/", response_model=CompletionResponse)
-async def completions(request: CompletionRequest):
-    '''
-    Completes a GPT4All model response.
-    '''
-    if settings.inference_mode == "gpu":
-        params = request.dict(exclude={'model', 'prompt', 'max_tokens', 'n'})
-        params["max_new_tokens"] = request.max_tokens
-        params["num_return_sequences"] = request.n
-
-        header = {"Content-Type": "application/json"}
-        if isinstance(request.prompt, list):
-            tasks = []
-            for prompt in request.prompt:
-                payload = {"parameters": params}
-                payload["inputs"] = prompt
-                task = gpu_infer(payload, header)
-                tasks.append(task)
-            results = await asyncio.gather(*tasks)
-
-            choices = []
-            for response in results:
-                scores = response["scores"] if "scores" in response else -1.0
-                choices.append(
-                    dict(
-                        CompletionChoice(
-                            text=response["generated_text"], index=0, logprobs=scores, finish_reason='stop'
-                        )
-                    )
-                )
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=choices,
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
-
-        else:
-            payload = {"parameters": params}
-            # If streaming, we need to return a StreamingResponse
-            payload["inputs"] = request.prompt
-
-            resp = await gpu_infer(payload, header)
-
-            output = resp["generated_text"]
-            # this returns all logprobs
-            scores = resp["scores"] if "scores" in resp else -1.0
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(text=output, index=0, logprobs=scores, finish_reason='stop'))],
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
-
-    else:
-
-        if request.model != settings.model:
-            raise HTTPException(status_code=400,
-                                detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
-
-        if isinstance(request.prompt, list):
-            if len(request.prompt) > 1:
-                raise HTTPException(status_code=400, detail="Can only infer one inference per request in CPU mode.")
-            else:
-                request.prompt = request.prompt[0]
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        output = model.generate(prompt=request.prompt,
-                                max_tokens=request.max_tokens,
-                                streaming=request.stream,
-                                top_k=request.top_k,
-                                top_p=request.top_p,
-                                temp=request.temperature,
-                                )
-
-        # If streaming, we need to return a StreamingResponse
-        if request.stream:
-            base_chunk = CompletionStreamResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[]
-            )
-            return StreamingResponse((response for response in stream_completion(output, base_chunk)),
-                                     media_type="text/event-stream")
-        else:
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(
-                    text=output,
-                    index=0,
-                    logprobs=-1,
-                    finish_reason='stop'
-                ))],
-                usage={
-                    'prompt_tokens': 0,  # TODO how to compute this?
-                    'completion_tokens': 0,
-                    'total_tokens': 0
-                }
-            )

gpt4all-api/gpt4all_api/app/api_v1/routes/embeddings.py

@@ -1,65 +0,0 @@
-from typing import List, Union
-from fastapi import APIRouter
-from api_v1.settings import settings
-from gpt4all import Embed4All
-from pydantic import BaseModel, Field
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class EmbeddingRequest(BaseModel):
-    model: str = Field(
-        settings.model, description="The model to generate an embedding from."
-    )
-    input: Union[str, List[str], List[int], List[List[int]]] = Field(
-        ..., description="Input text to embed, encoded as a string or array of tokens."
-    )
-
-
-class EmbeddingUsage(BaseModel):
-    prompt_tokens: int = 0
-    total_tokens: int = 0
-
-
-class Embedding(BaseModel):
-    index: int = 0
-    object: str = "embedding"
-    embedding: List[float]
-
-
-class EmbeddingResponse(BaseModel):
-    object: str = "list"
-    model: str
-    data: List[Embedding]
-    usage: EmbeddingUsage
-
-
-router = APIRouter(prefix="/embeddings", tags=["Embedding Endpoints"])
-
-embedder = Embed4All()
-
-
-def get_embedding(data: EmbeddingRequest) -> EmbeddingResponse:
-    """
-    Calculates the embedding for the given input using a specified model.
-
-    Args:
-        data (EmbeddingRequest): An EmbeddingRequest object containing the input data
-        and model name.
-
-    Returns:
-        EmbeddingResponse: An EmbeddingResponse object encapsulating the calculated embedding,
-        usage info, and the model name.
-    """
-    embedding = embedder.embed(data.input)
-    return EmbeddingResponse(
-        data=[Embedding(embedding=embedding)], usage=EmbeddingUsage(), model=data.model
-    )
-
-
-@router.post("/", response_model=EmbeddingResponse)
-def embeddings(data: EmbeddingRequest):
-    """
-    Creates a GPT4All embedding
-    """
-    return get_embedding(data)

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -1,39 +0,0 @@
-import requests
-from fastapi import APIRouter, HTTPException
-from pydantic import BaseModel, Field
-from typing import List, Dict
-
-# Define the router for the engines module
-router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
-
-# Define the models for the engines module
-class ListEnginesResponse(BaseModel):
-    data: List[Dict] = Field(..., description="All available models.")
-
-class EngineResponse(BaseModel):
-    data: List[Dict] = Field(..., description="All available models.")
-
-
-# Define the routes for the engines module
-@router.get("/", response_model=ListEnginesResponse)
-async def list_engines():
-    try:
-        response = requests.get('https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json')
-        response.raise_for_status()  # This will raise an HTTPError if the HTTP request returned an unsuccessful status code
-        engines = response.json()
-        return ListEnginesResponse(data=engines)
-    except requests.RequestException as e:
-        logger.error(f"Error fetching engine list: {e}")
-        raise HTTPException(status_code=500, detail="Error fetching engine list")
-
-# Define the routes for the engines module
-@router.get("/{engine_id}", response_model=EngineResponse)
-async def retrieve_engine(engine_id: str):
-    try:
-        # Implement logic to fetch a specific engine's details
-        # This is a placeholder, replace with your actual data retrieval logic
-        engine_details = {"id": engine_id, "name": "Engine Name", "description": "Engine Description"}
-        return EngineResponse(data=[engine_details])
-    except Exception as e:
-        logger.error(f"Error fetching engine details: {e}")
-        raise HTTPException(status_code=500, detail=f"Error fetching details for engine {engine_id}")

gpt4all-api/gpt4all_api/app/api_v1/routes/health.py

@@ -1,13 +0,0 @@
-import logging
-from fastapi import APIRouter
-from fastapi.responses import JSONResponse
-
-log = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/health", tags=["Health"])
-
-
-@router.get('/', response_class=JSONResponse)
-async def health_check():
-    """Runs a health check on this instance of the API."""
-    return JSONResponse({'status': 'ok'}, headers={'Access-Control-Allow-Origin': '*'})

gpt4all-api/gpt4all_api/app/api_v1/settings.py

@@ -1,19 +0,0 @@
-from pydantic import BaseSettings
-
-
-class Settings(BaseSettings):
-    app_environment = 'dev'
-    model: str = 'ggml-mpt-7b-chat.bin'
-    gpt4all_path: str = '/models'
-    inference_mode: str = "cpu"
-    hf_inference_server_host: str = "http://gpt4all_gpu:80/generate"
-    sentry_dns: str = None
-
-    temp: float = 0.18
-    top_p: float = 1.0
-    top_k: int = 50
-    repeat_penalty: float = 1.18
-
-
-
-settings = Settings()

gpt4all-api/gpt4all_api/app/docs.py

@@ -1,3 +0,0 @@
-desc = 'GPT4All API'
-
-endpoint_paths = {'health': '/health'}

gpt4all-api/gpt4all_api/app/main.py

@@ -1,84 +0,0 @@
-import logging
-import os
-
-import docs
-from api_v1 import events
-from api_v1.api import router as v1_router
-from api_v1.settings import settings
-from fastapi import FastAPI, HTTPException, Request
-from fastapi.logger import logger as fastapi_logger
-from starlette.middleware.cors import CORSMiddleware
-
-logger = logging.getLogger(__name__)
-
-app = FastAPI(title='GPT4All API', description=docs.desc)
-
-# CORS Configuration (in-case you want to deploy)
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["GET", "POST", "OPTIONS"],
-    allow_headers=["*"],
-)
-
-logger.info('Adding v1 endpoints..')
-
-# add v1
-app.include_router(v1_router, prefix='/v1')
-app.add_event_handler('startup', events.startup_event_handler(app))
-app.add_exception_handler(HTTPException, events.on_http_error)
-
-
-@app.on_event("startup")
-async def startup():
-    global model
-    if settings.inference_mode == "cpu":
-        logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
-        from gpt4all import GPT4All
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
-    else:
-        # is it possible to do this once the server is up?
-        ## TODO block until HF inference server is up.
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
-
-
-@app.on_event("shutdown")
-async def shutdown():
-    logger.info("Shutting down API")
-
-
-if settings.sentry_dns is not None:
-    import sentry_sdk
-
-    def traces_sampler(sampling_context):
-        if 'health' in sampling_context['transaction_context']['name']:
-            return False
-
-    sentry_sdk.init(
-        dsn=settings.sentry_dns, traces_sample_rate=0.1, traces_sampler=traces_sampler, send_default_pii=False
-    )
-
-# This is needed to get logs to show up in the app
-if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
-    gunicorn_error_logger = logging.getLogger("gunicorn.error")
-    gunicorn_logger = logging.getLogger("gunicorn")
-
-    root_logger = logging.getLogger()
-    fastapi_logger.setLevel(gunicorn_logger.level)
-    fastapi_logger.handlers = gunicorn_error_logger.handlers
-    root_logger.setLevel(gunicorn_logger.level)
-
-    uvicorn_logger = logging.getLogger("uvicorn.access")
-    uvicorn_logger.handlers = gunicorn_error_logger.handlers
-else:
-    # https://github.com/tiangolo/fastapi/issues/2019
-    LOG_FORMAT2 = (
-        "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
-    )
-    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -1,93 +0,0 @@
-"""
-Use the OpenAI python API to test gpt4all models.
-"""
-from typing import List, get_args
-import os
-from dotenv import load_dotenv
-
-import openai
-
-openai.api_base = "http://localhost:4891/v1"
-openai.api_key = "not needed for a local LLM"
-
-# Load the .env file
-env_path = 'gpt4all-api/gpt4all_api/.env'
-load_dotenv(dotenv_path=env_path)
-
-# Fetch MODEL_ID from .env file
-model_id = os.getenv('MODEL_BIN', 'default_model_id')
-embedding = os.getenv('EMBEDDING', 'default_embedding_model_id')
-print (model_id)
-print (embedding)
-
-def test_completion():
-    model = model_id
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-
-def test_streaming_completion():
-    model = model_id
-    prompt = "Who is Michael Jordan?"
-    tokens = []
-    for resp in openai.Completion.create(
-            model=model,
-            prompt=prompt,
-            max_tokens=50,
-            temperature=0.28,
-            top_p=0.95,
-            n=1,
-            echo=True,
-            stream=True):
-        tokens.append(resp.choices[0].text)
-
-    assert (len(tokens) > 0)
-    assert (len("".join(tokens)) > len(prompt))
-
-# Modified test batch, problems with keyerror in response
-def test_batched_completion():
-    model = model_id  # replace with your specific model ID
-    prompt = "Who is Michael Jordan?"
-    responses = []
-
-    # Loop to create completions one at a time
-    for _ in range(3):
-        response = openai.Completion.create(
-            model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-        )
-        responses.append(response)
-
-    # Assertions to check the responses
-    for response in responses:
-        assert len(response['choices'][0]['text']) > len(prompt)
-
-    assert len(responses) == 3
-
-def test_embedding():
-    model = embedding
-    prompt = "Who is Michael Jordan?"
-    response = openai.Embedding.create(model=model, input=prompt)
-    output = response["data"][0]["embedding"]
-    args = get_args(List[float])
-
-    assert response["model"] == model
-    assert isinstance(output, list)
-    assert all(isinstance(x, args) for x in output)
-
-def test_chat_completion():
-    model = model_id
-
-    response = openai.ChatCompletion.create(
-        model=model,
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": "Knock knock."},
-            {"role": "assistant", "content": "Who's there?"},
-            {"role": "user", "content": "Orange."},
-            ]
-    )
-
-    assert response.choices[0].message.role == "assistant"
-    assert len(response.choices[0].message.content) > 0

gpt4all-api/gpt4all_api/env

@@ -1,3 +0,0 @@
-# Add your GGUF compatible model LLM here. ie: MODEL_BIN="mistral-7b-instruct-v0.1.Q4_0", rename file ".env"
-# Make sure this LLM matches the model you placed inside the models folder
-MODEL_BIN=""

gpt4all-api/gpt4all_api/models/README.md

@@ -1 +0,0 @@
-### Drop GGUF compatible models here, make sure it matches MODEL_BIN on your .env file

gpt4all-api/gpt4all_api/requirements.txt

@@ -1,13 +0,0 @@
-aiohttp>=3.6.2
-aiofiles
-pydantic>=1.4.0,<2.0.0
-requests>=2.24.0
-ujson>=2.0.2
-fastapi>=0.95.0
-Jinja2>=3.0
-gpt4all>=1.0.0
-pytest
-openai==0.28.0
-black
-isort
-python-dotenv

gpt4all-api/makefile

@@ -1,46 +0,0 @@
-ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-APP_NAME:=gpt4all_api
-PYTHON:=python3.8
-SHELL := /bin/bash
-
-all: dependencies
-
-fresh: clean dependencies
-
-testenv: clean_testenv test_build
-	docker compose -f docker-compose.yaml up --build
-
-testenv_gpu: clean_testenv test_build
-	docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up --build
-
-testenv_d: clean_testenv test_build
-	docker compose env up --build -d
-
-test:
-	docker compose exec $(APP_NAME) pytest -svv --disable-warnings -p no:cacheprovider /app/tests
-
-test_build:
-    DOCKER_BUILDKIT=1 docker build -t $(APP_NAME) --progress plain -f $(APP_NAME)/Dockerfile.buildkit .
-
-clean_testenv:
-	docker compose down -v
-
-fresh_testenv: clean_testenv testenv
-
-venv:
-	if [ ! -d $(ROOT_DIR)/venv ]; then $(PYTHON) -m venv $(ROOT_DIR)/venv; fi
-
-dependencies: venv
-	source $(ROOT_DIR)/venv/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
-
-clean: clean_testenv
-	# Remove existing environment
-	rm -rf $(ROOT_DIR)/venv;
-	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
-
-
-black:
-	source $(ROOT_DIR)/venv/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
-
-isort:
-	source $(ROOT_DIR)/venv/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)

gpt4all-api/triton/README.md

@@ -0,0 +1,13 @@
+# To Run Inference Server
+
+docker run --gpus=1 --rm --net=host -v ${PWD}/model_store:/model_store nvcr.io/nvidia/tritonserver:23.01-py3 tritonserver --model-repository=/model_store
+
+python client.py --model=<model_name>
+
+
+## Dynamic Batching
+Need to figure out how to do batching such that we can have dynamic batching
+We're getting 1.3 infer/sec which seems slow....
+
+To test,
+perf_analyzer -m nomic-ai--gpt4all-j --input-data test_data.json --measurement-interval 25000 --request-rate-range=10 -b 8

gpt4all-api/triton/client.py

@@ -0,0 +1,75 @@
+import torch
+import tritonclient.grpc.aio as grpcclient
+
+
+def prepare_inference_inputs(
+    inputs_ids: torch.IntTensor, new_tokens: int = 1, temperature: float = 1.0
+):
+    batch_size = inputs_ids.shape[0]
+
+    input_ids_input = grpcclient.InferInput("input_ids", inputs_ids.shape, "INT32")
+    input_ids_input.set_data_from_numpy(inputs_ids.int().cpu().numpy())
+
+    new_tokens_input = grpcclient.InferInput(
+        "tensor_of_seq_len", [batch_size, new_tokens], "INT32"
+    )
+    new_tokens_input.set_data_from_numpy(
+        torch.zeros(batch_size, new_tokens, dtype=torch.int32).cpu().numpy()
+    )
+
+    temperature_input = grpcclient.InferInput("temperature", [batch_size, 1], "FP32")
+    temperature_input.set_data_from_numpy(
+        torch.full([batch_size, 1], temperature, dtype=torch.float32).cpu().numpy()
+    )
+
+    inputs = [input_ids_input, new_tokens_input, temperature_input]
+    outputs = [
+        grpcclient.InferRequestedOutput("logits"),
+        grpcclient.InferRequestedOutput("output_ids"),
+    ]
+    return inputs, outputs
+
+
+async def infer(
+    triton_client, model_name, input_ids, new_tokens: int = 1, temperature: float = 1.0
+):
+    inputs, outputs = prepare_inference_inputs(input_ids, new_tokens, temperature)
+
+    triton_model_name = model_name.replace("/", "--")
+
+    result = await triton_client.infer(
+        model_name=triton_model_name, inputs=inputs, outputs=outputs
+    )
+
+    logits = torch.tensor(result.as_numpy("logits").copy(), requires_grad=False)
+    output_ids = torch.tensor(result.as_numpy("output_ids").copy(), requires_grad=False)
+
+    return logits, output_ids
+
+def Client(url: str):
+  return grpcclient.InferenceServerClient(url=url)
+
+if __name__ == "__main__":
+    import argparse
+    from transformers import AutoTokenizer
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--url", type=str, default="localhost:8001")
+    parser.add_argument("--model", type=str, default="gpt2")
+
+    args = parser.parse_args()
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model, use_fast=False)
+
+    async def main():
+        async with Client(args.url) as triton_client:
+            while True:
+                prompt = input("Prompt: ")
+                input_ids = tokenizer.encode(prompt, return_tensors="pt")
+                last_logits, output_ids = await infer(
+                    triton_client, args.model, input_ids, new_tokens=256, temperature=1.0,
+                )
+                print(tokenizer.decode(output_ids[0]))
+
+    import asyncio
+    asyncio.run(main())

gpt4all-api/triton/convert_to_triton.py

@@ -0,0 +1,149 @@
+import argparse
+import os
+from string import Template
+
+import torch
+from torch import nn
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument(
+    "--model", type=str, required=True, help="Path to HF checkpoint with the base model"
+)
+
+parser.add_argument(
+    "--max-batch-size", type=int, default=64, help="Maximum batch size for inference"
+)
+
+parser.add_argument(
+    "--revision",
+    type=str,
+    required=False,
+    help="Optional branch/commit of the HF checkpoint",
+)
+
+parser.add_argument("--device", type=int, default=0)
+args = parser.parse_args()
+
+device = torch.device(args.device)
+
+
+class ModelLogits(nn.Module):
+    def __init__(self, model):
+        super().__init__()
+        self.model = model
+
+    @torch.inference_mode()
+    def forward(self, input_ids: torch.Tensor):
+        return self.model(input_ids).logits
+
+
+class InferModel(nn.Module):
+    def __init__(self, traced_model, eos_token_id):
+        super().__init__()
+        self.traced_model = traced_model
+        self.eos_token_id = eos_token_id
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        tensor_of_seq_len: torch.Tensor,
+        temperature: torch.Tensor,
+    ):
+        # this has mostly been adapted from huggingface generate
+        unfinished_sequences = torch.ones(input_ids.shape[0], dtype=torch.long, device=input_ids.device)
+        eos_token_id_tensor = torch.tensor([self.eos_token_id]).to(input_ids.device)
+
+        with torch.no_grad():
+            for _ in range(tensor_of_seq_len.shape[1] - 1):
+                logits = self.traced_model(input_ids).float()
+                next_token_logits = logits[:, -1, :]
+                next_token_logits = next_token_logits / temperature
+                
+                next_tokens = torch.multinomial(
+                    torch.softmax(next_token_logits, dim=-1), input_ids.shape[0] 
+                )
+
+                next_tokens = next_tokens * unfinished_sequences + self.eos_token_id * (1 - unfinished_sequences)
+
+                unfinished_sequences = unfinished_sequences.mul(
+                    next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod(dim=0)
+                )
+
+                # stop when each sentence is finished
+                if unfinished_sequences.max() == 0:
+                    return input_ids.int(), logits
+
+                input_ids = torch.cat([input_ids, next_tokens], dim=-1)
+
+                unfinished_sequences = unfinished_sequences.mul(
+                    next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod(dim=0)
+                )
+
+            # in TorchScript, the above logits var lifetime doesn't escape the loop's scope
+            logits = self.traced_model(input_ids).float()
+            next_token_logits = logits[:, -1, :]
+            next_token_logits = next_token_logits / temperature
+
+            next_tokens = torch.multinomial(
+                    torch.softmax(next_token_logits, dim=-1), input_ids.shape[0] 
+                )
+
+            next_tokens = next_tokens * unfinished_sequences + self.eos_token_id * (1 - unfinished_sequences)
+
+            input_ids = torch.cat([input_ids, next_tokens], dim=-1)
+
+            return input_ids.int(), logits
+            
+
+print(f"Converting {args.model} to TorchScript...")
+tokenizer = AutoTokenizer.from_pretrained(args.model, use_fast=False)
+model = ModelLogits(AutoModelForCausalLM.from_pretrained(args.model,
+                                                         trust_remote_code=True,
+                                                         revision=args.revision,
+                                                         torch_dtype=torch.float16,
+                                                         use_cache=False))
+
+model.eval()
+model.requires_grad_(False)
+model = model.to(device)
+
+
+input = tokenizer("annotator model's hash is 0x", return_tensors="pt").to(device)
+print(f"{model(input.input_ids)=}")
+
+traced_script_module = torch.jit.trace(model, input.input_ids)
+print("Tracing...")
+print(f"{traced_script_module(input.input_ids)=}")
+
+print("Scripting generation wrapper...")
+# need to script this as we have data conditional flow
+scripted_generator_model = torch.jit.script(InferModel(traced_script_module, tokenizer.eos_token_id))
+print(scripted_generator_model.code)
+
+print(f"{input.input_ids=}")
+# x = input.input_ids, torch.empty(1, 5), torch.full([1, 1], 1.0).cuda(), torch.full([1, 1], len(tokenizer) // 2).cuda(), torch.full([1, 1], 0.9).cuda()
+x = input.input_ids, torch.empty(1, 5),  torch.full([1, 1], 0.9).cuda()
+print(x[0].shape)
+
+print(f"{tokenizer.decode(scripted_generator_model(*x)[0][0])=}")
+
+sanitized_name = args.model.replace("/", "--")
+print("Model renamed to ", sanitized_name)
+
+print("Saving TorchScript model...")
+
+os.makedirs(f"model_store/{sanitized_name}/1", exist_ok=True)
+scripted_generator_model.save(f"model_store/{sanitized_name}/1/traced-model.pt")
+
+config_path = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), "triton_config.pbtxt"
+)
+with open(config_path) as f:
+    template = Template(f.read())
+config = template.substitute(
+    {"model_name": sanitized_name, "max_batch_size": args.max_batch_size}
+)
+with open(f"model_store/{sanitized_name}/config.pbtxt", "w") as f:
+    f.write(config)

gpt4all-api/triton/requirements.txt

@@ -0,0 +1,5 @@
+transformers
+triton
+einops
+pandas
+sentencepiece

gpt4all-api/triton/test_data.json

@@ -0,0 +1,34 @@
+{
+    "data":
+      [
+        {
+          "input_ids": {
+            "content": [17250, 11, 703, 389, 345, 30],
+            "shape": [6]
+          },
+          "tensor_of_seq_len": {
+            "content": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+            "shape": [17]
+          },
+          "temperature": {
+            "content": [1.0],
+            "shape": [1]
+          }
+        },
+        {
+            "input_ids": {
+              "content": [17250, 11, 703, 389, 345, 30],
+              "shape": [6]
+            },
+            "tensor_of_seq_len": {
+              "content": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+              "shape": [17]
+            },
+            "temperature": {
+              "content": [1.0],
+              "shape": [1]
+            }
+          }
+
+      ]
+  }

gpt4all-api/triton/triton_config.pbtxt

@@ -0,0 +1,69 @@
+name: "${model_name}"
+backend: "pytorch"
+default_model_filename: "traced-model.pt"
+max_batch_size: ${max_batch_size}
+
+dynamic_batching {
+}
+
+parameters {
+  key: "model_name"
+  value: {
+    string_value: "${model_name}"
+  }
+}
+
+instance_group [
+  {
+    count: 1
+    kind: KIND_GPU
+    gpus: [0]
+  }
+]
+
+input [
+  {
+    name: "input_ids"
+    data_type: TYPE_INT32
+    dims: [-1]
+  },
+  {
+    name: "tensor_of_seq_len"
+    data_type: TYPE_INT32
+    dims: [-1]
+  },
+  {
+    name: "temperature"
+    data_type: TYPE_FP32
+    dims: [-1]
+  }
+]
+
+output [
+  {
+    name: "output_ids"
+    data_type: TYPE_INT32
+    dims: [-1]
+  },
+  {
+    name: "logits"
+    data_type: TYPE_FP32
+    dims: [-1]
+  }
+]
+
+parameters {
+  key: "data_type"
+  value: {
+    string_value: "fp16"
+  }
+}
+
+parameters: {
+  key: "INFERENCE_MODE"
+  value: {
+    string_value: "true"
+  }
+}
+
+version_policy: {specific: {versions: [1]}}

gpt4all-backend/CMakeLists.txt

@@ -1,6 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if(APPLE)
   option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
@@ -10,9 +9,7 @@ if(APPLE)
     set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
   else()
     # Build for the host architecture on macOS
-    if(NOT CMAKE_OSX_ARCHITECTURES)
-      set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
-    endif()
+    set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
   endif()
 endif()
 
@@ -20,7 +17,7 @@ endif()
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 
 set(LLMODEL_VERSION_MAJOR 0)
-set(LLMODEL_VERSION_MINOR 5)
+set(LLMODEL_VERSION_MINOR 2)
 set(LLMODEL_VERSION_PATCH 0)
 set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
 project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
@@ -42,9 +39,6 @@ endif()
 include(llama.cpp.cmake)
 
 set(BUILD_VARIANTS default avxonly)
-if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-    set(BUILD_VARIANTS ${BUILD_VARIANTS} metal)
-endif()
 
 set(CMAKE_VERBOSE_MAKEFILE ON)
 
@@ -60,15 +54,10 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     set(LLAMA_F16C ${GPT4ALL_ALLOW_NON_AVX})
     set(LLAMA_FMA  ${GPT4ALL_ALLOW_NON_AVX})
 
-    if (BUILD_VARIANT STREQUAL metal)
-        set(LLAMA_METAL YES)
-    else()
-        set(LLAMA_METAL NO)
-    endif()
-
     # Include GGML
-    set(LLAMA_K_QUANTS YES)
     include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
+    include_ggml(llama.cpp-230511 -230511-${BUILD_VARIANT} ON)
+    include_ggml(llama.cpp-230519 -230519-${BUILD_VARIANT} ON)
 
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
@@ -76,14 +65,13 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         message(STATUS "Configuring model implementation target ${TARGET_NAME}")
         # Link to ggml/llama
         target_link_libraries(${TARGET_NAME}
-            PRIVATE ${BASE_LIB}-${BUILD_VARIANT})
+            PUBLIC ${BASE_LIB}-${BUILD_VARIANT})
         # Let it know about its build variant
         target_compile_definitions(${TARGET_NAME}
             PRIVATE GGML_BUILD_VARIANT="${BUILD_VARIANT}")
         # Enable IPO if possible
-# FIXME: Doesn't work with msvc reliably. See https://github.com/nomic-ai/gpt4all/issues/841
-#        set_property(TARGET ${TARGET_NAME}
-#                     PROPERTY INTERPROCEDURAL_OPTIMIZATION ${IPO_SUPPORTED})
+        set_property(TARGET ${TARGET_NAME}
+                     PROPERTY INTERPROCEDURAL_OPTIMIZATION ${IPO_SUPPORTED})
     endfunction()
 
     # Add each individual implementations
@@ -93,11 +81,25 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    if (NOT LLAMA_METAL)
-        add_library(gptj-${BUILD_VARIANT} SHARED
-            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        prepare_target(gptj llama-mainline)
-    endif()
+    add_library(llamamodel-230519-${BUILD_VARIANT} SHARED
+        llamamodel.cpp llmodel_shared.cpp)
+    target_compile_definitions(llamamodel-230519-${BUILD_VARIANT} PRIVATE
+        LLAMA_VERSIONS===2 LLAMA_DATE=230519)
+    prepare_target(llamamodel-230519 llama-230519)
+
+    add_library(llamamodel-230511-${BUILD_VARIANT} SHARED
+        llamamodel.cpp llmodel_shared.cpp)
+    target_compile_definitions(llamamodel-230511-${BUILD_VARIANT} PRIVATE
+        LLAMA_VERSIONS=<=1 LLAMA_DATE=230511)
+    prepare_target(llamamodel-230511 llama-230511)
+
+    add_library(gptj-${BUILD_VARIANT} SHARED
+        gptj.cpp utils.h utils.cpp llmodel_shared.cpp)
+    prepare_target(gptj ggml-230511)
+
+    add_library(mpt-${BUILD_VARIANT} SHARED
+        mpt.cpp utils.h utils.cpp llmodel_shared.cpp)
+    prepare_target(mpt ggml-230511)
 endforeach()
 
 add_library(llmodel

gpt4all-backend/dlhandle.h

@@ -18,7 +18,7 @@ public:
     };
 
     Dlhandle() : chandle(nullptr) {}
-    Dlhandle(const std::string& fpath, int flags = RTLD_LAZY | RTLD_LOCAL) {
+    Dlhandle(const std::string& fpath, int flags = RTLD_LAZY) {
         chandle = dlopen(fpath.c_str(), flags);
         if (!chandle) {
             throw Exception("dlopen(\""+fpath+"\"): "+dlerror());
@@ -53,8 +53,6 @@ public:
     }
 };
 #else
-#include <algorithm>
-#include <filesystem>
 #include <string>
 #include <exception>
 #include <stdexcept>
@@ -77,9 +75,7 @@ public:
 
     Dlhandle() : chandle(nullptr) {}
     Dlhandle(const std::string& fpath) {
-        std::string afpath = std::filesystem::absolute(fpath).string();
-        std::replace(afpath.begin(), afpath.end(), '/', '\\');
-        chandle = LoadLibraryExA(afpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
+        chandle = LoadLibraryA(fpath.c_str());
         if (!chandle) {
             throw Exception("dlopen(\""+fpath+"\"): Error");
         }

gpt4all-backend/gptj.cpp

@@ -2,13 +2,12 @@
 #include "gptj_impl.h"
 
 #include "utils.h"
-#include "llmodel_shared.h"
 
 #include <cassert>
-#include <cinttypes>
 #include <cmath>
 #include <cstdio>
 #include <cstring>
+#include <fstream>
 #include <map>
 #include <string>
 #include <vector>
@@ -31,6 +30,8 @@
 
 namespace {
 const char *modelType_ = "GPT-J";
+
+static const size_t MB = 1024*1024;
 }
 
 // default hparams (GPT-J 6B)
@@ -41,7 +42,7 @@ struct gptj_hparams {
     int32_t n_head  = 16;
     int32_t n_layer = 28;
     int32_t n_rot   = 64;
-    float norm_eps  = 1e-5;
+    int32_t f16     = 1;
 };
 
 struct gptj_layer {
@@ -64,6 +65,39 @@ struct gptj_layer {
     struct ggml_tensor * c_mlp_proj_b;
 };
 
+struct gptj_buffer {
+    uint8_t * addr = NULL;
+    size_t size = 0;
+
+    void resize(size_t size) {
+        delete[] addr;
+        addr = new uint8_t[size];
+        this->size = size;
+    }
+
+    ~gptj_buffer() {
+        fflush(stdout);
+        delete[] addr;
+    }
+};
+
+struct gptj_kv_cache {
+    struct ggml_tensor * k;
+    struct ggml_tensor * v;
+
+    struct ggml_context * ctx = NULL;
+
+    gptj_buffer buf;
+
+    int n; // number of tokens currently in the cache
+
+    ~gptj_kv_cache() {
+        if (ctx) {
+            ggml_free(ctx);
+        }
+    }
+};
+
 struct gptj_model {
     gptj_hparams hparams;
 
@@ -79,15 +113,13 @@ struct gptj_model {
     std::vector<gptj_layer> layers;
 
     // key + value memory
-    struct llm_kv_cache kv_self;
+    struct gptj_kv_cache kv_self;
 
     //
     struct ggml_context * ctx;
     std::map<std::string, struct ggml_tensor *> tensors;
 
-    llm_buffer eval_buf;
-    llm_buffer scr0_buf;
-    llm_buffer scr1_buf;
+    gptj_buffer buf;
 
     ~gptj_model() {
         if (ctx) {
@@ -98,7 +130,7 @@ struct gptj_model {
 
 static bool kv_cache_init(
         const struct gptj_hparams & hparams,
-              struct llm_kv_cache & cache,
+             struct gptj_kv_cache & cache,
                          ggml_type   wtype,
                                int   n_ctx) {
     const int n_embd  = hparams.n_embd;
@@ -107,7 +139,7 @@ static bool kv_cache_init(
     const int64_t n_mem      = (int64_t)n_layer*n_ctx;
     const int64_t n_elements = n_embd*n_mem;
 
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB);
 
     struct ggml_init_params params;
     params.mem_size   = cache.buf.size;
@@ -127,149 +159,200 @@ static bool kv_cache_init(
     return true;
 }
 
-// load the model's weights from a file path
-bool gptj_model_load(const std::string &fname, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
+// load the model's weights from a stream
+bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & model, gpt_vocab & vocab) {
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
-    if(mem_req != nullptr) {
-        *mem_req = 0;
-    }
 
-    // create the ggml context
-    struct gguf_init_params params = {
-        /*.no_alloc = */ false,
-        /*.ctx      = */ &model.ctx,
-    };
-
-    gguf_context *ggufctx = gguf_init_from_file(fname.c_str(), params);
-    if (!ggufctx) {
-        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
-        return false;
+    // verify magic
+    {
+        uint32_t magic;
+        fin.read((char *) &magic, sizeof(magic));
+        if (magic != 0x67676d6c) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
+            return false;
+        }
     }
 
     // load hparams
     {
         auto & hparams = model.hparams;
 
-        bool ok = false;
-        int keyidx;
-
-        do {
-            keyidx = gguf_find_key(ggufctx, "gptj.context_length");
-            if (keyidx == -1) { break; }
-            hparams.n_ctx = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.embedding_length");
-            if (keyidx == -1) { break; }
-            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.attention.head_count");
-            if (keyidx == -1) { break; }
-            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.block_count");
-            if (keyidx == -1) { break; }
-            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.rope.dimension_count");
-            if (keyidx == -1) { break; }
-            hparams.n_rot = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.attention.layer_norm_epsilon");
-            if (keyidx == -1) { break; }
-            hparams.norm_eps = gguf_get_val_f32(ggufctx, keyidx);
-
-            ok = true;
-        } while (false);
-
-        if (!ok) {
-            fprintf(stderr, "%s: required hparam missing!\n", __func__);
-            return false;
-        }
+        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
+        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
+        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
+        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
+        fin.read((char *) &hparams.n_rot,   sizeof(hparams.n_rot));
+        fin.read((char *) &hparams.f16,     sizeof(hparams.f16));
 
+        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
         printf("%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
         printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
         printf("%s: n_head  = %d\n", __func__, hparams.n_head);
         printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
         printf("%s: n_rot   = %d\n", __func__, hparams.n_rot);
+        printf("%s: f16     = %d\n", __func__, hparams.f16);
     }
 
     // load vocab
     {
-        auto & hparams = model.hparams;
+        int32_t n_vocab = 0;
+        fin.read((char *) &n_vocab, sizeof(n_vocab));
 
-        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
-            return false;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "gpt2") != 0) {
-            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
+        if (n_vocab != model.hparams.n_vocab) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
+                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
             return false;
         }
 
-        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
-        if (tokens_keyidx == -1) {
-            fprintf(stderr, "%s: gpt2 tokenizer vocab not found!\n", __func__);
-            return false;
-        }
+        std::string word;
+        for (int i = 0; i < n_vocab; i++) {
+            uint32_t len;
+            fin.read((char *) &len, sizeof(len));
 
-        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
-        printf("%s: gpt2 tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
+            word.resize(len);
+            fin.read((char *) word.data(), len);
 
-        for (int i = 0; i < hparams.n_vocab; i++) {
-            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
             vocab.token_to_id[word] = i;
             vocab.id_to_token[i] = word;
         }
     }
 
+    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
+    // in order to save memory and also to speed up the computation
+    ggml_type wtype = GGML_TYPE_COUNT;
+    switch (model.hparams.f16) {
+        case 0: wtype = GGML_TYPE_F32;  break;
+        case 1: wtype = GGML_TYPE_F16;  break;
+        case 2: wtype = GGML_TYPE_Q4_0; break;
+        case 3: wtype = GGML_TYPE_Q4_1; break;
+        case 5: wtype = GGML_TYPE_Q4_2; break;
+        default:
+                {
+                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
+                            __func__, fname.c_str(), model.hparams.f16);
+                    return false;
+                }
+    }
+
     auto & ctx = model.ctx;
 
-    size_t ctx_size = ggml_get_mem_size(ctx);
-    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size / (1024.0 * 1024.0));
+    size_t ctx_size = 0;
 
-    if (mem_req != nullptr) {
-        *mem_req = ctx_size;
-        gguf_free(ggufctx);
-        return false;
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_ctx   = hparams.n_ctx;
+        const int n_vocab = hparams.n_vocab;
+
+        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_g
+        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_b
+
+        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype); // wte
+
+        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype);         // lmh_g
+        ctx_size +=        n_vocab*ggml_type_sizef(GGML_TYPE_F32); // lmh_b
+
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_g
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_b
+
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_q_proj_w
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_k_proj_w
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_v_proj_w
+
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_proj_w
+
+        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_fc_w
+        ctx_size += n_layer*(       4*n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_fc_b
+
+        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_proj_w
+        ctx_size += n_layer*(         n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_proj_b
+
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_k
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_v
+
+        ctx_size += (5 + 10*n_layer)*256; // object overhead
+
+        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
+    }
+
+    // create the ggml context
+    {
+        struct ggml_init_params params = {
+            .mem_size   = ctx_size,
+            .mem_buffer = NULL,
+            .no_alloc = false
+        };
+
+        model.ctx = ggml_init(params);
+        if (!model.ctx) {
+            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
+            return false;
+        }
     }
 
     // prepare memory for the weights
     {
         const auto & hparams = model.hparams;
-        model.layers.resize(hparams.n_layer);
 
-        model.wte    = ggml_get_tensor(ctx, "token_embd.weight");
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_vocab = hparams.n_vocab;
 
-        model.ln_f_g = ggml_get_tensor(ctx, "output_norm.weight");
-        model.ln_f_b = ggml_get_tensor(ctx, "output_norm.bias");
+        model.layers.resize(n_layer);
 
-        model.lmh_g  = ggml_get_tensor(ctx, "output.weight");
-        model.lmh_b  = ggml_get_tensor(ctx, "output.bias");
+        model.wte    = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
 
-        auto name = [](int i, std::string n) {
-            static std::string key;
-            key = "blk." + std::to_string(i) + "." + n;
-            return key.c_str();
-        };
+        model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
 
-        for (int i = 0; i < hparams.n_layer; ++i) {
+        model.lmh_g  = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
+        model.lmh_b  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab);
+
+        // map by name
+        model.tensors["transformer.wte.weight"] = model.wte;
+
+        model.tensors["transformer.ln_f.weight"] = model.ln_f_g;
+        model.tensors["transformer.ln_f.bias"]   = model.ln_f_b;
+
+        model.tensors["lm_head.weight"] = model.lmh_g;
+        model.tensors["lm_head.bias"]   = model.lmh_b;
+
+        for (int i = 0; i < n_layer; ++i) {
             auto & layer = model.layers[i];
 
-            layer.ln_1_g          = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
-            layer.ln_1_b          = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
+            layer.ln_1_g          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
+            layer.ln_1_b          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
 
-            layer.c_attn_q_proj_w = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
-            layer.c_attn_k_proj_w = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
-            layer.c_attn_v_proj_w = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
+            layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
 
-            layer.c_attn_proj_w   = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
+            layer.c_attn_proj_w   = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
 
-            layer.c_mlp_fc_w      = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
-            layer.c_mlp_fc_b      = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
+            layer.c_mlp_fc_w      = ggml_new_tensor_2d(ctx, wtype,           n_embd, 4*n_embd);
+            layer.c_mlp_fc_b      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
 
-            layer.c_mlp_proj_w    = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
-            layer.c_mlp_proj_b    = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
+            layer.c_mlp_proj_w    = ggml_new_tensor_2d(ctx, wtype,         4*n_embd,   n_embd);
+            layer.c_mlp_proj_b    = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
+
+            // map by name
+            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.weight"]          = layer.ln_1_g;
+            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.bias"]            = layer.ln_1_b;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.q_proj.weight"]   = layer.c_attn_q_proj_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.k_proj.weight"]   = layer.c_attn_k_proj_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.v_proj.weight"]   = layer.c_attn_v_proj_w;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".attn.out_proj.weight"] = layer.c_attn_proj_w;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.weight"]     = layer.c_mlp_fc_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.bias"]       = layer.c_mlp_fc_b;
+
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.weight"]    = layer.c_mlp_proj_w;
+            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.bias"]      = layer.c_mlp_proj_b;
         }
     }
 
@@ -286,12 +369,110 @@ bool gptj_model_load(const std::string &fname, gptj_model & model, gpt_vocab & v
         printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
     }
 
-    model.scr0_buf.resize(256u * 1024 * 1024);
-    model.scr1_buf.resize(256u * 1024 * 1024);
+    // load weights
+    {
+        int n_tensors = 0;
+        size_t total_size = 0;
+
+        printf("%s: ", __func__);
+
+        while (true) {
+            int32_t n_dims;
+            int32_t length;
+            int32_t ftype;
+
+            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
+            fin.read(reinterpret_cast<char *>(&ftype),  sizeof(ftype));
+
+            if (fin.eof()) {
+                break;
+            }
+
+            int32_t nelements = 1;
+            int32_t ne[2] = { 1, 1 };
+            for (int i = 0; i < n_dims; ++i) {
+                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+                nelements *= ne[i];
+            }
+
+            std::string name(length, 0);
+            fin.read(&name[0], length);
+
+            if (model.tensors.find(name.data()) == model.tensors.end()) {
+                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
+                return false;
+            }
+
+            auto tensor = model.tensors[name.data()];
+            if (ggml_nelements(tensor) != nelements) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
+                return false;
+            }
+
+            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%lu, %lu], expected [%d, %d]\n",
+                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
+                return false;
+            }
+
+            if (0) {
+                static const char * ftype_str[] = { "f32", "f16", "q4_0", "q4_1", };
+                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ftype_str[ftype], ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
+            }
+
+            size_t bpe = 0;
+
+            switch (ftype) {
+                case 0: bpe = ggml_type_size(GGML_TYPE_F32);  break;
+                case 1: bpe = ggml_type_size(GGML_TYPE_F16);  break;
+                case 2: bpe = ggml_type_size(GGML_TYPE_Q4_0); assert(ne[0] % 64 == 0); break;
+                case 3: bpe = ggml_type_size(GGML_TYPE_Q4_1); assert(ne[0] % 64 == 0); break;
+                default:
+                        {
+                            fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype);
+                            return false;
+                        }
+            };
+
+            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
+                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
+                return false;
+            }
+
+            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
+
+            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
+            total_size += ggml_nbytes(tensor);
+            if (++n_tensors % 8 == 0) {
+                printf(".");
+                fflush(stdout);
+            }
+        }
+
+        printf(" done\n");
+
+        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
+    }
 
     return true;
 }
 
+// load the model's weights from a file path
+bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) {
+
+    auto fin = std::ifstream(fname, std::ios::binary);
+    if (!fin) {
+        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
+        return false;
+    }
+
+    bool loaded = gptj_model_load(fname, fin, model, vocab);
+    fin.close();
+    return loaded;
+}
+
 // evaluate the transformer
 //
 //   - model:     the model
@@ -320,37 +501,31 @@ bool gptj_eval(
     const int n_vocab = hparams.n_vocab;
     const int n_rot   = hparams.n_rot;
 
-    const size_t init_buf_size = 1024_MiB;
-    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
-        model.eval_buf.resize(init_buf_size);
+    const size_t init_buf_size = 1024u*MB;
+    if (!model.buf.addr || model.buf.size < init_buf_size)
+        model.buf.resize(init_buf_size);
 
-    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
+    if (mem_per_token > 0 && mem_per_token*N > model.buf.size) {
         const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
-        printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.eval_buf.size, buf_size_new);
+        printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
 
         // reallocate
-        model.eval_buf.resize(buf_size_new);
-        if (model.eval_buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
+        model.buf.resize(buf_size_new);
+        if (model.buf.addr == nullptr) {
+            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.buf.size);
             return false;
         }
     }
 
     struct ggml_init_params params = {
-        .mem_size   = model.eval_buf.size,
-        .mem_buffer = model.eval_buf.addr,
+        .mem_size   = model.buf.size,
+        .mem_buffer = model.buf.addr,
         .no_alloc = false
     };
 
     struct ggml_context * ctx0 = ggml_init(params);
-    struct ggml_cgraph * gf = ggml_new_graph(ctx0);
-
-    // KQ_pos - contains the positions
-    struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    int * data = (int *) KQ_pos->data;
-    for (int i = 0; i < N; ++i) {
-        data[i] = n_past + i;
-    }
+    struct ggml_cgraph gf = {};
+    gf.n_threads = n_threads;
 
     struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
     memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -360,10 +535,10 @@ bool gptj_eval(
 
     for (int il = 0; il < n_layer; ++il) {
         struct ggml_tensor * cur;
-        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
+
         // norm
         {
-            cur = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
+            cur = ggml_norm(ctx0, inpL);
 
             // cur = ln_1_g*cur + ln_1_b
             cur = ggml_add(ctx0,
@@ -377,44 +552,48 @@ bool gptj_eval(
 
         // self-attention
         {
-            struct ggml_tensor * Qcur = ggml_rope(
-                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N),
-                KQ_pos, n_rot, 0, 0
-            );
-            struct ggml_tensor * Kcur = ggml_rope(
-                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N),
-                KQ_pos, n_rot, 0, 0
-            );
+            struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur);
+            struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur);
+            struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur);
 
             // store key and value to memory
             {
-                struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur));
-
                 struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
-                        (   n_ctx)*ggml_element_size(model.kv_self.v),
-                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
+                struct ggml_tensor * v = ggml_view_1d(ctx0, model.kv_self.v, N*n_embd, (ggml_element_size(model.kv_self.v)*n_embd)*(il*n_ctx + n_past));
 
-                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
             }
 
             // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+            struct ggml_tensor * Q =
+                ggml_permute(ctx0,
+                        ggml_rope(ctx0,
+                            ggml_cpy(ctx0,
+                                Qcur,
+                                ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),
+                            n_past, n_rot, 0),
+                        0, 2, 1, 3);
 
             // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
             struct ggml_tensor * K =
                 ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0,
-                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                            n_embd/n_head, n_head, n_past + N),
+                        ggml_rope(ctx0,
+                            ggml_reshape_3d(ctx0,
+                                ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
+                                n_embd/n_head, n_head, n_past + N),
+                            n_past, n_rot, 1),
                         0, 2, 1, 3);
 
             // K * Q
             struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
 
             // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled = ggml_scale(ctx0, KQ, 1.0f/sqrt(float(n_embd)/n_head));
+            struct ggml_tensor * KQ_scaled =
+                ggml_scale(ctx0,
+                        KQ,
+                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
+                        );
 
             // KQ_masked = mask_past(KQ_scaled)
             struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past);
@@ -423,15 +602,17 @@ bool gptj_eval(
             struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
 
             // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V =
-                ggml_view_3d(ctx0, model.kv_self.v,
-                        n_past + N, n_embd/n_head, n_head,
-                        n_ctx*ggml_element_size(model.kv_self.v),
-                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
-                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
+            struct ggml_tensor * V_trans =
+                ggml_cpy(ctx0,
+                        ggml_permute(ctx0,
+                            ggml_reshape_3d(ctx0,
+                                ggml_view_1d(ctx0, model.kv_self.v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd),
+                                n_embd/n_head, n_head, n_past + N),
+                            1, 2, 0, 3),
+                        ggml_new_tensor_3d(ctx0, model.kv_self.v->type, n_past + N, n_embd/n_head, n_head));
 
             // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);
 
             // KQV_merged = KQV.permute(0, 2, 1, 3)
             struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
@@ -449,7 +630,6 @@ bool gptj_eval(
 
         struct ggml_tensor * inpFF = cur;
 
-        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
         // feed-forward network
         // this is independent of the self-attention result, so it could be done in parallel to the self-attention
         {
@@ -483,11 +663,9 @@ bool gptj_eval(
         inpL = ggml_add(ctx0, cur, inpL);
     }
 
-    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
     // norm
     {
-        inpL = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
+        inpL = ggml_norm(ctx0, inpL);
 
         // inpL = ln_f_g*inpL + ln_f_b
         inpL = ggml_add(ctx0,
@@ -497,8 +675,6 @@ bool gptj_eval(
                 ggml_repeat(ctx0, model.ln_f_b, inpL));
     }
 
-    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
-
     // lm_head
     {
         inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL);
@@ -511,22 +687,13 @@ bool gptj_eval(
     // logits -> probs
     //inpL = ggml_soft_max(ctx0, inpL);
 
-    ggml_build_forward_expand(gf, inpL);
-
     // run the computation
-    {
-        std::unique_ptr<uint8_t []> data;
-        auto plan = ggml_graph_plan(gf, n_threads);
-        if (plan.work_size > 0) {
-            data.reset(new uint8_t[plan.work_size]);
-            plan.work_data = data.get();
-        }
-        ggml_graph_compute(gf, &plan);
-    }
+    ggml_build_forward_expand(&gf, inpL);
+    ggml_graph_compute       (ctx0, &gf);
 
     //if (n_past%100 == 0) {
-    //    ggml_graph_print   (gf);
-    //    ggml_graph_dump_dot(gf, NULL, "gpt-2.dot");
+    //    ggml_graph_print   (&gf);
+    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
     //}
 
     //embd_w.resize(n_vocab*N);
@@ -668,38 +835,24 @@ struct GPTJPrivate {
 GPTJ::GPTJ()
     : d_ptr(new GPTJPrivate) {
     d_ptr->model = new gptj_model;
-    d_ptr->model->ctx = nullptr;
     d_ptr->modelLoaded = false;
 }
 
-size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx, int ngl) {
-    (void)n_ctx;
-    (void)ngl;
-    gptj_model dummy_model;
-    gpt_vocab dummy_vocab;
-    size_t mem_req;
-    gptj_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
-    return mem_req;
-}
-
-bool GPTJ::loadModel(const std::string &modelPath, int n_ctx, int ngl) {
-    (void)n_ctx;
-    (void)ngl;
-    d_ptr->modelLoaded = false;
-
+bool GPTJ::loadModel(const std::string &modelPath) {
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
 
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+
     // load the model
-    bool ok = gptj_model_load(modelPath, *d_ptr->model, d_ptr->vocab);
-    fflush(stdout);
-    if (!ok) {
+    if (!gptj_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab)) {
         std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
         return false;
     }
 
     d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
     d_ptr->modelLoaded = true;
+    fflush(stdout);
     return true;
 }
 
@@ -737,10 +890,8 @@ size_t GPTJ::restoreState(const uint8_t *src)
     return gptj_set_state_data(d_ptr->model, &d_ptr->rng, src);
 }
 
-std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &ctx, const std::string &str, bool special) const
+std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &, const std::string &str) const
 {
-    (void)ctx;
-    (void)special;
     return ::gpt_tokenize(d_ptr->vocab, str);
 }
 
@@ -756,7 +907,7 @@ LLModel::Token GPTJ::sampleToken(PromptContext &promptCtx) const
         d_ptr->rng);
 }
 
-std::string GPTJ::tokenToString(Token id) const
+std::string_view GPTJ::tokenToString(Token id) const
 {
     return d_ptr->vocab.id_to_token[id];
 }
@@ -785,16 +936,6 @@ const std::vector<LLModel::Token> &GPTJ::endTokens() const
     return fres;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -814,21 +955,10 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
-
-    bool isValid = gguf_get_version(ctx_gguf) <= 3;
-    isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
-
-    gguf_free(ctx_gguf);
-    return isValid;
+DLL_EXPORT bool magic_match(std::istream& f) {
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    return magic == 0x67676d6c;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/gptj_impl.h

@@ -15,11 +15,8 @@ public:
     GPTJ();
     ~GPTJ();
 
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
+    bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
@@ -30,13 +27,12 @@ private:
     GPTJPrivate *d_ptr;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
     Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token id) const override;
+    std::string_view tokenToString(Token) const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
-    const std::vector<Token> &endTokens() const override;
-    bool shouldAddBOS() const override { return false; }
+    const std::vector<Token>& endTokens() const override;
 };
 
 #endif // GPTJ_H

gpt4all-backend/llama.cpp.cmake

@@ -1,11 +1,3 @@
-#
-# Copyright (c) 2023 Nomic, Inc. All rights reserved.
-#
-# This software is licensed under the terms of the Software for Open Models License (SOM),
-# version 1.0, as detailed in the LICENSE_SOM.txt file. A copy of this license should accompany
-# this software. Except as expressly granted in the SOM license, all rights are reserved by Nomic, Inc.
-#
-
 cmake_minimum_required(VERSION 3.12) # Don't bump this version for no reason
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@@ -38,17 +30,10 @@ else()
     endif()
 endif()
 
-if (APPLE)
-    set(LLAMA_KOMPUTE_DEFAULT OFF)
-else()
-    set(LLAMA_KOMPUTE_DEFAULT ON)
-endif()
-
 
 #
 # Option list
 #
-# some of the options here are commented out so they can be set "dynamically" before calling include_ggml()
 
 # general
 option(LLAMA_STATIC                 "llama: static link libraries"                          OFF)
@@ -80,13 +65,8 @@ option(LLAMA_SANITIZE_UNDEFINED     "llama: enable undefined sanitizer"
 # 3rd party libs
 option(LLAMA_ACCELERATE             "llama: enable Accelerate framework"                    ON)
 option(LLAMA_OPENBLAS               "llama: use OpenBLAS"                                   OFF)
-#option(LLAMA_CUBLAS                 "llama: use cuBLAS"                                     OFF)
-#option(LLAMA_CLBLAST                "llama: use CLBlast"                                    OFF)
-#option(LLAMA_METAL                  "llama: use Metal"                                      OFF)
-option(LLAMA_KOMPUTE                "llama: use Kompute"                                    ${LLAMA_KOMPUTE_DEFAULT})
-set(LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor")
-set(LLAMA_CUDA_DMMV_X "32" CACHE STRING "llama: x stride for dmmv CUDA kernels")
-set(LLAMA_CUDA_DMMV_Y "1" CACHE STRING  "llama: y block size for dmmv CUDA kernels")
+option(LLAMA_CUBLAS                 "llama: use cuBLAS"                                     OFF)
+option(LLAMA_CLBLAST                "llama: use CLBlast"                                    OFF)
 
 #
 # Compile flags
@@ -159,160 +139,6 @@ if (LLAMA_OPENBLAS)
     endif()
 endif()
 
-if (LLAMA_KOMPUTE)
-    set(LLAMA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp-mainline)
-    if (NOT EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
-        message(FATAL_ERROR "Kompute not found")
-    endif()
-    message(STATUS "Kompute found")
-
-    add_compile_definitions(VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
-    find_package(Vulkan COMPONENTS glslc REQUIRED)
-    find_program(glslc_executable NAMES glslc HINTS Vulkan::glslc)
-    if (NOT glslc_executable)
-        message(FATAL_ERROR "glslc not found")
-    endif()
-
-    function(compile_shader)
-      set(options)
-      set(oneValueArgs)
-      set(multiValueArgs SOURCES)
-      cmake_parse_arguments(compile_shader "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-      foreach(source ${compile_shader_SOURCES})
-        get_filename_component(OP_FILE ${source} NAME)
-        set(spv_file ${CMAKE_CURRENT_BINARY_DIR}/${OP_FILE}.spv)
-        add_custom_command(
-            OUTPUT ${spv_file}
-            DEPENDS ${LLAMA_DIR}/${source}
-              ${LLAMA_DIR}/kompute-shaders/common.comp
-              ${LLAMA_DIR}/kompute-shaders/op_getrows.comp
-              ${LLAMA_DIR}/kompute-shaders/op_mul_mv_q_n_pre.comp
-              ${LLAMA_DIR}/kompute-shaders/op_mul_mv_q_n.comp
-            COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
-            COMMENT "Compiling ${source} to ${source}.spv"
-        )
-
-        get_filename_component(RAW_FILE_NAME ${spv_file} NAME)
-        set(FILE_NAME "shader${RAW_FILE_NAME}")
-        string(REPLACE ".comp.spv" ".h" HEADER_FILE ${FILE_NAME})
-        string(TOUPPER ${HEADER_FILE} HEADER_FILE_DEFINE)
-        string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}")
-        set(OUTPUT_HEADER_FILE "${HEADER_FILE}")
-        message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}")
-        if(CMAKE_GENERATOR MATCHES "Visual Studio")
-            add_custom_command(
-              OUTPUT ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              DEPENDS ${spv_file} xxd
-              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd"
-            )
-        else()
-            add_custom_command(
-              OUTPUT ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
-              DEPENDS ${spv_file} xxd
-              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
-            )
-        endif()
-      endforeach()
-    endfunction()
-
-    set(KOMPUTE_OPT_LOG_LEVEL Critical CACHE STRING "Kompute log level")
-    add_subdirectory(${LLAMA_DIR}/kompute)
-
-    # Compile our shaders
-    compile_shader(SOURCES
-      kompute-shaders/op_scale.comp
-      kompute-shaders/op_scale_8.comp
-      kompute-shaders/op_add.comp
-      kompute-shaders/op_addrow.comp
-      kompute-shaders/op_mul.comp
-      kompute-shaders/op_silu.comp
-      kompute-shaders/op_relu.comp
-      kompute-shaders/op_gelu.comp
-      kompute-shaders/op_softmax.comp
-      kompute-shaders/op_norm.comp
-      kompute-shaders/op_rmsnorm.comp
-      kompute-shaders/op_diagmask.comp
-      kompute-shaders/op_mul_mat_mat_f32.comp
-      kompute-shaders/op_mul_mat_f16.comp
-      kompute-shaders/op_mul_mat_q8_0.comp
-      kompute-shaders/op_mul_mat_q4_0.comp
-      kompute-shaders/op_mul_mat_q4_1.comp
-      kompute-shaders/op_mul_mat_q6_k.comp
-      kompute-shaders/op_getrows_f16.comp
-      kompute-shaders/op_getrows_q4_0.comp
-      kompute-shaders/op_getrows_q4_1.comp
-      kompute-shaders/op_getrows_q6_k.comp
-      kompute-shaders/op_rope_f16.comp
-      kompute-shaders/op_rope_f32.comp
-      kompute-shaders/op_cpy_f16_f16.comp
-      kompute-shaders/op_cpy_f16_f32.comp
-      kompute-shaders/op_cpy_f32_f16.comp
-      kompute-shaders/op_cpy_f32_f32.comp
-    )
-
-    # Create a custom target for our generated shaders
-    add_custom_target(generated_shaders DEPENDS
-      shaderop_scale.h
-      shaderop_scale_8.h
-      shaderop_add.h
-      shaderop_addrow.h
-      shaderop_mul.h
-      shaderop_silu.h
-      shaderop_relu.h
-      shaderop_gelu.h
-      shaderop_softmax.h
-      shaderop_norm.h
-      shaderop_rmsnorm.h
-      shaderop_diagmask.h
-      shaderop_mul_mat_mat_f32.h
-      shaderop_mul_mat_f16.h
-      shaderop_mul_mat_q8_0.h
-      shaderop_mul_mat_q4_0.h
-      shaderop_mul_mat_q4_1.h
-      shaderop_mul_mat_q6_k.h
-      shaderop_getrows_f16.h
-      shaderop_getrows_q4_0.h
-      shaderop_getrows_q4_1.h
-      shaderop_getrows_q6_k.h
-      shaderop_rope_f16.h
-      shaderop_rope_f32.h
-      shaderop_cpy_f16_f16.h
-      shaderop_cpy_f16_f32.h
-      shaderop_cpy_f32_f16.h
-      shaderop_cpy_f32_f32.h
-    )
-
-    # Create a custom command that depends on the generated_shaders
-    add_custom_command(
-        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp
-        COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp
-        DEPENDS generated_shaders
-        COMMENT "Ensuring shaders are generated before compiling ggml-kompute.cpp"
-    )
-
-    # Add the stamp to the main sources to ensure dependency tracking
-    set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml-kompute.cpp ${LLAMA_DIR}/ggml-kompute.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp)
-    add_compile_definitions(GGML_USE_KOMPUTE)
-    set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute)
-    set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CMAKE_BINARY_DIR})
-endif()
-
 if (LLAMA_ALL_WARNINGS)
     if (NOT MSVC)
         set(c_flags
@@ -366,13 +192,6 @@ endif()
 # TODO: probably these flags need to be tweaked on some architectures
 #       feel free to update the Makefile for your architecture and send a pull request or issue
 message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
-if (MSVC)
-  string(TOLOWER "${CMAKE_GENERATOR_PLATFORM}" CMAKE_GENERATOR_PLATFORM_LWR)
-  message(STATUS "CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}")
-else ()
-  set(CMAKE_GENERATOR_PLATFORM_LWR "")
-endif ()
-
 if (NOT MSVC)
     if (LLAMA_STATIC)
         add_link_options(-static)
@@ -388,158 +207,89 @@ if (NOT MSVC)
     endif()
 endif()
 
-if ((${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm") OR (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") OR ("${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "arm64"))
-    message(STATUS "ARM detected")
-    if (MSVC)
-        add_compile_definitions(__ARM_NEON)
-        add_compile_definitions(__ARM_FEATURE_FMA)
-        add_compile_definitions(__ARM_FEATURE_DOTPROD)
-        # add_compile_definitions(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) # MSVC doesn't support vdupq_n_f16, vld1q_f16, vst1q_f16
-        add_compile_definitions(__aarch64__) # MSVC defines _M_ARM64 instead
-    else()
-        include(CheckCXXCompilerFlag)
-        check_cxx_compiler_flag(-mfp16-format=ieee COMPILER_SUPPORTS_FP16_FORMAT_I3E)
-        if (NOT "${COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
-            add_compile_options(-mfp16-format=ieee)
-        endif()
-        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv6")
-            # Raspberry Pi 1, Zero
-            add_compile_options(-mfpu=neon-fp-armv8 -mno-unaligned-access)
-        endif()
-        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7")
-            # Raspberry Pi 2
-            add_compile_options(-mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations)
-        endif()
-        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv8")
-            # Raspberry Pi 3, 4, Zero 2 (32-bit)
-            add_compile_options(-mno-unaligned-access)
-        endif()
-    endif()
-elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$" OR "${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "^(x86_64|i686|amd64|x64)$" )
-    message(STATUS "x86 detected")
-    if (MSVC)
-        if (LLAMA_AVX512)
-            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX512>)
-            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
-            # MSVC has no compile-time flags enabling specific
-            # AVX512 extensions, neither it defines the
-            # macros corresponding to the extensions.
-            # Do it manually.
-            if (LLAMA_AVX512_VBMI)
-                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
-                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
-            endif()
-            if (LLAMA_AVX512_VNNI)
-                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
-                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
-            endif()
-        elseif (LLAMA_AVX2)
-            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX2>)
-            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
-        elseif (LLAMA_AVX)
-            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX>)
-            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
-        endif()
-    else()
-        if (LLAMA_F16C)
-            add_compile_options(-mf16c)
-        endif()
-        if (LLAMA_FMA)
-            add_compile_options(-mfma)
-        endif()
-        if (LLAMA_AVX)
-            add_compile_options(-mavx)
-        endif()
-        if (LLAMA_AVX2)
-            add_compile_options(-mavx2)
-        endif()
-        if (LLAMA_AVX512)
-            add_compile_options(-mavx512f)
-            add_compile_options(-mavx512bw)
-        endif()
-        if (LLAMA_AVX512_VBMI)
-            add_compile_options(-mavx512vbmi)
-        endif()
-        if (LLAMA_AVX512_VNNI)
-            add_compile_options(-mavx512vnni)
-        endif()
-    endif()
-elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
-    message(STATUS "PowerPC detected")
-    add_compile_options(-mcpu=native -mtune=native)
-    #TODO: Add  targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be)
-else()
-    message(STATUS "Unknown architecture")
-endif()
-
-#
-# POSIX conformance
-#
-
-# clock_gettime came in POSIX.1b (1993)
-# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional
-# posix_memalign came in POSIX.1-2001 / SUSv3
-# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985)
-add_compile_definitions(_XOPEN_SOURCE=600)
-
-# Somehow in OpenBSD whenever POSIX conformance is specified
-# some string functions rely on locale_t availability,
-# which was introduced in POSIX.1-2008, forcing us to go higher
-if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
-    remove_definitions(-D_XOPEN_SOURCE=600)
-    add_compile_definitions(_XOPEN_SOURCE=700)
-endif()
-
-# Data types, macros and functions related to controlling CPU affinity and
-# some memory allocation are available on Linux through GNU extensions in libc
-if (CMAKE_SYSTEM_NAME MATCHES "Linux")
-    add_compile_definitions(_GNU_SOURCE)
-endif()
-
-# RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1,
-# and on macOS its availability depends on enabling Darwin extensions
-# similarly on DragonFly, enabling BSD extensions is necessary
-if (
-    CMAKE_SYSTEM_NAME MATCHES "Darwin" OR
-    CMAKE_SYSTEM_NAME MATCHES "iOS" OR
-    CMAKE_SYSTEM_NAME MATCHES "tvOS" OR
-    CMAKE_SYSTEM_NAME MATCHES "DragonFly"
-)
-    add_compile_definitions(_DARWIN_C_SOURCE)
-endif()
-
-# alloca is a non-standard interface that is not visible on BSDs when
-# POSIX conformance is specified, but not all of them provide a clean way
-# to enable it in such cases
-if (CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
-    add_compile_definitions(__BSD_VISIBLE)
-endif()
-if (CMAKE_SYSTEM_NAME MATCHES "NetBSD")
-    add_compile_definitions(_NETBSD_SOURCE)
-endif()
-if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
-    add_compile_definitions(_BSD_SOURCE)
-endif()
-
 function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
     message(STATUS "Configuring ggml implementation target llama${SUFFIX} in ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}")
 
+    if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
+        message(STATUS "ARM detected")
+        if (MSVC)
+            # TODO: arm msvc?
+        else()
+            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
+                add_compile_options(-mcpu=native)
+            endif()
+            # TODO: armv6,7,8 version specific flags
+        endif()
+    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$")
+        message(STATUS "x86 detected")
+        if (MSVC)
+            if (LLAMA_AVX512)
+                add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX512>)
+                add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
+                # MSVC has no compile-time flags enabling specific
+                # AVX512 extensions, neither it defines the
+                # macros corresponding to the extensions.
+                # Do it manually.
+                if (LLAMA_AVX512_VBMI)
+                    add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
+                    add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
+                endif()
+                if (LLAMA_AVX512_VNNI)
+                    add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
+                    add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
+                endif()
+            elseif (LLAMA_AVX2)
+                add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX2>)
+                add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
+            elseif (LLAMA_AVX)
+                add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX>)
+                add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
+            endif()
+        else()
+            if (LLAMA_F16C)
+                add_compile_options(-mf16c)
+            endif()
+            if (LLAMA_FMA)
+                add_compile_options(-mfma)
+            endif()
+            if (LLAMA_AVX)
+                add_compile_options(-mavx)
+            endif()
+            if (LLAMA_AVX2)
+                add_compile_options(-mavx2)
+            endif()
+            if (LLAMA_AVX512)
+                add_compile_options(-mavx512f)
+                add_compile_options(-mavx512bw)
+            endif()
+            if (LLAMA_AVX512_VBMI)
+                add_compile_options(-mavx512vbmi)
+            endif()
+            if (LLAMA_AVX512_VNNI)
+                add_compile_options(-mavx512vnni)
+            endif()
+        endif()
+    else()
+        # TODO: support PowerPC
+        message(STATUS "Unknown architecture")
+    endif()
+
     #
     # Build libraries
     #
 
-    set(GGML_CUBLAS_USE NO)
-    if (LLAMA_CUBLAS)
+    if (LLAMA_CUBLAS AND EXISTS ${DIRECTORY}/ggml-cuda.h)
         cmake_minimum_required(VERSION 3.17)
 
         find_package(CUDAToolkit)
         if (CUDAToolkit_FOUND)
-            set(GGML_CUBLAS_USE YES)
             message(STATUS "cuBLAS found")
 
             enable_language(CUDA)
 
-            set(GGML_SOURCES_CUDA ${DIRECTORY}/ggml-cuda.cu ${DIRECTORY}/ggml-cuda.h)
+            set(GGML_CUDA_SOURCES ${DIRECTORY}/ggml-cuda.cu ${DIRECTORY}/ggml-cuda.h)
+
+            add_compile_definitions(GGML_USE_CUBLAS)
 
             if (LLAMA_STATIC)
                 set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static)
@@ -552,19 +302,14 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
         endif()
     endif()
 
-    set(GGML_CLBLAST_USE NO)
-    if (LLAMA_CLBLAST)
+    if (LLAMA_CLBLAST AND EXISTS ${DIRECTORY}/ggml-opencl.h)
         find_package(CLBlast)
         if (CLBlast_FOUND)
-            set(GGML_CLBLAST_USE YES)
             message(STATUS "CLBlast found")
 
-            set(GGML_OPENCL_SOURCE_FILE ggml-opencl.cpp)
-            if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/${GGML_OPENCL_SOURCE_FILE})
-                set(GGML_OPENCL_SOURCE_FILE ggml-opencl.c)
-            endif()
+            set(GGML_OPENCL_SOURCES ${DIRECTORY}/ggml-opencl.c ${DIRECTORY}/ggml-opencl.h)
 
-            set(GGML_OPENCL_SOURCES ${DIRECTORY}/${GGML_OPENCL_SOURCE_FILE} ${DIRECTORY}/ggml-opencl.h)
+            add_compile_definitions(GGML_USE_CLBLAST)
 
             set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} clblast)
         else()
@@ -572,47 +317,15 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
         endif()
     endif()
 
-    set(GGML_METAL_SOURCES)
-    if (LLAMA_METAL)
-        find_library(FOUNDATION_LIBRARY         Foundation              REQUIRED)
-        find_library(METAL_FRAMEWORK            Metal                   REQUIRED)
-        find_library(METALKIT_FRAMEWORK         MetalKit                REQUIRED)
-        find_library(METALPERFORMANCE_FRAMEWORK MetalPerformanceShaders REQUIRED)
-
-        set(GGML_METAL_SOURCES ${DIRECTORY}/ggml-metal.m ${DIRECTORY}/ggml-metal.h)
-        # get full path to the file
-        #add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
-
-        # copy ggml-metal.metal to bin directory
-        configure_file(${DIRECTORY}/ggml-metal.metal bin/ggml-metal.metal COPYONLY)
-
-        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS}
-            ${FOUNDATION_LIBRARY}
-            ${METAL_FRAMEWORK}
-            ${METALKIT_FRAMEWORK}
-            ${METALPERFORMANCE_FRAMEWORK}
-        )
-    endif()
-
     add_library(ggml${SUFFIX} OBJECT
                 ${DIRECTORY}/ggml.c
                 ${DIRECTORY}/ggml.h
-                ${DIRECTORY}/ggml-alloc.c
-                ${DIRECTORY}/ggml-alloc.h
-                ${DIRECTORY}/ggml-backend.c
-                ${DIRECTORY}/ggml-backend.h
-                ${DIRECTORY}/ggml-quants.h
-                ${DIRECTORY}/ggml-quants.c
-                ${GGML_SOURCES_CUDA}
-                ${GGML_METAL_SOURCES}
-                ${GGML_OPENCL_SOURCES}
-                ${GGML_SOURCES_KOMPUTE})
+                ${GGML_CUDA_SOURCES}
+                ${GGML_OPENCL_SOURCES})
 
-    if (LLAMA_METAL AND GGML_METAL_SOURCES)
-        target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)
-    endif()
     target_include_directories(ggml${SUFFIX} PUBLIC ${DIRECTORY})
     target_compile_features(ggml${SUFFIX} PUBLIC c_std_11) # don't bump
+    target_link_libraries(ggml${SUFFIX} PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
 
     if (BUILD_SHARED_LIBS)
         set_target_properties(ggml${SUFFIX} PROPERTIES POSITION_INDEPENDENT_CODE ON)
@@ -620,20 +333,19 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
 
     if (WITH_LLAMA)
         # Backwards compatibility with old llama.cpp versions
-#        set(LLAMA_UTIL_SOURCE_FILE llama-util.h)
+        set(LLAMA_UTIL_SOURCE_FILE llama-util.h)
         if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
             set(LLAMA_UTIL_SOURCE_FILE llama_util.h)
         endif()
 
-        add_library(llama${SUFFIX} STATIC
+        add_library(llama${SUFFIX}
                     ${DIRECTORY}/llama.cpp
-                    ${DIRECTORY}/llama.h)
+                    ${DIRECTORY}/llama.h
+                    ${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
 
-        if (LLAMA_METAL AND GGML_METAL_SOURCES)
-            target_compile_definitions(llama${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)
-        endif()
         target_include_directories(llama${SUFFIX} PUBLIC ${DIRECTORY})
         target_compile_features(llama${SUFFIX} PUBLIC cxx_std_11) # don't bump
+        target_link_libraries(llama${SUFFIX} PRIVATE ggml${SUFFIX} ${LLAMA_EXTRA_LIBS})
 
         if (BUILD_SHARED_LIBS)
             set_target_properties(llama${SUFFIX} PROPERTIES POSITION_INDEPENDENT_CODE ON)
@@ -641,7 +353,7 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
         endif()
     endif()
 
-    if (GGML_SOURCES_CUDA)
+    if (GGML_CUDA_SOURCES)
         message(STATUS "GGML CUDA sources found, configuring CUDA architecture")
         set_property(TARGET ggml${SUFFIX} PROPERTY CUDA_ARCHITECTURES OFF)
         set_property(TARGET ggml${SUFFIX} PROPERTY CUDA_SELECT_NVCC_ARCH_FLAGS "Auto")
@@ -649,97 +361,4 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
             set_property(TARGET llama${SUFFIX} PROPERTY CUDA_ARCHITECTURES OFF)
         endif()
     endif()
-
-    if (GGML_CUBLAS_USE)
-        target_compile_definitions(ggml${SUFFIX} PRIVATE
-            GGML_USE_CUBLAS
-            GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X}
-            GGML_CUDA_DMMV_Y=${LLAMA_CUDA_DMMV_Y})
-        if (WITH_LLAMA)
-            target_compile_definitions(llama${SUFFIX} PRIVATE
-                GGML_USE_CUBLAS
-                GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X}
-                GGML_CUDA_DMMV_Y=${LLAMA_CUDA_DMMV_Y})
-        endif()
-    endif()
-    if (GGML_CLBLAST_USE)
-        if (WITH_LLAMA)
-            target_compile_definitions(llama${SUFFIX} PRIVATE GGML_USE_CLBLAST)
-        endif()
-        target_compile_definitions(ggml${SUFFIX} PRIVATE GGML_USE_CLBLAST)
-    endif()
-
-    if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
-        message(STATUS "ARM detected")
-        if (MSVC)
-            # TODO: arm msvc?
-        else()
-            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
-                target_compile_options(ggml${SUFFIX} PRIVATE -mcpu=native)
-            endif()
-            # TODO: armv6,7,8 version specific flags
-        endif()
-    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$")
-        message(STATUS "x86 detected")
-        if (MSVC)
-            if (LLAMA_AVX512)
-                target_compile_options(ggml${SUFFIX} PRIVATE
-                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX512>
-                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
-                # MSVC has no compile-time flags enabling specific
-                # AVX512 extensions, neither it defines the
-                # macros corresponding to the extensions.
-                # Do it manually.
-                if (LLAMA_AVX512_VBMI)
-                    target_compile_definitions(ggml${SUFFIX} PRIVATE
-                        $<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>
-                        $<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
-                endif()
-                if (LLAMA_AVX512_VNNI)
-                    target_compile_definitions(ggml${SUFFIX} PRIVATE
-                        $<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>
-                        $<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
-                endif()
-            elseif (LLAMA_AVX2)
-                target_compile_options(ggml${SUFFIX} PRIVATE
-                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX2>
-                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
-            elseif (LLAMA_AVX)
-                target_compile_options(ggml${SUFFIX} PRIVATE
-                    $<$<COMPILE_LANGUAGE:C>:/arch:AVX>
-                    $<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
-            endif()
-        else()
-            if (LLAMA_F16C)
-                target_compile_options(ggml${SUFFIX} PRIVATE -mf16c)
-            endif()
-            if (LLAMA_FMA)
-                target_compile_options(ggml${SUFFIX} PRIVATE -mfma)
-            endif()
-            if (LLAMA_AVX)
-                target_compile_options(ggml${SUFFIX} PRIVATE -mavx)
-            endif()
-            if (LLAMA_AVX2)
-                target_compile_options(ggml${SUFFIX} PRIVATE -mavx2)
-            endif()
-            if (LLAMA_AVX512)
-                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512f)
-                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512bw)
-            endif()
-            if (LLAMA_AVX512_VBMI)
-                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512vbmi)
-            endif()
-            if (LLAMA_AVX512_VNNI)
-                target_compile_options(ggml${SUFFIX} PRIVATE -mavx512vnni)
-            endif()
-        endif()
-    else()
-        # TODO: support PowerPC
-        message(STATUS "Unknown architecture")
-    endif()
-
-    target_link_libraries(ggml${SUFFIX} PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
-    if (WITH_LLAMA)
-        target_link_libraries(llama${SUFFIX} PRIVATE ggml${SUFFIX} ${LLAMA_EXTRA_LIBS})
-    endif()
 endfunction()

gpt4all-backend/llamamodel.cpp

@@ -6,85 +6,65 @@
 #include <cstdio>
 #include <cstring>
 #include <fstream>
-#include <initializer_list>
-#include <iomanip>
-#include <iostream>
 #include <map>
-#include <numeric>
-#include <random>
-#include <sstream>
-#include <stdexcept>
 #include <string>
+#include <vector>
+#include <iostream>
+#if defined(_WIN32) && defined(_MSC_VER)
+    #define WIN32_LEAN_AND_MEAN
+    #ifndef NOMINMAX
+        #define NOMINMAX
+    #endif
+    #include <windows.h>
+    #include <io.h>
+    #include <stdio.h>
+#else
+    #include <unistd.h>
+#endif
+#include <random>
 #include <thread>
 #include <unordered_set>
-#include <vector>
 
 #include <llama.h>
 #include <ggml.h>
-#ifdef GGML_USE_KOMPUTE
-#include <ggml-kompute.h>
-#endif
 
-using namespace std::string_literals;
 
-// Maximum supported GGUF version
-static constexpr int GGUF_VER_MAX = 3;
-
-static const char * const modelType_ = "LLaMA";
-
-static const std::vector<const char *> KNOWN_ARCHES {
-    "baichuan", "bert", "bloom", "codeshell", "falcon", "gemma", "gpt2", "llama", "mpt", "nomic-bert", "orion",
-    "persimmon", "phi2", "plamo", "qwen", "qwen2", "refact", "stablelm", "starcoder"
-};
-
-static const std::vector<const char *> EMBEDDING_ARCHES {
-    "bert", "nomic-bert"
-};
-
-static bool is_embedding_arch(const std::string &arch) {
-    return std::find(EMBEDDING_ARCHES.begin(), EMBEDDING_ARCHES.end(), arch) < EMBEDDING_ARCHES.end();
-}
-
-static bool llama_verbose() {
-    const char* var = getenv("GPT4ALL_VERBOSE_LLAMACPP");
-    return var && *var;
-}
-
-static void llama_log_callback(enum ggml_log_level level, const char *text, void *userdata) {
-    (void)userdata;
-    if (llama_verbose() || level <= GGML_LOG_LEVEL_ERROR) {
-        fputs(text, stderr);
-    }
+namespace {
+const char *modelType_ = "LLaMA";
 }
 
 struct gpt_params {
     int32_t seed          = -1;   // RNG seed
     int32_t n_keep        = 0;    // number of tokens to keep from initial prompt
+#if LLAMA_DATE <= 230511
+    int32_t n_parts       = -1;   // amount of model parts (-1 = determine from model dimensions)
+#endif
 
+#if LLAMA_DATE >= 230519
     // sampling parameters
     float   tfs_z         = 1.0f; // 1.0 = disabled
     float   typical_p     = 1.0f; // 1.0 = disabled
+#endif
 
     std::string prompt = "";
 
-    enum ggml_type kv_type = GGML_TYPE_F16; // use f16 instead of f32 for memory kv
+    bool memory_f16        = true;  // use f16 instead of f32 for memory kv
 
     bool use_mmap          = true;  // use mmap for faster loads
     bool use_mlock         = false; // use mlock to keep model in memory
 };
 
+#if LLAMA_DATE >= 230519
 static int llama_sample_top_p_top_k(
         llama_context *ctx,
         const llama_token *last_n_tokens_data,
         int last_n_tokens_size,
         int top_k,
         float top_p,
-        float min_p,
         float temp,
-        float repeat_penalty,
-        int32_t pos) {
-    auto logits = llama_get_logits_ith(ctx, pos);
-    auto n_vocab = llama_n_vocab(llama_get_model(ctx));
+        float repeat_penalty) {
+    auto logits = llama_get_logits(ctx);
+    auto n_vocab = llama_n_vocab(ctx);
     // Populate initial list of all candidates
     std::vector<llama_token_data> candidates;
     candidates.reserve(n_vocab);
@@ -93,294 +73,63 @@ static int llama_sample_top_p_top_k(
     }
     llama_token_data_array candidates_p = {candidates.data(), candidates.size(), false};
     // Sample repeat penalty
-    llama_sample_repetition_penalties(nullptr, &candidates_p, last_n_tokens_data, last_n_tokens_size, repeat_penalty, 0.0f, 0.0f);
+    llama_sample_repetition_penalty(nullptr, &candidates_p, last_n_tokens_data, last_n_tokens_size, repeat_penalty);
     // Temperature sampling
     llama_sample_top_k(ctx, &candidates_p, top_k, 1);
     llama_sample_tail_free(ctx, &candidates_p, 1.0f, 1);
     llama_sample_typical(ctx, &candidates_p, 1.0f, 1);
     llama_sample_top_p(ctx, &candidates_p, top_p, 1);
-    llama_sample_min_p(ctx, &candidates_p, min_p, 1);
-    llama_sample_temp(ctx, &candidates_p, temp);
+    llama_sample_temperature(ctx, &candidates_p, temp);
     return llama_sample_token(ctx, &candidates_p);
 }
-
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != (GGUF_TYPE_STRING)) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
-static gguf_context *load_gguf(const char *fname) {
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ nullptr,
-    };
-    gguf_context *ctx = gguf_init_from_file(fname, params);
-    if (!ctx) {
-        std::cerr << __func__ << ": gguf_init_from_file failed\n";
-        return nullptr;
-    }
-
-    int gguf_ver = gguf_get_version(ctx);
-    if (gguf_ver > GGUF_VER_MAX) {
-        std::cerr << __func__ << ": unsupported gguf version: " << gguf_ver << "\n";
-        gguf_free(ctx);
-        return nullptr;
-    }
-
-    return ctx;
-}
-
-static int32_t get_arch_key_u32(std::string const &modelPath, std::string const &archKey) {
-    auto * ctx = load_gguf(modelPath.c_str());
-    if (!ctx)
-        return -1;
-    std::string arch = get_arch_name(ctx);
-
-    int32_t value = -1;
-    if (ctx) {
-        auto key = arch + "." + archKey;
-        int keyidx = gguf_find_key(ctx, key.c_str());
-        if (keyidx != -1) {
-            value = gguf_get_val_u32(ctx, keyidx);
-        } else {
-            std::cerr << __func__ << ": " << key << "not found in " << modelPath << "\n";
-        }
-    }
-
-    gguf_free(ctx);
-    return value;
-}
+#endif
 
 struct LLamaPrivate {
     const std::string modelPath;
-    bool modelLoaded = false;
-    int device = -1;
-    llama_model *model = nullptr;
+    bool modelLoaded;
     llama_context *ctx = nullptr;
-    llama_model_params model_params;
-    llama_context_params ctx_params;
+    llama_context_params params;
     int64_t n_threads = 0;
-    std::vector<LLModel::Token> end_tokens;
-    const char *backend_name = nullptr;
 };
 
 LLamaModel::LLamaModel()
-    : d_ptr(new LLamaPrivate) {}
-
-// default hparams (LLaMA 7B)
-struct llama_file_hparams {
-    uint32_t n_vocab = 32000;
-    uint32_t n_embd  = 4096;
-    uint32_t n_mult  = 256;
-    uint32_t n_head  = 32;
-    uint32_t n_layer = 32;
-    uint32_t n_rot   = 64;
-    enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16;
-};
-
-size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx, int ngl) {
-    // TODO(cebtenzzre): update to GGUF
-    (void)ngl; // FIXME(cetenzzre): use this value
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-    fin.seekg(0, std::ios_base::end);
-    size_t filesize = fin.tellg();
-    fin.seekg(0, std::ios_base::beg);
-    uint32_t magic = 0;
-    fin.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    if (magic != 0x67676a74) return 0;
-    uint32_t version = 0;
-    fin.read(reinterpret_cast<char*>(&version), sizeof(version));
-    llama_file_hparams hparams;
-    fin.read(reinterpret_cast<char*>(&hparams.n_vocab), sizeof(hparams.n_vocab));
-    fin.read(reinterpret_cast<char*>(&hparams.n_embd), sizeof(hparams.n_embd));
-    fin.read(reinterpret_cast<char*>(&hparams.n_head), sizeof(hparams.n_head));
-    fin.read(reinterpret_cast<char*>(&hparams.n_layer), sizeof(hparams.n_layer));
-    fin.read(reinterpret_cast<char*>(&hparams.n_rot), sizeof(hparams.n_rot));
-    fin.read(reinterpret_cast<char*>(&hparams.ftype), sizeof(hparams.ftype));
-    const size_t kvcache_element_size = 2; // fp16
-    const size_t est_kvcache_size = hparams.n_embd * hparams.n_layer * 2u * n_ctx * kvcache_element_size;
-    return filesize + est_kvcache_size;
-}
-
-bool LLamaModel::isModelBlacklisted(const std::string &modelPath) const {
-    auto * ctx = load_gguf(modelPath.c_str());
-    if (!ctx) {
-        std::cerr << __func__ << ": failed to load " << modelPath << "\n";
-        return false;
-    }
-
-    auto get_key = [ctx, &modelPath](const char *name) {
-        int keyidx = gguf_find_key(ctx, name);
-        if (keyidx == -1) {
-            throw std::logic_error(name + " not found in "s + modelPath);
-        }
-        return keyidx;
-    };
-
-    bool res = false;
-    try {
-        std::string name(gguf_get_val_str(ctx, get_key("general.name")));
-        int token_idx = get_key("tokenizer.ggml.tokens");
-        int n_vocab = gguf_get_arr_n(ctx, token_idx);
-
-        // check for known bad models
-        if (name == "open-orca_mistral-7b-openorca"
-            && n_vocab == 32002
-            && gguf_get_arr_str(ctx, token_idx, 32000) == "<dummy32000>"s // should be <|im_end|>
-        ) {
-            res = true;
-        }
-    } catch (const std::logic_error &e) {
-        std::cerr << __func__ << ": " << e.what() << "\n";
-    }
-
-    gguf_free(ctx);
-    return res;
-}
-
-bool LLamaModel::isEmbeddingModel(const std::string &modelPath) const {
-    auto *ctx_gguf = load_gguf(modelPath.c_str());
-    if (!ctx_gguf) {
-        std::cerr << __func__ << ": failed to load GGUF from " <<  modelPath << "\n";
-        return false;
-    }
-
-    std::string arch = get_arch_name(ctx_gguf);
-    gguf_free(ctx_gguf);
-    return is_embedding_arch(arch);
-}
-
-bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
-{
+    : d_ptr(new LLamaPrivate) {
     d_ptr->modelLoaded = false;
+}
 
-    // clean up after previous loadModel()
-    if (d_ptr->model) {
-        llama_free_model(d_ptr->model);
-        d_ptr->model = nullptr;
-    }
-    if (d_ptr->ctx) {
-        llama_free(d_ptr->ctx);
-        d_ptr->ctx = nullptr;
-    }
-
-    if (n_ctx < 8) {
-        std::cerr << "warning: minimum context size is 8, using minimum size.\n";
-        n_ctx = 8;
-    }
-
-    // -- load the model --
+bool LLamaModel::loadModel(const std::string &modelPath)
+{
+    // load the model
+    d_ptr->params = llama_context_default_params();
 
     gpt_params params;
-
-    d_ptr->model_params = llama_model_default_params();
-
-    d_ptr->model_params.use_mmap  = params.use_mmap;
+    d_ptr->params.n_ctx      = 2048;
+    d_ptr->params.seed       = params.seed;
+    d_ptr->params.f16_kv     = params.memory_f16;
+    d_ptr->params.use_mmap   = params.use_mmap;
 #if defined (__APPLE__)
-    d_ptr->model_params.use_mlock = true;
+    d_ptr->params.use_mlock  = true;
 #else
-    d_ptr->model_params.use_mlock = params.use_mlock;
+    d_ptr->params.use_mlock  = params.use_mlock;
+#endif
+#if LLAMA_DATE <= 230511
+    d_ptr->params.n_parts  = params.n_parts;
 #endif
 
-    d_ptr->model_params.progress_callback = &LLModel::staticProgressCallback;
-    d_ptr->model_params.progress_callback_user_data = this;
-
-    d_ptr->backend_name = "cpu"; // default
-
-#ifdef GGML_USE_KOMPUTE
-    if (d_ptr->device != -1) {
-        d_ptr->model_params.main_gpu = d_ptr->device;
-        d_ptr->model_params.n_gpu_layers = ngl;
-    }
-#elif defined(GGML_USE_METAL)
-    (void)ngl;
-
-    if (llama_verbose()) {
-        std::cerr << "llama.cpp: using Metal" << std::endl;
-        d_ptr->backend_name = "metal";
-    }
-
-    // always fully offload on Metal
-    // TODO(cebtenzzre): use this parameter to allow using more than 53% of system RAM to load a model
-    d_ptr->model_params.n_gpu_layers = 100;
-#else
-    (void)ngl;
-#endif
-
-    d_ptr->model = llama_load_model_from_file_gpt4all(modelPath.c_str(), &d_ptr->model_params);
-    if (!d_ptr->model) {
-        fflush(stdout);
-        d_ptr->device = -1;
-        std::cerr << "LLAMA ERROR: failed to load model from " << modelPath << std::endl;
+    d_ptr->ctx = llama_init_from_file(modelPath.c_str(), d_ptr->params);
+    if (!d_ptr->ctx) {
+        std::cerr << "LLAMA ERROR: failed to load model from " <<  modelPath << std::endl;
         return false;
     }
 
-    // -- initialize the context --
-
-    d_ptr->ctx_params = llama_context_default_params();
-
-    bool isEmbedding = is_embedding_arch(llama_model_arch(d_ptr->model));
-    const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
-    if (isEmbedding) {
-        d_ptr->ctx_params.n_batch = n_ctx;
-    } else {
-        if (n_ctx > n_ctx_train) {
-            std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
-                      << n_ctx << " specified)\n";
-        }
-    }
-
-    d_ptr->ctx_params.n_ctx   = n_ctx;
-    d_ptr->ctx_params.seed    = params.seed;
-    d_ptr->ctx_params.type_k  = params.kv_type;
-    d_ptr->ctx_params.type_v  = params.kv_type;
-
-    // The new batch API provides space for n_vocab*n_tokens logits. Tell llama.cpp early
-    // that we want this many logits so the state serializes consistently.
-    d_ptr->ctx_params.logits_all = true;
-
     d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->ctx_params.n_threads       = d_ptr->n_threads;
-    d_ptr->ctx_params.n_threads_batch = d_ptr->n_threads;
-
-    if (isEmbedding)
-        d_ptr->ctx_params.embeddings = true;
-
-    d_ptr->ctx = llama_new_context_with_model(d_ptr->model, d_ptr->ctx_params);
-    if (!d_ptr->ctx) {
-        fflush(stdout);
-        std::cerr << "LLAMA ERROR: failed to init context for model " <<  modelPath << std::endl;
-        llama_free_model(d_ptr->model);
-        d_ptr->model = nullptr;
-        d_ptr->device = -1;
-        return false;
-    }
-
-    d_ptr->end_tokens = {llama_token_eos(d_ptr->model)};
-
-#ifdef GGML_USE_KOMPUTE
-    if (usingGPUDevice() && ggml_vk_has_device()) {
-        std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
-        d_ptr->backend_name = "kompute";
-    }
-#endif
-
-    m_supportsEmbedding = isEmbedding;
-    m_supportsCompletion = !isEmbedding;
-
-    fflush(stdout);
     d_ptr->modelLoaded = true;
+    fflush(stderr);
     return true;
 }
 
 void LLamaModel::setThreadCount(int32_t n_threads) {
     d_ptr->n_threads = n_threads;
-    llama_set_n_threads(d_ptr->ctx, n_threads, n_threads);
 }
 
 int32_t LLamaModel::threadCount() const {
@@ -389,10 +138,7 @@ int32_t LLamaModel::threadCount() const {
 
 LLamaModel::~LLamaModel()
 {
-    if (d_ptr->ctx) {
-        llama_free(d_ptr->ctx);
-    }
-    llama_free_model(d_ptr->model);
+    llama_free(d_ptr->ctx);
 }
 
 bool LLamaModel::isModelLoaded() const
@@ -416,20 +162,18 @@ size_t LLamaModel::restoreState(const uint8_t *src)
     return llama_set_state_data(d_ptr->ctx, const_cast<uint8_t*>(src));
 }
 
-std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str, bool special) const
+std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str) const
 {
-    const bool wantBOS = ctx.n_past == 0 && ctx.tokens.empty();
-    const bool useBOS = wantBOS && shouldAddBOS();
-    auto strCat = wantBOS && !special ? " " + str : str; // insert leading space ourselves, llama.cpp fork doesn't anymore
-    std::vector<LLModel::Token> fres(strCat.size()+4);
-    auto fres_len = llama_tokenize(d_ptr->model, strCat.c_str(), strCat.length(), fres.data(), fres.size(), useBOS, special);
+    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
+    std::vector<LLModel::Token> fres(str.size()+4);
+    auto fres_len = llama_tokenize(d_ptr->ctx, str.c_str(), fres.data(), fres.size(), useBOS);
     fres.resize(fres_len);
     return fres;
 }
 
-std::string LLamaModel::tokenToString(Token id) const
+std::string_view LLamaModel::tokenToString(Token id) const
 {
-    return llama_token_to_piece(d_ptr->ctx, id);
+    return llama_token_to_str(d_ptr->ctx, id);
 }
 
 LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
@@ -437,33 +181,13 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
     const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
     return llama_sample_top_p_top_k(d_ptr->ctx,
         promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.min_p, promptCtx.temp,
-        promptCtx.repeat_penalty, promptCtx.n_last_batch_tokens - 1);
+        n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
+        promptCtx.repeat_penalty);
 }
 
 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
 {
-    llama_kv_cache_seq_rm(d_ptr->ctx, 0, ctx.n_past, -1);
-
-    llama_batch batch = llama_batch_init(tokens.size(), 0, 1);
-
-    batch.n_tokens = tokens.size();
-    ctx.n_last_batch_tokens = tokens.size();
-
-    for (int32_t i = 0; i < batch.n_tokens; i++) {
-        batch.token   [i] = tokens[i];
-        batch.pos     [i] = ctx.n_past + i;
-        batch.n_seq_id[i] = 1;
-        batch.seq_id  [i][0] = 0;
-        batch.logits  [i] = false;
-    }
-
-    // llama_decode will output logits only for the last token of the prompt
-    batch.logits[batch.n_tokens - 1] = true;
-
-    int res = llama_decode(d_ptr->ctx, batch);
-    llama_batch_free(batch);
-    return res == 0;
+    return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
 }
 
 int32_t LLamaModel::contextLength() const
@@ -473,455 +197,8 @@ int32_t LLamaModel::contextLength() const
 
 const std::vector<LLModel::Token> &LLamaModel::endTokens() const
 {
-    return d_ptr->end_tokens;
-}
-
-bool LLamaModel::shouldAddBOS() const
-{
-    int add_bos = llama_add_bos_token(d_ptr->model);
-    if (add_bos != -1) { return add_bos; }
-    auto vocab_type = llama_vocab_type(d_ptr->model);
-    return vocab_type == LLAMA_VOCAB_TYPE_SPM || vocab_type == LLAMA_VOCAB_TYPE_WPM;
-}
-
-int32_t LLamaModel::maxContextLength(std::string const &modelPath) const
-{
-    return get_arch_key_u32(modelPath, "context_length");
-}
-
-int32_t LLamaModel::layerCount(std::string const &modelPath) const
-{
-    return get_arch_key_u32(modelPath, "block_count");
-}
-
-std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryRequired) const
-{
-#ifdef GGML_USE_KOMPUTE
-    size_t count = 0;
-    auto * vkDevices = ggml_vk_available_devices(memoryRequired, &count);
-
-    if (vkDevices) {
-        std::vector<LLModel::GPUDevice> devices;
-        devices.reserve(count);
-
-        for (size_t i = 0; i < count; ++i) {
-            auto & dev = vkDevices[i];
-            devices.emplace_back(
-                /* index    = */ dev.index,
-                /* type     = */ dev.type,
-                /* heapSize = */ dev.heapSize,
-                /* name     = */ dev.name,
-                /* vendor   = */ dev.vendor
-            );
-            ggml_vk_device_destroy(&dev);
-        }
-
-        free(vkDevices);
-        return devices;
-    }
-#else
-    (void)memoryRequired;
-    std::cerr << __func__ << ": built without Kompute\n";
-#endif
-
-    return {};
-}
-
-bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string &name) const
-{
-#if defined(GGML_USE_KOMPUTE)
-    ggml_vk_device device;
-    bool ok = ggml_vk_get_device(&device, memoryRequired, name.c_str());
-    if (ok) {
-        d_ptr->device = device.index;
-        return true;
-    }
-#else
-    (void)memoryRequired;
-    (void)name;
-#endif
-    return false;
-}
-
-bool LLamaModel::initializeGPUDevice(int device, std::string *unavail_reason) const
-{
-#if defined(GGML_USE_KOMPUTE)
-    (void)unavail_reason;
-    d_ptr->device = device;
-    return true;
-#else
-    (void)device;
-    if (unavail_reason) {
-        *unavail_reason = "built without Kompute";
-    }
-    return false;
-#endif
-}
-
-bool LLamaModel::hasGPUDevice()
-{
-#if defined(GGML_USE_KOMPUTE)
-    return d_ptr->device != -1;
-#else
-    return false;
-#endif
-}
-
-bool LLamaModel::usingGPUDevice()
-{
-#if defined(GGML_USE_KOMPUTE)
-    return hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
-#elif defined(GGML_USE_METAL)
-    return true;
-#else
-    return false;
-#endif
-}
-
-void llama_batch_add(
-                 struct llama_batch & batch,
-                        llama_token   id,
-                          llama_pos   pos,
-    const std::vector<llama_seq_id> & seq_ids,
-                               bool   logits) {
-    batch.token   [batch.n_tokens] = id;
-    batch.pos     [batch.n_tokens] = pos;
-    batch.n_seq_id[batch.n_tokens] = seq_ids.size();
-    for (size_t i = 0; i < seq_ids.size(); ++i) {
-        batch.seq_id[batch.n_tokens][i] = seq_ids[i];
-    }
-    batch.logits  [batch.n_tokens] = logits;
-
-    batch.n_tokens++;
-}
-
-static void batch_add_seq(llama_batch &batch, const std::vector<LLModel::Token> &tokens, int seq_id) {
-    for (unsigned i = 0; i < tokens.size(); i++) {
-        llama_batch_add(batch, tokens[i], i, { seq_id }, i == tokens.size() - 1);
-    }
-}
-
-size_t LLamaModel::embeddingSize() const {
-    return llama_n_embd(d_ptr->model);
-}
-
-struct EmbModelSpec {
-    const char *docPrefix;
-    const char *queryPrefix;
-    std::vector<const char *> otherPrefixes = {};
-    bool matryoshkaCapable = false;
-    const char *recommendedDims = nullptr;
-};
-
-struct EmbModelGroup {
-    EmbModelSpec spec;
-    std::vector<const char *> names;
-};
-
-static const EmbModelSpec NOPREFIX_SPEC {"", ""};
-static const EmbModelSpec NOMIC_SPEC    {"search_document", "search_query", {"clustering", "classification"}};
-static const EmbModelSpec E5_SPEC       {"passage", "query"};
-
-static const EmbModelSpec NOMIC_1_5_SPEC {
-    "search_document", "search_query", {"clustering", "classification"}, true, "[768, 512, 384, 256, 128]",
-};
-static const EmbModelSpec LLM_EMBEDDER_SPEC {
-    "Represent this document for retrieval",
-    "Represent this query for retrieving relevant documents",
-};
-static const EmbModelSpec BGE_SPEC {
-    "", "Represent this sentence for searching relevant passages",
-};
-static const EmbModelSpec E5_MISTRAL_SPEC {
-    "", "Instruct: Given a query, retrieve relevant passages that answer the query\nQuery",
-};
-
-static const EmbModelGroup EMBEDDING_MODEL_SPECS[] {
-    {NOPREFIX_SPEC,     {"all-MiniLM-L6-v1", "all-MiniLM-L12-v1", "all-MiniLM-L6-v2", "all-MiniLM-L12-v2"}},
-    {NOMIC_SPEC,        {"nomic-embed-text-v1", "nomic-embed-text-v1-ablated", "nomic-embed-text-v1-unsupervised"}},
-    {NOMIC_1_5_SPEC,    {"nomic-embed-text-v1.5"}},
-    {LLM_EMBEDDER_SPEC, {"llm-embedder"}},
-    {BGE_SPEC,          {"bge-small-en", "bge-base-en", "bge-large-en",
-                         "bge-small-en-v1.5", "bge-base-en-v1.5", "bge-large-en-v1.5"}},
-    // NOTE: E5 Mistral is not yet implemented in llama.cpp, so it's not in EMBEDDING_ARCHES
-    {E5_SPEC,           {"e5-small", "e5-base", "e5-large",
-                         "e5-small-unsupervised", "e5-base-unsupervised", "e5-large-unsupervised",
-                         "e5-small-v2", "e5-base-v2", "e5-large-v2"}},
-    {E5_MISTRAL_SPEC,   {"e5-mistral-7b-instruct",
-                         "multilingual-e5-small", "multilingual-e5-base", "multilingual-e5-large",
-                         "multilingual-e5-large-instruct"}},
-};
-
-static const EmbModelSpec *getEmbedSpec(const std::string &modelName) {
-    static const auto &specs = EMBEDDING_MODEL_SPECS;
-    auto it = std::find_if(specs, std::end(specs),
-        [&modelName](auto &spec) {
-            auto &names = spec.names;
-            return std::find(names.begin(), names.end(), modelName) < names.end();
-        }
-    );
-    return it < std::end(specs) ? &it->spec : nullptr;
-}
-
-void LLamaModel::embed(
-    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
-    bool doMean, bool atlas
-) {
-    const EmbModelSpec *spec;
-    std::optional<std::string> prefix;
-    if (d_ptr->model && (spec = getEmbedSpec(llama_model_name(d_ptr->model))))
-        prefix = isRetrieval ? spec->queryPrefix : spec->docPrefix;
-
-    embed(texts, embeddings, prefix, dimensionality, tokenCount, doMean, atlas);
-}
-
-void LLamaModel::embed(
-    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
-    size_t *tokenCount, bool doMean, bool atlas, LLModel::EmbedCancelCallback *cancelCb
-) {
-    if (!d_ptr->model)
-        throw std::logic_error("no model is loaded");
-
-    const char *modelName = llama_model_name(d_ptr->model);
-    if (!m_supportsEmbedding)
-        throw std::logic_error("not an embedding model: "s + modelName);
-
-    auto *spec = getEmbedSpec(modelName);
-    if (!spec)
-        std::cerr << __func__ << ": warning: unknown model " << modelName << "\n";
-
-    const int32_t n_embd = llama_n_embd(d_ptr->model);
-    if (dimensionality < 0) {
-        dimensionality = n_embd;
-    } else if (spec && dimensionality != n_embd) {
-        auto msg = [dimensionality, modelName]() {
-            return "unsupported dimensionality " + std::to_string(dimensionality) + " for model " + modelName;
-        };
-        if (!spec->matryoshkaCapable)
-            throw std::out_of_range(msg() + " (supported: " + std::to_string(n_embd) + ")");
-        if (dimensionality == 0 || dimensionality > n_embd)
-            throw std::out_of_range(msg() + " (recommended: " + spec->recommendedDims + ")");
-    }
-
-    if (!prefix) {
-        if (!spec)
-            throw std::invalid_argument("unknown model "s + modelName + ", specify a prefix if applicable or an empty string");
-        prefix = spec->docPrefix;
-    } else if (spec && prefix != spec->docPrefix && prefix != spec->queryPrefix &&
-               std::find(spec->otherPrefixes.begin(), spec->otherPrefixes.end(), *prefix) == spec->otherPrefixes.end())
-    {
-        std::stringstream ss;
-        ss << std::quoted(*prefix) << " is not a valid task type for model " << modelName;
-        throw std::invalid_argument(ss.str());
-    }
-
-    embedInternal(texts, embeddings, *prefix, dimensionality, tokenCount, doMean, atlas, cancelCb, spec);
-}
-
-// MD5 hash of "nomic empty"
-static const char EMPTY_PLACEHOLDER[] = "24df574ea1c998de59d5be15e769658e";
-
-auto product(double a) -> std::function<double(double)> {
-    return [a](double b) { return a * b; };
-}
-
-template <typename T>
-double getL2NormScale(T *start, T *end) {
-    double magnitude = std::sqrt(std::inner_product(start, end, start, 0.0));
-    return 1.0 / std::max(magnitude, 1e-12);
-}
-
-void LLamaModel::embedInternal(
-    const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
-    size_t *tokenCount, bool doMean, bool atlas, LLModel::EmbedCancelCallback *cancelCb, const EmbModelSpec *spec
-) {
-    typedef std::vector<LLModel::Token> TokenString;
-    static constexpr int32_t atlasMaxLength = 8192;
-    static constexpr int chunkOverlap = 8; // Atlas overlaps chunks of input by 8 tokens
-
-    const llama_token bos_token = llama_token_bos(d_ptr->model);
-    const llama_token eos_token = llama_token_eos(d_ptr->model);
-
-    bool useBOS = shouldAddBOS();
-    bool useEOS = llama_vocab_type(d_ptr->model) == LLAMA_VOCAB_TYPE_WPM;
-
-    // no EOS, optional BOS
-    auto tokenize = [this, useBOS, useEOS, eos_token](std::string text, TokenString &tokens, bool wantBOS) {
-        if (!text.empty() && text[0] != ' ') {
-            text = ' ' + text; // normalize for SPM - our fork of llama.cpp doesn't add a space prefix
-        }
-        wantBOS &= useBOS;
-
-        tokens.resize(text.length()+4);
-        int32_t n_tokens = llama_tokenize(d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), wantBOS, false);
-        if (n_tokens) {
-            assert(useEOS == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
-            tokens.resize(n_tokens - useEOS); // erase EOS/SEP
-        } else {
-            tokens.clear();
-        }
-    };
-
-    // tokenize the texts
-    std::vector<TokenString> inputs;
-    for (unsigned i = 0; i < texts.size(); i++) {
-        auto &text = texts[i];
-        auto &inp = inputs.emplace_back();
-        tokenize(text, inp, false);
-        if (atlas && inp.size() > atlasMaxLength) {
-            if (doMean) {
-                throw std::length_error(
-                    "length of text at index " + std::to_string(i) + " is " + std::to_string(inp.size()) +
-                    " tokens which exceeds limit of " + std::to_string(atlasMaxLength)
-                );
-            }
-            inp.resize(atlasMaxLength);
-        } else if (inp.empty()) {
-            if (!atlas || !text.empty()) {
-                std::cerr << __func__ << ": warning: chunking tokenized text at index " << std::to_string(i)
-                          << " into zero tokens\n";
-            }
-            tokenize(EMPTY_PLACEHOLDER, inp, false);
-        }
-    }
-
-    // tokenize the prefix
-    TokenString prefixTokens;
-    if (prefix.empty()) {
-        prefixTokens.push_back(bos_token);
-    } else {
-        tokenize(prefix + ':', prefixTokens, true);
-    }
-
-    // n_ctx_train: max sequence length of model (RoPE scaling not implemented)
-    const uint32_t n_ctx_train = llama_n_ctx_train(d_ptr->model);
-    // n_batch (equals n_ctx): max tokens per call to llama_decode (one more more sequences)
-    const uint32_t n_batch = llama_n_batch(d_ptr->ctx);
-
-    // effective sequence length minus prefix and SEP token
-    const uint32_t max_len = std::min(n_ctx_train, n_batch) - (prefixTokens.size() + useEOS);
-    if (max_len <= chunkOverlap) {
-        throw std::logic_error("max chunk length of " + std::to_string(max_len) + " is smaller than overlap of " +
-                               std::to_string(chunkOverlap) + " tokens");
-    }
-
-    // split into max_len-sized chunks
-    struct split_batch { unsigned idx; TokenString batch; };
-    std::vector<split_batch> batches;
-    size_t totalTokens = 0;
-    for (unsigned i = 0; i < inputs.size(); i++) {
-        auto &input = inputs[i];
-        for (auto it = input.begin(); it < input.end(); it += max_len) {
-            if (it > input.begin()) { it -= chunkOverlap; }
-            auto end = std::min(it + max_len, input.end());
-            batches.push_back({ i, {} });
-            auto &batch = batches.back().batch;
-            batch = prefixTokens;
-            batch.insert(batch.end(), it, end);
-            totalTokens += end - it;
-            batch.push_back(eos_token);
-            if (!doMean) { break; /* limit text to one chunk */ }
-        }
-    }
-    inputs.clear();
-
-    if (cancelCb) {
-        // copy of batching code below, but just count tokens instead of running inference
-        unsigned nBatchTokens = 0;
-        std::vector<unsigned> batchSizes;
-        for (const auto &inp: batches) {
-            if (nBatchTokens + inp.batch.size() > n_batch) {
-                batchSizes.push_back(nBatchTokens);
-                nBatchTokens = 0;
-            }
-            nBatchTokens += inp.batch.size();
-        }
-        batchSizes.push_back(nBatchTokens);
-        if (cancelCb(batchSizes.data(), batchSizes.size(), d_ptr->backend_name)) {
-            throw std::runtime_error("operation was canceled");
-        }
-    }
-
-    // initialize batch
-    struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
-
-    // n_texts x n_embd matrix
-    const int32_t n_embd = llama_n_embd(d_ptr->model);
-    std::vector<double> embeddingsSum(texts.size() * n_embd);
-    std::vector<int> embeddingsSumTotal(texts.size());
-    std::vector<int> queued_indices; // text indices of batches to be processed
-
-    auto decode = [this, &queued_indices, n_embd, &batch, &embeddingsSum, &embeddingsSumTotal, spec, dimensionality]() {
-        if (llama_decode(d_ptr->ctx, batch) < 0)
-            throw std::runtime_error("llama_decode failed");
-
-        for (int i = 0; i < batch.n_tokens; ++i) {
-            if (!batch.logits[i]) { continue; }
-            int i_prompt = queued_indices[batch.seq_id[i][0]];
-            auto *out = &embeddingsSum[i_prompt * n_embd];
-
-            // sequence embeddings aren't available when pooling_type is NONE
-            auto *embd = llama_get_embeddings_seq(d_ptr->ctx, batch.seq_id[i][0]);
-            if (!embd) { embd = llama_get_embeddings_ith(d_ptr->ctx, i); }
-            assert(embd);
-
-            auto *embd_end = embd + n_embd;
-
-            // layer normalization for nomic-embed-text-v1.5
-            if (spec && spec->matryoshkaCapable) {
-                // normalize mean
-                double mean = std::accumulate(embd, embd_end, 0.0) / n_embd;
-                std::transform(embd, embd_end, embd, [mean](double f){ return f - mean; });
-
-                // unbiased sample variance, with Bessel's correction
-                double variance = std::inner_product(embd, embd_end, embd, 0.0) / (n_embd - 1);
-
-                // trim to matryoshka dim
-                embd_end = embd + dimensionality;
-
-                // normalize variance
-                std::transform(embd, embd_end, embd, product(1.0 / std::sqrt(variance + 1e-5)));
-            }
-
-            // L2 norm
-            auto scale = getL2NormScale(embd, embd_end);
-            std::transform(embd, embd_end, out, out, [scale](double e, double o){ return o + scale * e; });
-            embeddingsSumTotal[i_prompt]++;
-        }
-    };
-
-    // break into batches
-    for (const auto &inp: batches) {
-        // encode if at capacity
-        if (batch.n_tokens + inp.batch.size() > n_batch) {
-            decode();
-            batch.n_tokens = 0;
-            queued_indices.clear();
-        }
-
-        // add to batch
-        batch_add_seq(batch, inp.batch, queued_indices.size());
-        queued_indices.push_back(inp.idx);
-    }
-
-    // final batch
-    decode();
-
-    for (unsigned i = 0; i < texts.size(); i++) {
-        auto *embd = &embeddingsSum[i * n_embd];
-        auto *embd_end = embd + dimensionality;
-        int total = embeddingsSumTotal[i];
-
-        // average over chunks
-        std::transform(embd, embd_end, embd, product(1.0 / total));
-
-        // L2 norm and copy
-        auto scale = getL2NormScale(embd, embd_end);
-        std::transform(embd, embd_end, embeddings, product(scale));
-        embeddings += dimensionality;
-    }
-
-    if (tokenCount) { *tokenCount = totalTokens; }
+    static const std::vector<LLModel::Token> fres = {llama_token_eos()};
+    return fres;
 }
 
 #if defined(_WIN32)
@@ -943,29 +220,18 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char *fname) {
-    auto * ctx = load_gguf(fname);
-    std::string arch = get_arch_name(ctx);
-
-    bool valid = true;
-
-    if (std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), arch) == KNOWN_ARCHES.end()) {
-        // not supported by this version of llama.cpp
-        if (arch != "gptj") { // we support this via another module
-            std::cerr << __func__ << ": unsupported model architecture: " << arch << "\n";
-        }
-        valid = false;
-    }
-
-    if (valid && is_embedding_arch(arch) && gguf_find_key(ctx, (arch + ".pooling_type").c_str()) < 0)
-        valid = false; // old pre-llama.cpp embedding model, e.g. all-MiniLM-L6-v2-f16.gguf
-
-    gguf_free(ctx);
-    return valid;
+DLL_EXPORT bool magic_match(std::istream& f) {
+    // Check magic
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    if (magic != 0x67676a74) return false;
+    // Check version
+    uint32_t version = 0;
+    f.read(reinterpret_cast<char*>(&version), sizeof(version));
+    return version LLAMA_VERSIONS;
 }
 
 DLL_EXPORT LLModel *construct() {
-    llama_log_set(llama_log_callback, nullptr);
     return new LLamaModel;
 }
 }

gpt4all-backend/llamamodel_impl.h

@@ -4,66 +4,35 @@
 #ifndef LLAMAMODEL_H
 #define LLAMAMODEL_H
 
-#include <functional>
-#include <memory>
 #include <string>
+#include <functional>
 #include <vector>
 #include "llmodel.h"
 
 struct LLamaPrivate;
-struct EmbModelSpec;
-
 class LLamaModel : public LLModel {
 public:
     LLamaModel();
     ~LLamaModel();
 
-    bool supportsEmbedding() const override { return m_supportsEmbedding; }
-    bool supportsCompletion() const override { return m_supportsCompletion; }
-    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
-    bool isModelBlacklisted(const std::string &modelPath) const override;
-    bool isEmbeddingModel(const std::string &modelPath) const override;
+    bool loadModel(const std::string &modelPath) override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
-    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const override;
-    bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const override;
-    bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const override;
-    bool hasGPUDevice() override;
-    bool usingGPUDevice() override;
-
-    size_t embeddingSize() const override;
-    // user-specified prefix
-    void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
-               int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false,
-               EmbedCancelCallback *cancelCb = nullptr) override;
-    // automatic prefix
-    void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality = -1,
-               size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
 
 private:
-    std::unique_ptr<LLamaPrivate> d_ptr;
-    bool m_supportsEmbedding = false;
-    bool m_supportsCompletion = false;
+    LLamaPrivate *d_ptr;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
-    std::string tokenToString(Token id) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::string_view tokenToString(Token) const override;
+    Token sampleToken(PromptContext& ctx) const override;
+    bool evalTokens(PromptContext& ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
-    const std::vector<Token> &endTokens() const override;
-    bool shouldAddBOS() const override;
-    int32_t maxContextLength(std::string const &modelPath) const override;
-    int32_t layerCount(std::string const &modelPath) const override;
-
-    void embedInternal(const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
-                       size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb,
-                       const EmbModelSpec *spec);
+    const std::vector<Token>& endTokens() const override;
 };
 
 #endif // LLAMAMODEL_H

gpt4all-backend/llmodel.cpp

@@ -1,94 +1,81 @@
 #include "llmodel.h"
 #include "dlhandle.h"
-#include "sysinfo.h"
 
-#include <cassert>
-#include <cstdlib>
-#include <filesystem>
-#include <fstream>
 #include <iostream>
-#include <memory>
-#include <regex>
-#include <sstream>
 #include <string>
 #include <vector>
+#include <fstream>
+#include <filesystem>
+#include <cassert>
+#include <cstdlib>
+#include <sstream>
 
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
+std::string LLModel::m_implementations_search_path = ".";
 
-std::string s_implementations_search_path = ".";
-
-#if !(defined(__x86_64__) || defined(_M_X64))
-    // irrelevant on non-x86_64
-    #define cpu_supports_avx()  -1
-    #define cpu_supports_avx2() -1
-#elif defined(_MSC_VER)
-    // MSVC
-    static int get_cpu_info(int func_id, int reg_id) {
-        int info[4];
-        __cpuid(info, func_id);
-        return info[reg_id];
-    }
-
-    // AVX via EAX=1: Processor Info and Feature Bits, bit 28 of ECX
-    #define cpu_supports_avx()  (get_cpu_info(1, 2) & (1 << 28))
-    // AVX2 via EAX=7, ECX=0: Extended Features, bit 5 of EBX
-    #define cpu_supports_avx2() (get_cpu_info(7, 1) & (1 <<  5))
+static bool has_at_least_minimal_hardware() {
+#ifdef __x86_64__
+    #ifndef _MSC_VER
+        return __builtin_cpu_supports("avx");
+    #else
+        int cpuInfo[4];
+        __cpuid(cpuInfo, 1);
+        return cpuInfo[2] & (1 << 28);
+    #endif
 #else
-    // gcc/clang
-    #define cpu_supports_avx()  __builtin_cpu_supports("avx")
-    #define cpu_supports_avx2() __builtin_cpu_supports("avx2")
+    return true; // Don't know how to handle non-x86_64
 #endif
+}
 
-LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
-    : m_dlhandle(new Dlhandle(std::move(dlhandle_))) {
-    auto get_model_type = m_dlhandle->get<const char *()>("get_model_type");
+static bool requires_avxonly() {
+#ifdef __x86_64__
+    #ifndef _MSC_VER
+        return !__builtin_cpu_supports("avx2");
+    #else
+        int cpuInfo[4];
+        __cpuidex(cpuInfo, 7, 0);
+        return !(cpuInfo[1] & (1 << 5));
+    #endif
+#else
+    return false; // Don't know how to handle non-x86_64
+#endif
+}
+
+LLModel::Implementation::Implementation(Dlhandle &&dlhandle_) : dlhandle(new Dlhandle(std::move(dlhandle_))) {
+    auto get_model_type = dlhandle->get<const char *()>("get_model_type");
     assert(get_model_type);
-    m_modelType = get_model_type();
-    auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
+    modelType = get_model_type();
+    auto get_build_variant = dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
-    m_buildVariant = get_build_variant();
-    m_magicMatch = m_dlhandle->get<bool(const char*)>("magic_match");
-    assert(m_magicMatch);
-    m_construct = m_dlhandle->get<LLModel *()>("construct");
-    assert(m_construct);
+    buildVariant = get_build_variant();
+    magicMatch = dlhandle->get<bool(std::ifstream&)>("magic_match");
+    assert(magicMatch);
+    construct_ = dlhandle->get<LLModel *()>("construct");
+    assert(construct_);
 }
 
 LLModel::Implementation::Implementation(Implementation &&o)
-    : m_magicMatch(o.m_magicMatch)
-    , m_construct(o.m_construct)
-    , m_modelType(o.m_modelType)
-    , m_buildVariant(o.m_buildVariant)
-    , m_dlhandle(o.m_dlhandle) {
-    o.m_dlhandle = nullptr;
+    : construct_(o.construct_)
+    , modelType(o.modelType)
+    , buildVariant(o.buildVariant)
+    , magicMatch(o.magicMatch)
+    , dlhandle(o.dlhandle) {
+    o.dlhandle = nullptr;
 }
 
 LLModel::Implementation::~Implementation() {
-    delete m_dlhandle;
+    if (dlhandle) delete dlhandle;
 }
 
-static bool isImplementation(const Dlhandle &dl) {
+bool LLModel::Implementation::isImplementation(const Dlhandle &dl) {
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
-const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
-    if (cpu_supports_avx() == 0) {
-        throw std::runtime_error("CPU does not support AVX");
-    }
-
+const std::vector<LLModel::Implementation> &LLModel::implementationList() {
     // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
     // individual models without the cleanup of the static list interfering
-    static auto* libs = new std::vector<Implementation>([] () {
-        std::vector<Implementation> fres;
+    static auto* libs = new std::vector<LLModel::Implementation>([] () {
+        std::vector<LLModel::Implementation> fres;
 
-        std::string impl_name_re = "(gptj|llamamodel-mainline)";
-        if (cpu_supports_avx2() == 0) {
-            impl_name_re += "-avxonly";
-        } else {
-            impl_name_re += "-(default|metal)";
-        }
-        std::regex re(impl_name_re);
         auto search_in_directory = [&](const std::string& paths) {
             std::stringstream ss(paths);
             std::string path;
@@ -98,22 +85,20 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
                 // Iterate over all libraries
                 for (const auto& f : std::filesystem::directory_iterator(fs_path)) {
                     const std::filesystem::path& p = f.path();
-
                     if (p.extension() != LIB_FILE_EXT) continue;
-                    if (!std::regex_search(p.stem().string(), re)) continue;
-
                     // Add to list if model implementation
                     try {
                         Dlhandle dl(p.string());
-                        if (!isImplementation(dl))
+                        if (!Implementation::isImplementation(dl)) {
                             continue;
+                        }
                         fres.emplace_back(Implementation(std::move(dl)));
                     } catch (...) {}
                 }
             }
         };
 
-        search_in_directory(s_implementations_search_path);
+        search_in_directory(m_implementations_search_path);
 
         return fres;
     }());
@@ -121,127 +106,36 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     return *libs;
 }
 
-const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
-    bool buildVariantMatched = false;
+const LLModel::Implementation* LLModel::implementation(std::ifstream& f, const std::string& buildVariant) {
     for (const auto& i : implementationList()) {
-        if (buildVariant != i.m_buildVariant) continue;
-        buildVariantMatched = true;
-
-        if (!i.m_magicMatch(fname)) continue;
+        f.seekg(0);
+        if (!i.magicMatch(f)) continue;
+        if (buildVariant != i.buildVariant) continue;
         return &i;
     }
-
-    if (!buildVariantMatched)
-        throw std::runtime_error("Could not find any implementations for build variant: " + buildVariant);
-
-    return nullptr; // unsupported model format
+    return nullptr;
 }
 
-LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant, int n_ctx) {
-    // Get correct implementation
-    const Implementation* impl = nullptr;
+LLModel *LLModel::construct(const std::string &modelPath, std::string buildVariant) {
 
-    #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
-        if (buildVariant == "auto") {
-            size_t total_mem = getSystemTotalRAMInBytes();
-            impl = implementation(modelPath.c_str(), "metal");
-            if(impl) {
-                LLModel* metalimpl = impl->m_construct();
-                metalimpl->m_implementation = impl;
-                /* TODO(cebtenzzre): after we fix requiredMem, we should change this to happen at
-                 * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
-                 * most (all?) places where this is called, causing underestimation of required
-                 * memory. */
-                size_t req_mem = metalimpl->requiredMem(modelPath, n_ctx, 100);
-                float req_to_total = (float) req_mem / (float) total_mem;
-                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
-                if (req_to_total >= 0.53) {
-                    delete metalimpl;
-                    impl = nullptr;
-                } else {
-                    return metalimpl;
-                }
-            }
-        }
-    #else
-        (void)n_ctx;
-    #endif
+    if (!has_at_least_minimal_hardware())
+        return nullptr;
 
-    if (!impl) {
-        //TODO: Auto-detect CUDA/OpenCL
-        if (buildVariant == "auto") {
-            if (cpu_supports_avx2() == 0) {
-                buildVariant = "avxonly";
-            } else {
-                buildVariant = "default";
-            }
+    //TODO: Auto-detect CUDA/OpenCL
+    if (buildVariant == "auto") {
+        if (requires_avxonly()) {
+            buildVariant = "avxonly";
+        } else {
+            buildVariant = "default";
         }
-        impl = implementation(modelPath.c_str(), buildVariant);
-        if (!impl) return nullptr;
     }
-
+    // Read magic
+    std::ifstream f(modelPath, std::ios::binary);
+    if (!f) return nullptr;
+    // Get correct implementation
+    auto impl = implementation(f, buildVariant);
+    if (!impl) return nullptr;
+    f.close();
     // Construct and return llmodel implementation
-    auto fres = impl->m_construct();
-    fres->m_implementation = impl;
-    return fres;
-}
-
-LLModel *LLModel::Implementation::constructDefaultLlama() {
-    static std::unique_ptr<LLModel> llama([]() -> LLModel * {
-        const std::vector<LLModel::Implementation> *impls;
-        try {
-            impls = &implementationList();
-        } catch (const std::runtime_error &e) {
-            std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
-            return nullptr;
-        }
-
-        const LLModel::Implementation *impl = nullptr;
-        for (const auto &i: *impls) {
-            if (i.m_buildVariant == "metal" || i.m_modelType != "LLaMA") continue;
-            impl = &i;
-        }
-        if (!impl) {
-            std::cerr << __func__ << ": could not find llama.cpp implementation\n";
-            return nullptr;
-        }
-
-        auto fres = impl->m_construct();
-        fres->m_implementation = impl;
-        return fres;
-    }());
-    return llama.get();
-}
-
-std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices(size_t memoryRequired) {
-    auto *llama = constructDefaultLlama();
-    if (llama) { return llama->availableGPUDevices(memoryRequired); }
-    return {};
-}
-
-int32_t LLModel::Implementation::maxContextLength(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
-    return llama ? llama->maxContextLength(modelPath) : -1;
-}
-
-int32_t LLModel::Implementation::layerCount(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
-    return llama ? llama->layerCount(modelPath) : -1;
-}
-
-bool LLModel::Implementation::isEmbeddingModel(const std::string &modelPath) {
-    auto *llama = constructDefaultLlama();
-    return llama && llama->isEmbeddingModel(modelPath);
-}
-
-void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
-    s_implementations_search_path = path;
-}
-
-const std::string& LLModel::Implementation::implementationsSearchPath() {
-    return s_implementations_search_path;
-}
-
-bool LLModel::Implementation::hasSupportedCPU() {
-    return cpu_supports_avx() != 0;
+    return impl->construct();
 }

gpt4all-backend/llmodel.h

@@ -1,64 +1,41 @@
 #ifndef LLMODEL_H
 #define LLMODEL_H
 
-#include <cstdint>
-#include <fstream>
-#include <functional>
-#include <limits>
-#include <optional>
 #include <string>
-#include <string_view>
+#include <functional>
 #include <vector>
-
-#define LLMODEL_MAX_PROMPT_BATCH 128
+#include <string_view>
+#include <fstream>
+#include <cstdint>
+#include <limits>
 
 class Dlhandle;
+
 class LLModel {
 public:
     using Token = int32_t;
 
-    struct GPUDevice {
-        int index;
-        int type;
-        size_t heapSize;
-        std::string name;
-        std::string vendor;
-
-        GPUDevice(int index, int type, size_t heapSize, std::string name, std::string vendor):
-            index(index), type(type), heapSize(heapSize), name(std::move(name)), vendor(std::move(vendor)) {}
-    };
-
     class Implementation {
+        LLModel *(*construct_)();
+
     public:
-        Implementation(const Implementation &) = delete;
-        Implementation(Implementation &&);
+        Implementation(Dlhandle&&);
+        Implementation(const Implementation&) = delete;
+        Implementation(Implementation&&);
         ~Implementation();
 
-        std::string_view modelType() const { return m_modelType; }
-        std::string_view buildVariant() const { return m_buildVariant; }
+        static bool isImplementation(const Dlhandle&);
 
-        static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto", int n_ctx = 2048);
-        static std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0);
-        static int32_t maxContextLength(const std::string &modelPath);
-        static int32_t layerCount(const std::string &modelPath);
-        static bool isEmbeddingModel(const std::string &modelPath);
-        static void setImplementationsSearchPath(const std::string &path);
-        static const std::string &implementationsSearchPath();
-        static bool hasSupportedCPU();
+        std::string_view modelType, buildVariant;
+        bool (*magicMatch)(std::ifstream& f);
+        Dlhandle *dlhandle;
 
-    private:
-        Implementation(Dlhandle &&);
-
-        static const std::vector<Implementation> &implementationList();
-        static const Implementation *implementation(const char *fname, const std::string &buildVariant);
-        static LLModel *constructDefaultLlama();
-
-        bool (*m_magicMatch)(const char *fname);
-        LLModel *(*m_construct)();
-
-        std::string_view m_modelType;
-        std::string_view m_buildVariant;
-        Dlhandle *m_dlhandle;
+        // The only way an implementation should be constructed
+        LLModel *construct() const {
+            auto fres = construct_();
+            fres->m_implementation = this;
+            return fres;
+        }
     };
 
     struct PromptContext {
@@ -69,135 +46,61 @@ public:
         int32_t n_predict = 200;
         int32_t top_k = 40;
         float   top_p = 0.9f;
-        float   min_p = 0.0f;
         float   temp = 0.9f;
         int32_t n_batch = 9;
         float   repeat_penalty = 1.10f;
         int32_t repeat_last_n = 64;     // last n tokens to penalize
-        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context window
-        int32_t n_last_batch_tokens = 0;
+        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context
+            // window
     };
 
-    using ProgressCallback = std::function<bool(float progress)>;
-
     explicit LLModel() {}
     virtual ~LLModel() {}
 
-    virtual bool supportsEmbedding() const = 0;
-    virtual bool supportsCompletion() const = 0;
-    virtual bool loadModel(const std::string &modelPath, int n_ctx, int ngl) = 0;
-    virtual bool isModelBlacklisted(const std::string &modelPath) const { (void)modelPath; return false; };
-    virtual bool isEmbeddingModel(const std::string &modelPath) const { (void)modelPath; return false; }
+    virtual bool loadModel(const std::string &modelPath) = 0;
     virtual bool isModelLoaded() const = 0;
-    virtual size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) = 0;
     virtual size_t stateSize() const { return 0; }
-    virtual size_t saveState(uint8_t *dest) const { (void)dest; return 0; }
-    virtual size_t restoreState(const uint8_t *src) { (void)src; return 0; }
-
-    // This method requires the model to return true from supportsCompletion otherwise it will throw
-    // an error
+    virtual size_t saveState(uint8_t */*dest*/) const { return 0; }
+    virtual size_t restoreState(const uint8_t */*src*/) { return 0; }
     virtual void prompt(const std::string &prompt,
-                        const std::string &promptTemplate,
                         std::function<bool(int32_t)> promptCallback,
                         std::function<bool(int32_t, const std::string&)> responseCallback,
                         std::function<bool(bool)> recalculateCallback,
-                        PromptContext &ctx,
-                        bool special = false,
-                        std::string *fakeReply = nullptr);
+                        PromptContext &ctx);
 
-    using EmbedCancelCallback = bool(unsigned *batchSizes, unsigned nBatch, const char *backend);
-
-    virtual size_t embeddingSize() const {
-        throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
-    }
-    // user-specified prefix
-    virtual void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
-                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false,
-                       EmbedCancelCallback *cancelCb = nullptr);
-    // automatic prefix
-    virtual void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval,
-                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false);
-
-    virtual void setThreadCount(int32_t n_threads) { (void)n_threads; }
+    virtual void setThreadCount(int32_t /*n_threads*/) {}
     virtual int32_t threadCount() const { return 1; }
 
-    const Implementation &implementation() const {
+    const Implementation& implementation() const {
         return *m_implementation;
     }
 
-    virtual std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const {
-        (void)memoryRequired;
-        return {};
+    static const std::vector<Implementation>& implementationList();
+    static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
+    static LLModel *construct(const std::string &modelPath, std::string buildVariant = "default");
+
+    static inline void setImplementationsSearchPath(const std::string& path) {
+        m_implementations_search_path = path;
     }
-
-    virtual bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const {
-        (void)memoryRequired;
-        (void)name;
-        return false;
+    static inline const std::string& implementationsSearchPath() {
+        return m_implementations_search_path;
     }
 
-    virtual bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const {
-        (void)device;
-        if (unavail_reason) {
-            *unavail_reason = "model has no GPU support";
-        }
-        return false;
-    }
-
-    virtual bool hasGPUDevice() { return false; }
-    virtual bool usingGPUDevice() { return false; }
-
-    void setProgressCallback(ProgressCallback callback) { m_progressCallback = callback; }
-
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
-    virtual std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special = false) const = 0;
-    virtual std::string tokenToString(Token id) const = 0;
+    virtual std::vector<Token> tokenize(PromptContext &, const std::string&) const = 0;
+    virtual std::string_view tokenToString(Token) const = 0;
     virtual Token sampleToken(PromptContext &ctx) const = 0;
-    virtual bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const = 0;
+    virtual bool evalTokens(PromptContext &/*ctx*/, const std::vector<int32_t>& /*tokens*/) const = 0;
     virtual int32_t contextLength() const = 0;
-    virtual const std::vector<Token> &endTokens() const = 0;
-    virtual bool shouldAddBOS() const = 0;
-
-    virtual int32_t maxContextLength(std::string const &modelPath) const
-    {
-        (void)modelPath;
-        return -1;
-    }
-
-    virtual int32_t layerCount(std::string const &modelPath) const
-    {
-        (void)modelPath;
-        return -1;
-    }
+    virtual const std::vector<Token>& endTokens() const = 0;
 
     // This is a helper function called from the default implementation of 'prompt' but it can be
     // shared by all base classes so it isn't virtual
     void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate);
 
     const Implementation *m_implementation = nullptr;
-
-    ProgressCallback m_progressCallback;
-    static bool staticProgressCallback(float progress, void* ctx)
-    {
-        LLModel* model = static_cast<LLModel*>(ctx);
-        if (model && model->m_progressCallback)
-            return model->m_progressCallback(progress);
-        return true;
-    }
-
-    void decodePrompt(std::function<bool(int32_t)> promptCallback,
-                      std::function<bool(int32_t, const std::string&)> responseCallback,
-                      std::function<bool(bool)> recalculateCallback,
-                      PromptContext &promptCtx,
-                      std::vector<Token> embd_inp);
-    void generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
-                          std::function<bool(bool)> recalculateCallback,
-                          PromptContext &promptCtx);
-
-private:
-    friend class LLMImplementation;
+    static std::string m_implementations_search_path;
 };
-
 #endif // LLMODEL_H

gpt4all-backend/llmodel_c.cpp

@@ -1,119 +1,120 @@
 #include "llmodel_c.h"
 #include "llmodel.h"
 
-#include <cerrno>
 #include <cstring>
-#include <iostream>
-#include <memory>
-#include <optional>
+#include <cerrno>
 #include <utility>
 
+
 struct LLModelWrapper {
     LLModel *llModel = nullptr;
     LLModel::PromptContext promptContext;
-    ~LLModelWrapper() { delete llModel; }
 };
 
+
+thread_local static std::string last_error_message;
+
+
 llmodel_model llmodel_model_create(const char *model_path) {
-    const char *error;
-    auto fres = llmodel_model_create2(model_path, "auto", &error);
+    auto fres = llmodel_model_create2(model_path, "auto", nullptr);
     if (!fres) {
-        fprintf(stderr, "Unable to instantiate model: %s\n", error);
+        fprintf(stderr, "Invalid model file\n");
     }
     return fres;
 }
 
-static void llmodel_set_error(const char **errptr, const char *message) {
-    thread_local static std::string last_error_message;
-    if (errptr) {
-        last_error_message = message;
-        *errptr = last_error_message.c_str();
-    }
-}
-
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error) {
-    LLModel *llModel;
-    try {
-        llModel = LLModel::Implementation::construct(model_path, build_variant);
-    } catch (const std::exception& e) {
-        llmodel_set_error(error, e.what());
-        return nullptr;
-    }
-
-    if (!llModel) {
-        llmodel_set_error(error, "Model format not supported (no matching implementation found)");
-        return nullptr;
-    }
-
+llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error) {
     auto wrapper = new LLModelWrapper;
-    wrapper->llModel = llModel;
-    return wrapper;
+    llmodel_error new_error{};
+
+    try {
+        wrapper->llModel = LLModel::construct(model_path, build_variant);
+    } catch (const std::exception& e) {
+        new_error.code = EINVAL;
+        last_error_message = e.what();
+    }
+
+    if (!wrapper->llModel) {
+        delete std::exchange(wrapper, nullptr);
+        // Get errno and error message if none
+        if (new_error.code == 0) {
+            new_error.code = errno;
+            last_error_message = strerror(errno);
+        }
+        // Set message pointer
+        new_error.message = last_error_message.c_str();
+        // Set error argument
+        if (error) *error = new_error;
+    }
+    return reinterpret_cast<llmodel_model*>(wrapper);
 }
 
 void llmodel_model_destroy(llmodel_model model) {
-    delete static_cast<LLModelWrapper *>(model);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    delete wrapper->llModel;
 }
 
-size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl)
+bool llmodel_loadModel(llmodel_model model, const char *model_path)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
-    return wrapper->llModel->requiredMem(model_path, n_ctx, ngl);
-}
-
-bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl)
-{
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
-
-    std::string modelPath(model_path);
-    if (wrapper->llModel->isModelBlacklisted(modelPath)) {
-        size_t slash = modelPath.find_last_of("/\\");
-        auto basename = slash == std::string::npos ? modelPath : modelPath.substr(slash + 1);
-        std::cerr << "warning: model '" << basename << "' is out-of-date, please check for an updated version\n";
-    }
-    return wrapper->llModel->loadModel(modelPath, n_ctx, ngl);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->loadModel(model_path);
 }
 
 bool llmodel_isModelLoaded(llmodel_model model)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
     return wrapper->llModel->isModelLoaded();
 }
 
 uint64_t llmodel_get_state_size(llmodel_model model)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
     return wrapper->llModel->stateSize();
 }
 
 uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
     return wrapper->llModel->saveState(dest);
 }
 
 uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
     return wrapper->llModel->restoreState(src);
 }
 
+// Wrapper functions for the C callbacks
+bool prompt_wrapper(int32_t token_id, void *user_data) {
+    llmodel_prompt_callback callback = reinterpret_cast<llmodel_prompt_callback>(user_data);
+    return callback(token_id);
+}
+
+bool response_wrapper(int32_t token_id, const std::string &response, void *user_data) {
+    llmodel_response_callback callback = reinterpret_cast<llmodel_response_callback>(user_data);
+    return callback(token_id, response.c_str());
+}
+
+bool recalculate_wrapper(bool is_recalculating, void *user_data) {
+    llmodel_recalculate_callback callback = reinterpret_cast<llmodel_recalculate_callback>(user_data);
+    return callback(is_recalculating);
+}
+
 void llmodel_prompt(llmodel_model model, const char *prompt,
-                    const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
-                    llmodel_prompt_context *ctx,
-                    bool special,
-                    const char *fake_reply)
+                    llmodel_prompt_context *ctx)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
 
-    auto response_func = [response_callback](int32_t token_id, const std::string &response) {
-        return response_callback(token_id, response.c_str());
-    };
-
-    if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
-        wrapper->promptContext.tokens.resize(ctx->n_past);
+    // Create std::function wrappers that call the C function pointers
+    std::function<bool(int32_t)> prompt_func =
+        std::bind(&prompt_wrapper, std::placeholders::_1, reinterpret_cast<void*>(prompt_callback));
+    std::function<bool(int32_t, const std::string&)> response_func =
+        std::bind(&response_wrapper, std::placeholders::_1, std::placeholders::_2, reinterpret_cast<void*>(response_callback));
+    std::function<bool(bool)> recalc_func =
+        std::bind(&recalculate_wrapper, std::placeholders::_1, reinterpret_cast<void*>(recalculate_callback));
 
     // Copy the C prompt context
     wrapper->promptContext.n_past = ctx->n_past;
@@ -121,20 +122,14 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     wrapper->promptContext.n_predict = ctx->n_predict;
     wrapper->promptContext.top_k = ctx->top_k;
     wrapper->promptContext.top_p = ctx->top_p;
-    wrapper->promptContext.min_p = ctx->min_p;
     wrapper->promptContext.temp = ctx->temp;
     wrapper->promptContext.n_batch = ctx->n_batch;
     wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
     wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
     wrapper->promptContext.contextErase = ctx->context_erase;
 
-    std::string fake_reply_str;
-    if (fake_reply) { fake_reply_str = fake_reply; }
-    auto *fake_reply_p = fake_reply ? &fake_reply_str : nullptr;
-
     // Call the C++ prompt method
-    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, recalculate_callback,
-                             wrapper->promptContext, special, fake_reply_p);
+    wrapper->llModel->prompt(prompt, prompt_func, response_func, recalc_func, wrapper->promptContext);
 
     // Update the C context by giving access to the wrappers raw pointers to std::vector data
     // which involves no copies
@@ -149,7 +144,6 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->n_predict = wrapper->promptContext.n_predict;
     ctx->top_k = wrapper->promptContext.top_k;
     ctx->top_p = wrapper->promptContext.top_p;
-    ctx->min_p = wrapper->promptContext.min_p;
     ctx->temp = wrapper->promptContext.temp;
     ctx->n_batch = wrapper->promptContext.n_batch;
     ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
@@ -157,132 +151,24 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->context_erase = wrapper->promptContext.contextErase;
 }
 
-float *llmodel_embed(
-    llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix, int dimensionality,
-    size_t *token_count, bool do_mean, bool atlas, llmodel_emb_cancel_callback cancel_cb, const char **error
-) {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
-
-    if (!texts || !*texts) {
-        llmodel_set_error(error, "'texts' is NULL or empty");
-        return nullptr;
-    }
-
-    std::vector<std::string> textsVec;
-    while (*texts) { textsVec.emplace_back(*texts++); }
-
-    size_t embd_size;
-    float *embedding;
-
-    try {
-        embd_size = wrapper->llModel->embeddingSize();
-        if (dimensionality > 0 && dimensionality < int(embd_size))
-            embd_size = dimensionality;
-
-        embd_size *= textsVec.size();
-
-        std::optional<std::string> prefixStr;
-        if (prefix) { prefixStr = prefix; }
-
-        embedding = new float[embd_size];
-        wrapper->llModel->embed(textsVec, embedding, prefixStr, dimensionality, token_count, do_mean, atlas, cancel_cb);
-    } catch (std::exception const &e) {
-        llmodel_set_error(error, e.what());
-        return nullptr;
-    }
-
-    *embedding_size = embd_size;
-    return embedding;
-}
-
-void llmodel_free_embedding(float *ptr)
-{
-    delete[] ptr;
-}
-
 void llmodel_setThreadCount(llmodel_model model, int32_t n_threads)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
     wrapper->llModel->setThreadCount(n_threads);
 }
 
 int32_t llmodel_threadCount(llmodel_model model)
 {
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
     return wrapper->llModel->threadCount();
 }
 
 void llmodel_set_implementation_search_path(const char *path)
 {
-    LLModel::Implementation::setImplementationsSearchPath(path);
+    LLModel::setImplementationsSearchPath(path);
 }
 
 const char *llmodel_get_implementation_search_path()
 {
-    return LLModel::Implementation::implementationsSearchPath().c_str();
-}
-
-// RAII wrapper around a C-style struct
-struct llmodel_gpu_device_cpp: llmodel_gpu_device {
-    llmodel_gpu_device_cpp() = default;
-
-    llmodel_gpu_device_cpp(const llmodel_gpu_device_cpp  &) = delete;
-    llmodel_gpu_device_cpp(      llmodel_gpu_device_cpp &&) = delete;
-
-    const llmodel_gpu_device_cpp &operator=(const llmodel_gpu_device_cpp  &) = delete;
-          llmodel_gpu_device_cpp &operator=(      llmodel_gpu_device_cpp &&) = delete;
-
-    ~llmodel_gpu_device_cpp() {
-        free(const_cast<char *>(name));
-        free(const_cast<char *>(vendor));
-    }
-};
-
-static_assert(sizeof(llmodel_gpu_device_cpp) == sizeof(llmodel_gpu_device));
-
-struct llmodel_gpu_device *llmodel_available_gpu_devices(size_t memoryRequired, int *num_devices)
-{
-    static thread_local std::unique_ptr<llmodel_gpu_device_cpp[]> c_devices;
-
-    auto devices = LLModel::Implementation::availableGPUDevices(memoryRequired);
-    *num_devices = devices.size();
-
-    if (devices.empty()) { return nullptr; /* no devices */ }
-
-    c_devices = std::make_unique<llmodel_gpu_device_cpp[]>(devices.size());
-    for (unsigned i = 0; i < devices.size(); i++) {
-        const auto &dev  =   devices[i];
-              auto &cdev = c_devices[i];
-        cdev.index    = dev.index;
-        cdev.type     = dev.type;
-        cdev.heapSize = dev.heapSize;
-        cdev.name     = strdup(dev.name.c_str());
-        cdev.vendor   = strdup(dev.vendor.c_str());
-    }
-
-    return c_devices.get();
-}
-
-bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
-{
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
-    return wrapper->llModel->initializeGPUDevice(memoryRequired, std::string(device));
-}
-
-bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device)
-{
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
-    return wrapper->llModel->initializeGPUDevice(device->index);
-}
-
-bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
-{
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
-    return wrapper->llModel->initializeGPUDevice(device);
-}
-
-bool llmodel_has_gpu_device(llmodel_model model)
-{
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
-    return wrapper->llModel->hasGPUDevice();
+    return LLModel::implementationsSearchPath().c_str();
 }

gpt4all-backend/llmodel_c.h

@@ -23,6 +23,17 @@ extern "C" {
  */
 typedef void *llmodel_model;
 
+/**
+ * Structure containing any errors that may eventually occur
+ */
+struct llmodel_error {
+    const char *message;  // Human readable error description; Thread-local; guaranteed to survive until next llmodel C API call
+    int code;             // errno; 0 if none
+};
+#ifndef __cplusplus
+typedef struct llmodel_error llmodel_error;
+#endif
+
 /**
  * llmodel_prompt_context structure for holding the prompt context.
  * NOTE: The implementation takes care of all the memory handling of the raw logits pointer and the
@@ -39,25 +50,14 @@ struct llmodel_prompt_context {
     int32_t n_predict;      // number of tokens to predict
     int32_t top_k;          // top k logits to sample from
     float top_p;            // nucleus sampling probability threshold
-    float min_p;            // Min P sampling
     float temp;             // temperature to adjust model's output distribution
     int32_t n_batch;        // number of predictions to generate in parallel
     float repeat_penalty;   // penalty factor for repeated tokens
     int32_t repeat_last_n;  // last n tokens to penalize
     float context_erase;    // percent of context to erase if we exceed the context window
 };
-
-struct llmodel_gpu_device {
-    int index;
-    int type; // same as VkPhysicalDeviceType
-    size_t heapSize;
-    const char * name;
-    const char * vendor;
-};
-
 #ifndef __cplusplus
 typedef struct llmodel_prompt_context llmodel_prompt_context;
-typedef struct llmodel_gpu_device llmodel_gpu_device;
 #endif
 
 /**
@@ -82,15 +82,6 @@ typedef bool (*llmodel_response_callback)(int32_t token_id, const char *response
  */
 typedef bool (*llmodel_recalculate_callback)(bool is_recalculating);
 
-/**
- * Embedding cancellation callback for use with llmodel_embed.
- * @param batch_sizes The number of tokens in each batch that will be embedded.
- * @param n_batch The number of batches that will be embedded.
- * @param backend The backend that will be used for embedding. One of "cpu", "kompute", or "metal".
- * @return True to cancel llmodel_embed, false to continue.
- */
-typedef bool (*llmodel_emb_cancel_callback)(unsigned *batch_sizes, unsigned n_batch, const char *backend);
-
 /**
  * Create a llmodel instance.
  * Recognises correct model type from file at model_path
@@ -104,10 +95,10 @@ DEPRECATED llmodel_model llmodel_model_create(const char *model_path);
  * Recognises correct model type from file at model_path
  * @param model_path A string representing the path to the model file; will only be used to detect model type.
  * @param build_variant A string representing the implementation to use (auto, default, avxonly, ...),
- * @param error A pointer to a string; will only be set on error.
+ * @param error A pointer to a llmodel_error; will only be set on error.
  * @return A pointer to the llmodel_model instance; NULL on error.
  */
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error);
+llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error);
 
 /**
  * Destroy a llmodel instance.
@@ -116,25 +107,13 @@ llmodel_model llmodel_model_create2(const char *model_path, const char *build_va
  */
 void llmodel_model_destroy(llmodel_model model);
 
-/**
- * Estimate RAM requirement for a model file
- * @param model A pointer to the llmodel_model instance.
- * @param model_path A string representing the path to the model file.
- * @param n_ctx Maximum size of context window
- * @param ngl Number of GPU layers to use (Vulkan)
- * @return size greater than 0 if the model was parsed successfully, 0 if file could not be parsed.
- */
-size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl);
-
 /**
  * Load a model from a file.
  * @param model A pointer to the llmodel_model instance.
  * @param model_path A string representing the path to the model file.
- * @param n_ctx Maximum size of context window
- * @param ngl Number of GPU layers to use (Vulkan)
  * @return true if the model was loaded successfully, false otherwise.
  */
-bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl);
+bool llmodel_loadModel(llmodel_model model, const char *model_path);
 
 /**
  * Check if a model is loaded.
@@ -173,54 +152,16 @@ uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src);
  * Generate a response using the model.
  * @param model A pointer to the llmodel_model instance.
  * @param prompt A string representing the input prompt.
- * @param prompt_template A string representing the input prompt template.
  * @param prompt_callback A callback function for handling the processing of prompt.
  * @param response_callback A callback function for handling the generated response.
  * @param recalculate_callback A callback function for handling recalculation requests.
- * @param special True if special tokens in the prompt should be processed, false otherwise.
- * @param fake_reply A string to insert into context as the model's reply, or NULL to generate one.
  * @param ctx A pointer to the llmodel_prompt_context structure.
  */
 void llmodel_prompt(llmodel_model model, const char *prompt,
-                    const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
-                    llmodel_prompt_context *ctx,
-                    bool special,
-                    const char *fake_reply);
-
-/**
- * Generate an embedding using the model.
- * NOTE: If given NULL pointers for the model or text, or an empty text, a NULL pointer will be
- * returned. Bindings should signal an error when NULL is the return value.
- * @param model A pointer to the llmodel_model instance.
- * @param texts A pointer to a NULL-terminated array of strings representing the texts to generate an
- * embedding for.
- * @param embedding_size A pointer to a size_t type that will be set by the call indicating the length
- * of the returned floating point array.
- * @param prefix The model-specific prefix representing the embedding task, without the trailing colon. NULL for no
- * prefix.
- * @param dimensionality The embedding dimension, for use with Matryoshka-capable models. Set to -1 to for full-size.
- * @param token_count Return location for the number of prompt tokens processed, or NULL.
- * @param do_mean True to average multiple embeddings if the text is longer than the model can accept, False to
- * truncate.
- * @param atlas Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens with
- * long_text_mode="mean" will raise an error. Disabled by default.
- * @param cancel_cb Cancellation callback, or NULL. See the documentation of llmodel_emb_cancel_callback.
- * @param error Return location for a malloc()ed string that will be set on error, or NULL.
- * @return A pointer to an array of floating point values passed to the calling method which then will
- * be responsible for lifetime of this memory. NULL if an error occurred.
- */
-float *llmodel_embed(llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix,
-                     int dimensionality, size_t *token_count, bool do_mean, bool atlas,
-                     llmodel_emb_cancel_callback cancel_cb, const char **error);
-
-/**
- * Frees the memory allocated by the llmodel_embedding function.
- * @param ptr A pointer to the embedding as returned from llmodel_embedding.
- */
-void llmodel_free_embedding(float *ptr);
+                    llmodel_prompt_context *ctx);
 
 /**
  * Set the number of threads to be used by the model.
@@ -250,51 +191,6 @@ void llmodel_set_implementation_search_path(const char *path);
  */
 const char *llmodel_get_implementation_search_path();
 
-/**
- * Get a list of available GPU devices given the memory required.
- * @param memoryRequired The minimum amount of VRAM, in bytes
- * @return A pointer to an array of llmodel_gpu_device's whose number is given by num_devices.
- */
-struct llmodel_gpu_device* llmodel_available_gpu_devices(size_t memoryRequired, int* num_devices);
-
-/**
- * Initializes a GPU device based on a specified string criterion.
- *
- * This function initializes a GPU device based on a string identifier provided. The function
- * allows initialization based on general device type ("gpu"), vendor name ("amd", "nvidia", "intel"),
- * or any specific device name.
- *
- * @param memoryRequired The amount of memory (in bytes) required by the application or task
- *                       that will utilize the GPU device.
- * @param device A string specifying the desired criterion for GPU device selection. It can be:
- *               - "gpu": To initialize the best available GPU.
- *               - "amd", "nvidia", or "intel": To initialize the best available GPU from that vendor.
- *               - A specific GPU device name: To initialize a GPU with that exact name.
- *
- * @return True if the GPU device is successfully initialized based on the provided string
- *         criterion. Returns false if the desired GPU device could not be initialized.
- */
-bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device);
-
-/**
- * Initializes a GPU device by specifying a valid gpu device pointer.
- * @param device A gpu device pointer.
- * @return True if the GPU device is successfully initialized, false otherwise.
- */
-bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device);
-
-/**
- * Initializes a GPU device by its index.
- * @param device An integer representing the index of the GPU device to be initialized.
- * @return True if the GPU device is successfully initialized, false otherwise.
- */
-bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
-
-/**
- * @return True if a GPU device is successfully initialized, false otherwise.
- */
-bool llmodel_has_gpu_device(llmodel_model model);
-
 #ifdef __cplusplus
 }
 #endif

gpt4all-backend/llmodel_shared.cpp

@@ -2,21 +2,11 @@
 
 #include <cassert>
 #include <iostream>
-#include <regex>
-#include <string>
 #include <unordered_set>
 
-// TODO(cebtenzzre): replace this with llama_kv_cache_seq_shift for llamamodel (GPT-J needs this as-is)
 void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
-    int n_keep = shouldAddBOS();
-    const int32_t n_discard = (promptCtx.n_ctx - n_keep) * promptCtx.contextErase;
-
-    // Erase the first percentage of context from the tokens
-    std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
-    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
-
-    size_t i = n_keep;
-    promptCtx.n_past = n_keep;
+    size_t i = 0;
+    promptCtx.n_past = 0;
     while (i < promptCtx.tokens.size()) {
         size_t batch_end = std::min(i + promptCtx.n_batch, promptCtx.tokens.size());
         std::vector<int32_t> batch(promptCtx.tokens.begin() + i, promptCtx.tokens.begin() + batch_end);
@@ -36,139 +26,32 @@ stop_generating:
     recalculate(false);
 }
 
-static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err) {
-    static const std::regex placeholderRegex(R"(%[1-2](?![0-9]))");
-
-    auto it = std::sregex_iterator(tmpl.begin(), tmpl.end(), placeholderRegex);
-    placeholders.clear();
-    placeholders.insert(placeholders.end(), it, std::sregex_iterator());
-
-    if (placeholders.size() > 2) {
-        err = "ERROR: expected at most two placeholders, got " + std::to_string(placeholders.size());
-        return false;
-    }
-    if (placeholders.size() >= 1 && placeholders[0].str() != "%1") {
-        err = "ERROR: first placeholder must be %1, got " + placeholders[0].str();
-        return false;
-    }
-    if (placeholders.size() >= 2 && placeholders[1].str() != "%2") {
-        err = "ERROR: second placeholder must be %2, got " + placeholders[1].str();
-        return false;
-    }
-    return true;
-}
-
 void LLModel::prompt(const std::string &prompt,
-                     const std::string &promptTemplate,
                      std::function<bool(int32_t)> promptCallback,
                      std::function<bool(int32_t, const std::string&)> responseCallback,
                      std::function<bool(bool)> recalculateCallback,
-                     PromptContext &promptCtx,
-                     bool special,
-                     std::string *fakeReply)
+                     PromptContext &promptCtx)
 {
     if (!isModelLoaded()) {
-        std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
+        std::cerr << implementation().modelType << " ERROR: prompt won't work with an unloaded model!\n";
         return;
     }
 
-    if (!supportsCompletion()) {
-        std::string errorMessage = "ERROR: this model does not support text completion or chat!";
-        responseCallback(-1, errorMessage);
-        std::cerr << implementation().modelType() << " " << errorMessage << "\n";
-        return;
-    }
+    // tokenize the prompt
+    std::vector<Token> embd_inp = tokenize(promptCtx, prompt);
 
-    // parse the prompt template
-    std::vector<std::smatch> placeholders;
-    {
-        std::string err;
-        if (!parsePromptTemplate(promptTemplate, placeholders, err)) {
-            responseCallback(-1, err);
-            std::cerr << err << "\n";
-            return;
-        }
-    }
-
-    auto old_n_past = promptCtx.n_past; // prepare to fake n_past for tokenize
-
-    // tokenize the user prompt
-    std::vector<Token> embd_inp;
-    if (placeholders.empty()) {
-        // this is unusual, but well-defined
-        std::cerr << __func__ << ": prompt template has no placeholder\n";
-        embd_inp = tokenize(promptCtx, promptTemplate, true);
-    } else {
-        // template: beginning of user prompt
-        const auto &phUser = placeholders[0];
-        std::string userPrefix(phUser.prefix());
-        if (!userPrefix.empty()) {
-            embd_inp = tokenize(promptCtx, userPrefix, true);
-            promptCtx.n_past += embd_inp.size();
-        }
-
-        // user input (shouldn't have special token processing)
-        auto tokens = tokenize(promptCtx, prompt, special);
-        embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
-        promptCtx.n_past += tokens.size();
-
-        // template: end of user prompt + start of assistant prompt
-        size_t start = phUser.position() + phUser.length();
-        size_t end = placeholders.size() >= 2 ? placeholders[1].position() : promptTemplate.length();
-        auto userToAsst = promptTemplate.substr(start, end - start);
-        if (!userToAsst.empty()) {
-            tokens = tokenize(promptCtx, userToAsst, true);
-            embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
-            promptCtx.n_past += tokens.size();
-        }
-    }
-
-    promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
-
-    // decode the user prompt
-    decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
-
-    // decode the assistant's reply, either generated or spoofed
-    if (fakeReply == nullptr) {
-        generateResponse(responseCallback, recalculateCallback, promptCtx);
-    } else {
-        embd_inp = tokenize(promptCtx, *fakeReply, false);
-        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
-    }
-
-    // decode the rest of the prompt template
-    // template: end of assistant prompt
-    std::string asstSuffix;
-    if (placeholders.size() >= 2) {
-        size_t start = placeholders[1].position() + placeholders[1].length();
-        asstSuffix = promptTemplate.substr(start);
-    } else {
-        asstSuffix = "\n\n"; // default to a blank link, good for e.g. Alpaca
-    }
-    if (!asstSuffix.empty()) {
-        embd_inp = tokenize(promptCtx, asstSuffix, true);
-        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
-    }
-}
-
-void LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
-                           std::function<bool(int32_t, const std::string&)> responseCallback,
-                           std::function<bool(bool)> recalculateCallback,
-                           PromptContext &promptCtx,
-                           std::vector<Token> embd_inp) {
     // save the context size
     promptCtx.n_ctx = contextLength();
 
     if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
         responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed.");
-        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
-            " tokens and the context window is " << promptCtx.n_ctx << "!\n";
+        std::cerr << implementation().modelType << " ERROR: The prompt is" << embd_inp.size() <<
+            "tokens and the context window is" << promptCtx.n_ctx << "!\n";
         return;
     }
 
     promptCtx.n_predict = std::min(promptCtx.n_predict, promptCtx.n_ctx - (int) embd_inp.size());
     promptCtx.n_past = std::min(promptCtx.n_past, promptCtx.n_ctx);
-    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
 
     // process the prompt in batches
     size_t i = 0;
@@ -178,12 +61,17 @@ void LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
 
         // Check if the context has run out...
         if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
+            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
+            // Erase the first percentage of context from the tokens...
+            std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n";
+            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
+            promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
             assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
         }
 
         if (!evalTokens(promptCtx, batch)) {
-            std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
+            std::cerr << implementation().modelType << " ERROR: Failed to process prompt\n";
             return;
         }
 
@@ -192,17 +80,13 @@ void LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
             if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
                 promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(batch.at(t));
-            promptCtx.n_past += 1;
             if (!promptCallback(batch.at(t)))
                 return;
         }
+        promptCtx.n_past += batch.size();
         i = batch_end;
     }
-}
 
-void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
-                               std::function<bool(bool)> recalculateCallback,
-                               PromptContext &promptCtx) {
     std::string cachedResponse;
     std::vector<Token> cachedTokens;
     std::unordered_set<std::string> reversePrompts
@@ -216,21 +100,28 @@ void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)>
 
         // Check if the context has run out...
         if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
+            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
+            // Erase the first percentage of context from the tokens...
+            std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n";
+            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
+            promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
             assert(promptCtx.n_past + 1 <= promptCtx.n_ctx);
         }
 
         if (!evalTokens(promptCtx, { id })) {
-            std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
+            std::cerr << implementation().modelType << " ERROR: Failed to predict next token\n";
             return;
         }
 
+        promptCtx.n_past += 1;
+
         // display text
         for (const auto token : endTokens()) {
             if (id == token) return;
         }
 
-        const std::string str = tokenToString(id);
+        const std::string_view str = tokenToString(id);
 
         // Check if the provided str is part of our reverse prompts
         bool foundPartialReversePrompt = false;
@@ -259,7 +150,6 @@ void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)>
             if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
                 promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(t);
-            promptCtx.n_past += 1;
             //TODO: Conversion to std::string can be avoided here...
             if (!responseCallback(t, std::string(tokenToString(t))))
                 return;
@@ -267,32 +157,3 @@ void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)>
         cachedTokens.clear();
     }
 }
-
-void LLModel::embed(
-    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
-    size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb
-) {
-    (void)texts;
-    (void)embeddings;
-    (void)prefix;
-    (void)dimensionality;
-    (void)tokenCount;
-    (void)doMean;
-    (void)atlas;
-    (void)cancelCb;
-    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
-}
-
-void LLModel::embed(
-    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
-    bool doMean, bool atlas
-) {
-    (void)texts;
-    (void)embeddings;
-    (void)isRetrieval;
-    (void)dimensionality;
-    (void)tokenCount;
-    (void)doMean;
-    (void)atlas;
-    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
-}

gpt4all-backend/llmodel_shared.h

@@ -1,46 +0,0 @@
-#pragma once
-#include <cstdint>
-#include <cstddef>
-#include <vector>
-#include <ggml.h>
-
-struct llm_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-
-    void resize(size_t size) {
-        delete[] addr;
-        addr = new uint8_t[size];
-        this->size = size;
-    }
-
-    ~llm_buffer() {
-        delete[] addr;
-    }
-};
-
-struct llm_kv_cache {
-    struct ggml_tensor * k;
-    struct ggml_tensor * v;
-
-    struct ggml_context * ctx = NULL;
-
-    llm_buffer buf;
-
-    int n; // number of tokens currently in the cache
-
-    ~llm_kv_cache() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
-inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads) {
-    struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
-    if (plan.work_size > 0) {
-        buf.resize(plan.work_size);
-        plan.work_data = buf.addr;
-    }
-    ggml_graph_compute(graph, &plan);
-}

gpt4all-backend/mpt.cpp

@@ -0,0 +1,892 @@
+#define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#include "mpt_impl.h"
+
+#include "utils.h"
+
+#include <cassert>
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+#include <fstream>
+#include <map>
+#include <random>
+#include <string>
+#include <vector>
+#include <iostream>
+#if defined(_WIN32) && defined(_MSC_VER)
+    #define WIN32_LEAN_AND_MEAN
+    #ifndef NOMINMAX
+        #define NOMINMAX
+    #endif
+    #include <windows.h>
+    #include <io.h>
+    #include <stdio.h>
+#else
+    #include <unistd.h>
+#endif
+#include <sstream>
+#include <thread>
+#include <unordered_set>
+#include <regex>
+#include <ggml.h>
+
+
+namespace {
+const char *modelType_ = "MPT";
+
+static const size_t MB = 1024*1024;
+}
+
+// default hparams (MPT 7B)
+struct mpt_hparams {
+    int32_t n_vocab      = 50432;
+    int32_t n_ctx        = 2048;
+    int32_t n_embd       = 4096;
+    int32_t n_head       = 32;
+    int32_t n_layer      = 32;
+    float alibi_bias_max = 8;
+    float clip_qkv       = 0;
+    int32_t expand       = 4;
+    int32_t f16          = 1;
+};
+
+struct mpt_layer {
+    // normalization
+    struct ggml_tensor * norm_1_w;
+    struct ggml_tensor * norm_2_w;
+
+    // attention
+    struct ggml_tensor * attn_Wqkv_w;
+    struct ggml_tensor * attn_out_proj_w;
+
+    // ff
+    struct ggml_tensor * ffn_up_proj_w;
+    struct ggml_tensor * ffn_down_proj_w;
+};
+
+struct mpt_buffer {
+    uint8_t * addr = NULL;
+    size_t size = 0;
+
+    void resize(size_t size) {
+        delete[] addr;
+        addr = new uint8_t[size];
+        this->size = size;
+    }
+
+    ~mpt_buffer() {
+        fflush(stdout);
+        delete[] addr;
+    }
+};
+
+struct mpt_kv_cache {
+    struct ggml_tensor * k;
+    struct ggml_tensor * v;
+
+    struct ggml_context * ctx = NULL;
+
+    mpt_buffer buf;
+
+    int n; // number of tokens currently in the cache
+
+    ~mpt_kv_cache() {
+        if (ctx) {
+            ggml_free(ctx);
+        }
+    }
+};
+
+struct mpt_model {
+    mpt_hparams hparams;
+
+    // normalization
+    struct ggml_tensor * norm_f_w;
+
+    struct ggml_tensor * wte; // position embedding
+
+    // mpt does weight tying
+
+    std::vector<mpt_layer> layers;
+
+    struct mpt_kv_cache kv_self;
+    struct ggml_context * ctx;
+    std::map<std::string, struct ggml_tensor *> tensors;
+
+
+    mpt_buffer buf;
+
+    ~mpt_model() {
+        if (ctx) {
+            ggml_free(ctx);
+        }
+    }
+};
+
+static bool kv_cache_init(
+        const struct mpt_hparams & hparams,
+             struct mpt_kv_cache & cache,
+                         ggml_type   wtype,
+                               int   n_ctx) {
+    const int n_embd  = hparams.n_embd;
+    const int n_layer = hparams.n_layer;
+
+    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
+    const int64_t n_elements = n_embd*n_mem;
+
+    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB);
+
+    struct ggml_init_params params;
+    params.mem_size   = cache.buf.size;
+    params.mem_buffer = cache.buf.addr;
+    params.no_alloc   = false;
+
+    cache.ctx = ggml_init(params);
+
+    if (!cache.ctx) {
+        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
+        return false;
+    }
+
+    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
+
+    return true;
+}
+
+// load the model's weights from a stream
+bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & model, gpt_vocab & vocab) {
+    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+
+    // verify magic
+    {
+        uint32_t magic;
+        fin.read((char *) &magic, sizeof(magic));
+        if (magic != 0x67676d6d) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
+            return false;
+        }
+    }
+
+    // load hparams
+    {
+        auto & hparams = model.hparams;
+
+        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
+        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
+        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
+        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
+        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
+        fin.read((char *) &hparams.alibi_bias_max,  sizeof(hparams.alibi_bias_max));
+        fin.read((char *) &hparams.clip_qkv,  sizeof(hparams.clip_qkv));
+        fin.read((char *) &hparams.f16,   sizeof(hparams.f16));
+
+        printf("%s: n_vocab        = %d\n", __func__, hparams.n_vocab);
+        printf("%s: n_ctx          = %d\n", __func__, hparams.n_ctx);
+        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
+        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
+        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
+        printf("%s: alibi_bias_max = %f\n", __func__, hparams.alibi_bias_max);
+        printf("%s: clip_qkv       = %f\n", __func__, hparams.clip_qkv);
+        printf("%s: ftype          = %d\n", __func__, hparams.f16);
+    }
+
+    // load vocab
+    {
+        int32_t n_vocab = model.hparams.n_vocab;
+        fin.read((char *) &n_vocab, sizeof(n_vocab));
+
+        if (n_vocab != model.hparams.n_vocab) {
+            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
+                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
+            return false;
+        }
+
+        std::string word;
+        for (int i = 0; i < n_vocab; i++) {
+            uint32_t len;
+            fin.read((char *) &len, sizeof(len));
+            bool special = false;
+            if (len & (1<<31)) {
+                len = len &~ (1<<31);
+                special = true;
+            }
+
+            if (len > 0) {
+                word.resize(len);
+                fin.read((char *) word.data(), len);
+                vocab.token_to_id[word] = i;
+                vocab.id_to_token[i] = word;
+            }
+
+            if(special) {
+                vocab.add_special_token(word);
+            }
+        }
+    }
+
+    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
+    // in order to save memory and also to speed up the computation
+    ggml_type wtype = GGML_TYPE_COUNT;
+    switch (model.hparams.f16) {
+        case 0: wtype = GGML_TYPE_F32;  break;
+        case 1: wtype = GGML_TYPE_F16;  break;
+        case 2: wtype = GGML_TYPE_Q4_0; break;
+        case 3: wtype = GGML_TYPE_Q4_1; break;
+        case 5: wtype = GGML_TYPE_Q4_2; break;
+        default:
+                {
+                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
+                            __func__, fname.c_str(), model.hparams.f16);
+                    return false;
+                }
+    }
+
+    auto & ctx = model.ctx;
+
+    size_t ctx_size = 0;
+
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_ctx   = hparams.n_ctx;
+        const int n_vocab = hparams.n_vocab;
+        const int expand  = hparams.expand;
+
+
+        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_w
+
+        ctx_size += n_embd*n_vocab*ggml_type_sizef(GGML_TYPE_F32); // wte
+
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_1_w
+        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_2_w
+
+        ctx_size += n_layer*(3*n_embd*n_embd*ggml_type_sizef(wtype)); // attn_Wqkv_w
+        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // attn_out_proj_w
+
+        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype));  // ffn_up_proj_w
+        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype)); // ffn_down_proj_w
+
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_k
+        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_v
+
+        // TODO probably less now?
+        ctx_size += (5 + 10*n_layer)*256; // object overhead
+
+        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
+    }
+
+    // create the ggml context
+    {
+        struct ggml_init_params params = {
+            .mem_size   = ctx_size,
+            .mem_buffer = NULL,
+            .no_alloc   = false,
+        };
+
+        model.ctx = ggml_init(params);
+        if (!model.ctx) {
+            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
+            return false;
+        }
+    }
+
+    // prepare memory for the weights
+    {
+        const auto & hparams = model.hparams;
+
+        const int n_embd  = hparams.n_embd;
+        const int n_layer = hparams.n_layer;
+        const int n_vocab = hparams.n_vocab;
+        const int expand  = hparams.expand;
+
+        model.layers.resize(n_layer);
+
+        model.wte    = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab);
+        model.norm_f_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+        // map by name
+        model.tensors["transformer.wte.weight"] = model.wte;
+        model.tensors["transformer.norm_f.weight"] = model.norm_f_w;
+
+        for (int i = 0; i < n_layer; ++i) {
+            auto & layer = model.layers[i];
+
+            layer.norm_1_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+            layer.norm_2_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+
+            layer.attn_Wqkv_w     = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd * 3);
+            layer.attn_out_proj_w = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd);
+            layer.ffn_up_proj_w   = ggml_new_tensor_2d(ctx, wtype,        n_embd, expand*n_embd);
+            layer.ffn_down_proj_w = ggml_new_tensor_2d(ctx, wtype, expand*n_embd, n_embd);
+
+            // map by name
+            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_1.weight"]        = layer.norm_1_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_2.weight"]        = layer.norm_2_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.Wqkv.weight"]     = layer.attn_Wqkv_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.out_proj.weight"] = layer.attn_out_proj_w;
+
+            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.up_proj.weight"]   = layer.ffn_up_proj_w;
+            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.down_proj.weight"] = layer.ffn_down_proj_w;
+        }
+    }
+
+    // key + value memory
+    {
+        const auto & hparams = model.hparams;
+        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
+            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
+            ggml_free(ctx);
+            return false;
+        }
+
+        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
+        printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
+    }
+
+    // load weights
+    {
+        int n_tensors = 0;
+        size_t total_size = 0;
+
+        printf("%s: ", __func__);
+
+        while (true) {
+            int32_t n_dims;
+            int32_t length;
+            int32_t ttype;
+
+            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
+            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
+            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
+
+            if (fin.eof()) {
+                break;
+            }
+
+            int32_t nelements = 1;
+            int32_t ne[2] = { 1, 1 };
+            for (int i = 0; i < n_dims; ++i) {
+                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
+                nelements *= ne[i];
+            }
+
+            std::string name(length, 0);
+            fin.read(&name[0], length);
+
+            if (model.tensors.find(name.data()) == model.tensors.end()) {
+                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
+                return false;
+            }
+
+            auto tensor = model.tensors[name.data()];
+            if (ggml_nelements(tensor) != nelements) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
+                return false;
+            }
+
+            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
+                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
+                        __func__, name.data(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
+                return false;
+            }
+
+            // for debugging
+            if (0) {
+                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
+            }
+
+            const size_t bpe = ggml_type_size(ggml_type(ttype));
+
+            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
+                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
+                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
+                return false;
+            }
+
+            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
+
+            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ttype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
+            total_size += ggml_nbytes(tensor);
+            if (++n_tensors % 8 == 0) {
+                printf(".");
+                fflush(stdout);
+            }
+        }
+
+        printf(" done\n");
+
+        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
+    }
+
+    return true;
+}
+
+// load the model's weights from a file path
+bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vocab) {
+
+    auto fin = std::ifstream(fname, std::ios::binary);
+    if (!fin) {
+        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
+        return false;
+    }
+
+    bool loaded = mpt_model_load(fname, fin, model, vocab);
+    fin.close();
+    return loaded;
+}
+
+bool mpt_eval(
+        mpt_model & model,
+        const int n_threads,
+        const int n_past,
+        const std::vector<int>           & embd_inp,
+              std::vector<float>         & embd_w,
+              size_t                     & mem_per_token) {
+    const int N = embd_inp.size();
+
+    const auto & hparams = model.hparams;
+
+    const int n_embd  = hparams.n_embd;
+    const int n_layer = hparams.n_layer;
+    const int n_ctx   = hparams.n_ctx;
+    const int n_head  = hparams.n_head;
+    const int n_vocab = hparams.n_vocab;
+
+    const size_t init_buf_size = 1024u*MB;
+    if (!model.buf.addr || model.buf.size < init_buf_size)
+        model.buf.resize(init_buf_size);
+
+    if (mem_per_token > 0 && mem_per_token*N > model.buf.size) {
+        const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
+        // printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
+
+        // reallocate
+        model.buf.resize(buf_size_new);
+        if (model.buf.addr == nullptr) {
+            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.buf.size);
+            return false;
+        }
+    }
+
+    struct ggml_init_params params = {
+        .mem_size   = model.buf.size,
+        .mem_buffer = model.buf.addr,
+        .no_alloc = false
+    };
+
+    struct ggml_context * ctx0 = ggml_init(params);
+    struct ggml_cgraph gf = {};
+    gf.n_threads = n_threads;
+
+    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
+
+    // wte
+    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
+
+    for (int il = 0; il < n_layer; ++il) {
+
+        struct ggml_tensor * inpSA = inpL;
+        struct ggml_tensor * cur = inpSA;
+        // self-attention
+        {
+
+            // norm1
+            cur = ggml_norm(ctx0, cur);
+            cur = ggml_mul(ctx0,
+                    ggml_repeat(ctx0, model.layers[il].norm_1_w, cur),
+                    cur);
+            // compute QKV
+            cur = ggml_mul_mat(ctx0,
+                    model.layers[il].attn_Wqkv_w,
+                    cur);
+
+            // TODO: clip_qkv
+            struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 0*ggml_element_size(cur)*n_embd));
+            struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 1*ggml_element_size(cur)*n_embd));
+            struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 2*ggml_element_size(cur)*n_embd));
+
+            // TODO: qk_ln? (seems to be False in MPT-7B configs)
+            {
+                Vcur = ggml_transpose(ctx0, Vcur);
+
+                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
+                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
+                                        (   n_ctx)*ggml_element_size(model.kv_self.v),
+                                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
+
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
+                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
+            }
+            // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
+            struct ggml_tensor * Q =
+                ggml_permute(ctx0,
+                        ggml_reshape_3d(ctx0, Qcur, n_embd/n_head, n_head, N),
+                        0, 2, 1, 3);
+
+            struct ggml_tensor * K =
+                ggml_permute(ctx0,
+                        ggml_reshape_3d(ctx0,
+                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
+                            n_embd/n_head, n_head, n_past + N),
+                        0, 2, 1, 3);
+
+            // K * Q
+            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
+
+            // KQ_scaled = KQ / sqrt(n_embd/n_head)
+            struct ggml_tensor * KQ_scaled =
+                ggml_scale(ctx0,
+                        KQ,
+                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
+                        );
+
+
+            // Alibi
+            struct ggml_tensor * KQ_scaled_biased = ggml_alibi(ctx0, ggml_cont(ctx0, KQ_scaled), n_past, n_head);
+
+            // KQ_masked = mask_past(KQ_scaled)
+            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled_biased, n_past);
+
+            // KQ = soft_max(KQ_masked)
+            struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
+
+            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
+            struct ggml_tensor * V =
+                ggml_view_3d(ctx0, model.kv_self.v,
+                        n_past + N, n_embd/n_head, n_head,
+                        n_ctx*ggml_element_size(model.kv_self.v),
+                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
+                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
+
+            // KQV = transpose(V) * KQ_soft_max
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
+
+            // KQV_merged = KQV.permute(0, 2, 1, 3)
+            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
+
+            // cur = KQV_merged.contiguous().view(n_embd, N)
+            cur = ggml_cpy(ctx0,
+                    KQV_merged,
+                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+
+            // projection (no bias)
+            cur = ggml_mul_mat(ctx0,
+                    model.layers[il].attn_out_proj_w,
+                    cur);
+        }
+
+
+        // residual
+        struct ggml_tensor * resSA = ggml_add(ctx0, cur, inpSA);
+        // feed-forward network
+        {
+            cur = resSA;
+            // norm2
+            cur = ggml_norm(ctx0, cur);
+            cur = ggml_mul(ctx0,
+                    ggml_repeat(ctx0, model.layers[il].norm_2_w, cur),
+                    cur);
+            // ffn
+            cur = ggml_mul_mat(ctx0,
+                    model.layers[il].ffn_up_proj_w,
+                    cur);
+            cur = ggml_gelu(ctx0, cur);
+            cur = ggml_mul_mat(ctx0,
+                    model.layers[il].ffn_down_proj_w,
+                    cur);
+
+        }
+
+        // self-attention + FF
+        inpL = ggml_add(ctx0, cur, resSA);
+    }
+
+    struct ggml_tensor * out = inpL;
+    // -> logits
+    {
+        out = ggml_norm(ctx0, out);
+        out = ggml_mul(ctx0,
+                    ggml_repeat(ctx0, model.norm_f_w, out),
+                    out);
+        out = ggml_mul_mat(ctx0, model.wte, out);
+    }
+
+
+    // run the computation
+    ggml_build_forward_expand(&gf, out);
+    ggml_graph_compute       (ctx0, &gf);
+
+
+    // return result for just the last token
+    embd_w.resize(n_vocab);
+    memcpy(embd_w.data(), (float *) ggml_get_data(out) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
+
+    if (mem_per_token == 0) {
+        mem_per_token = ggml_used_mem(ctx0)/N;
+    }
+    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
+
+    ggml_free(ctx0);
+
+    return true;
+}
+
+
+#define MPT_MAX_RNG_STATE 64*1024
+
+size_t mpt_get_state_size(const mpt_model &model)
+{
+    // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state.
+    // for reference, std::mt19937(1337) serializes to 6701 bytes.
+    const size_t s_rng_size        = sizeof(size_t);
+    const size_t s_rng             = MPT_MAX_RNG_STATE;
+    const size_t s_kv_size         = sizeof(size_t);
+    const size_t s_kv_ntok         = sizeof(int);
+    const size_t s_kv              = model.kv_self.buf.size;
+    const size_t s_total = (
+        + s_rng_size
+        + s_rng
+        + s_kv_size
+        + s_kv_ntok
+        + s_kv
+    );
+    fflush(stdout);
+    return s_total;
+}
+
+size_t mpt_copy_state_data(const mpt_model &model, const std::mt19937 &rng, uint8_t *dest)
+{
+    uint8_t * out = dest;
+    fflush(stdout);
+    // copy rng
+    {
+        std::stringstream rng_ss;
+        rng_ss << rng;
+
+        const size_t rng_size = rng_ss.str().size();
+        char rng_buf[MPT_MAX_RNG_STATE];
+
+        memset(&rng_buf[0], 0, MPT_MAX_RNG_STATE);
+        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
+
+        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
+        memcpy(out, &rng_buf[0], MPT_MAX_RNG_STATE); out += MPT_MAX_RNG_STATE;
+    }
+
+    // copy kv cache
+    {
+        const size_t kv_size = model.kv_self.buf.size;
+        const int    kv_ntok = model.kv_self.n;
+
+        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
+        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
+
+        if (kv_size) {
+            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
+        }
+    }
+
+    const size_t written  = out - dest;
+    assert(written == mpt_get_state_size(model));
+    fflush(stdout);
+    return written;
+}
+
+size_t mpt_set_state_data(mpt_model *model, std::mt19937 *rng, const uint8_t *src)
+{
+    const uint8_t * in = src;
+
+    // set rng
+    {
+        size_t rng_size;
+        char   rng_buf[MPT_MAX_RNG_STATE];
+
+        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
+        memcpy(&rng_buf[0], in, MPT_MAX_RNG_STATE); in += MPT_MAX_RNG_STATE;
+
+        std::stringstream rng_ss;
+        rng_ss.str(std::string(&rng_buf[0], rng_size));
+        rng_ss >> *rng;
+
+        assert(rng_ss.fail() == false);
+    }
+
+    // set kv cache
+    {
+        size_t kv_size;
+        int kv_ntok;
+
+        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
+        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
+
+        if (kv_size) {
+            assert(model->kv_self.buf.size == kv_size);
+
+            void * k_data = model->kv_self.k->data; // remember data pointers
+            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
+
+            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
+
+            model->kv_self.k->data = k_data; // restore correct data pointers
+            model->kv_self.v->data = v_data;
+
+        }
+
+        model->kv_self.n = kv_ntok;
+    }
+
+    const size_t nread    = in - src;
+    assert(nread == mpt_get_state_size(*model));
+    fflush(stdout);
+    return nread;
+}
+
+struct MPTPrivate {
+    const std::string modelPath;
+    bool modelLoaded;
+    gpt_vocab vocab;
+    mpt_model *model = nullptr;
+    int64_t n_threads = 0;
+    size_t mem_per_token = 0;
+    std::mt19937 rng;
+    bool has_im_end = false;
+};
+
+MPT::MPT()
+    : d_ptr(new MPTPrivate) {
+    d_ptr->model = new mpt_model;
+    d_ptr->modelLoaded = false;
+}
+
+bool MPT::loadModel(const std::string &modelPath) {
+    std::mt19937 rng(time(NULL));
+    d_ptr->rng = rng;
+
+    auto fin = std::ifstream(modelPath, std::ios::binary);
+
+    // load the model
+    if (!mpt_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab)) {
+        std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
+        return false;
+    }
+
+    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    d_ptr->modelLoaded = true;
+    d_ptr->has_im_end = d_ptr->vocab.token_to_id.find("<|im_end|>") != d_ptr->vocab.token_to_id.end();
+    fflush(stdout);
+    return true;
+}
+
+void MPT::setThreadCount(int32_t n_threads) {
+    d_ptr->n_threads = n_threads;
+}
+
+int32_t MPT::threadCount() const
+{
+    return d_ptr->n_threads;
+}
+
+MPT::~MPT()
+{
+    delete d_ptr->model;
+}
+
+bool MPT::isModelLoaded() const
+{
+    return d_ptr->modelLoaded;
+}
+
+size_t MPT::stateSize() const
+{
+    return mpt_get_state_size(*d_ptr->model);
+}
+
+size_t MPT::saveState(uint8_t *dest) const
+{
+    return mpt_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
+}
+
+size_t MPT::restoreState(const uint8_t *src)
+{
+    return mpt_set_state_data(d_ptr->model, &d_ptr->rng, src);
+}
+
+std::vector<LLModel::Token> MPT::tokenize(PromptContext &, const std::string &str) const
+{
+    return ::gpt_tokenize(d_ptr->vocab, str);
+}
+
+std::string_view MPT::tokenToString(Token id) const
+{
+    return d_ptr->vocab.id_to_token[id];
+}
+
+LLModel::Token MPT::sampleToken(PromptContext &promptCtx) const
+{
+    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
+    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
+        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
+        n_prev_toks,
+        promptCtx.logits,
+        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
+        promptCtx.repeat_penalty,
+        d_ptr->rng);
+}
+
+bool MPT::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
+{
+    // determine the required inference memory per token:
+    static bool initialized = false;
+    if (!initialized) {
+        mpt_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
+            d_ptr->mem_per_token);
+        initialized = true;
+    }
+
+    return mpt_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
+}
+
+int32_t MPT::contextLength() const
+{
+    return d_ptr->model->hparams.n_ctx;
+}
+
+const std::vector<LLModel::Token> &MPT::endTokens() const
+{
+    static const std::vector<LLModel::Token> fres = {0, d_ptr->vocab.token_to_id["<|im_end|>"]};
+    return fres;
+}
+
+#if defined(_WIN32)
+#define DLL_EXPORT __declspec(dllexport)
+#else
+#define DLL_EXPORT __attribute__ ((visibility ("default")))
+#endif
+
+extern "C" {
+DLL_EXPORT bool is_g4a_backend_model_implementation() {
+    return true;
+}
+
+DLL_EXPORT const char *get_model_type() {
+    return modelType_;
+}
+
+DLL_EXPORT const char *get_build_variant() {
+    return GGML_BUILD_VARIANT;
+}
+
+DLL_EXPORT bool magic_match(std::istream& f) {
+    uint32_t magic = 0;
+    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
+    return magic == 0x67676d6d;
+}
+
+DLL_EXPORT LLModel *construct() {
+    return new MPT;
+}
+}

gpt4all-backend/mpt_impl.h

@@ -0,0 +1,38 @@
+#ifndef MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of mpt.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#endif
+#ifndef MPT_H
+#define MPT_H
+
+#include <string>
+#include <functional>
+#include <vector>
+#include "llmodel.h"
+
+struct MPTPrivate;
+class MPT : public LLModel {
+public:
+    MPT();
+    ~MPT();
+
+    bool loadModel(const std::string &modelPath) override;
+    bool isModelLoaded() const override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+private:
+    MPTPrivate *d_ptr;
+
+protected:
+    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::string_view tokenToString(Token) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    int32_t contextLength() const override;
+    const std::vector<Token>& endTokens() const override;
+};
+
+#endif // MPT_H

gpt4all-backend/scripts/convert_bert_hf_to_gguf.py

@@ -1,140 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import json
-import struct
-import sys
-from pathlib import Path
-
-import gguf
-import numpy as np
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-
-
-if not 2 <= len(sys.argv) < 4:
-    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = Path(sys.argv[1])
-
-with open(dir_model / "vocab.txt", encoding="utf-8") as f:
-    vocab = f.readlines()
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
-
-
-ARCH = gguf.MODEL_ARCH.BERT
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(dir_model)
-
-block_count = config.num_hidden_layers
-gguf_writer.add_name("BERT")
-gguf_writer.add_context_length(config.max_position_embeddings)
-gguf_writer.add_embedding_length(config.hidden_size)
-gguf_writer.add_feed_forward_length(config.intermediate_size)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_head_count(config.num_attention_heads)
-gguf_writer.add_file_type(ftype)
-
-print("gguf: get tokenizer metadata")
-
-try:
-    with open(dir_model / "tokenizer.json", encoding="utf-8") as f:
-        tokenizer_json = json.load(f)
-except FileNotFoundError as e:
-    print(f'Error: Missing {e.filename!r}', file=sys.stderr)
-    sys.exit(1)
-
-print("gguf: get wordpiece tokenizer vocab")
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-print(tokenizer.encode('I believe the meaning of life is'))
-
-tokens: list[bytearray] = []
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-
-# The number of tokens in tokenizer.json can differ from the expected vocab size.
-# This causes downstream issues with mismatched tensor sizes when running the inference
-for i in range(config.vocab_size):
-    try:
-        text = reverse_vocab[i]
-    except KeyError:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-
-    tokens.append(text)
-
-gguf_writer.add_tokenizer_model("bert")  # wordpiece
-gguf_writer.add_token_list(tokens)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-model = AutoModel.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
-print(model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
-        continue
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        l_type = 1
-    else:
-        l_type = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-backend/scripts/convert_gptj_to_gguf.py

@@ -1,165 +0,0 @@
-#!/usr/bin/env python3
-# Convert GPT-J-6B h5 transformer model to ggml format
-#
-# Load the model using GPTJForCausalLM.
-# Iterate over all variables and write them to a binary file.
-#
-# For each variable, write the following:
-#   - Number of dimensions (int)
-#   - Name length (int)
-#   - Dimensions (int[n_dims])
-#   - Name (char[name_length])
-#   - Data (float[n_dims])
-#
-# By default, the bigger matrices are converted to 16-bit floats.
-# This can be disabled by adding the "ftype" CLI argument.
-#
-# At the start of the ggml file we write the model parameters
-# and vocabulary.
-#
-
-from __future__ import annotations
-
-import sys
-import struct
-import json
-from pathlib import Path
-
-import gguf
-import numpy as np
-from transformers import AutoConfig, AutoTokenizer, GPTJForCausalLM
-from transformers.models.gpt2 import tokenization_gpt2
-
-
-if not 2 <= len(sys.argv) < 4:
-    print("Usage: python {} dir-model [ftype]\n".format(Path(__file__).name))
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = Path(sys.argv[1])
-fname_out = dir_model / "ggml-model.gguf"
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
-
-
-ARCH = gguf.MODEL_ARCH.GPTJ
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(dir_model)
-
-block_count = config.n_layer
-gguf_writer.add_name("GPT-J")
-gguf_writer.add_context_length(config.n_positions)
-gguf_writer.add_embedding_length(config.n_embd)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_feed_forward_length(4 * config.n_embd)
-gguf_writer.add_head_count(config.n_head)
-gguf_writer.add_rope_dimension_count(config.rotary_dim)
-gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
-gguf_writer.add_file_type(ftype)
-
-print("gguf: get gpt2 tokenizer vocab")
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-byte_encoder = tokenization_gpt2.bytes_to_unicode()
-byte_decoder = {v: k for k, v in byte_encoder.items()}
-
-tokens: list[bytearray] = []
-
-for i in range(config.vocab_size):
-    if i in reverse_vocab:
-        try:
-            text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
-        except KeyError:
-            text = bytearray()
-            for c in reverse_vocab[i]:
-                if ord(c) < 256:  # single byte character
-                    text.append(byte_decoder[c])
-                else:  # multibyte special token character
-                    text.extend(c.encode('utf-8'))
-    else:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-
-    tokens.append(text)
-
-
-gguf_writer.add_tokenizer_model("gpt2")
-gguf_writer.add_token_list(tokens)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-model = GPTJForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
-#print (model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-#print (list_vars)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    # we don't need these
-    if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
-        print("  Skipping variable:", name)
-        continue
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        ftype_cur = 1
-    elif ftype == 1 or data.dtype != np.float32:
-        print("  Converting to float32")
-        data = data.astype(np.float32)
-        ftype_cur = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-backend/scripts/convert_mpt_hf_to_ggml.py

@@ -0,0 +1,145 @@
+# Convert Hugging Face fine-tuned bloom-like models to ggml format
+#
+# Usage:
+#
+#   python3 models/convert-h5-to-ggml.py 
+#
+# This script is similar to "convert-pt-to-ggml.py"
+#
+
+import io
+import os
+import sys
+import struct
+import json
+import code
+import torch
+import numpy as np
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BloomForCausalLM
+
+# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a significant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+if len(sys.argv) < 3:
+    print("Usage: python convert-hf-to-ggml.py model_name dir-output [use-f32]")
+    print("  model_name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
+    print("  dir-output: directory where the output file will be written")
+    print("  use-f32:    if present, use float32 instead of float16")
+    sys.exit(1)
+
+model_name = sys.argv[1]
+dir_out = sys.argv[2]
+
+# make sure the output directory exists
+os.makedirs(dir_out, exist_ok=True)
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+ftype = 1
+if len(sys.argv) > 3:
+    ftype = 0
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
+hparams = config.to_dict()
+print("Loading model: ", model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32, low_cpu_mem_usage=True)
+print("Model loaded: ", model_name)
+
+
+fname_out = dir_out + f"/ggml-model-{model_name.split('/')[-1]}-{ftype_str[ftype]}.bin"
+fout = open(fname_out, "wb")
+vocab = tokenizer.vocab
+
+hparams["multiple_of"] = 1
+fout.write(struct.pack("I", 0x67676d6d)) # magic: ggml in hex
+fout.write(struct.pack("I", model.config.vocab_size))
+fout.write(struct.pack("I", model.config.max_seq_len))
+fout.write(struct.pack("I", model.config.n_layers))
+fout.write(struct.pack("I", model.config.n_heads))
+fout.write(struct.pack("I", model.config.d_model))
+fout.write(struct.pack("f", model.config.attn_config['alibi_bias_max']))
+clip_qkv = model.config.attn_config['clip_qkv']
+fout.write(struct.pack("f",  clip_qkv if clip_qkv is not None else 0))
+fout.write(struct.pack("I", ftype))
+
+# # Is this correct??
+# dot_token = tokenizer.encode(".")[0]
+# write tokens to ggml file 
+dot_token = tokenizer.encode('.')[0]
+fout.write(struct.pack("I", model.config.vocab_size))
+
+for i in range(model.config.vocab_size):
+    text = tokenizer.decode([dot_token, i]).encode('utf-8')
+    # remove the first byte (it's always '.')
+    text = text[1:]
+    enclen = len(text)
+    if i in tokenizer.all_special_ids:
+        print(f"special token: {text}")
+        enclen = enclen | 1<<31
+    fout.write(struct.pack("I", enclen))
+    fout.write(text)
+    
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable: " + name + " with shape: ", data.shape)
+
+    n_dims = len(data.shape);
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0;
+    if ftype != 0:
+        # Keep token embeddings in fp32
+        if name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
+            print("  Converting to float16")
+            data = data.astype(np.float16)
+            ftype_cur = 1
+        else:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+    else:
+        if data.dtype != np.float32:
+            print("  Converting to float32")
+            data = data.astype(np.float32)
+            ftype_cur = 0
+
+    # header
+    str = name.encode('utf-8')
+    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
+    for i in range(n_dims):
+        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
+    fout.write(str);
+
+    # data
+    data.tofile(fout)
+
+fout.close()
+
+print("Done. Output file: " + fname_out)
+print("")

gpt4all-backend/sysinfo.h

@@ -1,61 +0,0 @@
-#ifndef SYSINFO_H
-#define SYSINFO_H
-
-#include <fstream>
-#include <string>
-#include <sstream>
-#include <iomanip>
-
-#if defined(__linux__)
-#include <unistd.h>
-#elif defined(__APPLE__)
-#include <sys/types.h>
-#include <sys/sysctl.h>
-#elif defined(_WIN32)
-#include <windows.h>
-#endif
-
-static long long getSystemTotalRAMInBytes()
-{
-    long long totalRAM = 0;
-
-#if defined(__linux__)
-    std::ifstream file("/proc/meminfo");
-    std::string line;
-    while (std::getline(file, line)) {
-        if (line.find("MemTotal") != std::string::npos) {
-            std::string memTotalStr = line.substr(line.find(":") + 1);
-            memTotalStr.erase(0, memTotalStr.find_first_not_of(" "));
-            memTotalStr = memTotalStr.substr(0, memTotalStr.find(" "));
-            totalRAM = std::stoll(memTotalStr) * 1024;  // Convert from KB to bytes
-            break;
-        }
-    }
-    file.close();
-#elif defined(__APPLE__)
-    int mib[2] = {CTL_HW, HW_MEMSIZE};
-    size_t length = sizeof(totalRAM);
-    sysctl(mib, 2, &totalRAM, &length, NULL, 0);
-#elif defined(_WIN32)
-    MEMORYSTATUSEX memoryStatus;
-    memoryStatus.dwLength = sizeof(memoryStatus);
-    GlobalMemoryStatusEx(&memoryStatus);
-    totalRAM = memoryStatus.ullTotalPhys;
-#endif
-
-    return totalRAM;
-}
-
-static double getSystemTotalRAMInGB()
-{
-    return static_cast<double>(getSystemTotalRAMInBytes()) / (1024 * 1024 * 1024);
-}
-
-static std::string getSystemTotalRAMInGBString()
-{
-    std::stringstream ss;
-    ss << std::fixed << std::setprecision(2) << getSystemTotalRAMInGB() << " GB";
-    return ss.str();
-}
-
-#endif // SYSINFO_H

gpt4all-backend/utils.cpp

@@ -230,21 +230,8 @@ gpt_vocab::id gpt_sample_top_k_top_p(
     int n_logits = actualVocabSize;
 
     const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_size);
-    const auto * plogits = logits.data();
+    const auto * plogits = logits.data() + logits.size() - n_logits;
 
-    if (temp <= 0) {
-        // select the token with the highest logit directly
-        float max_logit = plogits[0];
-        gpt_vocab::id max_id = 0;
-
-        for (int i = 1; i < n_logits; ++i) {
-            if (plogits[i] > max_logit) {
-                max_logit = plogits[i];
-                max_id = i;
-            }
-        }
-        return max_id;
-    }
     std::vector<std::pair<double, gpt_vocab::id>> logits_id;
     logits_id.reserve(n_logits);
 

gpt4all-backend/utils.h

@@ -8,13 +8,6 @@
 #include <random>
 #include <thread>
 
-//
-// General purpose inline functions
-//
-constexpr inline unsigned long long operator ""_MiB(unsigned long long bytes) {
-    return bytes*1024*1024;
-}
-
 //
 // CLI argument parsing
 //

gpt4all-bindings/cli/README.md

@@ -1,44 +0,0 @@
-# GPT4All Command-Line Interface (CLI)
-
-GPT4All on the command-line.
-
-## Documentation
-<https://docs.gpt4all.io/gpt4all_cli.html>
-
-## Quickstart
-
-The CLI is based on the `gpt4all` Python bindings and the `typer` package.
-
-The following shows one way to get started with the CLI, the documentation has more information.
-Typically, you will want to replace `python` with `python3` on _Unix-like_ systems and `py -3` on
-_Windows_. Also, it's assumed you have all the necessary Python components already installed.
-
-The CLI is a self-contained Python script named [app.py] ([download][app.py-download]). As long as
-its package dependencies are present, you can download and run it from wherever you like.
-
-[app.py]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-bindings/cli/app.py
-[app.py-download]: https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-bindings/cli/app.py
-
-```shell
-# optional but recommended: create and use a virtual environment
-python -m venv gpt4all-cli
-```
-_Windows_ and _Unix-like_ systems differ slightly in how you activate a _virtual environment_:
-- _Unix-like_, typically: `. gpt4all-cli/bin/activate`
-- _Windows_: `gpt4all-cli\Scripts\activate`
-
-Then:
-```shell
-# pip-install the necessary packages; omit '--user' if using a virtual environment
-python -m pip install --user --upgrade gpt4all typer
-# run the CLI
-python app.py repl
-```
-By default, it will automatically download the `groovy` model to `.cache/gpt4all/` in your user
-directory, if necessary.
-
-If you have already saved a model beforehand, specify its path with the `-m`/`--model` argument,
-for example:
-```shell
-python app.py repl --model /home/user/my-gpt4all-models/gpt4all-13b-snoozy-q4_0.gguf
-```

gpt4all-bindings/cli/app.py

@@ -1,19 +1,8 @@
-#!/usr/bin/env python3
-"""GPT4All CLI
-
-The GPT4All CLI is a self-contained script based on the `gpt4all` and `typer` packages. It offers a
-REPL to communicate with a language model similar to the chat GUI application, but more basic.
-"""
-
-import importlib.metadata
-import io
 import sys
-from collections import namedtuple
-from typing_extensions import Annotated
-
 import typer
-from gpt4all import GPT4All
 
+from typing_extensions import Annotated
+from gpt4all import GPT4All
 
 MESSAGES = [
     {"role": "system", "content": "You are a helpful assistant."},
@@ -28,9 +17,7 @@ SPECIAL_COMMANDS = {
     "/help": lambda _: print("Special commands: /reset, /exit, /help and /clear"),
 }
 
-VersionInfo = namedtuple('VersionInfo', ['major', 'minor', 'micro'])
-VERSION_INFO = VersionInfo(1, 0, 2)
-VERSION = '.'.join(map(str, VERSION_INFO))  # convert to string form, like: '1.2.3'
+VERSION = "0.1.0"
 
 CLI_START_MESSAGE = f"""
     
@@ -46,6 +33,12 @@ Type /help for special commands.
                                                     
 """
 
+def _cli_override_response_callback(token_id, response):
+    resp = response.decode("utf-8")
+    print(resp, end="", flush=True)
+    return True
+
+
 # create typer app
 app = typer.Typer()
 
@@ -54,18 +47,13 @@ def repl(
     model: Annotated[
         str,
         typer.Option("--model", "-m", help="Model to use for chatbot"),
-    ] = "mistral-7b-instruct-v0.1.Q4_0.gguf",
+    ] = "ggml-gpt4all-j-v1.3-groovy",
     n_threads: Annotated[
         int,
         typer.Option("--n-threads", "-t", help="Number of threads to use for chatbot"),
     ] = None,
-    device: Annotated[
-        str,
-        typer.Option("--device", "-d", help="Device to use for chatbot, e.g. gpu, amd, nvidia, intel. Defaults to CPU."),
-    ] = None,
 ):
-    """The CLI read-eval-print loop."""
-    gpt4all_instance = GPT4All(model, device=device)
+    gpt4all_instance = GPT4All(model)
 
     # if threads are passed, set them
     if n_threads is not None:
@@ -80,23 +68,11 @@ def repl(
     else:
         print(f"\nUsing {gpt4all_instance.model.thread_count()} threads", end="")
 
+    # overwrite _response_callback on model
+    gpt4all_instance.model._response_callback = _cli_override_response_callback
+
     print(CLI_START_MESSAGE)
 
-    use_new_loop = False
-    try:
-        version = importlib.metadata.version('gpt4all')
-        version_major = int(version.split('.')[0])
-        if version_major >= 1:
-            use_new_loop = True
-    except:
-        pass  # fall back to old loop
-    if use_new_loop:
-        _new_loop(gpt4all_instance)
-    else:
-        _old_loop(gpt4all_instance)
-
-
-def _old_loop(gpt4all_instance):
     while True:
         message = input(" ⇢  ")
 
@@ -120,7 +96,6 @@ def _old_loop(gpt4all_instance):
             n_predict=200,
             top_k=40,
             top_p=0.9,
-            min_p=0.0,
             temp=0.9,
             n_batch=9,
             repeat_penalty=1.1,
@@ -128,59 +103,16 @@ def _old_loop(gpt4all_instance):
             context_erase=0.0,
             # required kwargs for cli ux (incremental response)
             verbose=False,
-            streaming=True,
+            std_passthrough=True,
         )
         # record assistant's response to messages
         MESSAGES.append(full_response.get("choices")[0].get("message"))
         print() # newline before next prompt
 
 
-def _new_loop(gpt4all_instance):
-    with gpt4all_instance.chat_session():
-        while True:
-            message = input(" ⇢  ")
-
-            # Check if special command and take action
-            if message in SPECIAL_COMMANDS:
-                SPECIAL_COMMANDS[message](MESSAGES)
-                continue
-
-            # if regular message, append to messages
-            MESSAGES.append({"role": "user", "content": message})
-
-            # execute chat completion and ignore the full response since 
-            # we are outputting it incrementally
-            response_generator = gpt4all_instance.generate(
-                message,
-                # preferential kwargs for chat ux
-                max_tokens=200,
-                temp=0.9,
-                top_k=40,
-                top_p=0.9,
-                min_p=0.0,
-                repeat_penalty=1.1,
-                repeat_last_n=64,
-                n_batch=9,
-                # required kwargs for cli ux (incremental response)
-                streaming=True,
-            )
-            response = io.StringIO()
-            for token in response_generator:
-                print(token, end='', flush=True)
-                response.write(token)
-
-            # record assistant's response to messages
-            response_message = {'role': 'assistant', 'content': response.getvalue()}
-            response.close()
-            gpt4all_instance.current_chat_session.append(response_message)
-            MESSAGES.append(response_message)
-            print() # newline before next prompt
-
-
 @app.command()
 def version():
-    """The CLI version command."""
-    print(f"gpt4all-cli v{VERSION}")
+    print("gpt4all-cli v0.1.0")
 
 
 if __name__ == "__main__":

gpt4all-bindings/cli/developer_notes.md

@@ -1,25 +0,0 @@
-# Developing the CLI
-## Documentation
-Documentation can be found in three places:
-- `app.py` docstrings & comments
-- a Readme: `gpt4all-bindings/cli/README.md`
-- the actual CLI documentation: `gpt4all-bindings/python/docs/gpt4all_cli.md`
-
-The _docstrings_ are meant for programmatic use. Since the CLI is primarily geared towards users and
-not to build on top, they're kept terse.
-
-The _Readme_ is mostly meant for users and includes:
-- a link to the _CLI documentation_ (on the [website])
-- a Quickstart section with some guidance on how to get started with a sane setup
-
-The _CLI documentation_ and other documentation are located in the above mentioned `docs/` folder.
-They're in Markdown format and built for the [website]. Of the three, they should be the most
-detailed.
-
-[website]: https://docs.gpt4all.io/gpt4all_cli.html
-
-
-## Versioning
-The version number should now follow the `gpt4all` PyPI package, so compatibility is more clear.
-
-The one place to change it is the `namedtuple` called `VERSION_INFO`.

gpt4all-bindings/csharp/.editorconfig

@@ -41,8 +41,6 @@ insert_final_newline = true
 
 # IDE0055: Fix formatting
 dotnet_diagnostic.IDE0055.severity = error
-dotnet_diagnostic.CS1573.severity = suggestion
-dotnet_diagnostic.CS1591.severity = suggestion
 
 # Sort using and Import directives with System.* appearing first
 dotnet_sort_system_directives_first = true
@@ -345,4 +343,4 @@ dotnet_diagnostic.IDE2004.severity = warning
 [src/{VisualStudio}/**/*.{cs,vb}]
 # CA1822: Make member static
 # There is a risk of accidentally breaking an internal API that partners rely on though IVT.
-dotnet_code_quality.CA1822.api_surface = private
+dotnet_code_quality.CA1822.api_surface = private

gpt4all-bindings/csharp/Directory.Build.props

@@ -5,7 +5,7 @@
         <Company></Company>
         <Copyright></Copyright>
         <NeutralLanguage>en-US</NeutralLanguage>
-        <Version>0.6.4-alpha</Version>
+        <Version>0.6.1-alpha</Version>
         <VersionSuffix>$(VersionSuffix)</VersionSuffix>
         <Version Condition=" '$(VersionSuffix)' != '' ">$(Version)$(VersionSuffix)</Version>
         <TreatWarningsAsErrors>true</TreatWarningsAsErrors>

gpt4all-bindings/csharp/Gpt4All.Samples/Gpt4All.Samples.csproj

@@ -1,33 +1,18 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
-    <PropertyGroup>
-        <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-        <GenerateDocumentationFile>true</GenerateDocumentationFile>
-    </PropertyGroup>
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
 
-    <ItemGroup>
-        <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
-    </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
+  </ItemGroup>
 
-    <ItemGroup>
-        <!-- Windows -->
-        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
-        <!-- Linux -->
-        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
-        <!-- MacOS -->
-        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
-    </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Properties\" />
+  </ItemGroup>
 
-    <ItemGroup>
-        <!-- Windows -->
-        <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-        <!-- Linux -->
-        <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-         <!-- MacOS -->
-        <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-        <Content Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.metal" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-    </ItemGroup>
 </Project>

gpt4all-bindings/csharp/Gpt4All.Tests/Constants.cs

@@ -1,9 +1,10 @@
-namespace Gpt4All.Tests;
-
-public static class Constants
+namespace Gpt4All.Tests
 {
-    public const string MODELS_BASE_DIR = "../../../models";
-    public const string LLAMA_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-l13b-snoozy.bin";
-    public const string GPTJ_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-j-v1.3-groovy.bin";
-    public const string MPT_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-mpt-7b-chat.bin";
+    public static class Constants
+    {
+        public const string MODELS_BASE_DIR = "../../../models";
+        public const string LLAMA_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-l13b-snoozy.bin";
+        public const string GPTJ_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-j-v1.3-groovy.bin";
+        public const string MPT_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-mpt-7b-chat.bin";
+    }
 }

gpt4all-bindings/csharp/Gpt4All.Tests/Gpt4All.Tests.csproj

@@ -1,60 +1,27 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
+        <TargetFramework>net6.0</TargetFramework>
         <Nullable>enable</Nullable>
 
         <IsPackable>false</IsPackable>
-        <GenerateDocumentationFile>true</GenerateDocumentationFile>
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2" />
-        <PackageReference Include="xunit" Version="2.4.2" />
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
+        <PackageReference Include="xunit" Version="2.4.1" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             <PrivateAssets>all</PrivateAssets>
         </PackageReference>
-        <PackageReference Include="coverlet.collector" Version="6.0.0">
+        <PackageReference Include="coverlet.collector" Version="3.1.0">
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             <PrivateAssets>all</PrivateAssets>
         </PackageReference>
     </ItemGroup>
 
     <ItemGroup>
-        <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
+      <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
     </ItemGroup>
 
-    <ItemGroup>
-        <!-- Windows -->
-        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
-        <!-- Linux -->
-        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
-        <!-- MacOS -->
-        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
-    </ItemGroup>
-
-    <ItemGroup>
-        <!-- Windows -->
-        <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-        <!-- Linux -->
-        <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-         <!-- MacOS -->
-        <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-    </ItemGroup>
-
-    <ItemGroup>
-      <PackageReference Update="Roslynator.Analyzers" Version="4.3.0">
-        <PrivateAssets>all</PrivateAssets>
-        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-      </PackageReference>
-      <PackageReference Update="Roslynator.CodeAnalysis.Analyzers" Version="4.3.0">
-        <PrivateAssets>all</PrivateAssets>
-        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-      </PackageReference>
-      <PackageReference Update="Roslynator.Formatting.Analyzers" Version="4.3.0">
-        <PrivateAssets>all</PrivateAssets>
-        <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-      </PackageReference>
-    </ItemGroup>
 </Project>

gpt4all-bindings/csharp/Gpt4All.Tests/ModelFactoryTests.cs

@@ -1,4 +1,4 @@
-using Xunit;
+using Xunit;
 
 namespace Gpt4All.Tests;
 
@@ -12,23 +12,20 @@ public class ModelFactoryTests
     }
 
     [Fact]
-    [Trait(Traits.SkipOnCI, "True")]
     public void CanLoadLlamaModel()
     {
-        using var model = _modelFactory.LoadModel(Constants.LLAMA_MODEL_PATH);
+        using var model = _modelFactory.LoadLlamaModel(Constants.LLAMA_MODEL_PATH);
     }
 
     [Fact]
-    [Trait(Traits.SkipOnCI, "True")]
     public void CanLoadGptjModel()
     {
-        using var model = _modelFactory.LoadModel(Constants.GPTJ_MODEL_PATH);
+        using var model = _modelFactory.LoadGptjModel(Constants.GPTJ_MODEL_PATH);
     }
 
     [Fact]
-    [Trait(Traits.SkipOnCI, "True")]
     public void CanLoadMptModel()
     {
-        using var model = _modelFactory.LoadModel(Constants.MPT_MODEL_PATH);
+        using var model = _modelFactory.LoadMptModel(Constants.MPT_MODEL_PATH);
     }
 }

gpt4all-bindings/csharp/Gpt4All.Tests/NativeLibraryLoaderTests.cs

@@ -1,56 +0,0 @@
-using System.IO;
-using Gpt4All.LibraryLoader;
-using Xunit;
-
-namespace Gpt4All.Tests;
-
-public class NativeLibraryLoaderTests
-{
-    [Fact]
-    public void NativeLibraryShouldLoad()
-    {
-        var result = NativeLibraryLoader.LoadNativeLibrary(bypassLoading: false);
-        Assert.True(result.IsSuccess);
-    }
-
-    private const string LLModelLib = "libllmodel.{0}";
-
-    [PlatformSpecificFact(Platforms.Windows)]
-    public void NativeLibraryShouldLoad_Windows()
-    {
-        var libraryLoader = new WindowsLibraryLoader();
-
-        var libraryPath = Path.Combine(
-            Environment.CurrentDirectory,
-            string.Format(LLModelLib, "dll"));
-
-        var result = libraryLoader.OpenLibrary(libraryPath);
-        Assert.True(result.IsSuccess);
-    }
-
-    [PlatformSpecificFact(Platforms.Linux)]
-    public void NativeLibraryShouldLoad_Linux()
-    {
-        var libraryLoader = new LinuxLibraryLoader();
-
-        var libraryPath = Path.Combine(
-            Environment.CurrentDirectory,
-            string.Format(LLModelLib, "so"));
-
-        var result = libraryLoader.OpenLibrary(libraryPath);
-        Assert.True(result.IsSuccess);
-    }
-
-    [PlatformSpecificFact(Platforms.MacOS)]
-    public void NativeLibraryShouldLoad_MacOS()
-    {
-        var libraryLoader = new MacOsLibraryLoader();
-
-        var libraryPath = Path.Combine(
-            Environment.CurrentDirectory,
-            string.Format(LLModelLib, "dylib"));
-
-        var result = libraryLoader.OpenLibrary(libraryPath);
-        Assert.True(result.IsSuccess);
-    }
-}

gpt4all-bindings/csharp/Gpt4All.Tests/PlatformSpecificFactAttribute.cs

@@ -1,27 +0,0 @@
-using Xunit;
-
-namespace Gpt4All.Tests;
-
-public static class Platforms
-{
-    public const string Windows = "windows";
-    public const string Linux = "linux";
-    public const string MacOS = "macOS";
-}
-
-/// <summary>
-/// This attribute ensures the Fact is only run on the specified platform.
-/// </summary>
-/// <remarks>
-/// <see cref="OperatingSystem.IsOSPlatform(string)"/> for info about the platform string.
-/// </remarks>
-public class PlatformSpecificFactAttribute : FactAttribute
-{
-    public PlatformSpecificFactAttribute(string platform)
-    {
-        if (!OperatingSystem.IsOSPlatform(platform))
-        {
-            Skip = $"Test only runs on {platform}.";
-        }
-    }
-}

gpt4all-bindings/csharp/Gpt4All.Tests/Traits.cs

@@ -1,6 +0,0 @@
-namespace Gpt4All.Tests;
-
-public static class Traits
-{
-    public const string SkipOnCI = "SKIP_ON_CI";
-}

gpt4all-bindings/csharp/Gpt4All/Bindings/ILLModel.cs

@@ -5,6 +5,8 @@
 /// </summary>
 public interface ILLModel : IDisposable
 {
+    ModelType ModelType { get; }
+
     ulong GetStateSizeBytes();
 
     int GetThreadCount();

gpt4all-bindings/csharp/Gpt4All/Bindings/LLModel.cs

@@ -1,212 +1,247 @@
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
-
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Arguments for the response processing callback
-/// </summary>
-/// <param name="TokenId">The token id of the response</param>
-/// <param name="Response"> The response string. NOTE: a token_id of -1 indicates the string is an error string</param>
-/// <return>
-/// A bool indicating whether the model should keep generating
-/// </return>
-public record ModelResponseEventArgs(int TokenId, string Response)
-{
-    public bool IsError => TokenId == -1;
-}
-
-/// <summary>
-/// Arguments for the prompt processing callback
-/// </summary>
-/// <param name="TokenId">The token id of the prompt</param>
-/// <return>
-/// A bool indicating whether the model should keep processing
-/// </return>
-public record ModelPromptEventArgs(int TokenId)
-{
-}
-
-/// <summary>
-/// Arguments for the recalculating callback
-/// </summary>
-/// <param name="IsRecalculating"> whether the model is recalculating the context.</param>
-/// <return>
-/// A bool indicating whether the model should keep generating
-/// </return>
-public record ModelRecalculatingEventArgs(bool IsRecalculating);
-
-/// <summary>
-/// Base class and universal wrapper for GPT4All language models built around llmodel C-API.
-/// </summary>
-public class LLModel : ILLModel
-{
-    protected readonly IntPtr _handle;
-    private readonly ILogger _logger;
-    private bool _disposed;
-
-    internal LLModel(IntPtr handle, ILogger? logger = null)
-    {
-        _handle = handle;
-        _logger = logger ?? NullLogger.Instance;
-    }
-
-    /// <summary>
-    /// Create a new model from a pointer
-    /// </summary>
-    /// <param name="handle">Pointer to underlying model</param>
-    public static LLModel Create(IntPtr handle, ILogger? logger = null)
-    {
-        return new LLModel(handle, logger: logger);
-    }
-
-    /// <summary>
-    /// Generate a response using the model
-    /// </summary>
-    /// <param name="text">The input promp</param>
-    /// <param name="context">The context</param>
-    /// <param name="promptCallback">A callback function for handling the processing of prompt</param>
-    /// <param name="responseCallback">A callback function for handling the generated response</param>
-    /// <param name="recalculateCallback">A callback function for handling recalculation requests</param>
-    /// <param name="cancellationToken"></param>
-    public void Prompt(
-        string text,
-        LLModelPromptContext context,
-        Func<ModelPromptEventArgs, bool>? promptCallback = null,
-        Func<ModelResponseEventArgs, bool>? responseCallback = null,
-        Func<ModelRecalculatingEventArgs, bool>? recalculateCallback = null,
-        CancellationToken cancellationToken = default)
-    {
-        GC.KeepAlive(promptCallback);
-        GC.KeepAlive(responseCallback);
-        GC.KeepAlive(recalculateCallback);
-        GC.KeepAlive(cancellationToken);
-
-        _logger.LogInformation("Prompt input='{Prompt}' ctx={Context}", text, context.Dump());
-
-        NativeMethods.llmodel_prompt(
-            _handle,
-            text,
-            (tokenId) =>
-            {
-                if (cancellationToken.IsCancellationRequested) return false;
-                if (promptCallback == null) return true;
-                var args = new ModelPromptEventArgs(tokenId);
-                return promptCallback(args);
-            },
-            (tokenId, response) =>
-            {
-                if (cancellationToken.IsCancellationRequested)
-                {
-                    _logger.LogDebug("ResponseCallback evt=CancellationRequested");
-                    return false;
-                }
-
-                if (responseCallback == null) return true;
-                var args = new ModelResponseEventArgs(tokenId, response);
-                return responseCallback(args);
-            },
-            (isRecalculating) =>
-            {
-                if (cancellationToken.IsCancellationRequested) return false;
-                if (recalculateCallback == null) return true;
-                var args = new ModelRecalculatingEventArgs(isRecalculating);
-                return recalculateCallback(args);
-            },
-            ref context.UnderlyingContext
-        );
-    }
-
-    /// <summary>
-    ///  Set the number of threads to be used by the model.
-    /// </summary>
-    /// <param name="threadCount">The new thread count</param>
-    public void SetThreadCount(int threadCount)
-    {
-        NativeMethods.llmodel_setThreadCount(_handle, threadCount);
-    }
-
-    /// <summary>
-    /// Get  the number of threads used by the model.
-    /// </summary>
-    /// <returns>the number of threads used by the model</returns>
-    public int GetThreadCount()
-    {
-        return NativeMethods.llmodel_threadCount(_handle);
-    }
-
-    /// <summary>
-    /// Get the size of the internal state of the model.
-    /// </summary>
-    /// <remarks>
-    /// This state data is specific to the type of model you have created.
-    /// </remarks>
-    /// <returns>the size in bytes of the internal state of the model</returns>
-    public ulong GetStateSizeBytes()
-    {
-        return NativeMethods.llmodel_get_state_size(_handle);
-    }
-
-    /// <summary>
-    /// Saves the internal state of the model to the specified destination address.
-    /// </summary>
-    /// <param name="source">A pointer to the src</param>
-    /// <returns>The number of bytes copied</returns>
-    public unsafe ulong SaveStateData(byte* source)
-    {
-        return NativeMethods.llmodel_save_state_data(_handle, source);
-    }
-
-    /// <summary>
-    /// Restores the internal state of the model using data from the specified address.
-    /// </summary>
-    /// <param name="destination">A pointer to destination</param>
-    /// <returns>the number of bytes read</returns>
-    public unsafe ulong RestoreStateData(byte* destination)
-    {
-        return NativeMethods.llmodel_restore_state_data(_handle, destination);
-    }
-
-    /// <summary>
-    /// Check if the model is loaded.
-    /// </summary>
-    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
-    public bool IsLoaded()
-    {
-        return NativeMethods.llmodel_isModelLoaded(_handle);
-    }
-
-    /// <summary>
-    /// Load the model from a file.
-    /// </summary>
-    /// <param name="modelPath">The path to the model file.</param>
-    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
-    public bool Load(string modelPath)
-    {
-        return NativeMethods.llmodel_loadModel(_handle, modelPath, 2048, 100);
-    }
-
-    protected void Destroy()
-    {
-        NativeMethods.llmodel_model_destroy(_handle);
-    }
-    protected virtual void Dispose(bool disposing)
-    {
-        if (_disposed) return;
-
-        if (disposing)
-        {
-            // dispose managed state
-        }
-
-        Destroy();
-
-        _disposed = true;
-    }
-
-    public void Dispose()
-    {
-        Dispose(disposing: true);
-        GC.SuppressFinalize(this);
-    }
-}
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace Gpt4All.Bindings;
+
+/// <summary>
+/// Arguments for the response processing callback
+/// </summary>
+/// <param name="TokenId">The token id of the response</param>
+/// <param name="Response"> The response string. NOTE: a token_id of -1 indicates the string is an error string</param>
+/// <return>
+/// A bool indicating whether the model should keep generating
+/// </return>
+public record ModelResponseEventArgs(int TokenId, string Response)
+{
+    public bool IsError => TokenId == -1;
+}
+
+/// <summary>
+/// Arguments for the prompt processing callback
+/// </summary>
+/// <param name="TokenId">The token id of the prompt</param>
+/// <return>
+/// A bool indicating whether the model should keep processing
+/// </return>
+public record ModelPromptEventArgs(int TokenId)
+{
+}
+
+/// <summary>
+/// Arguments for the recalculating callback
+/// </summary>
+/// <param name="IsRecalculating"> whether the model is recalculating the context.</param>
+/// <return>
+/// A bool indicating whether the model should keep generating
+/// </return>
+public record ModelRecalculatingEventArgs(bool IsRecalculating);
+
+/// <summary>
+/// Base class and universal wrapper for GPT4All language models built around llmodel C-API.
+/// </summary>
+public class LLModel : ILLModel
+{
+    protected readonly IntPtr _handle;
+    private readonly ModelType _modelType;
+    private readonly ILogger _logger;
+    private bool _disposed;
+
+    public ModelType ModelType => _modelType;
+
+    internal LLModel(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    {
+        _handle = handle;
+        _modelType = modelType;
+        _logger = logger ?? NullLogger.Instance;
+    }
+
+    /// <summary>
+    /// Create a new model from a pointer
+    /// </summary>
+    /// <param name="handle">Pointer to underlying model</param>
+    /// <param name="modelType">The model type</param>
+    public static LLModel Create(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    {
+        return new LLModel(handle, modelType, logger: logger);
+    }
+
+    /// <summary>
+    /// Generate a response using the model
+    /// </summary>
+    /// <param name="text">The input promp</param>
+    /// <param name="context">The context</param>
+    /// <param name="promptCallback">A callback function for handling the processing of prompt</param>
+    /// <param name="responseCallback">A callback function for handling the generated response</param>
+    /// <param name="recalculateCallback">A callback function for handling recalculation requests</param>
+    /// <param name="cancellationToken"></param>
+    public void Prompt(
+        string text,
+        LLModelPromptContext context,
+        Func<ModelPromptEventArgs, bool>? promptCallback = null,
+        Func<ModelResponseEventArgs, bool>? responseCallback = null,
+        Func<ModelRecalculatingEventArgs, bool>? recalculateCallback = null,
+        CancellationToken cancellationToken = default)
+    {
+        GC.KeepAlive(promptCallback);
+        GC.KeepAlive(responseCallback);
+        GC.KeepAlive(recalculateCallback);
+        GC.KeepAlive(cancellationToken);
+
+        _logger.LogInformation("Prompt input='{Prompt}' ctx={Context}", text, context.Dump());
+
+        NativeMethods.llmodel_prompt(
+            _handle,
+            text,
+            (tokenId) =>
+            {
+                if (cancellationToken.IsCancellationRequested) return false;
+                if (promptCallback == null) return true;
+                var args = new ModelPromptEventArgs(tokenId);
+                return promptCallback(args);
+            },
+            (tokenId, response) =>
+            {
+                if (cancellationToken.IsCancellationRequested)
+                {
+                    _logger.LogDebug("ResponseCallback evt=CancellationRequested");
+                    return false;
+                }
+
+                if (responseCallback == null) return true;
+                var args = new ModelResponseEventArgs(tokenId, response);
+                return responseCallback(args);
+            },
+            (isRecalculating) =>
+            {
+                if (cancellationToken.IsCancellationRequested) return false;
+                if (recalculateCallback == null) return true;
+                var args = new ModelRecalculatingEventArgs(isRecalculating);
+                return recalculateCallback(args);
+            },
+            ref context.UnderlyingContext
+        );
+    }
+
+    /// <summary>
+    ///  Set the number of threads to be used by the model.
+    /// </summary>
+    /// <param name="threadCount">The new thread count</param>
+    public void SetThreadCount(int threadCount)
+    {
+        NativeMethods.llmodel_setThreadCount(_handle, threadCount);
+    }
+
+    /// <summary>
+    /// Get  the number of threads used by the model.
+    /// </summary>
+    /// <returns>the number of threads used by the model</returns>
+    public int GetThreadCount()
+    {
+        return NativeMethods.llmodel_threadCount(_handle);
+    }
+
+    /// <summary>
+    /// Get the size of the internal state of the model.
+    /// </summary>
+    /// <remarks>
+    /// This state data is specific to the type of model you have created.
+    /// </remarks>
+    /// <returns>the size in bytes of the internal state of the model</returns>
+    public ulong GetStateSizeBytes()
+    {
+        return NativeMethods.llmodel_get_state_size(_handle);
+    }
+
+    /// <summary>
+    /// Saves the internal state of the model to the specified destination address.
+    /// </summary>
+    /// <param name="source">A pointer to the src</param>
+    /// <returns>The number of bytes copied</returns>
+    public unsafe ulong SaveStateData(byte* source)
+    {
+        return NativeMethods.llmodel_save_state_data(_handle, source);
+    }
+
+    /// <summary>
+    /// Restores the internal state of the model using data from the specified address.
+    /// </summary>
+    /// <param name="destination">A pointer to destination</param>
+    /// <returns>the number of bytes read</returns>
+    public unsafe ulong RestoreStateData(byte* destination)
+    {
+        return NativeMethods.llmodel_restore_state_data(_handle, destination);
+    }
+
+    /// <summary>
+    /// Check if the model is loaded.
+    /// </summary>
+    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
+    public bool IsLoaded()
+    {
+        return NativeMethods.llmodel_isModelLoaded(_handle);
+    }
+
+    /// <summary>
+    /// Load the model from a file.
+    /// </summary>
+    /// <param name="modelPath">The path to the model file.</param>
+    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
+    public bool Load(string modelPath)
+    {
+        return NativeMethods.llmodel_loadModel(_handle, modelPath);
+    }
+
+    protected void Destroy()
+    {
+        NativeMethods.llmodel_model_destroy(_handle);
+    }
+
+    protected void DestroyLLama()
+    {
+        NativeMethods.llmodel_llama_destroy(_handle);
+    }
+
+    protected void DestroyGptj()
+    {
+        NativeMethods.llmodel_gptj_destroy(_handle);
+    }
+
+    protected void DestroyMtp()
+    {
+        NativeMethods.llmodel_mpt_destroy(_handle);
+    }
+
+    protected virtual void Dispose(bool disposing)
+    {
+        if (_disposed) return;
+
+        if (disposing)
+        {
+            // dispose managed state
+        }
+
+        switch (_modelType)
+        {
+            case ModelType.LLAMA:
+                DestroyLLama();
+                break;
+            case ModelType.GPTJ:
+                DestroyGptj();
+                break;
+            case ModelType.MPT:
+                DestroyMtp();
+                break;
+            default:
+                Destroy();
+                break;
+        }
+
+        _disposed = true;
+    }
+
+    public void Dispose()
+    {
+        Dispose(disposing: true);
+        GC.SuppressFinalize(this);
+    }
+}

gpt4all-bindings/csharp/Gpt4All/Bindings/LLPromptContext.cs

@@ -1,147 +1,138 @@
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Wrapper around the llmodel_prompt_context structure for holding the prompt context.
-/// </summary>
-/// <remarks>
-/// The implementation takes care of all the memory handling of the raw logits pointer and the
-/// raw tokens pointer.Attempting to resize them or modify them in any way can lead to undefined behavior
-/// </remarks>
-public unsafe class LLModelPromptContext
-{
-    private llmodel_prompt_context _ctx;
-
-    internal ref llmodel_prompt_context UnderlyingContext => ref _ctx;
-
-    public LLModelPromptContext()
-    {
-        _ctx = new();
-    }
-
-    /// <summary>
-    /// logits of current context
-    /// </summary>
-    public Span<float> Logits => new(_ctx.logits, (int)_ctx.logits_size);
-
-    /// <summary>
-    /// the size of the raw logits vector
-    /// </summary>
-    public nuint LogitsSize
-    {
-        get => _ctx.logits_size;
-        set => _ctx.logits_size = value;
-    }
-
-    /// <summary>
-    /// current tokens in the context window
-    /// </summary>
-    public Span<int> Tokens => new(_ctx.tokens, (int)_ctx.tokens_size);
-
-    /// <summary>
-    /// the size of the raw tokens vector
-    /// </summary>
-    public nuint TokensSize
-    {
-        get => _ctx.tokens_size;
-        set => _ctx.tokens_size = value;
-    }
-
-    /// <summary>
-    /// top k logits to sample from
-    /// </summary>
-    public int TopK
-    {
-        get => _ctx.top_k;
-        set => _ctx.top_k = value;
-    }
-
-    /// <summary>
-    /// nucleus sampling probability threshold
-    /// </summary>
-    public float TopP
-    {
-        get => _ctx.top_p;
-        set => _ctx.top_p = value;
-    }
-
-    /// <summary>
-    /// min p sampling probability threshold
-    /// </summary>
-    public float MinP
-    {
-        get => _ctx.min_p;
-        set => _ctx.min_p = value;
-    }
-
-    /// <summary>
-    /// temperature to adjust model's output distribution
-    /// </summary>
-    public float Temperature
-    {
-        get => _ctx.temp;
-        set => _ctx.temp = value;
-    }
-
-    /// <summary>
-    /// number of tokens in past conversation
-    /// </summary>
-    public int PastNum
-    {
-        get => _ctx.n_past;
-        set => _ctx.n_past = value;
-    }
-
-    /// <summary>
-    /// number of predictions to generate in parallel
-    /// </summary>
-    public int Batches
-    {
-        get => _ctx.n_batch;
-        set => _ctx.n_batch = value;
-    }
-
-    /// <summary>
-    /// number of tokens to predict
-    /// </summary>
-    public int TokensToPredict
-    {
-        get => _ctx.n_predict;
-        set => _ctx.n_predict = value;
-    }
-
-    /// <summary>
-    /// penalty factor for repeated tokens
-    /// </summary>
-    public float RepeatPenalty
-    {
-        get => _ctx.repeat_penalty;
-        set => _ctx.repeat_penalty = value;
-    }
-
-    /// <summary>
-    /// last n tokens to penalize
-    /// </summary>
-    public int RepeatLastN
-    {
-        get => _ctx.repeat_last_n;
-        set => _ctx.repeat_last_n = value;
-    }
-
-    /// <summary>
-    /// number of tokens possible in context window
-    /// </summary>
-    public int ContextSize
-    {
-        get => _ctx.n_ctx;
-        set => _ctx.n_ctx = value;
-    }
-
-    /// <summary>
-    /// percent of context to erase if we exceed the context window
-    /// </summary>
-    public float ContextErase
-    {
-        get => _ctx.context_erase;
-        set => _ctx.context_erase = value;
-    }
-}
+namespace Gpt4All.Bindings;
+
+/// <summary>
+/// Wrapper around the llmodel_prompt_context structure for holding the prompt context.
+/// </summary>
+/// <remarks>
+/// The implementation takes care of all the memory handling of the raw logits pointer and the
+/// raw tokens pointer.Attempting to resize them or modify them in any way can lead to undefined behavior
+/// </remarks>
+public unsafe class LLModelPromptContext
+{
+    private llmodel_prompt_context _ctx;
+
+    internal ref llmodel_prompt_context UnderlyingContext => ref _ctx;
+
+    public LLModelPromptContext()
+    {
+        _ctx = new();
+    }
+
+    /// <summary>
+    /// logits of current context
+    /// </summary>
+    public Span<float> Logits => new(_ctx.logits, (int)_ctx.logits_size);
+
+    /// <summary>
+    /// the size of the raw logits vector
+    /// </summary>
+    public nuint LogitsSize
+    {
+        get => _ctx.logits_size;
+        set => _ctx.logits_size = value;
+    }
+
+    /// <summary>
+    /// current tokens in the context window
+    /// </summary>
+    public Span<int> Tokens => new(_ctx.tokens, (int)_ctx.tokens_size);
+
+    /// <summary>
+    /// the size of the raw tokens vector
+    /// </summary>
+    public nuint TokensSize
+    {
+        get => _ctx.tokens_size;
+        set => _ctx.tokens_size = value;
+    }
+
+    /// <summary>
+    /// top k logits to sample from
+    /// </summary>
+    public int TopK
+    {
+        get => _ctx.top_k;
+        set => _ctx.top_k = value;
+    }
+
+    /// <summary>
+    /// nucleus sampling probability threshold
+    /// </summary>
+    public float TopP
+    {
+        get => _ctx.top_p;
+        set => _ctx.top_p = value;
+    }
+
+    /// <summary>
+    /// temperature to adjust model's output distribution
+    /// </summary>
+    public float Temperature
+    {
+        get => _ctx.temp;
+        set => _ctx.temp = value;
+    }
+
+    /// <summary>
+    /// number of tokens in past conversation
+    /// </summary>
+    public int PastNum
+    {
+        get => _ctx.n_past;
+        set => _ctx.n_past = value;
+    }
+
+    /// <summary>
+    /// number of predictions to generate in parallel
+    /// </summary>
+    public int Batches
+    {
+        get => _ctx.n_batch;
+        set => _ctx.n_batch = value;
+    }
+
+    /// <summary>
+    /// number of tokens to predict
+    /// </summary>
+    public int TokensToPredict
+    {
+        get => _ctx.n_predict;
+        set => _ctx.n_predict = value;
+    }
+
+    /// <summary>
+    /// penalty factor for repeated tokens
+    /// </summary>
+    public float RepeatPenalty
+    {
+        get => _ctx.repeat_penalty;
+        set => _ctx.repeat_penalty = value;
+    }
+
+    /// <summary>
+    /// last n tokens to penalize
+    /// </summary>
+    public int RepeatLastN
+    {
+        get => _ctx.repeat_last_n;
+        set => _ctx.repeat_last_n = value;
+    }
+
+    /// <summary>
+    /// number of tokens possible in context window
+    /// </summary>
+    public int ContextSize
+    {
+        get => _ctx.n_ctx;
+        set => _ctx.n_ctx = value;
+    }
+
+    /// <summary>
+    /// percent of context to erase if we exceed the context window
+    /// </summary>
+    public float ContextErase
+    {
+        get => _ctx.context_erase;
+        set => _ctx.context_erase = value;
+    }
+}

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeMethods.cs

@@ -1,112 +1,126 @@
-using System.Runtime.InteropServices;
-
-namespace Gpt4All.Bindings;
-
-public unsafe partial struct llmodel_prompt_context
-{
-    public float* logits;
-
-    [NativeTypeName("size_t")]
-    public nuint logits_size;
-
-    [NativeTypeName("int32_t *")]
-    public int* tokens;
-
-    [NativeTypeName("size_t")]
-    public nuint tokens_size;
-
-    [NativeTypeName("int32_t")]
-    public int n_past;
-
-    [NativeTypeName("int32_t")]
-    public int n_ctx;
-
-    [NativeTypeName("int32_t")]
-    public int n_predict;
-
-    [NativeTypeName("int32_t")]
-    public int top_k;
-
-    public float top_p;
-
-    public float min_p;
-
-    public float temp;
-
-    [NativeTypeName("int32_t")]
-    public int n_batch;
-
-    public float repeat_penalty;
-
-    [NativeTypeName("int32_t")]
-    public int repeat_last_n;
-
-    public float context_erase;
-}
-#pragma warning disable CA2101
-internal static unsafe partial class NativeMethods
-{
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelResponseCallback(int token_id, [MarshalAs(UnmanagedType.LPUTF8Str)] string response);
-
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelPromptCallback(int token_id);
-
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelRecalculateCallback(bool isRecalculating);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_model_create2(
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string build_variant,
-        out IntPtr error);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_model_destroy([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public static extern bool llmodel_loadModel(
-        [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path,
-        [NativeTypeName("int32_t")] int n_ctx,
-        [NativeTypeName("int32_t")] int ngl);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-
-    [return: MarshalAs(UnmanagedType.I1)]
-    public static extern bool llmodel_isModelLoaded([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_get_state_size([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_save_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("uint8_t *")] byte* dest);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_restore_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("const uint8_t *")] byte* src);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    public static extern void llmodel_prompt(
-        [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string prompt,
-        LlmodelPromptCallback prompt_callback,
-        LlmodelResponseCallback response_callback,
-        LlmodelRecalculateCallback recalculate_callback,
-        ref llmodel_prompt_context ctx);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_setThreadCount([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("int32_t")] int n_threads);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("int32_t")]
-    public static extern int llmodel_threadCount([NativeTypeName("llmodel_model")] IntPtr model);
-}
-#pragma warning restore CA2101
+using System.Runtime.InteropServices;
+
+namespace Gpt4All.Bindings;
+
+public unsafe partial struct llmodel_prompt_context
+{
+    public float* logits;
+
+    [NativeTypeName("size_t")]
+    public nuint logits_size;
+
+    [NativeTypeName("int32_t *")]
+    public int* tokens;
+
+    [NativeTypeName("size_t")]
+    public nuint tokens_size;
+
+    [NativeTypeName("int32_t")]
+    public int n_past;
+
+    [NativeTypeName("int32_t")]
+    public int n_ctx;
+
+    [NativeTypeName("int32_t")]
+    public int n_predict;
+
+    [NativeTypeName("int32_t")]
+    public int top_k;
+
+    public float top_p;
+
+    public float temp;
+
+    [NativeTypeName("int32_t")]
+    public int n_batch;
+
+    public float repeat_penalty;
+
+    [NativeTypeName("int32_t")]
+    public int repeat_last_n;
+
+    public float context_erase;
+}
+
+internal static unsafe partial class NativeMethods
+{
+    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public delegate bool LlmodelResponseCallback(int token_id, [MarshalAs(UnmanagedType.LPUTF8Str)] string response);
+
+    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public delegate bool LlmodelPromptCallback(int token_id);
+
+    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public delegate bool LlmodelRecalculateCallback(bool isRecalculating);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("llmodel_model")]
+    public static extern IntPtr llmodel_gptj_create();
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    public static extern void llmodel_gptj_destroy([NativeTypeName("llmodel_model")] IntPtr gptj);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("llmodel_model")]
+    public static extern IntPtr llmodel_mpt_create();
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    public static extern void llmodel_mpt_destroy([NativeTypeName("llmodel_model")] IntPtr mpt);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("llmodel_model")]
+    public static extern IntPtr llmodel_llama_create();
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    public static extern void llmodel_llama_destroy([NativeTypeName("llmodel_model")] IntPtr llama);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
+    [return: NativeTypeName("llmodel_model")]
+    public static extern IntPtr llmodel_model_create(
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    public static extern void llmodel_model_destroy([NativeTypeName("llmodel_model")] IntPtr model);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    public static extern bool llmodel_loadModel(
+        [NativeTypeName("llmodel_model")] IntPtr model,
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+
+    [return: MarshalAs(UnmanagedType.I1)]
+    public static extern bool llmodel_isModelLoaded([NativeTypeName("llmodel_model")] IntPtr model);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("uint64_t")]
+    public static extern ulong llmodel_get_state_size([NativeTypeName("llmodel_model")] IntPtr model);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("uint64_t")]
+    public static extern ulong llmodel_save_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("uint8_t *")] byte* dest);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("uint64_t")]
+    public static extern ulong llmodel_restore_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("const uint8_t *")] byte* src);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
+    public static extern void llmodel_prompt(
+        [NativeTypeName("llmodel_model")] IntPtr model,
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string prompt,
+        LlmodelPromptCallback prompt_callback,
+        LlmodelResponseCallback response_callback,
+        LlmodelRecalculateCallback recalculate_callback,
+        ref llmodel_prompt_context ctx);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    public static extern void llmodel_setThreadCount([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("int32_t")] int n_threads);
+
+    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
+    [return: NativeTypeName("int32_t")]
+    public static extern int llmodel_threadCount([NativeTypeName("llmodel_model")] IntPtr model);
+}

gpt4all-bindings/csharp/Gpt4All/Extensions/LLPromptContextExtensions.cs

@@ -16,7 +16,6 @@ internal static class LLPromptContextExtensions
             n_predict = {ctx.n_predict}
             top_k = {ctx.top_k}
             top_p = {ctx.top_p}
-            min_p = {ctx.min_p}
             temp = {ctx.temp}
             n_batch = {ctx.n_batch}
             repeat_penalty = {ctx.repeat_penalty}

gpt4all-bindings/csharp/Gpt4All/Extensions/PredictRequestOptionsExtensions.cs

@@ -12,7 +12,6 @@ public static class PredictRequestOptionsExtensions
             TokensSize = opts.TokensSize,
             TopK = opts.TopK,
             TopP = opts.TopP,
-            MinP = opts.MinP,
             PastNum = opts.PastConversationTokensNum,
             RepeatPenalty = opts.RepeatPenalty,
             Temperature = opts.Temperature,

gpt4all-bindings/csharp/Gpt4All/Gpt4All.cs

@@ -1,11 +1,8 @@
 using System.Diagnostics;
-using System.Runtime.CompilerServices;
 using Gpt4All.Bindings;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 
-[assembly: InternalsVisibleTo("Gpt4All.Tests")]
-
 namespace Gpt4All;
 
 public class Gpt4All : IGpt4AllModel

gpt4all-bindings/csharp/Gpt4All/Gpt4All.csproj

@@ -1,23 +1,27 @@
 <Project Sdk="Microsoft.NET.Sdk">
-    <PropertyGroup>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-        <GenerateDocumentationFile>true</GenerateDocumentationFile>
-        <TargetFramework>net8.0</TargetFramework>
-    </PropertyGroup>
-    <ItemGroup>
-        <!-- Windows -->
-        <None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
-        <!-- Linux -->
-        <None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
-        <!-- MacOS -->
-        <None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
-        <Content Include="..\runtimes\osx\native\*.metal" Pack="true" PackagePath="contentFiles\any\any;content">
-            <PackageCopyToOutput>true</PackageCopyToOutput>
-        </Content>
-    </ItemGroup>
-    <ItemGroup>
-        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
-    </ItemGroup>
+
+  <PropertyGroup>
+	<TargetFrameworks>net6.0</TargetFrameworks>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+	<ItemGroup>
+		<!-- Windows -->
+		<None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
+		<!-- Linux -->
+		<None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<!-- Windows -->
+		<None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+		<!-- Linux -->
+		<None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
+	</ItemGroup>
+
+	<ItemGroup>
+	  <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
+	</ItemGroup>
 </Project>

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/ILibraryLoader.cs

@@ -1,6 +0,0 @@
-namespace Gpt4All.LibraryLoader;
-
-public interface ILibraryLoader
-{
-    LoadResult OpenLibrary(string? fileName);
-}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LinuxLibraryLoader.cs

@@ -1,53 +0,0 @@
-using System.Runtime.InteropServices;
-
-namespace Gpt4All.LibraryLoader;
-
-internal class LinuxLibraryLoader : ILibraryLoader
-{
-#pragma warning disable CA2101
-    [DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
-#pragma warning restore CA2101
-    public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags);
-
-#pragma warning disable CA2101
-    [DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
-#pragma warning restore CA2101
-    public static extern IntPtr NativeOpenLibraryLibdl2(string? filename, int flags);
-
-    [DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
-    public static extern IntPtr GetLoadError();
-
-    [DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
-    public static extern IntPtr GetLoadError2();
-
-    public LoadResult OpenLibrary(string? fileName)
-    {
-        IntPtr loadedLib;
-        try
-        {
-            // open with rtls lazy flag
-            loadedLib = NativeOpenLibraryLibdl2(fileName, 0x00001);
-        }
-        catch (DllNotFoundException)
-        {
-            loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001);
-        }
-
-        if (loadedLib == IntPtr.Zero)
-        {
-            string errorMessage;
-            try
-            {
-                errorMessage = Marshal.PtrToStringAnsi(GetLoadError2()) ?? "Unknown error";
-            }
-            catch (DllNotFoundException)
-            {
-                errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error";
-            }
-
-            return LoadResult.Failure(errorMessage);
-        }
-
-        return LoadResult.Success;
-    }
-}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LoadResult.cs

@@ -1,20 +0,0 @@
-namespace Gpt4All.LibraryLoader;
-
-public class LoadResult
-{
-    private LoadResult(bool isSuccess, string? errorMessage)
-    {
-        IsSuccess = isSuccess;
-        ErrorMessage = errorMessage;
-    }
-
-    public static LoadResult Success { get; } = new(true, null);
-
-    public static LoadResult Failure(string errorMessage)
-    {
-        return new(false, errorMessage);
-    }
-
-    public bool IsSuccess { get; }
-    public string? ErrorMessage { get; }
-}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/MacOsLibraryLoader.cs

@@ -1,28 +0,0 @@
-using System.Runtime.InteropServices;
-
-namespace Gpt4All.LibraryLoader;
-
-internal class MacOsLibraryLoader : ILibraryLoader
-{
-#pragma warning disable CA2101
-    [DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
-#pragma warning restore CA2101
-    public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags);
-
-    [DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
-    public static extern IntPtr GetLoadError();
-
-    public LoadResult OpenLibrary(string? fileName)
-    {
-        var loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001);
-
-        if (loadedLib == IntPtr.Zero)
-        {
-            var errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error";
-
-            return LoadResult.Failure(errorMessage);
-        }
-
-        return LoadResult.Success;
-    }
-}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/NativeLibraryLoader.cs

@@ -1,81 +0,0 @@
-#if !IOS && !MACCATALYST && !TVOS && !ANDROID
-using System.Runtime.InteropServices;
-#endif
-
-namespace Gpt4All.LibraryLoader;
-
-public static class NativeLibraryLoader
-{
-    private static ILibraryLoader? defaultLibraryLoader;
-
-    /// <summary>
-    /// Sets the library loader used to load the native libraries. Overwrite this only if you want some custom loading.
-    /// </summary>
-    /// <param name="libraryLoader">The library loader to be used.</param>
-    public static void SetLibraryLoader(ILibraryLoader libraryLoader)
-    {
-        defaultLibraryLoader = libraryLoader;
-    }
-
-    internal static LoadResult LoadNativeLibrary(string? path = default, bool bypassLoading = true)
-    {
-        // If the user has handled loading the library themselves, we don't need to do anything.
-        if (bypassLoading)
-        {
-            return LoadResult.Success;
-        }
-
-        var architecture = RuntimeInformation.OSArchitecture switch
-        {
-            Architecture.X64 => "x64",
-            Architecture.X86 => "x86",
-            Architecture.Arm => "arm",
-            Architecture.Arm64 => "arm64",
-            _ => throw new PlatformNotSupportedException(
-                $"Unsupported OS platform, architecture: {RuntimeInformation.OSArchitecture}")
-        };
-
-        var (platform, extension) = Environment.OSVersion.Platform switch
-        {
-            _ when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => ("win", "dll"),
-            _ when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => ("linux", "so"),
-            _ when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => ("osx", "dylib"),
-            _ => throw new PlatformNotSupportedException(
-                $"Unsupported OS platform, architecture: {RuntimeInformation.OSArchitecture}")
-        };
-
-        // If the user hasn't set the path, we'll try to find it ourselves.
-        if (string.IsNullOrEmpty(path))
-        {
-            var libraryName = "libllmodel";
-            var assemblySearchPath = new[]
-            {
-                AppDomain.CurrentDomain.RelativeSearchPath,
-                Path.GetDirectoryName(typeof(NativeLibraryLoader).Assembly.Location),
-                Path.GetDirectoryName(Environment.GetCommandLineArgs()[0])
-            }.FirstOrDefault(it => !string.IsNullOrEmpty(it));
-            // Search for the library dll within the assembly search path. If it doesn't exist, for whatever reason, use the default path.
-            path = Directory.EnumerateFiles(assemblySearchPath ?? string.Empty, $"{libraryName}.{extension}", SearchOption.AllDirectories).FirstOrDefault() ?? Path.Combine("runtimes", $"{platform}-{architecture}", $"{libraryName}.{extension}");
-        }
-
-        if (defaultLibraryLoader != null)
-        {
-            return defaultLibraryLoader.OpenLibrary(path);
-        }
-
-        if (!File.Exists(path))
-        {
-            throw new FileNotFoundException($"Native Library not found in path {path}. " +
-                                            $"Verify you have have included the native Gpt4All library in your application.");
-        }
-
-        ILibraryLoader libraryLoader = platform switch
-        {
-            "win" => new WindowsLibraryLoader(),
-            "osx" => new MacOsLibraryLoader(),
-            "linux" => new LinuxLibraryLoader(),
-            _ => throw new PlatformNotSupportedException($"Currently {platform} platform is not supported")
-        };
-        return libraryLoader.OpenLibrary(path);
-    }
-}

gpt4all-bindings/csharp/Gpt4All/LibraryLoader/WindowsLibraryLoader.cs

@@ -1,24 +0,0 @@
-using System.ComponentModel;
-using System.Runtime.InteropServices;
-
-namespace Gpt4All.LibraryLoader;
-
-internal class WindowsLibraryLoader : ILibraryLoader
-{
-    public LoadResult OpenLibrary(string? fileName)
-    {
-        var loadedLib = LoadLibrary(fileName);
-
-        if (loadedLib == IntPtr.Zero)
-        {
-            var errorCode = Marshal.GetLastWin32Error();
-            var errorMessage = new Win32Exception(errorCode).Message;
-            return LoadResult.Failure(errorMessage);
-        }
-
-        return LoadResult.Success;
-    }
-
-    [DllImport("kernel32", SetLastError = true, CharSet = CharSet.Auto)]
-    private static extern IntPtr LoadLibrary([MarshalAs(UnmanagedType.LPWStr)] string? lpFileName);
-}

gpt4all-bindings/csharp/Gpt4All/Model/Gpt4AllModelFactory.cs

@@ -1,62 +1,61 @@
-using System.Diagnostics;
-using Microsoft.Extensions.Logging.Abstractions;
-using Microsoft.Extensions.Logging;
-using Gpt4All.Bindings;
-using Gpt4All.LibraryLoader;
-using System.Runtime.InteropServices;
-
-namespace Gpt4All;
-
-public class Gpt4AllModelFactory : IGpt4AllModelFactory
-{
-    private readonly ILoggerFactory _loggerFactory;
-    private readonly ILogger _logger;
-    private static bool bypassLoading;
-    private static string? libraryPath;
-
-    private static readonly Lazy<LoadResult> libraryLoaded = new(() =>
-    {
-        return NativeLibraryLoader.LoadNativeLibrary(Gpt4AllModelFactory.libraryPath, Gpt4AllModelFactory.bypassLoading);
-    }, true);
-
-    public Gpt4AllModelFactory(string? libraryPath = default, bool bypassLoading = true, ILoggerFactory? loggerFactory = null)
-    {
-        _loggerFactory = loggerFactory ?? NullLoggerFactory.Instance;
-        _logger = _loggerFactory.CreateLogger<Gpt4AllModelFactory>();
-        Gpt4AllModelFactory.libraryPath = libraryPath;
-        Gpt4AllModelFactory.bypassLoading = bypassLoading;
-
-        if (!libraryLoaded.Value.IsSuccess)
-        {
-            throw new Exception($"Failed to load native gpt4all library. Error: {libraryLoaded.Value.ErrorMessage}");
-        }
-    }
-
-    private Gpt4All CreateModel(string modelPath)
-    {
-        _logger.LogInformation("Creating model path={ModelPath}", modelPath);
-        IntPtr error;
-        var handle = NativeMethods.llmodel_model_create2(modelPath, "auto", out error);
-        if (error != IntPtr.Zero)
-        {
-            throw new Exception(Marshal.PtrToStringAnsi(error));
-        }
-        _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
-        _logger.LogInformation("Model loading started");
-        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath, 2048, 100);
-        _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
-        if (!loadedSuccessfully)
-        {
-            throw new Exception($"Failed to load model: '{modelPath}'");
-        }
-
-        var logger = _loggerFactory.CreateLogger<LLModel>();
-        var underlyingModel = LLModel.Create(handle, logger: logger);
-
-        Debug.Assert(underlyingModel.IsLoaded());
-
-        return new Gpt4All(underlyingModel, logger: logger);
-    }
-
-    public IGpt4AllModel LoadModel(string modelPath) => CreateModel(modelPath);
-}
+using System.Diagnostics;
+using Microsoft.Extensions.Logging;
+using Gpt4All.Bindings;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace Gpt4All;
+
+public class Gpt4AllModelFactory : IGpt4AllModelFactory
+{
+    private readonly ILoggerFactory _loggerFactory;
+    private readonly ILogger _logger;
+
+    public Gpt4AllModelFactory(ILoggerFactory? loggerFactory = null)
+    {
+        _loggerFactory = loggerFactory ?? NullLoggerFactory.Instance;
+        _logger = _loggerFactory.CreateLogger<Gpt4AllModelFactory>();
+    }
+
+    private IGpt4AllModel CreateModel(string modelPath, ModelType? modelType = null)
+    {
+        var modelType_ = modelType ?? ModelFileUtils.GetModelTypeFromModelFileHeader(modelPath);
+
+        _logger.LogInformation("Creating model path={ModelPath} type={ModelType}", modelPath, modelType_);
+
+        var handle = modelType_ switch
+        {
+            ModelType.LLAMA => NativeMethods.llmodel_llama_create(),
+            ModelType.GPTJ => NativeMethods.llmodel_gptj_create(),
+            ModelType.MPT => NativeMethods.llmodel_mpt_create(),
+            _ => NativeMethods.llmodel_model_create(modelPath),
+        };
+
+        _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
+        _logger.LogInformation("Model loading started");
+
+        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath);
+
+        _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
+
+        if (loadedSuccessfully == false)
+        {
+            throw new Exception($"Failed to load model: '{modelPath}'");
+        }
+
+        var logger = _loggerFactory.CreateLogger<LLModel>();
+
+        var underlyingModel = LLModel.Create(handle, modelType_, logger: logger);
+
+        Debug.Assert(underlyingModel.IsLoaded());
+
+        return new Gpt4All(underlyingModel, logger: logger);
+    }
+
+    public IGpt4AllModel LoadModel(string modelPath) => CreateModel(modelPath, modelType: null);
+
+    public IGpt4AllModel LoadMptModel(string modelPath) => CreateModel(modelPath, ModelType.MPT);
+
+    public IGpt4AllModel LoadGptjModel(string modelPath) => CreateModel(modelPath, ModelType.GPTJ);
+
+    public IGpt4AllModel LoadLlamaModel(string modelPath) => CreateModel(modelPath, ModelType.LLAMA);
+}

gpt4all-bindings/csharp/Gpt4All/Model/IGpt4AllModelFactory.cs

@@ -1,6 +1,12 @@
-namespace Gpt4All;
-
-public interface IGpt4AllModelFactory
-{
-    IGpt4AllModel LoadModel(string modelPath);
-}
+namespace Gpt4All;
+
+public interface IGpt4AllModelFactory
+{
+    IGpt4AllModel LoadGptjModel(string modelPath);
+
+    IGpt4AllModel LoadLlamaModel(string modelPath);
+
+    IGpt4AllModel LoadModel(string modelPath);
+
+    IGpt4AllModel LoadMptModel(string modelPath);
+}