ci: explicitly build with 4 jobs on Linux (#2799)

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/config.yml

@@ -1,194 +1,19 @@
 version: 2.1
+setup: true
 orbs:
-  win: circleci/windows@5.0
-  python: circleci/python@1.2
-
-jobs:
-  build-py-docs:
-    docker:
-      - image: circleci/python:3.8
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          # TODO: eventually this will be cleaned up so we aren't building
-          # new dependencies each time unnecessarily.
-          # This will be introduced once we setup branch and path filtering
-          command: |
-            sudo apt-get update
-            sudo apt-get -y install python3 python3-pip
-            sudo pip3 install awscli --upgrade
-            sudo pip3 install mkdocs mkdocs-material mkautodoc 'mkdocstrings[python]'
-      - run:
-          name: Make Documentation
-          command: |
-            cd gpt4all-bindings/python/
-            mkdocs build
-      - run:
-          name: Deploy Documentation
-          command: |
-            cd gpt4all-bindings/python/
-            aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat
-      - run:
-          name: Invalidate docs.gpt4all.io cloudfront
-          command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"
-
-            
-
-  build-py-linux:
-    docker:
-      - image: circleci/python:3.8
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y cmake build-essential
-            pip install setuptools wheel cmake
-      - run:
-          name: Build C library
-          command: |
-            git submodule init
-            git submodule update
-            cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake ..
-            cmake --build . --parallel
-      - run:
-          name: Build wheel
-          command: |
-            cd gpt4all-bindings/python/
-            python setup.py bdist_wheel --plat-name=manylinux1_x86_64
-      - persist_to_workspace:
-          root: gpt4all-bindings/python/dist
-          paths:
-            - "*.whl"
-
-  build-py-macos:
-    macos:
-      xcode: "14.2.0"
-    resource_class: macos.m1.large.gen1
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            brew install cmake
-            pip install setuptools wheel cmake
-      - run:
-          name: Build C library
-          command: |
-            git submodule init
-            git submodule update
-            cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake .. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
-            cmake --build . --parallel
-      - run:
-          name: Build wheel
-          command: |
-            cd gpt4all-bindings/python
-            python setup.py bdist_wheel --plat-name=macosx_10_9_universal2
-      - persist_to_workspace:
-          root: gpt4all-bindings/python/dist
-          paths:
-            - "*.whl"
-
-  build-py-windows:
-    executor:
-      name: win/default
-    steps:
-      - checkout
-      - run:
-          name: Install MinGW64
-          command: choco install -y mingw --force --no-progress
-      - run:
-          name: Add MinGW64 to PATH
-          command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
-      - run:
-          name: Install dependencies
-          command: choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
-      - run:
-          name: Install Python dependencies
-          command: pip install setuptools wheel cmake
-      - run:
-          name: Build C library
-          command: |
-            git submodule init
-            git submodule update
-            cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake -G "MinGW Makefiles" .. 
-            cmake --build . --parallel
-      - run:
-          name: Build wheel
-          # TODO: As part of this task, we need to move mingw64 binaries into package.
-          # This is terrible and needs a more robust solution eventually.
-          command: |
-            cd gpt4all-bindings/python
-            cd gpt4all
-            mkdir llmodel_DO_NOT_MODIFY
-            mkdir llmodel_DO_NOT_MODIFY/build/
-            cp 'C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
-            cd ..
-            python setup.py bdist_wheel --plat-name=win_amd64
-      - persist_to_workspace:
-          root: gpt4all-bindings/python/dist
-          paths:
-            - "*.whl"
-
-  store-and-upload-wheels:
-    docker:
-      - image: circleci/python:3.8
-    steps:
-      - setup_remote_docker
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Install dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y cmake build-essential
-            pip install setuptools wheel twine
-      - run:
-          name: Upload Python package
-          command: |
-            twine upload /tmp/workspace/*.whl --username __token__ --password $PYPI_CRED
-      - store_artifacts:
-          path: /tmp/workspace
+  path-filtering: circleci/path-filtering@0.0.1
 
 workflows:
-  version: 2
-  deploy-docs:
+  version: 2.1
+  generate-config:
     jobs:
-      - build-py-docs:
-          filters:
-            branches:
-              only:
-                - main
-  # build-py-deploy:
-  #   jobs:
-  #     - build-py-linux:
-  #         filters:
-  #           branches:
-  #             only:
-  #     - build-py-macos:
-  #         filters:
-  #           branches:
-  #             only:
-  #     - build-py-windows:
-  #         filters:
-  #           branches:
-  #             only:
-  #     - store-and-upload-wheels:
-  #         filters:
-  #           branches:
-  #             only:
-  #         requires:
-  #           - build-py-windows
-  #           - build-py-linux
-  #           - build-py-macos
+      - path-filtering/filter:
+          base-revision: main
+          config-path: .circleci/continue_config.yml
+          mapping: |
+            .circleci/.* run-all-workflows true
+            gpt4all-backend/.* run-all-workflows true
+            gpt4all-bindings/python/.* run-python-workflow true
+            gpt4all-bindings/typescript/.* run-ts-workflow true
+            gpt4all-chat/.* run-chat-workflow true
+            .* run-default-workflow true

.circleci/grab_notary_id.py

@@ -0,0 +1,17 @@
+import re
+import sys
+
+ID_REG = r"id: (.*)"
+
+def main() -> None:
+    notary_log = sys.argv[1]
+    with open(notary_log, "r") as f:
+        notary_output = f.read()
+        id_m = re.search(ID_REG, notary_output)
+        if id_m:
+            print(id_m.group(1))
+        else:
+            raise RuntimeError("Unable to parse ID from notarization logs")
+
+if __name__ == "__main__":
+    main()

.codespellrc

@@ -1,4 +1,3 @@
 [codespell]
-skip = .git,*.pdf,*.svg
-#
-# ignore-words-list =
+ignore-words-list = blong, afterall, som, assistent, crasher
+skip = .git,*.pdf,*.svg,*.lock,*.ts

.github/ISSUE_TEMPLATE/bindings-bug.md

@@ -0,0 +1,35 @@
+---
+name: "\U0001F6E0 Bindings Bug Report"
+about: A bug report for the GPT4All Bindings
+labels: ["bindings", "bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Example Code
+
+<!-- Please provide a minimal code example that can be used to experience this issue. Delete this section if it does not apply. -->
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. -->
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? -->
+
+### Your Environment
+
+- Bindings version (e.g. "Version" from `pip show gpt4all`):
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,70 +0,0 @@
-name: "\U0001F41B Bug Report"
-description: Submit a bug report to help us improve GPT4All
-labels: ["02 Bug Report"]
-body:
-  - type: markdown
-    attributes:
-      value: >
-        Thank you for taking the time to file a bug report. Before creating a new
-        issue, please make sure to take a few moments to check the issue tracker
-        for existing issues about the bug.
-
-  - type: textarea
-    id: system-info
-    attributes:
-      label: System Info
-      description: Please share your system info with us.
-      placeholder: GPT4All version, platform, python version, etc...
-    validations:
-      required: true
-
-  - type: checkboxes
-    id: information-scripts-examples
-    attributes:
-      label: Information
-      description: "The problem arises when using:"
-      options:
-        - label: "The official example notebooks/scripts"
-        - label: "My own modified scripts"
-
-  - type: checkboxes
-    id: related-components
-    attributes:
-      label: Related Components
-      description: "Select the components related to the issue (if applicable):"
-      options:
-        - label: "backend"
-        - label: "bindings"
-        - label: "python-bindings"
-        - label: "chat-ui"
-        - label: "models"
-        - label: "circleci"
-        - label: "docker"
-        - label: "api"
-
-  - type: textarea
-    id: reproduction
-    validations:
-      required: true
-    attributes:
-      label: Reproduction
-      description: |
-        Please provide a [code sample](https://stackoverflow.com/help/minimal-reproducible-example) that reproduces the problem you ran into. It can be a Colab link or just a code snippet.
-        If you have code snippets, error messages, stack traces please provide them here as well.
-        Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting
-        Avoid screenshots when possible, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code.
-
-      placeholder: |
-        Steps to reproduce the behavior:
-
-          1.
-          2.
-          3.
-
-  - type: textarea
-    id: expected-behavior
-    validations:
-      required: true
-    attributes:
-      label: Expected behavior
-      description: "A clear and concise description of what you would expect to happen."

.github/ISSUE_TEMPLATE/chat-bug.md

@@ -0,0 +1,31 @@
+---
+name: "\U0001F4AC GPT4All Bug Report"
+about: A bug report for GPT4All Chat
+labels: ["chat", "bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. -->
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? -->
+
+### Your Environment
+
+- GPT4All version:
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -1,2 +1 @@
-blank_issues_enabled: false
-version: 2.1
+version: 2.1

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,9 @@
+---
+name: "\U0001F4C4 Documentation"
+about: An issue related to the GPT4All documentation
+labels: ["documentation"]
+---
+
+### Documentation
+
+<!-- Please describe the issue with the documentation as clearly as possible. -->

.github/ISSUE_TEMPLATE/documentation.yml

@@ -1,19 +0,0 @@
-name: Documentation
-description: Report an issue related to the GPT4All documentation.
-title: "DOC: <Please write a comprehensive title after the 'DOC: ' prefix>"
-labels: [03 - Documentation]
-
-body:
-- type: textarea
-  attributes: 
-    label: "Issue with current documentation:"
-    description: >
-      Please make sure to leave a reference to the document/code you're
-      referring to.
-
-- type: textarea
-  attributes:
-    label: "Idea or request for content:"
-    description: >
-      Please describe as clearly as possible what topics you think are missing
-      from the current documentation.

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,10 @@
+---
+name: "\U0001F680 Feature Request"
+about: Submit a proposal/request for a new GPT4All feature
+title: "[Feature] Feature request title..."
+labels: ["enhancement"]
+---
+
+### Feature Request
+
+<!-- A clear and concise description of the feature proposal. -->

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -1,30 +0,0 @@
-name: "\U0001F680 Feature Request"
-description: Submit a proposal/request for a new GPT4All feature
-labels: ["02 Feature Request"]
-body:
-  - type: textarea
-    id: feature-request
-    validations:
-      required: true
-    attributes:
-      label: Feature request
-      description: |
-        A clear and concise description of the feature proposal. Please provide links to any relevant GitHub repos, papers, or other resources if relevant.
-
-  - type: textarea
-    id: motivation
-    validations:
-      required: true
-    attributes:
-      label: Motivation
-      description: |
-        Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too.
-
-  - type: textarea
-    id: contribution
-    validations:
-      required: true
-    attributes:
-      label: Your contribution
-      description: |
-        Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/nomic-ai/gpt4all/blob/main/CONTRIBUTING.md)

.github/ISSUE_TEMPLATE/other-bug.md

@@ -0,0 +1,32 @@
+---
+name: "\U0001F41B Other Bug Report"
+about: A bug in another component of GPT4All
+labels: ["bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. If this bug involves original code, please provide a minimal version that can reproduce the issue. -->
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? -->
+
+### Your Environment
+
+- GPT4All version (if applicable):
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->
+

.github/ISSUE_TEMPLATE/other.yml

@@ -1,18 +0,0 @@
-name: Other Issue
-description: Raise an issue that wouldn't be covered by the other templates.
-title: "Issue: <Please write a comprehensive title after the 'Issue: ' prefix>"
-labels: [04 - Other]
-
-body:
-  - type: textarea
-    attributes:
-      label: "Issue you'd like to raise."
-      description: >
-        Please describe the issue you'd like to raise as clearly as possible.
-        Make sure to include any relevant links or references.
-
-  - type: textarea
-    attributes:
-      label: "Suggestion:"
-      description: >
-        Please outline a suggestion to improve the issue here.

.github/workflows/codespell.yml

@@ -14,6 +14,6 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v2

.gitignore

@@ -1,3 +1,6 @@
+*.arrow
+squad_*
+*sbert_embedded*
 *.pkl
 ckpts*
 .deepspeed_env
@@ -178,3 +181,9 @@ CMakeLists.txt.user
 gpt4all-chat/models/*
 build_*
 build-*
+
+# IntelliJ
+.idea/
+
+# LLM models
+*.gguf

.gitmodules

@@ -1,9 +1,7 @@
-[submodule "llama.cpp-230519"]
-	path = gpt4all-backend/llama.cpp-230519
-	url = https://github.com/ggerganov/llama.cpp.git
-[submodule "llama.cpp-230511"]
-	path = gpt4all-backend/llama.cpp-230511
-	url = https://github.com/manyoso/llama.cpp.git
 [submodule "llama.cpp-mainline"]
 	path = gpt4all-backend/llama.cpp-mainline
-	url = https://github.com/ggerganov/llama.cpp.git
+	url = https://github.com/nomic-ai/llama.cpp.git
+	branch = master
+[submodule "gpt4all-chat/usearch"]
+	path = gpt4all-chat/usearch
+	url = https://github.com/nomic-ai/usearch.git

MAINTAINERS.md

@@ -0,0 +1,82 @@
+# MAINTAINERS
+
+## Rules
+
+* All content inside GPT4All shall have a documented maintainer
+* If a maintainer decides to retire or resign a call for volunteers will go
+  out
+* If no further maintainer can be found in a reasonable time frame, then the
+  content will be marked deprecated and removed in time
+
+## Job
+
+Maintainers will be...
+
+1. Responsible for overseeing content under their stewardship
+2. Responsible for triaging new issues, reviewing PRs, assigning priority
+   to tasks
+3. Responsible for keeping content in sufficient quality in a timely fashion
+
+## List
+
+Adam Treat ([@manyoso](https://github.com/manyoso))<br/>
+E-mail: adam@nomic.ai<br/>
+Discord: `@gonzochess75`
+- Overall project maintainer
+- Chat UI
+
+Jared Van Bortel ([@cebtenzzre](https://github.com/cebtenzzre))<br/>
+E-mail: jared@nomic.ai<br/>
+Discord: `@cebtenzzre`
+- gpt4all-backend
+- Python binding
+- Python CLI app
+
+Jacob Nguyen ([@jacoobes](https://github.com/jacoobes))<br/>
+Discord: `@jacoobes`<br/>
+E-mail: `jacoobes@sern.dev`
+- TypeScript binding
+
+Dominik ([@cosmic-snow](https://github.com/cosmic-snow))<br/>
+E-mail: cosmic-snow@mailfence.com<br/>
+Discord: `@cosmic__snow`
+- Community documentation (GitHub Wiki)
+
+Max Cembalest ([@mcembalest](https://github.com/mcembalest))<br/>
+E-mail: max@nomic.ai<br/>
+Discord: `@maxcembalest.`
+- Official documentation (gpt4all-bindings/python/docs -> https://docs.gpt4all.io/)
+
+Thiago Ramos ([@thiagojramos](https://github.com/thiagojramos))<br/>
+E-mail: thiagojramos@outlook.com<br/>
+- pt\_BR translation
+
+Victor Emanuel ([@SINAPSA-IC](https://github.com/SINAPSA-IC))<br/>
+E-mail: contact@sinapsaro.ro<br/>
+Discord: `@sinapsa_ic_56124_99632`
+- ro\_RO translation
+
+不知火 Shiranui ([@supersonictw](https://github.com/supersonictw))<br/>
+E-mail: supersonic@livemail.tw<br/>
+Discord: `@supersonictw`
+- zh\_TW translation
+
+Jeremy Tayco ([@jstayco](https://github.com/jstayco))<br/>
+E-mail: jstayco@protonmail.ch<br/>
+Discord: `@vertana`
+- es\_MX translation
+
+Riccardo Giovanetti ([@Harvester62](https://github.com/Harvester62))<br/>
+E-mail: riccardo.giovanetti@gmail.com<br/>
+Discord: `@harvester62`
+- it\_IT translation
+
+Tim ([@Tim453](https://github.com/Tim453))<br/>
+E-mail: tim453@mailbox.org<br/>
+Discord: `@Tim453`
+- Flatpak
+
+Jack ([@wuodoo](https://github.com/wuodoo))<br/>
+E-mail: 2296103047@qq.com><br/>
+Discord: `@mikage`
+- zh\_CN translation

README.md

@@ -1,75 +1,88 @@
 <h1 align="center">GPT4All</h1>
-<p align="center">Open-source assistant-style large language models that run locally on your CPU</p>
+
+<p align="center">GPT4All runs large language models (LLMs) privately on everyday desktops & laptops. <br> <br> No API calls or GPUs required - you can just download the application and <a href="https://docs.gpt4all.io/gpt4all_desktop/quickstart.html#quickstart">get started</a>
+
+https://github.com/nomic-ai/gpt4all/assets/70534565/513a0f15-4964-4109-89e4-4f9a9011f311
 
 <p align="center">
-<a href="https://gpt4all.io">GPT4All Website</a>
+  <a href="https://gpt4all.io/installers/gpt4all-installer-win64.exe">
+    <img src="gpt4all-bindings/python/docs/assets/windows.png" width="80" height="80"><br>
+    Download for Windows
+  </a>
 </p>
 
 <p align="center">
-<a href="https://docs.gpt4all.io">GPT4All Documentation</a>
+  <a href="https://gpt4all.io/installers/gpt4all-installer-darwin.dmg">
+    <img src="gpt4all-bindings/python/docs/assets/mac.png" width="85" height="100"><br>
+    Download for MacOS
+  </a>
 </p>
 
 <p align="center">
-<a href="https://discord.gg/mGZE39AS3e">Discord</a>
+  <a href="https://gpt4all.io/installers/gpt4all-installer-linux.run">
+    <img src="gpt4all-bindings/python/docs/assets/ubuntu.svg" width="120" height="120"><br>
+    Download for Ubuntu
+  </a>
 </p>
 
 <p align="center">
-<a href="https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html">🦜️🔗 Official Langchain Backend</a> 
+  <a href='https://flathub.org/apps/io.gpt4all.gpt4all'>
+    <img width='240' alt='Get it on Flathub' src='https://flathub.org/api/badge?locale=en'><br>
+    Get it on Flathub (community maintained)
+  </a>
 </p>
 
+<p align="center">
+  <a href="https://gpt4all.io">Website</a> &bull; <a href="https://docs.gpt4all.io">Documentation</a> &bull; <a href="https://discord.gg/mGZE39AS3e">Discord</a>
+</p>
+<p align="center">
+  <a href="https://forms.nomic.ai/gpt4all-release-notes-signup">Subscribe to the newsletter</a>
+</p>
 <p align="center">
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
 </p>
-
 <p align="center">
-  <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
-</p>
-<p align="center">
-Run on an M1 Mac (not sped up!)
+ <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg" alt="phorm.ai"></a>
 </p>
 
-## GPT4All: An ecosystem of open-source on-edge large language models.
-GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs.
+## Install GPT4All Python
 
-Learn more in the [documentation](https://docs.gpt4all.io).
+`gpt4all` gives you access to LLMs with our Python client around [`llama.cpp`](https://github.com/ggerganov/llama.cpp) implementations. 
 
-The goal is simple - be the best instruction tuned assistant-style language model that any person or enterprise can freely use, distribute and build on.
+Nomic contributes to open source software like [`llama.cpp`](https://github.com/ggerganov/llama.cpp) to make LLMs accessible and efficient **for all**.
 
-A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models. 
+```bash
+pip install gpt4all
+```
+
+```python
+from gpt4all import GPT4All
+model = GPT4All("Meta-Llama-3-8B-Instruct.Q4_0.gguf") # downloads / loads a 4.66GB LLM
+with model.chat_session():
+    print(model.generate("How can I run LLMs efficiently on my laptop?", max_tokens=1024))
+```
 
 
-### Chat Client
-Run any GPT4All model natively on your home desktop with the auto-updating desktop chat client. See <a href="https://gpt4all.io">GPT4All Website</a> for a full list of open-source models you can run with this powerful desktop application.
+## Integrations
 
-Direct Installer Links:
+:parrot::link: [Langchain](https://python.langchain.com/v0.2/docs/integrations/providers/gpt4all/)
+:card_file_box: [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
+:telescope: [OpenLIT (OTel-native Monitoring)](https://github.com/openlit/openlit) - [Docs](https://docs.openlit.io/latest/integrations/gpt4all)
 
-* [Mac/OSX](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
-
-* [Windows](https://gpt4all.io/installers/gpt4all-installer-win64.exe)
-
-* [Ubuntu](https://gpt4all.io/installers/gpt4all-installer-linux.run)
-
-If you have older hardware that only supports avx and not avx2 you can use these.
-
-* [Mac/OSX - avx-only](https://gpt4all.io/installers/gpt4all-installer-darwin-avx-only.dmg)
-
-* [Windows - avx-only](https://gpt4all.io/installers/gpt4all-installer-win64-avx-only.exe)
-
-* [Ubuntu - avx-only](https://gpt4all.io/installers/gpt4all-installer-linux-avx-only.run)
-
-Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io/)
-
-### Chat Client building and running
-
-* Follow the visual instructions on the chat client [build_and_run](gpt4all-chat/build_and_run.md) page
-
-### Bindings
-
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
+## Release History
+- **July 2nd, 2024**: V3.0.0 Release
+    - Fresh redesign of the chat application UI
+    - Improved user workflow for LocalDocs
+    - Expanded access to more model architectures
+- **October 19th, 2023**: GGUF Support Launches with Support for:
+    - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
+    - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4\_0 and Q4\_1 quantizations in GGUF.
+    - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
+- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on NVIDIA and AMD GPUs.
+- **July 2023**: Stable support for LocalDocs, a feature that allows you to privately and locally chat with your data.
+- **June 28th, 2023**: [Docker-based API server] launches allowing inference of local LLMs from an OpenAI-compatible HTTP endpoint.
 
+[Docker-based API server]: https://github.com/nomic-ai/gpt4all/tree/cef74c2be20f5b697055d5b8b506861c7b997fab/gpt4all-api
 
 ## Contributing
 GPT4All welcomes contributions, involvement, and discussion from the open source community!
@@ -79,20 +92,6 @@ Check project discord, with project owners, or through existing issues/PRs to av
 Please make sure to tag all of the above with relevant project identifiers or your contribution could potentially get lost.
 Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 
-## Technical Reports
-
-<p align="center">
-<a href="https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf">:green_book: Technical Report 3: GPT4All Snoozy and Groovy </a>
-</p>
-
-<p align="center">
-<a href="https://static.nomic.ai/gpt4all/2023_GPT4All-J_Technical_Report_2.pdf">:green_book: Technical Report 2: GPT4All-J </a>
-</p>
-
-<p align="center">
-<a href="https://s3.amazonaws.com/static.nomic.ai/gpt4all/2023_GPT4All_Technical_Report.pdf">:green_book: Technical Report 1: GPT4All</a>
-</p>
-
 ## Citation
 
 If you utilize this repository, models or data in a downstream project, please consider citing it with:

gpt4all-api/README.md

@@ -1,2 +0,0 @@
-# GPT4All API
-This directory will contain code to build out a RESTful API for GPT4All models. Exact details TBD, but as an MVP, user should be able to send requests to list, download, and generate text with different models.

gpt4all-api/triton/README.md

@@ -1,13 +0,0 @@
-# To Run Inference Server
-
-docker run --gpus=1 --rm --net=host -v ${PWD}/model_store:/model_store nvcr.io/nvidia/tritonserver:23.01-py3 tritonserver --model-repository=/model_store
-
-python client.py --model=<model_name>
-
-
-## Dynamic Batching
-Need to figure out how to do batching such that we can have dynamic batching
-We're getting 1.3 infer/sec which seems slow....
-
-To test,
-perf_analyzer -m nomic-ai--gpt4all-j --input-data test_data.json --measurement-interval 25000 --request-rate-range=10 -b 8

gpt4all-api/triton/client.py

@@ -1,75 +0,0 @@
-import torch
-import tritonclient.grpc.aio as grpcclient
-
-
-def prepare_inference_inputs(
-    inputs_ids: torch.IntTensor, new_tokens: int = 1, temperature: float = 1.0
-):
-    batch_size = inputs_ids.shape[0]
-
-    input_ids_input = grpcclient.InferInput("input_ids", inputs_ids.shape, "INT32")
-    input_ids_input.set_data_from_numpy(inputs_ids.int().cpu().numpy())
-
-    new_tokens_input = grpcclient.InferInput(
-        "tensor_of_seq_len", [batch_size, new_tokens], "INT32"
-    )
-    new_tokens_input.set_data_from_numpy(
-        torch.zeros(batch_size, new_tokens, dtype=torch.int32).cpu().numpy()
-    )
-
-    temperature_input = grpcclient.InferInput("temperature", [batch_size, 1], "FP32")
-    temperature_input.set_data_from_numpy(
-        torch.full([batch_size, 1], temperature, dtype=torch.float32).cpu().numpy()
-    )
-
-    inputs = [input_ids_input, new_tokens_input, temperature_input]
-    outputs = [
-        grpcclient.InferRequestedOutput("logits"),
-        grpcclient.InferRequestedOutput("output_ids"),
-    ]
-    return inputs, outputs
-
-
-async def infer(
-    triton_client, model_name, input_ids, new_tokens: int = 1, temperature: float = 1.0
-):
-    inputs, outputs = prepare_inference_inputs(input_ids, new_tokens, temperature)
-
-    triton_model_name = model_name.replace("/", "--")
-
-    result = await triton_client.infer(
-        model_name=triton_model_name, inputs=inputs, outputs=outputs
-    )
-
-    logits = torch.tensor(result.as_numpy("logits").copy(), requires_grad=False)
-    output_ids = torch.tensor(result.as_numpy("output_ids").copy(), requires_grad=False)
-
-    return logits, output_ids
-
-def Client(url: str):
-  return grpcclient.InferenceServerClient(url=url)
-
-if __name__ == "__main__":
-    import argparse
-    from transformers import AutoTokenizer
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--url", type=str, default="localhost:8001")
-    parser.add_argument("--model", type=str, default="gpt2")
-
-    args = parser.parse_args()
-
-    tokenizer = AutoTokenizer.from_pretrained(args.model, use_fast=False)
-
-    async def main():
-        async with Client(args.url) as triton_client:
-            while True:
-                prompt = input("Prompt: ")
-                input_ids = tokenizer.encode(prompt, return_tensors="pt")
-                last_logits, output_ids = await infer(
-                    triton_client, args.model, input_ids, new_tokens=256, temperature=1.0,
-                )
-                print(tokenizer.decode(output_ids[0]))
-
-    import asyncio
-    asyncio.run(main())

gpt4all-api/triton/convert_to_triton.py

@@ -1,149 +0,0 @@
-import argparse
-import os
-from string import Template
-
-import torch
-from torch import nn
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-parser = argparse.ArgumentParser()
-
-parser.add_argument(
-    "--model", type=str, required=True, help="Path to HF checkpoint with the base model"
-)
-
-parser.add_argument(
-    "--max-batch-size", type=int, default=64, help="Maximum batch size for inference"
-)
-
-parser.add_argument(
-    "--revision",
-    type=str,
-    required=False,
-    help="Optional branch/commit of the HF checkpoint",
-)
-
-parser.add_argument("--device", type=int, default=0)
-args = parser.parse_args()
-
-device = torch.device(args.device)
-
-
-class ModelLogits(nn.Module):
-    def __init__(self, model):
-        super().__init__()
-        self.model = model
-
-    @torch.inference_mode()
-    def forward(self, input_ids: torch.Tensor):
-        return self.model(input_ids).logits
-
-
-class InferModel(nn.Module):
-    def __init__(self, traced_model, eos_token_id):
-        super().__init__()
-        self.traced_model = traced_model
-        self.eos_token_id = eos_token_id
-
-    def forward(
-        self,
-        input_ids: torch.Tensor,
-        tensor_of_seq_len: torch.Tensor,
-        temperature: torch.Tensor,
-    ):
-        # this has mostly been adapted from huggingface generate
-        unfinished_sequences = torch.ones(input_ids.shape[0], dtype=torch.long, device=input_ids.device)
-        eos_token_id_tensor = torch.tensor([self.eos_token_id]).to(input_ids.device)
-
-        with torch.no_grad():
-            for _ in range(tensor_of_seq_len.shape[1] - 1):
-                logits = self.traced_model(input_ids).float()
-                next_token_logits = logits[:, -1, :]
-                next_token_logits = next_token_logits / temperature
-                
-                next_tokens = torch.multinomial(
-                    torch.softmax(next_token_logits, dim=-1), input_ids.shape[0] 
-                )
-
-                next_tokens = next_tokens * unfinished_sequences + self.eos_token_id * (1 - unfinished_sequences)
-
-                unfinished_sequences = unfinished_sequences.mul(
-                    next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod(dim=0)
-                )
-
-                # stop when each sentence is finished
-                if unfinished_sequences.max() == 0:
-                    return input_ids.int(), logits
-
-                input_ids = torch.cat([input_ids, next_tokens], dim=-1)
-
-                unfinished_sequences = unfinished_sequences.mul(
-                    next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod(dim=0)
-                )
-
-            # in TorchScript, the above logits var lifetime doesn't escape the loop's scope
-            logits = self.traced_model(input_ids).float()
-            next_token_logits = logits[:, -1, :]
-            next_token_logits = next_token_logits / temperature
-
-            next_tokens = torch.multinomial(
-                    torch.softmax(next_token_logits, dim=-1), input_ids.shape[0] 
-                )
-
-            next_tokens = next_tokens * unfinished_sequences + self.eos_token_id * (1 - unfinished_sequences)
-
-            input_ids = torch.cat([input_ids, next_tokens], dim=-1)
-
-            return input_ids.int(), logits
-            
-
-print(f"Converting {args.model} to TorchScript...")
-tokenizer = AutoTokenizer.from_pretrained(args.model, use_fast=False)
-model = ModelLogits(AutoModelForCausalLM.from_pretrained(args.model,
-                                                         trust_remote_code=True,
-                                                         revision=args.revision,
-                                                         torch_dtype=torch.float16,
-                                                         use_cache=False))
-
-model.eval()
-model.requires_grad_(False)
-model = model.to(device)
-
-
-input = tokenizer("annotator model's hash is 0x", return_tensors="pt").to(device)
-print(f"{model(input.input_ids)=}")
-
-traced_script_module = torch.jit.trace(model, input.input_ids)
-print("Tracing...")
-print(f"{traced_script_module(input.input_ids)=}")
-
-print("Scripting generation wrapper...")
-# need to script this as we have data conditional flow
-scripted_generator_model = torch.jit.script(InferModel(traced_script_module, tokenizer.eos_token_id))
-print(scripted_generator_model.code)
-
-print(f"{input.input_ids=}")
-# x = input.input_ids, torch.empty(1, 5), torch.full([1, 1], 1.0).cuda(), torch.full([1, 1], len(tokenizer) // 2).cuda(), torch.full([1, 1], 0.9).cuda()
-x = input.input_ids, torch.empty(1, 5),  torch.full([1, 1], 0.9).cuda()
-print(x[0].shape)
-
-print(f"{tokenizer.decode(scripted_generator_model(*x)[0][0])=}")
-
-sanitized_name = args.model.replace("/", "--")
-print("Model renamed to ", sanitized_name)
-
-print("Saving TorchScript model...")
-
-os.makedirs(f"model_store/{sanitized_name}/1", exist_ok=True)
-scripted_generator_model.save(f"model_store/{sanitized_name}/1/traced-model.pt")
-
-config_path = os.path.join(
-    os.path.dirname(os.path.realpath(__file__)), "triton_config.pbtxt"
-)
-with open(config_path) as f:
-    template = Template(f.read())
-config = template.substitute(
-    {"model_name": sanitized_name, "max_batch_size": args.max_batch_size}
-)
-with open(f"model_store/{sanitized_name}/config.pbtxt", "w") as f:
-    f.write(config)

gpt4all-api/triton/requirements.txt

@@ -1,5 +0,0 @@
-transformers
-triton
-einops
-pandas
-sentencepiece

gpt4all-api/triton/test_data.json

@@ -1,34 +0,0 @@
-{
-    "data":
-      [
-        {
-          "input_ids": {
-            "content": [17250, 11, 703, 389, 345, 30],
-            "shape": [6]
-          },
-          "tensor_of_seq_len": {
-            "content": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
-            "shape": [17]
-          },
-          "temperature": {
-            "content": [1.0],
-            "shape": [1]
-          }
-        },
-        {
-            "input_ids": {
-              "content": [17250, 11, 703, 389, 345, 30],
-              "shape": [6]
-            },
-            "tensor_of_seq_len": {
-              "content": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
-              "shape": [17]
-            },
-            "temperature": {
-              "content": [1.0],
-              "shape": [1]
-            }
-          }
-
-      ]
-  }

gpt4all-api/triton/triton_config.pbtxt

@@ -1,69 +0,0 @@
-name: "${model_name}"
-backend: "pytorch"
-default_model_filename: "traced-model.pt"
-max_batch_size: ${max_batch_size}
-
-dynamic_batching {
-}
-
-parameters {
-  key: "model_name"
-  value: {
-    string_value: "${model_name}"
-  }
-}
-
-instance_group [
-  {
-    count: 1
-    kind: KIND_GPU
-    gpus: [0]
-  }
-]
-
-input [
-  {
-    name: "input_ids"
-    data_type: TYPE_INT32
-    dims: [-1]
-  },
-  {
-    name: "tensor_of_seq_len"
-    data_type: TYPE_INT32
-    dims: [-1]
-  },
-  {
-    name: "temperature"
-    data_type: TYPE_FP32
-    dims: [-1]
-  }
-]
-
-output [
-  {
-    name: "output_ids"
-    data_type: TYPE_INT32
-    dims: [-1]
-  },
-  {
-    name: "logits"
-    data_type: TYPE_FP32
-    dims: [-1]
-  }
-]
-
-parameters {
-  key: "data_type"
-  value: {
-    string_value: "fp16"
-  }
-}
-
-parameters: {
-  key: "INFERENCE_MODE"
-  value: {
-    string_value: "true"
-  }
-}
-
-version_policy: {specific: {versions: [1]}}

gpt4all-backend/CMakeLists.txt

@@ -1,15 +1,26 @@
-cmake_minimum_required(VERSION 3.16)
+cmake_minimum_required(VERSION 3.21)  # for PROJECT_IS_TOP_LEVEL
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-if(APPLE)
-  option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
-  if(BUILD_UNIVERSAL)
+if (APPLE)
+    option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
+else()
+    option(LLMODEL_KOMPUTE "llmodel: use Kompute"              ON)
+    option(LLMODEL_VULKAN  "llmodel: use Vulkan"               OFF)
+    option(LLMODEL_CUDA    "llmodel: use CUDA"                 ON)
+    option(LLMODEL_ROCM    "llmodel: use ROCm"                 OFF)
+endif()
+
+if (APPLE)
+  if (BUILD_UNIVERSAL)
     # Build a Universal binary on macOS
     # This requires that the found Qt library is compiled as Universal binaries.
     set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
   else()
     # Build for the host architecture on macOS
-    set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
+    if (NOT CMAKE_OSX_ARCHITECTURES)
+      set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
+    endif()
   endif()
 endif()
 
@@ -17,7 +28,7 @@ endif()
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 
 set(LLMODEL_VERSION_MAJOR 0)
-set(LLMODEL_VERSION_MINOR 2)
+set(LLMODEL_VERSION_MINOR 5)
 set(LLMODEL_VERSION_PATCH 0)
 set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
 project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
@@ -36,28 +47,72 @@ else()
     message(STATUS "Interprocedural optimization support detected")
 endif()
 
+set(DIRECTORY llama.cpp-mainline)
 include(llama.cpp.cmake)
 
-set(BUILD_VARIANTS default avxonly)
+set(BUILD_VARIANTS)
+if (APPLE)
+    list(APPEND BUILD_VARIANTS metal)
+endif()
+if (LLMODEL_KOMPUTE)
+    list(APPEND BUILD_VARIANTS kompute kompute-avxonly)
+else()
+    list(PREPEND BUILD_VARIANTS cpu cpu-avxonly)
+endif()
+if (LLMODEL_VULKAN)
+    list(APPEND BUILD_VARIANTS vulkan vulkan-avxonly)
+endif()
+if (LLMODEL_CUDA)
+    if (DEFINED CMAKE_CUDA_ARCHITECTURES)
+        set(GGML_CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}")
+    endif()
+
+    include(CheckLanguage)
+    check_language(CUDA)
+    if (NOT CMAKE_CUDA_COMPILER)
+        message(WARNING "CUDA Toolkit not found. To build without CUDA, use -DLLMODEL_CUDA=OFF.")
+    endif()
+    enable_language(CUDA)
+    list(APPEND BUILD_VARIANTS cuda cuda-avxonly)
+endif()
+if (LLMODEL_ROCM)
+    enable_language(HIP)
+    list(APPEND BUILD_VARIANTS rocm rocm-avxonly)
+endif()
 
 set(CMAKE_VERBOSE_MAKEFILE ON)
 
 # Go through each build variant
 foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     # Determine flags
-    if (BUILD_VARIANT STREQUAL avxonly)
-        set(GPT4ALL_ALLOW_NON_AVX NO)
+    if (BUILD_VARIANT MATCHES avxonly)
+        set(GPT4ALL_ALLOW_NON_AVX OFF)
     else()
-        set(GPT4ALL_ALLOW_NON_AVX YES)
+        set(GPT4ALL_ALLOW_NON_AVX ON)
+    endif()
+    set(GGML_AVX2 ${GPT4ALL_ALLOW_NON_AVX})
+    set(GGML_F16C ${GPT4ALL_ALLOW_NON_AVX})
+    set(GGML_FMA  ${GPT4ALL_ALLOW_NON_AVX})
+
+    set(GGML_METAL   OFF)
+    set(GGML_KOMPUTE OFF)
+    set(GGML_VULKAN  OFF)
+    set(GGML_CUDA    OFF)
+    set(GGML_ROCM    OFF)
+    if (BUILD_VARIANT MATCHES metal)
+        set(GGML_METAL   ON)
+    elseif (BUILD_VARIANT MATCHES kompute)
+        set(GGML_KOMPUTE ON)
+    elseif (BUILD_VARIANT MATCHES vulkan)
+        set(GGML_VULKAN  ON)
+    elseif (BUILD_VARIANT MATCHES cuda)
+        set(GGML_CUDA    ON)
+    elseif (BUILD_VARIANT MATCHES rocm)
+        set(GGML_HIPBLAS ON)
     endif()
-    set(LLAMA_AVX2 ${GPT4ALL_ALLOW_NON_AVX})
-    set(LLAMA_F16C ${GPT4ALL_ALLOW_NON_AVX})
-    set(LLAMA_FMA  ${GPT4ALL_ALLOW_NON_AVX})
 
     # Include GGML
-    include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
-    include_ggml(llama.cpp-230511 -230511-${BUILD_VARIANT} ON)
-    include_ggml(llama.cpp-230519 -230519-${BUILD_VARIANT} ON)
+    include_ggml(-mainline-${BUILD_VARIANT})
 
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
@@ -65,13 +120,14 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         message(STATUS "Configuring model implementation target ${TARGET_NAME}")
         # Link to ggml/llama
         target_link_libraries(${TARGET_NAME}
-            PUBLIC ${BASE_LIB}-${BUILD_VARIANT})
+            PRIVATE ${BASE_LIB}-${BUILD_VARIANT})
         # Let it know about its build variant
         target_compile_definitions(${TARGET_NAME}
             PRIVATE GGML_BUILD_VARIANT="${BUILD_VARIANT}")
         # Enable IPO if possible
-        set_property(TARGET ${TARGET_NAME}
-                     PROPERTY INTERPROCEDURAL_OPTIMIZATION ${IPO_SUPPORTED})
+# FIXME: Doesn't work with msvc reliably. See https://github.com/nomic-ai/gpt4all/issues/841
+#        set_property(TARGET ${TARGET_NAME}
+#                     PROPERTY INTERPROCEDURAL_OPTIMIZATION ${IPO_SUPPORTED})
     endfunction()
 
     # Add each individual implementations
@@ -81,31 +137,15 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    add_library(llamamodel-230519-${BUILD_VARIANT} SHARED
-        llamamodel.cpp llmodel_shared.cpp)
-    target_compile_definitions(llamamodel-230519-${BUILD_VARIANT} PRIVATE
-        LLAMA_VERSIONS===2 LLAMA_DATE=230519)
-    prepare_target(llamamodel-230519 llama-230519)
-
-    add_library(llamamodel-230511-${BUILD_VARIANT} SHARED
-        llamamodel.cpp llmodel_shared.cpp)
-    target_compile_definitions(llamamodel-230511-${BUILD_VARIANT} PRIVATE
-        LLAMA_VERSIONS=<=1 LLAMA_DATE=230511)
-    prepare_target(llamamodel-230511 llama-230511)
-
-    add_library(gptj-${BUILD_VARIANT} SHARED
-        gptj.cpp utils.h utils.cpp llmodel_shared.cpp)
-    prepare_target(gptj ggml-230511)
-
-    add_library(mpt-${BUILD_VARIANT} SHARED
-        mpt.cpp utils.h utils.cpp llmodel_shared.cpp)
-    prepare_target(mpt ggml-230511)
+    if (NOT PROJECT_IS_TOP_LEVEL AND BUILD_VARIANT STREQUAL cuda)
+        set(CUDAToolkit_BIN_DIR ${CUDAToolkit_BIN_DIR} PARENT_SCOPE)
+    endif()
 endforeach()
 
 add_library(llmodel
     llmodel.h llmodel.cpp llmodel_shared.cpp
     llmodel_c.h llmodel_c.cpp
-    dlhandle.h
+    dlhandle.cpp
 )
 target_compile_definitions(llmodel PRIVATE LIB_FILE_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}")
 

gpt4all-backend/dlhandle.cpp

@@ -0,0 +1,73 @@
+#include "dlhandle.h"
+
+#include <string>
+
+#ifndef _WIN32
+#   include <dlfcn.h>
+#else
+#   include <cassert>
+#   include <sstream>
+#   define WIN32_LEAN_AND_MEAN
+#   ifndef NOMINMAX
+#       define NOMINMAX
+#   endif
+#   include <windows.h>
+#endif
+
+using namespace std::string_literals;
+namespace fs = std::filesystem;
+
+
+#ifndef _WIN32
+
+Dlhandle::Dlhandle(const fs::path &fpath)
+{
+    chandle = dlopen(fpath.c_str(), RTLD_LAZY | RTLD_LOCAL);
+    if (!chandle) {
+        throw Exception("dlopen: "s + dlerror());
+    }
+}
+
+Dlhandle::~Dlhandle()
+{
+    if (chandle) dlclose(chandle);
+}
+
+void *Dlhandle::get_internal(const char *symbol) const
+{
+    return dlsym(chandle, symbol);
+}
+
+#else // defined(_WIN32)
+
+Dlhandle::Dlhandle(const fs::path &fpath)
+{
+    fs::path afpath = fs::absolute(fpath);
+
+    // Suppress the "Entry Point Not Found" dialog, caused by outdated nvcuda.dll from the GPU driver
+    UINT lastErrorMode = GetErrorMode();
+    SetErrorMode(lastErrorMode | SEM_FAILCRITICALERRORS);
+
+    chandle = LoadLibraryExW(afpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
+
+    SetErrorMode(lastErrorMode);
+
+    if (!chandle) {
+        DWORD err = GetLastError();
+        std::ostringstream ss;
+        ss << "LoadLibraryExW failed with error 0x" << std::hex << err;
+        throw Exception(ss.str());
+    }
+}
+
+Dlhandle::~Dlhandle()
+{
+    if (chandle) FreeLibrary(HMODULE(chandle));
+}
+
+void *Dlhandle::get_internal(const char *symbol) const
+{
+    return GetProcAddress(HMODULE(chandle), symbol);
+}
+
+#endif // defined(_WIN32)

gpt4all-backend/dlhandle.h

@@ -1,15 +1,15 @@
-#ifndef DLHANDLE_H
-#define DLHANDLE_H
-#ifndef _WIN32
-#include <string>
+#pragma once
+
+#include <filesystem>
 #include <stdexcept>
+#include <string>
 #include <utility>
-#include <dlfcn.h>
 
+namespace fs = std::filesystem;
 
 
 class Dlhandle {
-    void *chandle;
+    void *chandle = nullptr;
 
 public:
     class Exception : public std::runtime_error {
@@ -17,88 +17,31 @@ public:
         using std::runtime_error::runtime_error;
     };
 
-    Dlhandle() : chandle(nullptr) {}
-    Dlhandle(const std::string& fpath, int flags = RTLD_LAZY) {
-        chandle = dlopen(fpath.c_str(), flags);
-        if (!chandle) {
-            throw Exception("dlopen(\""+fpath+"\"): "+dlerror());
-        }
-    }
-    Dlhandle(const Dlhandle& o) = delete;
-    Dlhandle(Dlhandle&& o) : chandle(o.chandle) {
+    Dlhandle() = default;
+    Dlhandle(const fs::path &fpath);
+    Dlhandle(const Dlhandle &o) = delete;
+    Dlhandle(Dlhandle &&o)
+        : chandle(o.chandle)
+    {
         o.chandle = nullptr;
     }
-    ~Dlhandle() {
-        if (chandle) dlclose(chandle);
-    }
 
-    auto operator =(Dlhandle&& o) {
+    ~Dlhandle();
+
+    Dlhandle &operator=(Dlhandle &&o) {
         chandle = std::exchange(o.chandle, nullptr);
+        return *this;
     }
 
-    bool is_valid() const {
-        return chandle != nullptr;
-    }
-    operator bool() const {
-        return is_valid();
+    template <typename T>
+    T *get(const std::string &symbol) const {
+        return reinterpret_cast<T *>(get_internal(symbol.c_str()));
     }
 
-    template<typename T>
-    T* get(const std::string& fname) const {
-        auto fres = reinterpret_cast<T*>(dlsym(chandle, fname.c_str()));
-        return (dlerror()==NULL)?fres:nullptr;
-    }
-    auto get_fnc(const std::string& fname) const {
-        return get<void*(...)>(fname);
+    auto get_fnc(const std::string &symbol) const {
+        return get<void*(...)>(symbol);
     }
+
+private:
+    void *get_internal(const char *symbol) const;
 };
-#else
-#include <string>
-#include <exception>
-#include <stdexcept>
-#ifndef NOMINMAX
-    #define NOMINMAX
-#endif
-#include <windows.h>
-#include <libloaderapi.h>
-
-
-
-class Dlhandle {
-    HMODULE chandle;
-
-public:
-    class Exception : public std::runtime_error {
-    public:
-        using std::runtime_error::runtime_error;
-    };
-
-    Dlhandle() : chandle(nullptr) {}
-    Dlhandle(const std::string& fpath) {
-        chandle = LoadLibraryA(fpath.c_str());
-        if (!chandle) {
-            throw Exception("dlopen(\""+fpath+"\"): Error");
-        }
-    }
-    Dlhandle(const Dlhandle& o) = delete;
-    Dlhandle(Dlhandle&& o) : chandle(o.chandle) {
-        o.chandle = nullptr;
-    }
-    ~Dlhandle() {
-        if (chandle) FreeLibrary(chandle);
-    }
-
-    bool is_valid() const {
-        return chandle != nullptr;
-    }
-
-    template<typename T>
-    T* get(const std::string& fname) const {
-        return reinterpret_cast<T*>(GetProcAddress(chandle, fname.c_str()));
-    }
-    auto get_fnc(const std::string& fname) const {
-        return get<void*(...)>(fname);
-    }
-};
-#endif
-#endif // DLHANDLE_H

gpt4all-backend/gptj.cpp

@@ -1,967 +0,0 @@
-#define GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "gptj_impl.h"
-
-#include "utils.h"
-
-#include <cassert>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <fstream>
-#include <map>
-#include <string>
-#include <vector>
-#include <iostream>
-#if defined(_WIN32) && defined(_MSC_VER)
-    #define WIN32_LEAN_AND_MEAN
-    #ifndef NOMINMAX
-        #define NOMINMAX
-    #endif
-    #include <windows.h>
-    #include <io.h>
-    #include <stdio.h>
-#else
-    #include <unistd.h>
-#endif
-#include <sstream>
-#include <unordered_set>
-#include <ggml.h>
-
-
-namespace {
-const char *modelType_ = "GPT-J";
-
-static const size_t MB = 1024*1024;
-}
-
-// default hparams (GPT-J 6B)
-struct gptj_hparams {
-    int32_t n_vocab = 50400;
-    int32_t n_ctx   = 2048;
-    int32_t n_embd  = 4096;
-    int32_t n_head  = 16;
-    int32_t n_layer = 28;
-    int32_t n_rot   = 64;
-    int32_t f16     = 1;
-};
-
-struct gptj_layer {
-    // normalization
-    struct ggml_tensor * ln_1_g;
-    struct ggml_tensor * ln_1_b;
-
-    // attention
-    struct ggml_tensor * c_attn_q_proj_w;
-    struct ggml_tensor * c_attn_k_proj_w;
-    struct ggml_tensor * c_attn_v_proj_w;
-
-    struct ggml_tensor * c_attn_proj_w;
-
-    // ff
-    struct ggml_tensor * c_mlp_fc_w;
-    struct ggml_tensor * c_mlp_fc_b;
-
-    struct ggml_tensor * c_mlp_proj_w;
-    struct ggml_tensor * c_mlp_proj_b;
-};
-
-struct gptj_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-
-    void resize(size_t size) {
-        delete[] addr;
-        addr = new uint8_t[size];
-        this->size = size;
-    }
-
-    ~gptj_buffer() {
-        fflush(stdout);
-        delete[] addr;
-    }
-};
-
-struct gptj_kv_cache {
-    struct ggml_tensor * k;
-    struct ggml_tensor * v;
-
-    struct ggml_context * ctx = NULL;
-
-    gptj_buffer buf;
-
-    int n; // number of tokens currently in the cache
-
-    ~gptj_kv_cache() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
-struct gptj_model {
-    gptj_hparams hparams;
-
-    // normalization
-    struct ggml_tensor * ln_f_g;
-    struct ggml_tensor * ln_f_b;
-
-    struct ggml_tensor * wte; // position embedding
-
-    struct ggml_tensor * lmh_g; // language model head
-    struct ggml_tensor * lmh_b; // language model bias
-
-    std::vector<gptj_layer> layers;
-
-    // key + value memory
-    struct gptj_kv_cache kv_self;
-
-    //
-    struct ggml_context * ctx;
-    std::map<std::string, struct ggml_tensor *> tensors;
-
-    gptj_buffer buf;
-
-    ~gptj_model() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
-static bool kv_cache_init(
-        const struct gptj_hparams & hparams,
-             struct gptj_kv_cache & cache,
-                         ggml_type   wtype,
-                               int   n_ctx) {
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-
-    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
-    const int64_t n_elements = n_embd*n_mem;
-
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB);
-
-    struct ggml_init_params params;
-    params.mem_size   = cache.buf.size;
-    params.mem_buffer = cache.buf.addr;
-    params.no_alloc   = false;
-
-    cache.ctx = ggml_init(params);
-
-    if (!cache.ctx) {
-        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
-        return false;
-    }
-
-    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-
-    return true;
-}
-
-// load the model's weights from a stream
-bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & model, gpt_vocab & vocab) {
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
-
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *) &magic, sizeof(magic));
-        if (magic != 0x67676d6c) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
-            return false;
-        }
-    }
-
-    // load hparams
-    {
-        auto & hparams = model.hparams;
-
-        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
-        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
-        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
-        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *) &hparams.n_rot,   sizeof(hparams.n_rot));
-        fin.read((char *) &hparams.f16,     sizeof(hparams.f16));
-
-        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
-        printf("%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
-        printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_head  = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
-        printf("%s: n_rot   = %d\n", __func__, hparams.n_rot);
-        printf("%s: f16     = %d\n", __func__, hparams.f16);
-    }
-
-    // load vocab
-    {
-        int32_t n_vocab = 0;
-        fin.read((char *) &n_vocab, sizeof(n_vocab));
-
-        if (n_vocab != model.hparams.n_vocab) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
-                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
-            return false;
-        }
-
-        std::string word;
-        for (int i = 0; i < n_vocab; i++) {
-            uint32_t len;
-            fin.read((char *) &len, sizeof(len));
-
-            word.resize(len);
-            fin.read((char *) word.data(), len);
-
-            vocab.token_to_id[word] = i;
-            vocab.id_to_token[i] = word;
-        }
-    }
-
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = GGML_TYPE_COUNT;
-    switch (model.hparams.f16) {
-        case 0: wtype = GGML_TYPE_F32;  break;
-        case 1: wtype = GGML_TYPE_F16;  break;
-        case 2: wtype = GGML_TYPE_Q4_0; break;
-        case 3: wtype = GGML_TYPE_Q4_1; break;
-        case 5: wtype = GGML_TYPE_Q4_2; break;
-        default:
-                {
-                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
-                            __func__, fname.c_str(), model.hparams.f16);
-                    return false;
-                }
-    }
-
-    auto & ctx = model.ctx;
-
-    size_t ctx_size = 0;
-
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_ctx   = hparams.n_ctx;
-        const int n_vocab = hparams.n_vocab;
-
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_g
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_b
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype); // wte
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype);         // lmh_g
-        ctx_size +=        n_vocab*ggml_type_sizef(GGML_TYPE_F32); // lmh_b
-
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_g
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_b
-
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_q_proj_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_k_proj_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_v_proj_w
-
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_proj_w
-
-        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_fc_w
-        ctx_size += n_layer*(       4*n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_fc_b
-
-        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_proj_w
-        ctx_size += n_layer*(         n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_proj_b
-
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_k
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_v
-
-        ctx_size += (5 + 10*n_layer)*256; // object overhead
-
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
-    }
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size   = ctx_size,
-            .mem_buffer = NULL,
-            .no_alloc = false
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx) {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            return false;
-        }
-    }
-
-    // prepare memory for the weights
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_vocab = hparams.n_vocab;
-
-        model.layers.resize(n_layer);
-
-        model.wte    = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
-
-        model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-        model.lmh_g  = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
-        model.lmh_b  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab);
-
-        // map by name
-        model.tensors["transformer.wte.weight"] = model.wte;
-
-        model.tensors["transformer.ln_f.weight"] = model.ln_f_g;
-        model.tensors["transformer.ln_f.bias"]   = model.ln_f_b;
-
-        model.tensors["lm_head.weight"] = model.lmh_g;
-        model.tensors["lm_head.bias"]   = model.lmh_b;
-
-        for (int i = 0; i < n_layer; ++i) {
-            auto & layer = model.layers[i];
-
-            layer.ln_1_g          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
-            layer.ln_1_b          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
-
-            layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
-            layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
-            layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
-
-            layer.c_attn_proj_w   = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
-
-            layer.c_mlp_fc_w      = ggml_new_tensor_2d(ctx, wtype,           n_embd, 4*n_embd);
-            layer.c_mlp_fc_b      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
-
-            layer.c_mlp_proj_w    = ggml_new_tensor_2d(ctx, wtype,         4*n_embd,   n_embd);
-            layer.c_mlp_proj_b    = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
-
-            // map by name
-            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.weight"]          = layer.ln_1_g;
-            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.bias"]            = layer.ln_1_b;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.q_proj.weight"]   = layer.c_attn_q_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.k_proj.weight"]   = layer.c_attn_k_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.v_proj.weight"]   = layer.c_attn_v_proj_w;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.out_proj.weight"] = layer.c_attn_proj_w;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.weight"]     = layer.c_mlp_fc_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.bias"]       = layer.c_mlp_fc_b;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.weight"]    = layer.c_mlp_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.bias"]      = layer.c_mlp_proj_b;
-        }
-    }
-
-    // key + value memory
-    {
-        const auto & hparams = model.hparams;
-        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
-            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
-            ggml_free(ctx);
-            return false;
-        }
-
-        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
-        printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
-    }
-
-    // load weights
-    {
-        int n_tensors = 0;
-        size_t total_size = 0;
-
-        printf("%s: ", __func__);
-
-        while (true) {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ftype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ftype),  sizeof(ftype));
-
-            if (fin.eof()) {
-                break;
-            }
-
-            int32_t nelements = 1;
-            int32_t ne[2] = { 1, 1 };
-            for (int i = 0; i < n_dims; ++i) {
-                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-
-            if (model.tensors.find(name.data()) == model.tensors.end()) {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                return false;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                return false;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%lu, %lu], expected [%d, %d]\n",
-                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
-                return false;
-            }
-
-            if (0) {
-                static const char * ftype_str[] = { "f32", "f16", "q4_0", "q4_1", };
-                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ftype_str[ftype], ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
-            }
-
-            size_t bpe = 0;
-
-            switch (ftype) {
-                case 0: bpe = ggml_type_size(GGML_TYPE_F32);  break;
-                case 1: bpe = ggml_type_size(GGML_TYPE_F16);  break;
-                case 2: bpe = ggml_type_size(GGML_TYPE_Q4_0); assert(ne[0] % 64 == 0); break;
-                case 3: bpe = ggml_type_size(GGML_TYPE_Q4_1); assert(ne[0] % 64 == 0); break;
-                default:
-                        {
-                            fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype);
-                            return false;
-                        }
-            };
-
-            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
-                return false;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
-            total_size += ggml_nbytes(tensor);
-            if (++n_tensors % 8 == 0) {
-                printf(".");
-                fflush(stdout);
-            }
-        }
-
-        printf(" done\n");
-
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
-    }
-
-    return true;
-}
-
-// load the model's weights from a file path
-bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) {
-
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin) {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
-        return false;
-    }
-
-    bool loaded = gptj_model_load(fname, fin, model, vocab);
-    fin.close();
-    return loaded;
-}
-
-// evaluate the transformer
-//
-//   - model:     the model
-//   - n_threads: number of threads to use
-//   - n_past:    the context size so far
-//   - embd_inp:  the embeddings of the tokens in the context
-//   - embd_w:    the predicted logits for the next token
-//
-// The GPT-J model requires about 16MB of memory per input token.
-//
-bool gptj_eval(
-        gptj_model & model,
-        const int n_threads,
-        const int n_past,
-        const std::vector<gpt_vocab::id> & embd_inp,
-              std::vector<float>         & embd_w,
-              size_t                     & mem_per_token) {
-    const int N = embd_inp.size();
-
-    const auto & hparams = model.hparams;
-
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_ctx   = hparams.n_ctx;
-    const int n_head  = hparams.n_head;
-    const int n_vocab = hparams.n_vocab;
-    const int n_rot   = hparams.n_rot;
-
-    const size_t init_buf_size = 1024u*MB;
-    if (!model.buf.addr || model.buf.size < init_buf_size)
-        model.buf.resize(init_buf_size);
-
-    if (mem_per_token > 0 && mem_per_token*N > model.buf.size) {
-        const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
-        printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
-
-        // reallocate
-        model.buf.resize(buf_size_new);
-        if (model.buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.buf.size);
-            return false;
-        }
-    }
-
-    struct ggml_init_params params = {
-        .mem_size   = model.buf.size,
-        .mem_buffer = model.buf.addr,
-        .no_alloc = false
-    };
-
-    struct ggml_context * ctx0 = ggml_init(params);
-    struct ggml_cgraph gf = {};
-    gf.n_threads = n_threads;
-
-    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
-
-    // wte
-    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
-
-    for (int il = 0; il < n_layer; ++il) {
-        struct ggml_tensor * cur;
-
-        // norm
-        {
-            cur = ggml_norm(ctx0, inpL);
-
-            // cur = ln_1_g*cur + ln_1_b
-            cur = ggml_add(ctx0,
-                    ggml_mul(ctx0,
-                        ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),
-                        cur),
-                    ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));
-        }
-
-        struct ggml_tensor * inpSA = cur;
-
-        // self-attention
-        {
-            struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur);
-            struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur);
-            struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur);
-
-            // store key and value to memory
-            {
-                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_1d(ctx0, model.kv_self.v, N*n_embd, (ggml_element_size(model.kv_self.v)*n_embd)*(il*n_ctx + n_past));
-
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
-            }
-
-            // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q =
-                ggml_permute(ctx0,
-                        ggml_rope(ctx0,
-                            ggml_cpy(ctx0,
-                                Qcur,
-                                ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),
-                            n_past, n_rot, 0),
-                        0, 2, 1, 3);
-
-            // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
-            struct ggml_tensor * K =
-                ggml_permute(ctx0,
-                        ggml_rope(ctx0,
-                            ggml_reshape_3d(ctx0,
-                                ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                                n_embd/n_head, n_head, n_past + N),
-                            n_past, n_rot, 1),
-                        0, 2, 1, 3);
-
-            // K * Q
-            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
-
-            // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled =
-                ggml_scale(ctx0,
-                        KQ,
-                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
-                        );
-
-            // KQ_masked = mask_past(KQ_scaled)
-            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past);
-
-            // KQ = soft_max(KQ_masked)
-            struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
-
-            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V_trans =
-                ggml_cpy(ctx0,
-                        ggml_permute(ctx0,
-                            ggml_reshape_3d(ctx0,
-                                ggml_view_1d(ctx0, model.kv_self.v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd),
-                                n_embd/n_head, n_head, n_past + N),
-                            1, 2, 0, 3),
-                        ggml_new_tensor_3d(ctx0, model.kv_self.v->type, n_past + N, n_embd/n_head, n_head));
-
-            // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);
-
-            // KQV_merged = KQV.permute(0, 2, 1, 3)
-            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            // cur = KQV_merged.contiguous().view(n_embd, N)
-            cur = ggml_cpy(ctx0,
-                    KQV_merged,
-                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-            // projection (no bias)
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].c_attn_proj_w,
-                    cur);
-        }
-
-        struct ggml_tensor * inpFF = cur;
-
-        // feed-forward network
-        // this is independent of the self-attention result, so it could be done in parallel to the self-attention
-        {
-            // note here we pass inpSA instead of cur
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].c_mlp_fc_w,
-                    inpSA);
-
-            cur = ggml_add(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),
-                    cur);
-
-            // GELU activation
-            cur = ggml_gelu(ctx0, cur);
-
-            // projection
-            // cur = proj_w*cur + proj_b
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].c_mlp_proj_w,
-                    cur);
-
-            cur = ggml_add(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),
-                    cur);
-        }
-
-        // self-attention + FF
-        cur  = ggml_add(ctx0, cur, inpFF);
-
-        // input for next layer
-        inpL = ggml_add(ctx0, cur, inpL);
-    }
-
-    // norm
-    {
-        inpL = ggml_norm(ctx0, inpL);
-
-        // inpL = ln_f_g*inpL + ln_f_b
-        inpL = ggml_add(ctx0,
-                ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.ln_f_g, inpL),
-                    inpL),
-                ggml_repeat(ctx0, model.ln_f_b, inpL));
-    }
-
-    // lm_head
-    {
-        inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL);
-
-        inpL = ggml_add(ctx0,
-                ggml_repeat(ctx0, model.lmh_b, inpL),
-                inpL);
-    }
-
-    // logits -> probs
-    //inpL = ggml_soft_max(ctx0, inpL);
-
-    // run the computation
-    ggml_build_forward_expand(&gf, inpL);
-    ggml_graph_compute       (ctx0, &gf);
-
-    //if (n_past%100 == 0) {
-    //    ggml_graph_print   (&gf);
-    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
-    //}
-
-    //embd_w.resize(n_vocab*N);
-    //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
-
-    // return result for just the last token
-    embd_w.resize(n_vocab);
-    memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
-
-    if (mem_per_token == 0) {
-        mem_per_token = ggml_used_mem(ctx0)/N;
-    }
-    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
-
-    ggml_free(ctx0);
-
-    return true;
-}
-
-#define GPTJ_MAX_RNG_STATE 64*1024
-
-size_t gptj_get_state_size(const gptj_model &model)
-{
-    // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state.
-    // for reference, std::mt19937(1337) serializes to 6701 bytes.
-    const size_t s_rng_size        = sizeof(size_t);
-    const size_t s_rng             = GPTJ_MAX_RNG_STATE;
-    const size_t s_kv_size         = sizeof(size_t);
-    const size_t s_kv_ntok         = sizeof(int);
-    const size_t s_kv              = model.kv_self.buf.size;
-    const size_t s_total = (
-        + s_rng_size
-        + s_rng
-        + s_kv_size
-        + s_kv_ntok
-        + s_kv
-    );
-    fflush(stdout);
-    return s_total;
-}
-
-size_t gptj_copy_state_data(const gptj_model &model, const std::mt19937 &rng, uint8_t *dest)
-{
-    uint8_t * out = dest;
-    fflush(stdout);
-    // copy rng
-    {
-        std::stringstream rng_ss;
-        rng_ss << rng;
-
-        const size_t rng_size = rng_ss.str().size();
-        char rng_buf[GPTJ_MAX_RNG_STATE];
-
-        memset(&rng_buf[0], 0, GPTJ_MAX_RNG_STATE);
-        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
-
-        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
-        memcpy(out, &rng_buf[0], GPTJ_MAX_RNG_STATE); out += GPTJ_MAX_RNG_STATE;
-    }
-
-    // copy kv cache
-    {
-        const size_t kv_size = model.kv_self.buf.size;
-        const int    kv_ntok = model.kv_self.n;
-
-        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
-        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
-
-        if (kv_size) {
-            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
-        }
-    }
-
-    const size_t written  = out - dest;
-    assert(written == gptj_get_state_size(model));
-    fflush(stdout);
-    return written;
-}
-
-size_t gptj_set_state_data(gptj_model *model, std::mt19937 *rng, const uint8_t *src)
-{
-    const uint8_t * in = src;
-
-    // set rng
-    {
-        size_t rng_size;
-        char   rng_buf[GPTJ_MAX_RNG_STATE];
-
-        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
-        memcpy(&rng_buf[0], in, GPTJ_MAX_RNG_STATE); in += GPTJ_MAX_RNG_STATE;
-
-        std::stringstream rng_ss;
-        rng_ss.str(std::string(&rng_buf[0], rng_size));
-        rng_ss >> *rng;
-
-        assert(rng_ss.fail() == false);
-    }
-
-    // set kv cache
-    {
-        size_t kv_size;
-        int kv_ntok;
-
-        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
-        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
-
-        if (kv_size) {
-            assert(model->kv_self.buf.size == kv_size);
-
-            void * k_data = model->kv_self.k->data; // remember data pointers
-            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
-
-            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
-
-            model->kv_self.k->data = k_data; // restore correct data pointers
-            model->kv_self.v->data = v_data;
-
-        }
-
-        model->kv_self.n = kv_ntok;
-    }
-
-    const size_t nread    = in - src;
-    assert(nread == gptj_get_state_size(*model));
-    fflush(stdout);
-    return nread;
-}
-
-struct GPTJPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    gpt_vocab vocab;
-    gptj_model *model = nullptr;
-    int64_t n_threads = 0;
-    size_t mem_per_token = 0;
-    std::mt19937 rng;
-};
-
-GPTJ::GPTJ()
-    : d_ptr(new GPTJPrivate) {
-    d_ptr->model = new gptj_model;
-    d_ptr->modelLoaded = false;
-}
-
-bool GPTJ::loadModel(const std::string &modelPath) {
-    std::mt19937 rng(time(NULL));
-    d_ptr->rng = rng;
-
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-
-    // load the model
-    if (!gptj_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab)) {
-        std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
-        return false;
-    }
-
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    fflush(stdout);
-    return true;
-}
-
-void GPTJ::setThreadCount(int32_t n_threads) {
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t GPTJ::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-GPTJ::~GPTJ()
-{
-    delete d_ptr->model;
-}
-
-bool GPTJ::isModelLoaded() const
-{
-    return d_ptr->modelLoaded;
-}
-
-size_t GPTJ::stateSize() const
-{
-    return gptj_get_state_size(*d_ptr->model);
-}
-
-size_t GPTJ::saveState(uint8_t *dest) const
-{
-    return gptj_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
-}
-
-size_t GPTJ::restoreState(const uint8_t *src)
-{
-    return gptj_set_state_data(d_ptr->model, &d_ptr->rng, src);
-}
-
-std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &, const std::string &str) const
-{
-    return ::gpt_tokenize(d_ptr->vocab, str);
-}
-
-LLModel::Token GPTJ::sampleToken(PromptContext &promptCtx) const
-{
-    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
-    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
-        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks,
-        promptCtx.logits,
-        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty,
-        d_ptr->rng);
-}
-
-std::string_view GPTJ::tokenToString(Token id) const
-{
-    return d_ptr->vocab.id_to_token[id];
-}
-
-bool GPTJ::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    // determine the required inference memory per token:
-    static bool initialized = false;
-    if (!initialized) {
-        gptj_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
-            d_ptr->mem_per_token);
-        initialized = true;
-    }
-
-    return gptj_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
-}
-
-int32_t GPTJ::contextLength() const
-{
-    return d_ptr->model->hparams.n_ctx;
-}
-
-const std::vector<LLModel::Token> &GPTJ::endTokens() const
-{
-    static const std::vector<LLModel::Token> fres = {50256};
-    return fres;
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    return magic == 0x67676d6c;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new GPTJ;
-}
-}

gpt4all-backend/gptj_impl.h

@@ -1,38 +0,0 @@
-#ifndef GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of gptj.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef GPTJ_H
-#define GPTJ_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include "llmodel.h"
-
-struct GPTJPrivate;
-class GPTJ : public LLModel {
-public:
-    GPTJ();
-    ~GPTJ();
-
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    GPTJPrivate *d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string_view tokenToString(Token) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // GPTJ_H

gpt4all-backend/llamamodel_impl.h

@@ -4,35 +4,68 @@
 #ifndef LLAMAMODEL_H
 #define LLAMAMODEL_H
 
-#include <string>
-#include <functional>
-#include <vector>
 #include "llmodel.h"
 
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
 struct LLamaPrivate;
+struct EmbModelSpec;
+
 class LLamaModel : public LLModel {
 public:
     LLamaModel();
     ~LLamaModel();
 
-    bool loadModel(const std::string &modelPath) override;
+    bool supportsEmbedding() const override { return m_supportsEmbedding; }
+    bool supportsCompletion() const override { return m_supportsCompletion; }
+    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
+    bool isModelBlacklisted(const std::string &modelPath) const override;
+    bool isEmbeddingModel(const std::string &modelPath) const override;
     bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
+    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0) const override;
+    bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const override;
+    bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const override;
+    bool usingGPUDevice() const override;
+    const char *backendName() const override;
+    const char *gpuDeviceName() const override;
+
+    size_t embeddingSize() const override;
+    // user-specified prefix
+    void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
+               int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false,
+               EmbedCancelCallback *cancelCb = nullptr) override;
+    // automatic prefix
+    void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality = -1,
+               size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
 
 private:
-    LLamaPrivate *d_ptr;
+    std::unique_ptr<LLamaPrivate> d_ptr;
+    bool m_supportsEmbedding = false;
+    bool m_supportsCompletion = false;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string_view tokenToString(Token) const override;
-    Token sampleToken(PromptContext& ctx) const override;
-    bool evalTokens(PromptContext& ctx, const std::vector<int32_t> &tokens) const override;
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) override;
+    std::string tokenToString(Token id) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
+    const std::vector<Token> &endTokens() const override;
+    bool shouldAddBOS() const override;
+    int32_t maxContextLength(std::string const &modelPath) const override;
+    int32_t layerCount(std::string const &modelPath) const override;
+
+    void embedInternal(const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
+                       size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb,
+                       const EmbModelSpec *spec);
 };
 
 #endif // LLAMAMODEL_H

gpt4all-backend/llmodel.cpp

@@ -1,104 +1,171 @@
 #include "llmodel.h"
+
 #include "dlhandle.h"
 
-#include <iostream>
-#include <string>
-#include <vector>
-#include <fstream>
-#include <filesystem>
 #include <cassert>
 #include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <iterator>
+#include <memory>
+#include <optional>
+#include <regex>
 #include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
 
-std::string LLModel::m_implementations_search_path = ".";
-
-static bool has_at_least_minimal_hardware() {
-#ifdef __x86_64__
-    #ifndef _MSC_VER
-        return __builtin_cpu_supports("avx");
-    #else
-        int cpuInfo[4];
-        __cpuid(cpuInfo, 1);
-        return cpuInfo[2] & (1 << 28);
-    #endif
-#else
-    return true; // Don't know how to handle non-x86_64
+#ifdef _WIN32
+#   define WIN32_LEAN_AND_MEAN
+#   ifndef NOMINMAX
+#       define NOMINMAX
+#   endif
+#   include <windows.h>
 #endif
-}
 
-static bool requires_avxonly() {
-#ifdef __x86_64__
-    #ifndef _MSC_VER
-        return !__builtin_cpu_supports("avx2");
-    #else
-        int cpuInfo[4];
-        __cpuidex(cpuInfo, 7, 0);
-        return !(cpuInfo[1] & (1 << 5));
-    #endif
-#else
-    return false; // Don't know how to handle non-x86_64
+#ifdef _MSC_VER
+#   include <intrin.h>
 #endif
-}
 
-LLModel::Implementation::Implementation(Dlhandle &&dlhandle_) : dlhandle(new Dlhandle(std::move(dlhandle_))) {
-    auto get_model_type = dlhandle->get<const char *()>("get_model_type");
+#if defined(__APPLE__) && defined(__aarch64__)
+#   include "sysinfo.h" // for getSystemTotalRAMInBytes
+#endif
+
+namespace fs = std::filesystem;
+
+#ifndef __APPLE__
+static const std::string DEFAULT_BACKENDS[] = {"kompute", "cpu"};
+#elif defined(__aarch64__)
+static const std::string DEFAULT_BACKENDS[] = {"metal", "cpu"};
+#else
+static const std::string DEFAULT_BACKENDS[] = {"cpu"};
+#endif
+
+std::string s_implementations_search_path = ".";
+
+#if !(defined(__x86_64__) || defined(_M_X64))
+    // irrelevant on non-x86_64
+    #define cpu_supports_avx()  -1
+    #define cpu_supports_avx2() -1
+#elif defined(_MSC_VER)
+    // MSVC
+    static int get_cpu_info(int func_id, int reg_id) {
+        int info[4];
+        __cpuid(info, func_id);
+        return info[reg_id];
+    }
+
+    // AVX via EAX=1: Processor Info and Feature Bits, bit 28 of ECX
+    #define cpu_supports_avx()  !!(get_cpu_info(1, 2) & (1 << 28))
+    // AVX2 via EAX=7, ECX=0: Extended Features, bit 5 of EBX
+    #define cpu_supports_avx2() !!(get_cpu_info(7, 1) & (1 <<  5))
+#else
+    // gcc/clang
+    #define cpu_supports_avx()  !!__builtin_cpu_supports("avx")
+    #define cpu_supports_avx2() !!__builtin_cpu_supports("avx2")
+#endif
+
+LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
+    : m_dlhandle(new Dlhandle(std::move(dlhandle_))) {
+    auto get_model_type = m_dlhandle->get<const char *()>("get_model_type");
     assert(get_model_type);
-    modelType = get_model_type();
-    auto get_build_variant = dlhandle->get<const char *()>("get_build_variant");
+    m_modelType = get_model_type();
+    auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
-    buildVariant = get_build_variant();
-    magicMatch = dlhandle->get<bool(std::ifstream&)>("magic_match");
-    assert(magicMatch);
-    construct_ = dlhandle->get<LLModel *()>("construct");
-    assert(construct_);
+    m_buildVariant = get_build_variant();
+    m_getFileArch = m_dlhandle->get<char *(const char *)>("get_file_arch");
+    assert(m_getFileArch);
+    m_isArchSupported = m_dlhandle->get<bool(const char *)>("is_arch_supported");
+    assert(m_isArchSupported);
+    m_construct = m_dlhandle->get<LLModel *()>("construct");
+    assert(m_construct);
 }
 
 LLModel::Implementation::Implementation(Implementation &&o)
-    : construct_(o.construct_)
-    , modelType(o.modelType)
-    , buildVariant(o.buildVariant)
-    , magicMatch(o.magicMatch)
-    , dlhandle(o.dlhandle) {
-    o.dlhandle = nullptr;
+    : m_getFileArch(o.m_getFileArch)
+    , m_isArchSupported(o.m_isArchSupported)
+    , m_construct(o.m_construct)
+    , m_modelType(o.m_modelType)
+    , m_buildVariant(o.m_buildVariant)
+    , m_dlhandle(o.m_dlhandle) {
+    o.m_dlhandle = nullptr;
 }
 
-LLModel::Implementation::~Implementation() {
-    if (dlhandle) delete dlhandle;
+LLModel::Implementation::~Implementation()
+{
+    delete m_dlhandle;
 }
 
-bool LLModel::Implementation::isImplementation(const Dlhandle &dl) {
+static bool isImplementation(const Dlhandle &dl)
+{
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
-const std::vector<LLModel::Implementation> &LLModel::implementationList() {
+// Add the CUDA Toolkit to the DLL search path on Windows.
+// This is necessary for chat.exe to find CUDA when started from Qt Creator.
+static void addCudaSearchPath()
+{
+#ifdef _WIN32
+    if (const auto *cudaPath = _wgetenv(L"CUDA_PATH")) {
+        auto libDir = std::wstring(cudaPath) + L"\\bin";
+        if (!AddDllDirectory(libDir.c_str())) {
+            auto err = GetLastError();
+            std::wcerr << L"AddDllDirectory(\"" << libDir << L"\") failed with error 0x" << std::hex << err << L"\n";
+        }
+    }
+#endif
+}
+
+const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList()
+{
+    if (cpu_supports_avx() == 0) {
+        throw std::runtime_error("CPU does not support AVX");
+    }
+
     // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
     // individual models without the cleanup of the static list interfering
-    static auto* libs = new std::vector<LLModel::Implementation>([] () {
-        std::vector<LLModel::Implementation> fres;
+    static auto* libs = new std::vector<Implementation>([] () {
+        std::vector<Implementation> fres;
 
+        addCudaSearchPath();
+
+        std::string impl_name_re = "llamamodel-mainline-(cpu|metal|kompute|vulkan|cuda)";
+        if (cpu_supports_avx2() == 0) {
+            impl_name_re += "-avxonly";
+        }
+        std::regex re(impl_name_re);
         auto search_in_directory = [&](const std::string& paths) {
             std::stringstream ss(paths);
             std::string path;
             // Split the paths string by the delimiter and process each path.
             while (std::getline(ss, path, ';')) {
-                std::filesystem::path fs_path(path);
+                std::u8string u8_path(path.begin(), path.end());
                 // Iterate over all libraries
-                for (const auto& f : std::filesystem::directory_iterator(fs_path)) {
-                    const std::filesystem::path& p = f.path();
+                for (const auto &f : fs::directory_iterator(u8_path)) {
+                    const fs::path &p = f.path();
+
                     if (p.extension() != LIB_FILE_EXT) continue;
+                    if (!std::regex_search(p.stem().string(), re)) continue;
+
                     // Add to list if model implementation
+                    Dlhandle dl;
                     try {
-                        Dlhandle dl(p.string());
-                        if (!Implementation::isImplementation(dl)) {
-                            continue;
-                        }
-                        fres.emplace_back(Implementation(std::move(dl)));
-                    } catch (...) {}
+                        dl = Dlhandle(p);
+                    } catch (const Dlhandle::Exception &e) {
+                        std::cerr << "Failed to load " << p.filename().string() << ": " << e.what() << "\n";
+                        continue;
+                    }
+                    if (!isImplementation(dl)) {
+                        std::cerr << "Not an implementation: " << p.filename().string() << "\n";
+                        continue;
+                    }
+                    fres.emplace_back(Implementation(std::move(dl)));
                 }
             }
         };
 
-        search_in_directory(m_implementations_search_path);
+        search_in_directory(s_implementations_search_path);
 
         return fres;
     }());
@@ -106,36 +173,175 @@ const std::vector<LLModel::Implementation> &LLModel::implementationList() {
     return *libs;
 }
 
-const LLModel::Implementation* LLModel::implementation(std::ifstream& f, const std::string& buildVariant) {
-    for (const auto& i : implementationList()) {
-        f.seekg(0);
-        if (!i.magicMatch(f)) continue;
-        if (buildVariant != i.buildVariant) continue;
-        return &i;
+static std::string applyCPUVariant(const std::string &buildVariant)
+{
+    if (buildVariant != "metal" && cpu_supports_avx2() == 0) {
+        return buildVariant + "-avxonly";
     }
+    return buildVariant;
+}
+
+const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant)
+{
+    bool buildVariantMatched = false;
+    std::optional<std::string> archName;
+    for (const auto& i : implementationList()) {
+        if (buildVariant != i.m_buildVariant) continue;
+        buildVariantMatched = true;
+
+        char *arch = i.m_getFileArch(fname);
+        if (!arch) continue;
+        archName = arch;
+
+        bool archSupported = i.m_isArchSupported(arch);
+        free(arch);
+        if (archSupported) return &i;
+    }
+
+    if (!buildVariantMatched)
+        return nullptr;
+    if (!archName)
+        throw UnsupportedModelError("Unsupported file format");
+
+    throw BadArchError(std::move(*archName));
+}
+
+LLModel *LLModel::Implementation::construct(const std::string &modelPath, const std::string &backend, int n_ctx)
+{
+    std::vector<std::string> desiredBackends;
+    if (backend != "auto") {
+        desiredBackends.push_back(backend);
+    } else {
+        desiredBackends.insert(desiredBackends.end(), DEFAULT_BACKENDS, std::end(DEFAULT_BACKENDS));
+    }
+
+    for (const auto &desiredBackend: desiredBackends) {
+        const auto *impl = implementation(modelPath.c_str(), applyCPUVariant(desiredBackend));
+
+        if (impl) {
+            // Construct llmodel implementation
+            auto *fres = impl->m_construct();
+            fres->m_implementation = impl;
+
+#if defined(__APPLE__) && defined(__aarch64__) // FIXME: See if metal works for intel macs
+            /* TODO(cebtenzzre): after we fix requiredMem, we should change this to happen at
+             * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
+             * most (all?) places where this is called, causing underestimation of required
+             * memory. */
+            if (backend == "auto" && desiredBackend == "metal") {
+                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
+                size_t req_mem = fres->requiredMem(modelPath, n_ctx, 100);
+                if (req_mem >= size_t(0.53f * getSystemTotalRAMInBytes())) {
+                    delete fres;
+                    continue;
+                }
+            }
+#else
+            (void)n_ctx;
+#endif
+
+            return fres;
+        }
+    }
+
+    throw MissingImplementationError("Could not find any implementations for backend: " + backend);
+}
+
+LLModel *LLModel::Implementation::constructGlobalLlama(const std::optional<std::string> &backend)
+{
+    static std::unordered_map<std::string, std::unique_ptr<LLModel>> implCache;
+
+    const std::vector<Implementation> *impls;
+    try {
+        impls = &implementationList();
+    } catch (const std::runtime_error &e) {
+        std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
+        return nullptr;
+    }
+
+    std::vector<std::string> desiredBackends;
+    if (backend) {
+        desiredBackends.push_back(backend.value());
+    } else {
+        desiredBackends.insert(desiredBackends.end(), DEFAULT_BACKENDS, std::end(DEFAULT_BACKENDS));
+    }
+
+    const Implementation *impl = nullptr;
+
+    for (const auto &desiredBackend: desiredBackends) {
+        auto cacheIt = implCache.find(desiredBackend);
+        if (cacheIt != implCache.end())
+            return cacheIt->second.get(); // cached
+
+        for (const auto &i: *impls) {
+            if (i.m_modelType == "LLaMA" && i.m_buildVariant == applyCPUVariant(desiredBackend)) {
+                impl = &i;
+                break;
+            }
+        }
+
+        if (impl) {
+            auto *fres = impl->m_construct();
+            fres->m_implementation = impl;
+            implCache[desiredBackend] = std::unique_ptr<LLModel>(fres);
+            return fres;
+        }
+    }
+
+    std::cerr << __func__ << ": could not find Llama implementation for backend: " << backend.value_or("default") << "\n";
     return nullptr;
 }
 
-LLModel *LLModel::construct(const std::string &modelPath, std::string buildVariant) {
-
-    if (!has_at_least_minimal_hardware())
-        return nullptr;
-
-    //TODO: Auto-detect CUDA/OpenCL
-    if (buildVariant == "auto") {
-        if (requires_avxonly()) {
-            buildVariant = "avxonly";
-        } else {
-            buildVariant = "default";
+std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices(size_t memoryRequired)
+{
+    std::vector<LLModel::GPUDevice> devices;
+#ifndef __APPLE__
+    static const std::string backends[] = {"kompute", "cuda"};
+    for (const auto &backend: backends) {
+        auto *llama = constructGlobalLlama(backend);
+        if (llama) {
+            auto backendDevs = llama->availableGPUDevices(memoryRequired);
+            devices.insert(devices.end(), backendDevs.begin(), backendDevs.end());
         }
     }
-    // Read magic
-    std::ifstream f(modelPath, std::ios::binary);
-    if (!f) return nullptr;
-    // Get correct implementation
-    auto impl = implementation(f, buildVariant);
-    if (!impl) return nullptr;
-    f.close();
-    // Construct and return llmodel implementation
-    return impl->construct();
+#endif
+    return devices;
+}
+
+int32_t LLModel::Implementation::maxContextLength(const std::string &modelPath)
+{
+    auto *llama = constructGlobalLlama();
+    return llama ? llama->maxContextLength(modelPath) : -1;
+}
+
+int32_t LLModel::Implementation::layerCount(const std::string &modelPath)
+{
+    auto *llama = constructGlobalLlama();
+    return llama ? llama->layerCount(modelPath) : -1;
+}
+
+bool LLModel::Implementation::isEmbeddingModel(const std::string &modelPath)
+{
+    auto *llama = constructGlobalLlama();
+    return llama && llama->isEmbeddingModel(modelPath);
+}
+
+void LLModel::Implementation::setImplementationsSearchPath(const std::string& path)
+{
+    s_implementations_search_path = path;
+}
+
+const std::string& LLModel::Implementation::implementationsSearchPath()
+{
+    return s_implementations_search_path;
+}
+
+bool LLModel::Implementation::hasSupportedCPU()
+{
+    return cpu_supports_avx() != 0;
+}
+
+int LLModel::Implementation::cpuSupportsAVX2()
+{
+    return cpu_supports_avx2();
 }

gpt4all-backend/llmodel.h

@@ -1,106 +1,264 @@
 #ifndef LLMODEL_H
 #define LLMODEL_H
 
-#include <string>
-#include <functional>
-#include <vector>
-#include <string_view>
-#include <fstream>
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
 #include <cstdint>
-#include <limits>
+#include <functional>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+#include <vector>
 
 class Dlhandle;
 
+using namespace std::string_literals;
+
+#define LLMODEL_MAX_PROMPT_BATCH 128
+
 class LLModel {
 public:
     using Token = int32_t;
 
-    class Implementation {
-        LLModel *(*construct_)();
-
+    class BadArchError: public std::runtime_error {
     public:
-        Implementation(Dlhandle&&);
-        Implementation(const Implementation&) = delete;
-        Implementation(Implementation&&);
+        BadArchError(std::string arch)
+            : runtime_error("Unsupported model architecture: " + arch)
+            , m_arch(std::move(arch))
+            {}
+
+        const std::string &arch() const noexcept { return m_arch; }
+
+    private:
+        std::string m_arch;
+    };
+
+    class MissingImplementationError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
+    class UnsupportedModelError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
+    struct GPUDevice {
+        const char *backend;
+        int index;
+        int type;
+        size_t heapSize;
+        std::string name;
+        std::string vendor;
+
+        GPUDevice(const char *backend, int index, int type, size_t heapSize, std::string name, std::string vendor):
+            backend(backend), index(index), type(type), heapSize(heapSize), name(std::move(name)),
+            vendor(std::move(vendor)) {}
+
+        std::string selectionName() const
+        {
+            assert(backend == "cuda"s || backend == "kompute"s);
+            return backendName() + ": " + name;
+        }
+
+        std::string backendName() const { return backendIdToName(backend); }
+
+        static std::string backendIdToName(const std::string &backend) { return s_backendNames.at(backend); }
+
+        static std::string updateSelectionName(const std::string &name) {
+            if (name == "Auto" || name == "CPU" || name == "Metal")
+                return name;
+            auto it = std::find_if(s_backendNames.begin(), s_backendNames.end(), [&name](const auto &entry) {
+                return name.starts_with(entry.second + ": ");
+            });
+            if (it != s_backendNames.end())
+                return name;
+            return "Vulkan: " + name; // previously, there were only Vulkan devices
+        }
+
+    private:
+        static inline const std::unordered_map<std::string, std::string> s_backendNames {
+            {"cpu", "CPU"}, {"metal", "Metal"}, {"cuda", "CUDA"}, {"kompute", "Vulkan"},
+        };
+    };
+
+    class Implementation {
+    public:
+        Implementation(const Implementation &) = delete;
+        Implementation(Implementation &&);
         ~Implementation();
 
-        static bool isImplementation(const Dlhandle&);
+        std::string_view modelType() const { return m_modelType; }
+        std::string_view buildVariant() const { return m_buildVariant; }
 
-        std::string_view modelType, buildVariant;
-        bool (*magicMatch)(std::ifstream& f);
-        Dlhandle *dlhandle;
+        static LLModel *construct(const std::string &modelPath, const std::string &backend = "auto", int n_ctx = 2048);
+        static std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0);
+        static int32_t maxContextLength(const std::string &modelPath);
+        static int32_t layerCount(const std::string &modelPath);
+        static bool isEmbeddingModel(const std::string &modelPath);
+        static void setImplementationsSearchPath(const std::string &path);
+        static const std::string &implementationsSearchPath();
+        static bool hasSupportedCPU();
+        // 0 for no, 1 for yes, -1 for non-x86_64
+        static int cpuSupportsAVX2();
 
-        // The only way an implementation should be constructed
-        LLModel *construct() const {
-            auto fres = construct_();
-            fres->m_implementation = this;
-            return fres;
-        }
+    private:
+        Implementation(Dlhandle &&);
+
+        static const std::vector<Implementation> &implementationList();
+        static const Implementation *implementation(const char *fname, const std::string &buildVariant);
+        static LLModel *constructGlobalLlama(const std::optional<std::string> &backend = std::nullopt);
+
+        char *(*m_getFileArch)(const char *fname);
+        bool (*m_isArchSupported)(const char *arch);
+        LLModel *(*m_construct)();
+
+        std::string_view m_modelType;
+        std::string_view m_buildVariant;
+        Dlhandle *m_dlhandle;
     };
 
     struct PromptContext {
-        std::vector<float> logits;      // logits of current context
         std::vector<int32_t> tokens;    // current tokens in the context window
         int32_t n_past = 0;             // number of tokens in past conversation
         int32_t n_ctx = 0;              // number of tokens possible in context window
         int32_t n_predict = 200;
         int32_t top_k = 40;
         float   top_p = 0.9f;
+        float   min_p = 0.0f;
         float   temp = 0.9f;
         int32_t n_batch = 9;
         float   repeat_penalty = 1.10f;
         int32_t repeat_last_n = 64;     // last n tokens to penalize
-        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context
-            // window
+        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context window
     };
 
+    using ProgressCallback = std::function<bool(float progress)>;
+
     explicit LLModel() {}
     virtual ~LLModel() {}
 
-    virtual bool loadModel(const std::string &modelPath) = 0;
+    virtual bool supportsEmbedding() const = 0;
+    virtual bool supportsCompletion() const = 0;
+    virtual bool loadModel(const std::string &modelPath, int n_ctx, int ngl) = 0;
+    virtual bool isModelBlacklisted(const std::string &modelPath) const { (void)modelPath; return false; };
+    virtual bool isEmbeddingModel(const std::string &modelPath) const { (void)modelPath; return false; }
     virtual bool isModelLoaded() const = 0;
+    virtual size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) = 0;
     virtual size_t stateSize() const { return 0; }
-    virtual size_t saveState(uint8_t */*dest*/) const { return 0; }
-    virtual size_t restoreState(const uint8_t */*src*/) { return 0; }
+    virtual size_t saveState(uint8_t *dest) const { (void)dest; return 0; }
+    virtual size_t restoreState(const uint8_t *src) { (void)src; return 0; }
+
+    // This method requires the model to return true from supportsCompletion otherwise it will throw
+    // an error
     virtual void prompt(const std::string &prompt,
+                        const std::string &promptTemplate,
                         std::function<bool(int32_t)> promptCallback,
                         std::function<bool(int32_t, const std::string&)> responseCallback,
                         std::function<bool(bool)> recalculateCallback,
-                        PromptContext &ctx);
+                        PromptContext &ctx,
+                        bool special = false,
+                        std::string *fakeReply = nullptr);
 
-    virtual void setThreadCount(int32_t /*n_threads*/) {}
+    using EmbedCancelCallback = bool(unsigned *batchSizes, unsigned nBatch, const char *backend);
+
+    virtual size_t embeddingSize() const {
+        throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+    }
+    // user-specified prefix
+    virtual void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
+                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false,
+                       EmbedCancelCallback *cancelCb = nullptr);
+    // automatic prefix
+    virtual void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval,
+                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false);
+
+    virtual void setThreadCount(int32_t n_threads) { (void)n_threads; }
     virtual int32_t threadCount() const { return 1; }
 
-    const Implementation& implementation() const {
+    const Implementation &implementation() const {
         return *m_implementation;
     }
 
-    static const std::vector<Implementation>& implementationList();
-    static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
-    static LLModel *construct(const std::string &modelPath, std::string buildVariant = "default");
+    virtual std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const {
+        (void)memoryRequired;
+        return {};
+    }
 
-    static inline void setImplementationsSearchPath(const std::string& path) {
-        m_implementations_search_path = path;
+    virtual bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const {
+        (void)memoryRequired;
+        (void)name;
+        return false;
     }
-    static inline const std::string& implementationsSearchPath() {
-        return m_implementations_search_path;
+
+    virtual bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const {
+        (void)device;
+        if (unavail_reason) {
+            *unavail_reason = "model has no GPU support";
+        }
+        return false;
     }
 
+    virtual bool usingGPUDevice() const { return false; }
+    virtual const char *backendName() const { return "cpu"; }
+    virtual const char *gpuDeviceName() const { return nullptr; }
+
+    void setProgressCallback(ProgressCallback callback) { m_progressCallback = callback; }
+
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
-    virtual std::vector<Token> tokenize(PromptContext &, const std::string&) const = 0;
-    virtual std::string_view tokenToString(Token) const = 0;
+    virtual std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special = false) = 0;
+    virtual std::string tokenToString(Token id) const = 0;
     virtual Token sampleToken(PromptContext &ctx) const = 0;
-    virtual bool evalTokens(PromptContext &/*ctx*/, const std::vector<int32_t>& /*tokens*/) const = 0;
+    virtual bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const = 0;
     virtual int32_t contextLength() const = 0;
-    virtual const std::vector<Token>& endTokens() const = 0;
+    virtual const std::vector<Token> &endTokens() const = 0;
+    virtual bool shouldAddBOS() const = 0;
+
+    virtual int32_t maxContextLength(std::string const &modelPath) const
+    {
+        (void)modelPath;
+        return -1;
+    }
+
+    virtual int32_t layerCount(std::string const &modelPath) const
+    {
+        (void)modelPath;
+        return -1;
+    }
 
     // This is a helper function called from the default implementation of 'prompt' but it can be
     // shared by all base classes so it isn't virtual
     void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate);
 
     const Implementation *m_implementation = nullptr;
-    static std::string m_implementations_search_path;
+
+    ProgressCallback m_progressCallback;
+    static bool staticProgressCallback(float progress, void* ctx)
+    {
+        LLModel* model = static_cast<LLModel*>(ctx);
+        if (model && model->m_progressCallback)
+            return model->m_progressCallback(progress);
+        return true;
+    }
+
+    bool decodePrompt(std::function<bool(int32_t)> promptCallback,
+                      std::function<bool(int32_t, const std::string&)> responseCallback,
+                      std::function<bool(bool)> recalculateCallback,
+                      PromptContext &promptCtx,
+                      std::vector<Token> embd_inp);
+    void generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
+                          std::function<bool(bool)> recalculateCallback,
+                          PromptContext &promptCtx);
+
+    Token m_tokenize_last_token = -1; // not serialized
+
+    friend class LLMImplementation;
 };
+
 #endif // LLMODEL_H

gpt4all-backend/llmodel_c.cpp

@@ -1,120 +1,121 @@
 #include "llmodel_c.h"
+
 #include "llmodel.h"
 
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
 #include <cstring>
-#include <cerrno>
-#include <utility>
-
+#include <exception>
+#include <functional>
+#include <iostream>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
 
 struct LLModelWrapper {
     LLModel *llModel = nullptr;
     LLModel::PromptContext promptContext;
+    ~LLModelWrapper() { delete llModel; }
 };
 
-
-thread_local static std::string last_error_message;
-
-
-llmodel_model llmodel_model_create(const char *model_path) {
-    auto fres = llmodel_model_create2(model_path, "auto", nullptr);
+llmodel_model llmodel_model_create(const char *model_path)
+{
+    const char *error;
+    auto fres = llmodel_model_create2(model_path, "auto", &error);
     if (!fres) {
-        fprintf(stderr, "Invalid model file\n");
+        fprintf(stderr, "Unable to instantiate model: %s\n", error);
     }
     return fres;
 }
 
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error) {
-    auto wrapper = new LLModelWrapper;
-    llmodel_error new_error{};
-
-    try {
-        wrapper->llModel = LLModel::construct(model_path, build_variant);
-    } catch (const std::exception& e) {
-        new_error.code = EINVAL;
-        last_error_message = e.what();
-    }
-
-    if (!wrapper->llModel) {
-        delete std::exchange(wrapper, nullptr);
-        // Get errno and error message if none
-        if (new_error.code == 0) {
-            new_error.code = errno;
-            last_error_message = strerror(errno);
-        }
-        // Set message pointer
-        new_error.message = last_error_message.c_str();
-        // Set error argument
-        if (error) *error = new_error;
-    }
-    return reinterpret_cast<llmodel_model*>(wrapper);
-}
-
-void llmodel_model_destroy(llmodel_model model) {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    delete wrapper->llModel;
-}
-
-bool llmodel_loadModel(llmodel_model model, const char *model_path)
+static void llmodel_set_error(const char **errptr, const char *message)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->loadModel(model_path);
+    thread_local static std::string last_error_message;
+    if (errptr) {
+        last_error_message = message;
+        *errptr = last_error_message.c_str();
+    }
+}
+
+llmodel_model llmodel_model_create2(const char *model_path, const char *backend, const char **error)
+{
+    LLModel *llModel;
+    try {
+        llModel = LLModel::Implementation::construct(model_path, backend);
+    } catch (const std::exception& e) {
+        llmodel_set_error(error, e.what());
+        return nullptr;
+    }
+
+    auto wrapper = new LLModelWrapper;
+    wrapper->llModel = llModel;
+    return wrapper;
+}
+
+void llmodel_model_destroy(llmodel_model model)
+{
+    delete static_cast<LLModelWrapper *>(model);
+}
+
+size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl)
+{
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->requiredMem(model_path, n_ctx, ngl);
+}
+
+bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl)
+{
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+
+    std::string modelPath(model_path);
+    if (wrapper->llModel->isModelBlacklisted(modelPath)) {
+        size_t slash = modelPath.find_last_of("/\\");
+        auto basename = slash == std::string::npos ? modelPath : modelPath.substr(slash + 1);
+        std::cerr << "warning: model '" << basename << "' is out-of-date, please check for an updated version\n";
+    }
+    return wrapper->llModel->loadModel(modelPath, n_ctx, ngl);
 }
 
 bool llmodel_isModelLoaded(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->isModelLoaded();
 }
 
 uint64_t llmodel_get_state_size(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->stateSize();
 }
 
 uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->saveState(dest);
 }
 
 uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->restoreState(src);
 }
 
-// Wrapper functions for the C callbacks
-bool prompt_wrapper(int32_t token_id, void *user_data) {
-    llmodel_prompt_callback callback = reinterpret_cast<llmodel_prompt_callback>(user_data);
-    return callback(token_id);
-}
-
-bool response_wrapper(int32_t token_id, const std::string &response, void *user_data) {
-    llmodel_response_callback callback = reinterpret_cast<llmodel_response_callback>(user_data);
-    return callback(token_id, response.c_str());
-}
-
-bool recalculate_wrapper(bool is_recalculating, void *user_data) {
-    llmodel_recalculate_callback callback = reinterpret_cast<llmodel_recalculate_callback>(user_data);
-    return callback(is_recalculating);
-}
-
 void llmodel_prompt(llmodel_model model, const char *prompt,
+                    const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
-                    llmodel_prompt_context *ctx)
+                    llmodel_prompt_context *ctx,
+                    bool special,
+                    const char *fake_reply)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
 
-    // Create std::function wrappers that call the C function pointers
-    std::function<bool(int32_t)> prompt_func =
-        std::bind(&prompt_wrapper, std::placeholders::_1, reinterpret_cast<void*>(prompt_callback));
-    std::function<bool(int32_t, const std::string&)> response_func =
-        std::bind(&response_wrapper, std::placeholders::_1, std::placeholders::_2, reinterpret_cast<void*>(response_callback));
-    std::function<bool(bool)> recalc_func =
-        std::bind(&recalculate_wrapper, std::placeholders::_1, reinterpret_cast<void*>(recalculate_callback));
+    auto response_func = [response_callback](int32_t token_id, const std::string &response) {
+        return response_callback(token_id, response.c_str());
+    };
 
     // Copy the C prompt context
     wrapper->promptContext.n_past = ctx->n_past;
@@ -122,19 +123,23 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     wrapper->promptContext.n_predict = ctx->n_predict;
     wrapper->promptContext.top_k = ctx->top_k;
     wrapper->promptContext.top_p = ctx->top_p;
+    wrapper->promptContext.min_p = ctx->min_p;
     wrapper->promptContext.temp = ctx->temp;
     wrapper->promptContext.n_batch = ctx->n_batch;
     wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
     wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
     wrapper->promptContext.contextErase = ctx->context_erase;
 
+    std::string fake_reply_str;
+    if (fake_reply) { fake_reply_str = fake_reply; }
+    auto *fake_reply_p = fake_reply ? &fake_reply_str : nullptr;
+
     // Call the C++ prompt method
-    wrapper->llModel->prompt(prompt, prompt_func, response_func, recalc_func, wrapper->promptContext);
+    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, recalculate_callback,
+                             wrapper->promptContext, special, fake_reply_p);
 
     // Update the C context by giving access to the wrappers raw pointers to std::vector data
     // which involves no copies
-    ctx->logits = wrapper->promptContext.logits.data();
-    ctx->logits_size = wrapper->promptContext.logits.size();
     ctx->tokens = wrapper->promptContext.tokens.data();
     ctx->tokens_size = wrapper->promptContext.tokens.size();
 
@@ -144,6 +149,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->n_predict = wrapper->promptContext.n_predict;
     ctx->top_k = wrapper->promptContext.top_k;
     ctx->top_p = wrapper->promptContext.top_p;
+    ctx->min_p = wrapper->promptContext.min_p;
     ctx->temp = wrapper->promptContext.temp;
     ctx->n_batch = wrapper->promptContext.n_batch;
     ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
@@ -151,24 +157,139 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->context_erase = wrapper->promptContext.contextErase;
 }
 
+float *llmodel_embed(
+    llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix, int dimensionality,
+    size_t *token_count, bool do_mean, bool atlas, llmodel_emb_cancel_callback cancel_cb, const char **error
+) {
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+
+    if (!texts || !*texts) {
+        llmodel_set_error(error, "'texts' is NULL or empty");
+        return nullptr;
+    }
+
+    std::vector<std::string> textsVec;
+    while (*texts) { textsVec.emplace_back(*texts++); }
+
+    size_t embd_size;
+    float *embedding;
+
+    try {
+        embd_size = wrapper->llModel->embeddingSize();
+        if (dimensionality > 0 && dimensionality < int(embd_size))
+            embd_size = dimensionality;
+
+        embd_size *= textsVec.size();
+
+        std::optional<std::string> prefixStr;
+        if (prefix) { prefixStr = prefix; }
+
+        embedding = new float[embd_size];
+        wrapper->llModel->embed(textsVec, embedding, prefixStr, dimensionality, token_count, do_mean, atlas, cancel_cb);
+    } catch (std::exception const &e) {
+        llmodel_set_error(error, e.what());
+        return nullptr;
+    }
+
+    *embedding_size = embd_size;
+    return embedding;
+}
+
+void llmodel_free_embedding(float *ptr)
+{
+    delete[] ptr;
+}
+
 void llmodel_setThreadCount(llmodel_model model, int32_t n_threads)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     wrapper->llModel->setThreadCount(n_threads);
 }
 
 int32_t llmodel_threadCount(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->threadCount();
 }
 
 void llmodel_set_implementation_search_path(const char *path)
 {
-    LLModel::setImplementationsSearchPath(path);
+    LLModel::Implementation::setImplementationsSearchPath(path);
 }
 
 const char *llmodel_get_implementation_search_path()
 {
-    return LLModel::implementationsSearchPath().c_str();
+    return LLModel::Implementation::implementationsSearchPath().c_str();
+}
+
+// RAII wrapper around a C-style struct
+struct llmodel_gpu_device_cpp: llmodel_gpu_device {
+    llmodel_gpu_device_cpp() = default;
+
+    llmodel_gpu_device_cpp(const llmodel_gpu_device_cpp  &) = delete;
+    llmodel_gpu_device_cpp(      llmodel_gpu_device_cpp &&) = delete;
+
+    const llmodel_gpu_device_cpp &operator=(const llmodel_gpu_device_cpp  &) = delete;
+          llmodel_gpu_device_cpp &operator=(      llmodel_gpu_device_cpp &&) = delete;
+
+    ~llmodel_gpu_device_cpp() {
+        free(const_cast<char *>(name));
+        free(const_cast<char *>(vendor));
+    }
+};
+
+static_assert(sizeof(llmodel_gpu_device_cpp) == sizeof(llmodel_gpu_device));
+
+struct llmodel_gpu_device *llmodel_available_gpu_devices(size_t memoryRequired, int *num_devices)
+{
+    static thread_local std::unique_ptr<llmodel_gpu_device_cpp[]> c_devices;
+
+    auto devices = LLModel::Implementation::availableGPUDevices(memoryRequired);
+    *num_devices = devices.size();
+
+    if (devices.empty()) { return nullptr; /* no devices */ }
+
+    c_devices = std::make_unique<llmodel_gpu_device_cpp[]>(devices.size());
+    for (unsigned i = 0; i < devices.size(); i++) {
+        const auto &dev  =   devices[i];
+              auto &cdev = c_devices[i];
+        cdev.backend  = dev.backend;
+        cdev.index    = dev.index;
+        cdev.type     = dev.type;
+        cdev.heapSize = dev.heapSize;
+        cdev.name     = strdup(dev.name.c_str());
+        cdev.vendor   = strdup(dev.vendor.c_str());
+    }
+
+    return c_devices.get();
+}
+
+bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
+{
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->initializeGPUDevice(memoryRequired, std::string(device));
+}
+
+bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device)
+{
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->initializeGPUDevice(device->index);
+}
+
+bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
+{
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->initializeGPUDevice(device);
+}
+
+const char *llmodel_model_backend_name(llmodel_model model)
+{
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->backendName();
+}
+
+const char *llmodel_model_gpu_device_name(llmodel_model model)
+{
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->gpuDeviceName();
 }

gpt4all-backend/llmodel_c.h

@@ -1,9 +1,9 @@
 #ifndef LLMODEL_C_H
 #define LLMODEL_C_H
 
-#include <stdint.h>
-#include <stddef.h>
 #include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
 
 #ifdef __GNUC__
 #define DEPRECATED __attribute__ ((deprecated))
@@ -23,17 +23,6 @@ extern "C" {
  */
 typedef void *llmodel_model;
 
-/**
- * Structure containing any errors that may eventually occur
- */
-struct llmodel_error {
-    const char *message;  // Human readable error description; Thread-local; guaranteed to survive until next llmodel C API call
-    int code;             // errno; 0 if none
-};
-#ifndef __cplusplus
-typedef struct llmodel_error llmodel_error;
-#endif
-
 /**
  * llmodel_prompt_context structure for holding the prompt context.
  * NOTE: The implementation takes care of all the memory handling of the raw logits pointer and the
@@ -41,8 +30,6 @@ typedef struct llmodel_error llmodel_error;
  * behavior.
  */
 struct llmodel_prompt_context {
-    float *logits;          // logits of current context
-    size_t logits_size;     // the size of the raw logits vector
     int32_t *tokens;        // current tokens in the context window
     size_t tokens_size;     // the size of the raw tokens vector
     int32_t n_past;         // number of tokens in past conversation
@@ -50,14 +37,26 @@ struct llmodel_prompt_context {
     int32_t n_predict;      // number of tokens to predict
     int32_t top_k;          // top k logits to sample from
     float top_p;            // nucleus sampling probability threshold
+    float min_p;            // Min P sampling
     float temp;             // temperature to adjust model's output distribution
     int32_t n_batch;        // number of predictions to generate in parallel
     float repeat_penalty;   // penalty factor for repeated tokens
     int32_t repeat_last_n;  // last n tokens to penalize
     float context_erase;    // percent of context to erase if we exceed the context window
 };
+
+struct llmodel_gpu_device {
+    const char * backend;
+    int index;
+    int type; // same as VkPhysicalDeviceType
+    size_t heapSize;
+    const char * name;
+    const char * vendor;
+};
+
 #ifndef __cplusplus
 typedef struct llmodel_prompt_context llmodel_prompt_context;
+typedef struct llmodel_gpu_device llmodel_gpu_device;
 #endif
 
 /**
@@ -82,6 +81,15 @@ typedef bool (*llmodel_response_callback)(int32_t token_id, const char *response
  */
 typedef bool (*llmodel_recalculate_callback)(bool is_recalculating);
 
+/**
+ * Embedding cancellation callback for use with llmodel_embed.
+ * @param batch_sizes The number of tokens in each batch that will be embedded.
+ * @param n_batch The number of batches that will be embedded.
+ * @param backend The backend that will be used for embedding. One of "cpu", "kompute", "cuda", or "metal".
+ * @return True to cancel llmodel_embed, false to continue.
+ */
+typedef bool (*llmodel_emb_cancel_callback)(unsigned *batch_sizes, unsigned n_batch, const char *backend);
+
 /**
  * Create a llmodel instance.
  * Recognises correct model type from file at model_path
@@ -94,11 +102,11 @@ DEPRECATED llmodel_model llmodel_model_create(const char *model_path);
  * Create a llmodel instance.
  * Recognises correct model type from file at model_path
  * @param model_path A string representing the path to the model file; will only be used to detect model type.
- * @param build_variant A string representing the implementation to use (auto, default, avxonly, ...),
- * @param error A pointer to a llmodel_error; will only be set on error.
+ * @param backend A string representing the implementation to use. One of 'auto', 'cpu', 'metal', 'kompute', or 'cuda'.
+ * @param error A pointer to a string; will only be set on error.
  * @return A pointer to the llmodel_model instance; NULL on error.
  */
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error);
+llmodel_model llmodel_model_create2(const char *model_path, const char *backend, const char **error);
 
 /**
  * Destroy a llmodel instance.
@@ -107,13 +115,25 @@ llmodel_model llmodel_model_create2(const char *model_path, const char *build_va
  */
 void llmodel_model_destroy(llmodel_model model);
 
+/**
+ * Estimate RAM requirement for a model file
+ * @param model A pointer to the llmodel_model instance.
+ * @param model_path A string representing the path to the model file.
+ * @param n_ctx Maximum size of context window
+ * @param ngl Number of GPU layers to use (Vulkan)
+ * @return size greater than 0 if the model was parsed successfully, 0 if file could not be parsed.
+ */
+size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl);
+
 /**
  * Load a model from a file.
  * @param model A pointer to the llmodel_model instance.
  * @param model_path A string representing the path to the model file.
+ * @param n_ctx Maximum size of context window
+ * @param ngl Number of GPU layers to use (Vulkan)
  * @return true if the model was loaded successfully, false otherwise.
  */
-bool llmodel_loadModel(llmodel_model model, const char *model_path);
+bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl);
 
 /**
  * Check if a model is loaded.
@@ -152,16 +172,54 @@ uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src);
  * Generate a response using the model.
  * @param model A pointer to the llmodel_model instance.
  * @param prompt A string representing the input prompt.
+ * @param prompt_template A string representing the input prompt template.
  * @param prompt_callback A callback function for handling the processing of prompt.
  * @param response_callback A callback function for handling the generated response.
  * @param recalculate_callback A callback function for handling recalculation requests.
+ * @param special True if special tokens in the prompt should be processed, false otherwise.
+ * @param fake_reply A string to insert into context as the model's reply, or NULL to generate one.
  * @param ctx A pointer to the llmodel_prompt_context structure.
  */
 void llmodel_prompt(llmodel_model model, const char *prompt,
+                    const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
-                    llmodel_prompt_context *ctx);
+                    llmodel_prompt_context *ctx,
+                    bool special,
+                    const char *fake_reply);
+
+/**
+ * Generate an embedding using the model.
+ * NOTE: If given NULL pointers for the model or text, or an empty text, a NULL pointer will be
+ * returned. Bindings should signal an error when NULL is the return value.
+ * @param model A pointer to the llmodel_model instance.
+ * @param texts A pointer to a NULL-terminated array of strings representing the texts to generate an
+ * embedding for.
+ * @param embedding_size A pointer to a size_t type that will be set by the call indicating the length
+ * of the returned floating point array.
+ * @param prefix The model-specific prefix representing the embedding task, without the trailing colon. NULL for no
+ * prefix.
+ * @param dimensionality The embedding dimension, for use with Matryoshka-capable models. Set to -1 to for full-size.
+ * @param token_count Return location for the number of prompt tokens processed, or NULL.
+ * @param do_mean True to average multiple embeddings if the text is longer than the model can accept, False to
+ * truncate.
+ * @param atlas Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens with
+ * long_text_mode="mean" will raise an error. Disabled by default.
+ * @param cancel_cb Cancellation callback, or NULL. See the documentation of llmodel_emb_cancel_callback.
+ * @param error Return location for a malloc()ed string that will be set on error, or NULL.
+ * @return A pointer to an array of floating point values passed to the calling method which then will
+ * be responsible for lifetime of this memory. NULL if an error occurred.
+ */
+float *llmodel_embed(llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix,
+                     int dimensionality, size_t *token_count, bool do_mean, bool atlas,
+                     llmodel_emb_cancel_callback cancel_cb, const char **error);
+
+/**
+ * Frees the memory allocated by the llmodel_embedding function.
+ * @param ptr A pointer to the embedding as returned from llmodel_embedding.
+ */
+void llmodel_free_embedding(float *ptr);
 
 /**
  * Set the number of threads to be used by the model.
@@ -191,6 +249,56 @@ void llmodel_set_implementation_search_path(const char *path);
  */
 const char *llmodel_get_implementation_search_path();
 
+/**
+ * Get a list of available GPU devices given the memory required.
+ * @param memoryRequired The minimum amount of VRAM, in bytes
+ * @return A pointer to an array of llmodel_gpu_device's whose number is given by num_devices.
+ */
+struct llmodel_gpu_device* llmodel_available_gpu_devices(size_t memoryRequired, int* num_devices);
+
+/**
+ * Initializes a GPU device based on a specified string criterion.
+ *
+ * This function initializes a GPU device based on a string identifier provided. The function
+ * allows initialization based on general device type ("gpu"), vendor name ("amd", "nvidia", "intel"),
+ * or any specific device name.
+ *
+ * @param memoryRequired The amount of memory (in bytes) required by the application or task
+ *                       that will utilize the GPU device.
+ * @param device A string specifying the desired criterion for GPU device selection. It can be:
+ *               - "gpu": To initialize the best available GPU.
+ *               - "amd", "nvidia", or "intel": To initialize the best available GPU from that vendor.
+ *               - A specific GPU device name: To initialize a GPU with that exact name.
+ *
+ * @return True if the GPU device is successfully initialized based on the provided string
+ *         criterion. Returns false if the desired GPU device could not be initialized.
+ */
+bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device);
+
+/**
+ * Initializes a GPU device by specifying a valid gpu device pointer.
+ * @param device A gpu device pointer.
+ * @return True if the GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device);
+
+/**
+ * Initializes a GPU device by its index.
+ * @param device An integer representing the index of the GPU device to be initialized.
+ * @return True if the GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
+
+/**
+ * @return The name of the llama.cpp backend currently in use. One of "cpu", "kompute", or "metal".
+ */
+const char *llmodel_model_backend_name(llmodel_model model);
+
+/**
+ * @return The name of the GPU device currently in use, or NULL for backends other than Kompute.
+ */
+const char *llmodel_model_gpu_device_name(llmodel_model model);
+
 #ifdef __cplusplus
 }
 #endif

gpt4all-backend/llmodel_shared.cpp

@@ -1,12 +1,34 @@
 #include "llmodel.h"
 
+#include <algorithm>
 #include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
 #include <iostream>
+#include <optional>
+#include <regex>
+#include <sstream>
+#include <stdexcept>
+#include <string>
 #include <unordered_set>
+#include <vector>
 
-void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
-    size_t i = 0;
-    promptCtx.n_past = 0;
+// TODO(cebtenzzre): replace this with llama_kv_cache_seq_shift for llamamodel (GPT-J needs this as-is)
+// FIXME(jared): if recalculate returns false, we leave n_past<tokens.size() and do not tell the caller to stop
+// FIXME(jared): if we get here during chat name or follow-up generation, bad things will happen when we try to restore
+// the old prompt context afterwards
+void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate)
+{
+    int n_keep = shouldAddBOS();
+    const int32_t n_discard = (promptCtx.n_ctx - n_keep) * promptCtx.contextErase;
+
+    // Erase the first percentage of context from the tokens
+    std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
+    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
+
+    size_t i = n_keep;
+    promptCtx.n_past = n_keep;
     while (i < promptCtx.tokens.size()) {
         size_t batch_end = std::min(i + promptCtx.n_batch, promptCtx.tokens.size());
         std::vector<int32_t> batch(promptCtx.tokens.begin() + i, promptCtx.tokens.begin() + batch_end);
@@ -26,32 +48,153 @@ stop_generating:
     recalculate(false);
 }
 
+static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err)
+{
+    static const std::regex placeholderRegex(R"(%[1-2](?![0-9]))");
+
+    auto it = std::sregex_iterator(tmpl.begin(), tmpl.end(), placeholderRegex);
+    placeholders.clear();
+    placeholders.insert(placeholders.end(), it, std::sregex_iterator());
+
+    if (placeholders.size() > 2) {
+        err = "ERROR: expected at most two placeholders, got " + std::to_string(placeholders.size());
+        return false;
+    }
+    if (placeholders.size() >= 1 && placeholders[0].str() != "%1") {
+        err = "ERROR: first placeholder must be %1, got " + placeholders[0].str();
+        return false;
+    }
+    if (placeholders.size() >= 2 && placeholders[1].str() != "%2") {
+        err = "ERROR: second placeholder must be %2, got " + placeholders[1].str();
+        return false;
+    }
+    return true;
+}
+
 void LLModel::prompt(const std::string &prompt,
+                     const std::string &promptTemplate,
                      std::function<bool(int32_t)> promptCallback,
                      std::function<bool(int32_t, const std::string&)> responseCallback,
                      std::function<bool(bool)> recalculateCallback,
-                     PromptContext &promptCtx)
+                     PromptContext &promptCtx,
+                     bool special,
+                     std::string *fakeReply)
 {
     if (!isModelLoaded()) {
-        std::cerr << implementation().modelType << " ERROR: prompt won't work with an unloaded model!\n";
+        std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
         return;
     }
 
-    // tokenize the prompt
-    std::vector<Token> embd_inp = tokenize(promptCtx, prompt);
+    if (!supportsCompletion()) {
+        std::string errorMessage = "ERROR: this model does not support text completion or chat!";
+        responseCallback(-1, errorMessage);
+        std::cerr << implementation().modelType() << " " << errorMessage << "\n";
+        return;
+    }
 
+    // make sure token cache matches decode offset
+    if (promptCtx.tokens.size() < promptCtx.n_past) {
+        std::ostringstream ss;
+        ss << "expected n_past to be at most " << promptCtx.tokens.size() << ", got " << promptCtx.n_past;
+        throw std::out_of_range(ss.str());
+    }
+    if (promptCtx.n_past < promptCtx.tokens.size())
+        promptCtx.tokens.resize(promptCtx.n_past);
+    m_tokenize_last_token = promptCtx.tokens.empty() ? -1 : promptCtx.tokens.back(); // not serialized
+
+    // parse the prompt template
+    std::vector<std::smatch> placeholders;
+    {
+        std::string err;
+        if (!parsePromptTemplate(promptTemplate, placeholders, err)) {
+            responseCallback(-1, err);
+            std::cerr << err << "\n";
+            return;
+        }
+    }
+
+    auto old_n_past = promptCtx.n_past; // prepare to fake n_past for tokenize
+
+    // tokenize the user prompt
+    std::vector<Token> embd_inp;
+    if (placeholders.empty()) {
+        // this is unusual, but well-defined
+        std::cerr << __func__ << ": prompt template has no placeholder\n";
+        embd_inp = tokenize(promptCtx, promptTemplate, true);
+    } else {
+        // template: beginning of user prompt
+        const auto &phUser = placeholders[0];
+        std::string userPrefix(phUser.prefix());
+        if (!userPrefix.empty()) {
+            embd_inp = tokenize(promptCtx, userPrefix, true);
+            promptCtx.n_past += embd_inp.size();
+        }
+
+        // user input (shouldn't have special token processing)
+        auto tokens = tokenize(promptCtx, prompt, special);
+        embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+        promptCtx.n_past += tokens.size();
+
+        // template: end of user prompt + start of assistant prompt
+        size_t start = phUser.position() + phUser.length();
+        size_t end = placeholders.size() >= 2 ? placeholders[1].position() : promptTemplate.length();
+        auto userToAsst = promptTemplate.substr(start, end - start);
+        if (!userToAsst.empty()) {
+            tokens = tokenize(promptCtx, userToAsst, true);
+            embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+            promptCtx.n_past += tokens.size();
+        }
+    }
+
+    promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
+
+    // decode the user prompt
+    if (!decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp))
+        return; // error
+
+    // decode the assistant's reply, either generated or spoofed
+    if (fakeReply == nullptr) {
+        generateResponse(responseCallback, recalculateCallback, promptCtx);
+    } else {
+        embd_inp = tokenize(promptCtx, *fakeReply, false);
+        if (!decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp))
+            return; // error
+    }
+
+    // decode the rest of the prompt template
+    // template: end of assistant prompt
+    std::string asstSuffix;
+    if (placeholders.size() >= 2) {
+        size_t start = placeholders[1].position() + placeholders[1].length();
+        asstSuffix = promptTemplate.substr(start);
+    } else {
+        asstSuffix = "\n\n"; // default to a blank link, good for e.g. Alpaca
+    }
+    if (!asstSuffix.empty()) {
+        embd_inp = tokenize(promptCtx, asstSuffix, true);
+        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+    }
+}
+
+// returns false on error
+bool LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
+                           std::function<bool(int32_t, const std::string&)> responseCallback,
+                           std::function<bool(bool)> recalculateCallback,
+                           PromptContext &promptCtx,
+                           std::vector<Token> embd_inp) {
     // save the context size
     promptCtx.n_ctx = contextLength();
 
     if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
         responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed.");
-        std::cerr << implementation().modelType << " ERROR: The prompt is" << embd_inp.size() <<
-            "tokens and the context window is" << promptCtx.n_ctx << "!\n";
-        return;
+        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
+            " tokens and the context window is " << promptCtx.n_ctx << "!\n";
+        return false;
     }
 
     promptCtx.n_predict = std::min(promptCtx.n_predict, promptCtx.n_ctx - (int) embd_inp.size());
     promptCtx.n_past = std::min(promptCtx.n_past, promptCtx.n_ctx);
+    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
 
     // process the prompt in batches
     size_t i = 0;
@@ -61,32 +204,31 @@ void LLModel::prompt(const std::string &prompt,
 
         // Check if the context has run out...
         if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
-            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
-            // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n";
-            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
-            promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
             assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
         }
 
         if (!evalTokens(promptCtx, batch)) {
-            std::cerr << implementation().modelType << " ERROR: Failed to process prompt\n";
-            return;
+            std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
+            return false;
         }
 
         size_t tokens = batch_end - i;
         for (size_t t = 0; t < tokens; ++t) {
-            if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
-                promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(batch.at(t));
+            promptCtx.n_past += 1;
             if (!promptCallback(batch.at(t)))
-                return;
+                return false;
         }
-        promptCtx.n_past += batch.size();
         i = batch_end;
     }
 
+    return true;
+}
+
+void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
+                               std::function<bool(bool)> recalculateCallback,
+                               PromptContext &promptCtx) {
     std::string cachedResponse;
     std::vector<Token> cachedTokens;
     std::unordered_set<std::string> reversePrompts
@@ -100,28 +242,21 @@ void LLModel::prompt(const std::string &prompt,
 
         // Check if the context has run out...
         if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
-            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
-            // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n";
-            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
-            promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
             assert(promptCtx.n_past + 1 <= promptCtx.n_ctx);
         }
 
         if (!evalTokens(promptCtx, { id })) {
-            std::cerr << implementation().modelType << " ERROR: Failed to predict next token\n";
+            std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
             return;
         }
 
-        promptCtx.n_past += 1;
-
         // display text
         for (const auto token : endTokens()) {
             if (id == token) return;
         }
 
-        const std::string_view str = tokenToString(id);
+        const std::string str = tokenToString(id);
 
         // Check if the provided str is part of our reverse prompts
         bool foundPartialReversePrompt = false;
@@ -147,9 +282,8 @@ void LLModel::prompt(const std::string &prompt,
 
         // Empty the cache
         for (auto t : cachedTokens) {
-            if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
-                promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(t);
+            promptCtx.n_past += 1;
             //TODO: Conversion to std::string can be avoided here...
             if (!responseCallback(t, std::string(tokenToString(t))))
                 return;
@@ -157,3 +291,32 @@ void LLModel::prompt(const std::string &prompt,
         cachedTokens.clear();
     }
 }
+
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)prefix;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    (void)cancelCb;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+}
+
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
+    bool doMean, bool atlas
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)isRetrieval;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+}

gpt4all-backend/llmodel_shared.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <ggml.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+struct llm_buffer {
+    uint8_t * addr = NULL;
+    size_t size = 0;
+
+    void resize(size_t size) {
+        delete[] addr;
+        addr = new uint8_t[size];
+        this->size = size;
+    }
+
+    ~llm_buffer() {
+        delete[] addr;
+    }
+};
+
+struct llm_kv_cache {
+    struct ggml_tensor * k;
+    struct ggml_tensor * v;
+
+    struct ggml_context * ctx = NULL;
+
+    llm_buffer buf;
+
+    int n; // number of tokens currently in the cache
+
+    ~llm_kv_cache() {
+        if (ctx) {
+            ggml_free(ctx);
+        }
+    }
+};
+
+inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads)
+{
+    struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
+    if (plan.work_size > 0) {
+        buf.resize(plan.work_size);
+        plan.work_data = buf.addr;
+    }
+    ggml_graph_compute(graph, &plan);
+}

gpt4all-backend/mpt.cpp

@@ -1,892 +0,0 @@
-#define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "mpt_impl.h"
-
-#include "utils.h"
-
-#include <cassert>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <fstream>
-#include <map>
-#include <random>
-#include <string>
-#include <vector>
-#include <iostream>
-#if defined(_WIN32) && defined(_MSC_VER)
-    #define WIN32_LEAN_AND_MEAN
-    #ifndef NOMINMAX
-        #define NOMINMAX
-    #endif
-    #include <windows.h>
-    #include <io.h>
-    #include <stdio.h>
-#else
-    #include <unistd.h>
-#endif
-#include <sstream>
-#include <thread>
-#include <unordered_set>
-#include <regex>
-#include <ggml.h>
-
-
-namespace {
-const char *modelType_ = "MPT";
-
-static const size_t MB = 1024*1024;
-}
-
-// default hparams (MPT 7B)
-struct mpt_hparams {
-    int32_t n_vocab      = 50432;
-    int32_t n_ctx        = 2048;
-    int32_t n_embd       = 4096;
-    int32_t n_head       = 32;
-    int32_t n_layer      = 32;
-    float alibi_bias_max = 8;
-    float clip_qkv       = 0;
-    int32_t expand       = 4;
-    int32_t f16          = 1;
-};
-
-struct mpt_layer {
-    // normalization
-    struct ggml_tensor * norm_1_w;
-    struct ggml_tensor * norm_2_w;
-
-    // attention
-    struct ggml_tensor * attn_Wqkv_w;
-    struct ggml_tensor * attn_out_proj_w;
-
-    // ff
-    struct ggml_tensor * ffn_up_proj_w;
-    struct ggml_tensor * ffn_down_proj_w;
-};
-
-struct mpt_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-
-    void resize(size_t size) {
-        delete[] addr;
-        addr = new uint8_t[size];
-        this->size = size;
-    }
-
-    ~mpt_buffer() {
-        fflush(stdout);
-        delete[] addr;
-    }
-};
-
-struct mpt_kv_cache {
-    struct ggml_tensor * k;
-    struct ggml_tensor * v;
-
-    struct ggml_context * ctx = NULL;
-
-    mpt_buffer buf;
-
-    int n; // number of tokens currently in the cache
-
-    ~mpt_kv_cache() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
-struct mpt_model {
-    mpt_hparams hparams;
-
-    // normalization
-    struct ggml_tensor * norm_f_w;
-
-    struct ggml_tensor * wte; // position embedding
-
-    // mpt does weight tying
-
-    std::vector<mpt_layer> layers;
-
-    struct mpt_kv_cache kv_self;
-    struct ggml_context * ctx;
-    std::map<std::string, struct ggml_tensor *> tensors;
-
-
-    mpt_buffer buf;
-
-    ~mpt_model() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
-static bool kv_cache_init(
-        const struct mpt_hparams & hparams,
-             struct mpt_kv_cache & cache,
-                         ggml_type   wtype,
-                               int   n_ctx) {
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-
-    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
-    const int64_t n_elements = n_embd*n_mem;
-
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB);
-
-    struct ggml_init_params params;
-    params.mem_size   = cache.buf.size;
-    params.mem_buffer = cache.buf.addr;
-    params.no_alloc   = false;
-
-    cache.ctx = ggml_init(params);
-
-    if (!cache.ctx) {
-        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
-        return false;
-    }
-
-    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-
-    return true;
-}
-
-// load the model's weights from a stream
-bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & model, gpt_vocab & vocab) {
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
-
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *) &magic, sizeof(magic));
-        if (magic != 0x67676d6d) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
-            return false;
-        }
-    }
-
-    // load hparams
-    {
-        auto & hparams = model.hparams;
-
-        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
-        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
-        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
-        fin.read((char *) &hparams.alibi_bias_max,  sizeof(hparams.alibi_bias_max));
-        fin.read((char *) &hparams.clip_qkv,  sizeof(hparams.clip_qkv));
-        fin.read((char *) &hparams.f16,   sizeof(hparams.f16));
-
-        printf("%s: n_vocab        = %d\n", __func__, hparams.n_vocab);
-        printf("%s: n_ctx          = %d\n", __func__, hparams.n_ctx);
-        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
-        printf("%s: alibi_bias_max = %f\n", __func__, hparams.alibi_bias_max);
-        printf("%s: clip_qkv       = %f\n", __func__, hparams.clip_qkv);
-        printf("%s: ftype          = %d\n", __func__, hparams.f16);
-    }
-
-    // load vocab
-    {
-        int32_t n_vocab = model.hparams.n_vocab;
-        fin.read((char *) &n_vocab, sizeof(n_vocab));
-
-        if (n_vocab != model.hparams.n_vocab) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
-                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
-            return false;
-        }
-
-        std::string word;
-        for (int i = 0; i < n_vocab; i++) {
-            uint32_t len;
-            fin.read((char *) &len, sizeof(len));
-            bool special = false;
-            if (len & (1<<31)) {
-                len = len &~ (1<<31);
-                special = true;
-            }
-
-            if (len > 0) {
-                word.resize(len);
-                fin.read((char *) word.data(), len);
-                vocab.token_to_id[word] = i;
-                vocab.id_to_token[i] = word;
-            }
-
-            if(special) {
-                vocab.add_special_token(word);
-            }
-        }
-    }
-
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = GGML_TYPE_COUNT;
-    switch (model.hparams.f16) {
-        case 0: wtype = GGML_TYPE_F32;  break;
-        case 1: wtype = GGML_TYPE_F16;  break;
-        case 2: wtype = GGML_TYPE_Q4_0; break;
-        case 3: wtype = GGML_TYPE_Q4_1; break;
-        case 5: wtype = GGML_TYPE_Q4_2; break;
-        default:
-                {
-                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
-                            __func__, fname.c_str(), model.hparams.f16);
-                    return false;
-                }
-    }
-
-    auto & ctx = model.ctx;
-
-    size_t ctx_size = 0;
-
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_ctx   = hparams.n_ctx;
-        const int n_vocab = hparams.n_vocab;
-        const int expand  = hparams.expand;
-
-
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_w
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(GGML_TYPE_F32); // wte
-
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_1_w
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_2_w
-
-        ctx_size += n_layer*(3*n_embd*n_embd*ggml_type_sizef(wtype)); // attn_Wqkv_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // attn_out_proj_w
-
-        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype));  // ffn_up_proj_w
-        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype)); // ffn_down_proj_w
-
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_k
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_v
-
-        // TODO probably less now?
-        ctx_size += (5 + 10*n_layer)*256; // object overhead
-
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
-    }
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size   = ctx_size,
-            .mem_buffer = NULL,
-            .no_alloc   = false,
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx) {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            return false;
-        }
-    }
-
-    // prepare memory for the weights
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_vocab = hparams.n_vocab;
-        const int expand  = hparams.expand;
-
-        model.layers.resize(n_layer);
-
-        model.wte    = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab);
-        model.norm_f_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-        // map by name
-        model.tensors["transformer.wte.weight"] = model.wte;
-        model.tensors["transformer.norm_f.weight"] = model.norm_f_w;
-
-        for (int i = 0; i < n_layer; ++i) {
-            auto & layer = model.layers[i];
-
-            layer.norm_1_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.norm_2_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            layer.attn_Wqkv_w     = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd * 3);
-            layer.attn_out_proj_w = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd);
-            layer.ffn_up_proj_w   = ggml_new_tensor_2d(ctx, wtype,        n_embd, expand*n_embd);
-            layer.ffn_down_proj_w = ggml_new_tensor_2d(ctx, wtype, expand*n_embd, n_embd);
-
-            // map by name
-            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_1.weight"]        = layer.norm_1_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_2.weight"]        = layer.norm_2_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.Wqkv.weight"]     = layer.attn_Wqkv_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.out_proj.weight"] = layer.attn_out_proj_w;
-
-            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.up_proj.weight"]   = layer.ffn_up_proj_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.down_proj.weight"] = layer.ffn_down_proj_w;
-        }
-    }
-
-    // key + value memory
-    {
-        const auto & hparams = model.hparams;
-        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
-            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
-            ggml_free(ctx);
-            return false;
-        }
-
-        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
-        printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
-    }
-
-    // load weights
-    {
-        int n_tensors = 0;
-        size_t total_size = 0;
-
-        printf("%s: ", __func__);
-
-        while (true) {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ttype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
-
-            if (fin.eof()) {
-                break;
-            }
-
-            int32_t nelements = 1;
-            int32_t ne[2] = { 1, 1 };
-            for (int i = 0; i < n_dims; ++i) {
-                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-
-            if (model.tensors.find(name.data()) == model.tensors.end()) {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                return false;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                return false;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
-                        __func__, name.data(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
-                return false;
-            }
-
-            // for debugging
-            if (0) {
-                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
-            }
-
-            const size_t bpe = ggml_type_size(ggml_type(ttype));
-
-            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
-                return false;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ttype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
-            total_size += ggml_nbytes(tensor);
-            if (++n_tensors % 8 == 0) {
-                printf(".");
-                fflush(stdout);
-            }
-        }
-
-        printf(" done\n");
-
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
-    }
-
-    return true;
-}
-
-// load the model's weights from a file path
-bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vocab) {
-
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin) {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
-        return false;
-    }
-
-    bool loaded = mpt_model_load(fname, fin, model, vocab);
-    fin.close();
-    return loaded;
-}
-
-bool mpt_eval(
-        mpt_model & model,
-        const int n_threads,
-        const int n_past,
-        const std::vector<int>           & embd_inp,
-              std::vector<float>         & embd_w,
-              size_t                     & mem_per_token) {
-    const int N = embd_inp.size();
-
-    const auto & hparams = model.hparams;
-
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_ctx   = hparams.n_ctx;
-    const int n_head  = hparams.n_head;
-    const int n_vocab = hparams.n_vocab;
-
-    const size_t init_buf_size = 1024u*MB;
-    if (!model.buf.addr || model.buf.size < init_buf_size)
-        model.buf.resize(init_buf_size);
-
-    if (mem_per_token > 0 && mem_per_token*N > model.buf.size) {
-        const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
-        // printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
-
-        // reallocate
-        model.buf.resize(buf_size_new);
-        if (model.buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.buf.size);
-            return false;
-        }
-    }
-
-    struct ggml_init_params params = {
-        .mem_size   = model.buf.size,
-        .mem_buffer = model.buf.addr,
-        .no_alloc = false
-    };
-
-    struct ggml_context * ctx0 = ggml_init(params);
-    struct ggml_cgraph gf = {};
-    gf.n_threads = n_threads;
-
-    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
-
-    // wte
-    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
-
-    for (int il = 0; il < n_layer; ++il) {
-
-        struct ggml_tensor * inpSA = inpL;
-        struct ggml_tensor * cur = inpSA;
-        // self-attention
-        {
-
-            // norm1
-            cur = ggml_norm(ctx0, cur);
-            cur = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].norm_1_w, cur),
-                    cur);
-            // compute QKV
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].attn_Wqkv_w,
-                    cur);
-
-            // TODO: clip_qkv
-            struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 0*ggml_element_size(cur)*n_embd));
-            struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 1*ggml_element_size(cur)*n_embd));
-            struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 2*ggml_element_size(cur)*n_embd));
-
-            // TODO: qk_ln? (seems to be False in MPT-7B configs)
-            {
-                Vcur = ggml_transpose(ctx0, Vcur);
-
-                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
-                                        (   n_ctx)*ggml_element_size(model.kv_self.v),
-                                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
-
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
-            }
-            // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q =
-                ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0, Qcur, n_embd/n_head, n_head, N),
-                        0, 2, 1, 3);
-
-            struct ggml_tensor * K =
-                ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0,
-                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                            n_embd/n_head, n_head, n_past + N),
-                        0, 2, 1, 3);
-
-            // K * Q
-            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
-
-            // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled =
-                ggml_scale(ctx0,
-                        KQ,
-                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
-                        );
-
-
-            // Alibi
-            struct ggml_tensor * KQ_scaled_biased = ggml_alibi(ctx0, ggml_cont(ctx0, KQ_scaled), n_past, n_head);
-
-            // KQ_masked = mask_past(KQ_scaled)
-            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled_biased, n_past);
-
-            // KQ = soft_max(KQ_masked)
-            struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
-
-            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V =
-                ggml_view_3d(ctx0, model.kv_self.v,
-                        n_past + N, n_embd/n_head, n_head,
-                        n_ctx*ggml_element_size(model.kv_self.v),
-                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
-                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
-
-            // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
-
-            // KQV_merged = KQV.permute(0, 2, 1, 3)
-            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            // cur = KQV_merged.contiguous().view(n_embd, N)
-            cur = ggml_cpy(ctx0,
-                    KQV_merged,
-                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-            // projection (no bias)
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].attn_out_proj_w,
-                    cur);
-        }
-
-
-        // residual
-        struct ggml_tensor * resSA = ggml_add(ctx0, cur, inpSA);
-        // feed-forward network
-        {
-            cur = resSA;
-            // norm2
-            cur = ggml_norm(ctx0, cur);
-            cur = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].norm_2_w, cur),
-                    cur);
-            // ffn
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].ffn_up_proj_w,
-                    cur);
-            cur = ggml_gelu(ctx0, cur);
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].ffn_down_proj_w,
-                    cur);
-
-        }
-
-        // self-attention + FF
-        inpL = ggml_add(ctx0, cur, resSA);
-    }
-
-    struct ggml_tensor * out = inpL;
-    // -> logits
-    {
-        out = ggml_norm(ctx0, out);
-        out = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.norm_f_w, out),
-                    out);
-        out = ggml_mul_mat(ctx0, model.wte, out);
-    }
-
-
-    // run the computation
-    ggml_build_forward_expand(&gf, out);
-    ggml_graph_compute       (ctx0, &gf);
-
-
-    // return result for just the last token
-    embd_w.resize(n_vocab);
-    memcpy(embd_w.data(), (float *) ggml_get_data(out) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
-
-    if (mem_per_token == 0) {
-        mem_per_token = ggml_used_mem(ctx0)/N;
-    }
-    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
-
-    ggml_free(ctx0);
-
-    return true;
-}
-
-
-#define MPT_MAX_RNG_STATE 64*1024
-
-size_t mpt_get_state_size(const mpt_model &model)
-{
-    // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state.
-    // for reference, std::mt19937(1337) serializes to 6701 bytes.
-    const size_t s_rng_size        = sizeof(size_t);
-    const size_t s_rng             = MPT_MAX_RNG_STATE;
-    const size_t s_kv_size         = sizeof(size_t);
-    const size_t s_kv_ntok         = sizeof(int);
-    const size_t s_kv              = model.kv_self.buf.size;
-    const size_t s_total = (
-        + s_rng_size
-        + s_rng
-        + s_kv_size
-        + s_kv_ntok
-        + s_kv
-    );
-    fflush(stdout);
-    return s_total;
-}
-
-size_t mpt_copy_state_data(const mpt_model &model, const std::mt19937 &rng, uint8_t *dest)
-{
-    uint8_t * out = dest;
-    fflush(stdout);
-    // copy rng
-    {
-        std::stringstream rng_ss;
-        rng_ss << rng;
-
-        const size_t rng_size = rng_ss.str().size();
-        char rng_buf[MPT_MAX_RNG_STATE];
-
-        memset(&rng_buf[0], 0, MPT_MAX_RNG_STATE);
-        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
-
-        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
-        memcpy(out, &rng_buf[0], MPT_MAX_RNG_STATE); out += MPT_MAX_RNG_STATE;
-    }
-
-    // copy kv cache
-    {
-        const size_t kv_size = model.kv_self.buf.size;
-        const int    kv_ntok = model.kv_self.n;
-
-        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
-        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
-
-        if (kv_size) {
-            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
-        }
-    }
-
-    const size_t written  = out - dest;
-    assert(written == mpt_get_state_size(model));
-    fflush(stdout);
-    return written;
-}
-
-size_t mpt_set_state_data(mpt_model *model, std::mt19937 *rng, const uint8_t *src)
-{
-    const uint8_t * in = src;
-
-    // set rng
-    {
-        size_t rng_size;
-        char   rng_buf[MPT_MAX_RNG_STATE];
-
-        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
-        memcpy(&rng_buf[0], in, MPT_MAX_RNG_STATE); in += MPT_MAX_RNG_STATE;
-
-        std::stringstream rng_ss;
-        rng_ss.str(std::string(&rng_buf[0], rng_size));
-        rng_ss >> *rng;
-
-        assert(rng_ss.fail() == false);
-    }
-
-    // set kv cache
-    {
-        size_t kv_size;
-        int kv_ntok;
-
-        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
-        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
-
-        if (kv_size) {
-            assert(model->kv_self.buf.size == kv_size);
-
-            void * k_data = model->kv_self.k->data; // remember data pointers
-            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
-
-            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
-
-            model->kv_self.k->data = k_data; // restore correct data pointers
-            model->kv_self.v->data = v_data;
-
-        }
-
-        model->kv_self.n = kv_ntok;
-    }
-
-    const size_t nread    = in - src;
-    assert(nread == mpt_get_state_size(*model));
-    fflush(stdout);
-    return nread;
-}
-
-struct MPTPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    gpt_vocab vocab;
-    mpt_model *model = nullptr;
-    int64_t n_threads = 0;
-    size_t mem_per_token = 0;
-    std::mt19937 rng;
-    bool has_im_end = false;
-};
-
-MPT::MPT()
-    : d_ptr(new MPTPrivate) {
-    d_ptr->model = new mpt_model;
-    d_ptr->modelLoaded = false;
-}
-
-bool MPT::loadModel(const std::string &modelPath) {
-    std::mt19937 rng(time(NULL));
-    d_ptr->rng = rng;
-
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-
-    // load the model
-    if (!mpt_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab)) {
-        std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
-        return false;
-    }
-
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    d_ptr->has_im_end = d_ptr->vocab.token_to_id.find("<|im_end|>") != d_ptr->vocab.token_to_id.end();
-    fflush(stdout);
-    return true;
-}
-
-void MPT::setThreadCount(int32_t n_threads) {
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t MPT::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-MPT::~MPT()
-{
-    delete d_ptr->model;
-}
-
-bool MPT::isModelLoaded() const
-{
-    return d_ptr->modelLoaded;
-}
-
-size_t MPT::stateSize() const
-{
-    return mpt_get_state_size(*d_ptr->model);
-}
-
-size_t MPT::saveState(uint8_t *dest) const
-{
-    return mpt_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
-}
-
-size_t MPT::restoreState(const uint8_t *src)
-{
-    return mpt_set_state_data(d_ptr->model, &d_ptr->rng, src);
-}
-
-std::vector<LLModel::Token> MPT::tokenize(PromptContext &, const std::string &str) const
-{
-    return ::gpt_tokenize(d_ptr->vocab, str);
-}
-
-std::string_view MPT::tokenToString(Token id) const
-{
-    return d_ptr->vocab.id_to_token[id];
-}
-
-LLModel::Token MPT::sampleToken(PromptContext &promptCtx) const
-{
-    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
-    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
-        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks,
-        promptCtx.logits,
-        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty,
-        d_ptr->rng);
-}
-
-bool MPT::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    // determine the required inference memory per token:
-    static bool initialized = false;
-    if (!initialized) {
-        mpt_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
-            d_ptr->mem_per_token);
-        initialized = true;
-    }
-
-    return mpt_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
-}
-
-int32_t MPT::contextLength() const
-{
-    return d_ptr->model->hparams.n_ctx;
-}
-
-const std::vector<LLModel::Token> &MPT::endTokens() const
-{
-    static const std::vector<LLModel::Token> fres = {0, d_ptr->vocab.token_to_id["<|im_end|>"]};
-    return fres;
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    return magic == 0x67676d6d;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new MPT;
-}
-}

gpt4all-backend/mpt_impl.h

@@ -1,38 +0,0 @@
-#ifndef MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of mpt.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef MPT_H
-#define MPT_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include "llmodel.h"
-
-struct MPTPrivate;
-class MPT : public LLModel {
-public:
-    MPT();
-    ~MPT();
-
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    MPTPrivate *d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string_view tokenToString(Token) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // MPT_H

gpt4all-backend/scripts/convert_mpt_hf_to_ggml.py

@@ -1,145 +0,0 @@
-# Convert Hugging Face fine-tuned bloom-like models to ggml format
-#
-# Usage:
-#
-#   python3 models/convert-h5-to-ggml.py 
-#
-# This script is similar to "convert-pt-to-ggml.py"
-#
-
-import io
-import os
-import sys
-import struct
-import json
-import code
-import torch
-import numpy as np
-
-from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BloomForCausalLM
-
-# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
-def bytes_to_unicode():
-    """
-    Returns list of utf-8 byte and a corresponding list of unicode strings.
-    The reversible bpe codes work on unicode strings.
-    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
-    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
-    This is a significant percentage of your normal, say, 32K bpe vocab.
-    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
-    And avoids mapping to whitespace/control characters the bpe code barfs on.
-    """
-    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
-    cs = bs[:]
-    n = 0
-    for b in range(2**8):
-        if b not in bs:
-            bs.append(b)
-            cs.append(2**8+n)
-            n += 1
-    cs = [chr(n) for n in cs]
-    return dict(zip(bs, cs))
-
-if len(sys.argv) < 3:
-    print("Usage: python convert-hf-to-ggml.py model_name dir-output [use-f32]")
-    print("  model_name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
-    print("  dir-output: directory where the output file will be written")
-    print("  use-f32:    if present, use float32 instead of float16")
-    sys.exit(1)
-
-model_name = sys.argv[1]
-dir_out = sys.argv[2]
-
-# make sure the output directory exists
-os.makedirs(dir_out, exist_ok=True)
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-ftype = 1
-if len(sys.argv) > 3:
-    ftype = 0
-
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
-hparams = config.to_dict()
-print("Loading model: ", model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32, low_cpu_mem_usage=True)
-print("Model loaded: ", model_name)
-
-
-fname_out = dir_out + f"/ggml-model-{model_name.split('/')[-1]}-{ftype_str[ftype]}.bin"
-fout = open(fname_out, "wb")
-vocab = tokenizer.vocab
-
-hparams["multiple_of"] = 1
-fout.write(struct.pack("I", 0x67676d6d)) # magic: ggml in hex
-fout.write(struct.pack("I", model.config.vocab_size))
-fout.write(struct.pack("I", model.config.max_seq_len))
-fout.write(struct.pack("I", model.config.n_layers))
-fout.write(struct.pack("I", model.config.n_heads))
-fout.write(struct.pack("I", model.config.d_model))
-fout.write(struct.pack("f", model.config.attn_config['alibi_bias_max']))
-clip_qkv = model.config.attn_config['clip_qkv']
-fout.write(struct.pack("f",  clip_qkv if clip_qkv is not None else 0))
-fout.write(struct.pack("I", ftype))
-
-# # Is this correct??
-# dot_token = tokenizer.encode(".")[0]
-# write tokens to ggml file 
-dot_token = tokenizer.encode('.')[0]
-fout.write(struct.pack("I", model.config.vocab_size))
-
-for i in range(model.config.vocab_size):
-    text = tokenizer.decode([dot_token, i]).encode('utf-8')
-    # remove the first byte (it's always '.')
-    text = text[1:]
-    enclen = len(text)
-    if i in tokenizer.all_special_ids:
-        print(f"special token: {text}")
-        enclen = enclen | 1<<31
-    fout.write(struct.pack("I", enclen))
-    fout.write(text)
-    
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable: " + name + " with shape: ", data.shape)
-
-    n_dims = len(data.shape);
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0;
-    if ftype != 0:
-        # Keep token embeddings in fp32
-        if name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
-            print("  Converting to float16")
-            data = data.astype(np.float16)
-            ftype_cur = 1
-        else:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-    else:
-        if data.dtype != np.float32:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-
-    # header
-    str = name.encode('utf-8')
-    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str);
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-backend/sysinfo.h

@@ -0,0 +1,65 @@
+#ifndef SYSINFO_H
+#define SYSINFO_H
+
+#include <fstream>
+#include <iomanip>
+#include <sstream>
+#include <string>
+
+#if defined(__linux__)
+#   include <unistd.h>
+#elif defined(__APPLE__)
+#   include <sys/types.h>
+#   include <sys/sysctl.h>
+#elif defined(_WIN32)
+#   define WIN32_LEAN_AND_MEAN
+#   ifndef NOMINMAX
+#       define NOMINMAX
+#   endif
+#   include <windows.h>
+#endif
+
+static long long getSystemTotalRAMInBytes()
+{
+    long long totalRAM = 0;
+
+#if defined(__linux__)
+    std::ifstream file("/proc/meminfo");
+    std::string line;
+    while (std::getline(file, line)) {
+        if (line.find("MemTotal") != std::string::npos) {
+            std::string memTotalStr = line.substr(line.find(":") + 1);
+            memTotalStr.erase(0, memTotalStr.find_first_not_of(" "));
+            memTotalStr = memTotalStr.substr(0, memTotalStr.find(" "));
+            totalRAM = std::stoll(memTotalStr) * 1024;  // Convert from KB to bytes
+            break;
+        }
+    }
+    file.close();
+#elif defined(__APPLE__)
+    int mib[2] = {CTL_HW, HW_MEMSIZE};
+    size_t length = sizeof(totalRAM);
+    sysctl(mib, 2, &totalRAM, &length, NULL, 0);
+#elif defined(_WIN32)
+    MEMORYSTATUSEX memoryStatus;
+    memoryStatus.dwLength = sizeof(memoryStatus);
+    GlobalMemoryStatusEx(&memoryStatus);
+    totalRAM = memoryStatus.ullTotalPhys;
+#endif
+
+    return totalRAM;
+}
+
+static double getSystemTotalRAMInGB()
+{
+    return static_cast<double>(getSystemTotalRAMInBytes()) / (1024 * 1024 * 1024);
+}
+
+static std::string getSystemTotalRAMInGBString()
+{
+    std::stringstream ss;
+    ss << std::fixed << std::setprecision(2) << getSystemTotalRAMInGB() << " GB";
+    return ss.str();
+}
+
+#endif // SYSINFO_H

gpt4all-backend/utils.cpp

@@ -1,9 +1,15 @@
 #include "utils.h"
 
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
 #include <fstream>
+#include <iterator>
 #include <regex>
+#include <utility>
 
-void replace(std::string & str, const std::string & needle, const std::string & replacement) {
+void replace(std::string & str, const std::string & needle, const std::string & replacement)
+{
     size_t pos = 0;
     while ((pos = str.find(needle, pos)) != std::string::npos) {
         str.replace(pos, needle.length(), replacement);
@@ -11,7 +17,8 @@ void replace(std::string & str, const std::string & needle, const std::string &
     }
 }
 
-std::map<std::string, int32_t> json_parse(const std::string & fname) {
+std::map<std::string, int32_t> json_parse(const std::string & fname)
+{
     std::map<std::string, int32_t> result;
 
     // read file into string
@@ -102,7 +109,8 @@ std::map<std::string, int32_t> json_parse(const std::string & fname) {
     return result;
 }
 
-std::vector<gpt_vocab::id> gpt_tokenize_inner(const gpt_vocab & vocab, const std::string & text) {
+std::vector<gpt_vocab::id> gpt_tokenize_inner(const gpt_vocab & vocab, const std::string & text)
+{
     std::vector<std::string> words;
 
     // first split the text into words
@@ -157,12 +165,14 @@ std::vector<gpt_vocab::id> gpt_tokenize_inner(const gpt_vocab & vocab, const std
     return tokens;
 }
 
-std::string regex_escape(const std::string &s) {
+std::string regex_escape(const std::string &s)
+{
   static const std::regex metacharacters(R"([\.\^\$\-\+\(\)\[\]\{\}\|\?\*])");
   return std::regex_replace(s, metacharacters, "\\$&");
 }
 
-std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {
+std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text)
+{
     // Generate the subpattern from the special_tokens vector if it's not empty
     if (!vocab.special_tokens.empty()) {
         std::vector<gpt_vocab::id> out;
@@ -198,7 +208,8 @@ std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::stri
 }
 
 
-bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {
+bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab)
+{
     printf("%s: loading vocab from '%s'\n", __func__, fname.c_str());
 
     vocab.token_to_id = ::json_parse(fname);
@@ -230,8 +241,21 @@ gpt_vocab::id gpt_sample_top_k_top_p(
     int n_logits = actualVocabSize;
 
     const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_size);
-    const auto * plogits = logits.data() + logits.size() - n_logits;
+    const auto * plogits = logits.data();
 
+    if (temp <= 0) {
+        // select the token with the highest logit directly
+        float max_logit = plogits[0];
+        gpt_vocab::id max_id = 0;
+
+        for (int i = 1; i < n_logits; ++i) {
+            if (plogits[i] > max_logit) {
+                max_logit = plogits[i];
+                max_id = i;
+            }
+        }
+        return max_id;
+    }
     std::vector<std::pair<double, gpt_vocab::id>> logits_id;
     logits_id.reserve(n_logits);
 
@@ -312,4 +336,4 @@ gpt_vocab::id gpt_sample_top_k_top_p(
     int idx = dist(rng);
 
     return logits_id[idx].second;
-}
+}

gpt4all-backend/utils.h

@@ -2,11 +2,22 @@
 
 #pragma once
 
-#include <string>
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
 #include <map>
-#include <vector>
 #include <random>
+#include <string>
 #include <thread>
+#include <vector>
+
+//
+// General purpose inline functions
+//
+constexpr inline unsigned long long operator ""_MiB(unsigned long long bytes)
+{
+    return bytes*1024*1024;
+}
 
 //
 // CLI argument parsing

gpt4all-bindings/README.md

@@ -1,3 +1,21 @@
-# GPT4All Bindings
-This directory will contain language specific bindings on top of the C/C++ model backends.
-We will have one directory per language binding (e.g. Python, Typescript, Golang, etc.).
+# GPT4All Language Bindings
+These are the language bindings for the GPT4All backend. They provide functionality to load GPT4All models (and other llama.cpp models), generate text, and (in the case of the Python bindings) embed text as a vector representation.
+
+See their respective folders for language-specific documentation.
+
+### Languages
+- [Python](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python) (Nomic official, maintained by [@cebtenzzre](https://github.com/cebtenzzre))
+- [Node.js/Typescript](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript) (community, maintained by [@jacoobes](https://github.com/jacoobes) and [@iimez](https://github.com/iimez))
+
+<br/>
+<br/>
+
+<details><summary><b>Archived Bindings</b></summary>
+<br/>
+
+The following bindings have been removed from this repository due to lack of maintenance. If adopted, they can be brought back&mdash;feel free to message a developer on Dicsord if you are interested in maintaining one of them. Below are links to their last available version (not necessarily the last working version).
+- C#: [41c9013f](https://github.com/nomic-ai/gpt4all/tree/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-bindings/csharp)
+- Java: [41c9013f](https://github.com/nomic-ai/gpt4all/tree/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-bindings/java)
+- Go: [41c9013f](https://github.com/nomic-ai/gpt4all/tree/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-bindings/golang)
+
+</details>

gpt4all-bindings/cli/README.md

@@ -0,0 +1,43 @@
+# GPT4All Command-Line Interface (CLI)
+
+GPT4All on the command-line.
+
+More details on the [wiki](https://github.com/nomic-ai/gpt4all/wiki/Python-CLI).
+
+## Quickstart
+
+The CLI is based on the `gpt4all` Python bindings and the `typer` package.
+
+The following shows one way to get started with the CLI, the documentation has more information.
+Typically, you will want to replace `python` with `python3` on _Unix-like_ systems and `py -3` on
+_Windows_. Also, it's assumed you have all the necessary Python components already installed.
+
+The CLI is a self-contained Python script named [app.py] ([download][app.py-download]). As long as
+its package dependencies are present, you can download and run it from wherever you like.
+
+[app.py]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-bindings/cli/app.py
+[app.py-download]: https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-bindings/cli/app.py
+
+```shell
+# optional but recommended: create and use a virtual environment
+python -m venv gpt4all-cli
+```
+_Windows_ and _Unix-like_ systems differ slightly in how you activate a _virtual environment_:
+- _Unix-like_, typically: `. gpt4all-cli/bin/activate`
+- _Windows_: `gpt4all-cli\Scripts\activate`
+
+Then:
+```shell
+# pip-install the necessary packages; omit '--user' if using a virtual environment
+python -m pip install --user --upgrade gpt4all typer
+# run the CLI
+python app.py repl
+```
+By default, it will automatically download the `Mistral Instruct` model to `.cache/gpt4all/` in your
+user directory, if necessary.
+
+If you have already saved a model beforehand, specify its path with the `-m`/`--model` argument,
+for example:
+```shell
+python app.py repl --model /home/user/my-gpt4all-models/mistral-7b-instruct-v0.1.Q4_0.gguf
+```

gpt4all-bindings/cli/app.py

@@ -1,9 +1,20 @@
-import sys
-import typer
+#!/usr/bin/env python3
+"""GPT4All CLI
 
+The GPT4All CLI is a self-contained script based on the `gpt4all` and `typer` packages. It offers a
+REPL to communicate with a language model similar to the chat GUI application, but more basic.
+"""
+
+import importlib.metadata
+import io
+import sys
+from collections import namedtuple
 from typing_extensions import Annotated
+
+import typer
 from gpt4all import GPT4All
 
+
 MESSAGES = [
     {"role": "system", "content": "You are a helpful assistant."},
     {"role": "user", "content": "Hello there."},
@@ -17,7 +28,9 @@ SPECIAL_COMMANDS = {
     "/help": lambda _: print("Special commands: /reset, /exit, /help and /clear"),
 }
 
-VERSION = "0.1.0"
+VersionInfo = namedtuple('VersionInfo', ['major', 'minor', 'micro'])
+VERSION_INFO = VersionInfo(1, 0, 2)
+VERSION = '.'.join(map(str, VERSION_INFO))  # convert to string form, like: '1.2.3'
 
 CLI_START_MESSAGE = f"""
     
@@ -33,12 +46,6 @@ Type /help for special commands.
                                                     
 """
 
-def _cli_override_response_callback(token_id, response):
-    resp = response.decode("utf-8")
-    print(resp, end="", flush=True)
-    return True
-
-
 # create typer app
 app = typer.Typer()
 
@@ -47,13 +54,18 @@ def repl(
     model: Annotated[
         str,
         typer.Option("--model", "-m", help="Model to use for chatbot"),
-    ] = "ggml-gpt4all-j-v1.3-groovy",
+    ] = "mistral-7b-instruct-v0.1.Q4_0.gguf",
     n_threads: Annotated[
         int,
         typer.Option("--n-threads", "-t", help="Number of threads to use for chatbot"),
     ] = None,
+    device: Annotated[
+        str,
+        typer.Option("--device", "-d", help="Device to use for chatbot, e.g. gpu, amd, nvidia, intel. Defaults to CPU."),
+    ] = None,
 ):
-    gpt4all_instance = GPT4All(model)
+    """The CLI read-eval-print loop."""
+    gpt4all_instance = GPT4All(model, device=device)
 
     # if threads are passed, set them
     if n_threads is not None:
@@ -68,11 +80,23 @@ def repl(
     else:
         print(f"\nUsing {gpt4all_instance.model.thread_count()} threads", end="")
 
-    # overwrite _response_callback on model
-    gpt4all_instance.model._response_callback = _cli_override_response_callback
-
     print(CLI_START_MESSAGE)
 
+    use_new_loop = False
+    try:
+        version = importlib.metadata.version('gpt4all')
+        version_major = int(version.split('.')[0])
+        if version_major >= 1:
+            use_new_loop = True
+    except:
+        pass  # fall back to old loop
+    if use_new_loop:
+        _new_loop(gpt4all_instance)
+    else:
+        _old_loop(gpt4all_instance)
+
+
+def _old_loop(gpt4all_instance):
     while True:
         message = input(" ⇢  ")
 
@@ -96,6 +120,7 @@ def repl(
             n_predict=200,
             top_k=40,
             top_p=0.9,
+            min_p=0.0,
             temp=0.9,
             n_batch=9,
             repeat_penalty=1.1,
@@ -103,16 +128,59 @@ def repl(
             context_erase=0.0,
             # required kwargs for cli ux (incremental response)
             verbose=False,
-            std_passthrough=True,
+            streaming=True,
         )
         # record assistant's response to messages
         MESSAGES.append(full_response.get("choices")[0].get("message"))
         print() # newline before next prompt
 
 
+def _new_loop(gpt4all_instance):
+    with gpt4all_instance.chat_session():
+        while True:
+            message = input(" ⇢  ")
+
+            # Check if special command and take action
+            if message in SPECIAL_COMMANDS:
+                SPECIAL_COMMANDS[message](MESSAGES)
+                continue
+
+            # if regular message, append to messages
+            MESSAGES.append({"role": "user", "content": message})
+
+            # execute chat completion and ignore the full response since 
+            # we are outputting it incrementally
+            response_generator = gpt4all_instance.generate(
+                message,
+                # preferential kwargs for chat ux
+                max_tokens=200,
+                temp=0.9,
+                top_k=40,
+                top_p=0.9,
+                min_p=0.0,
+                repeat_penalty=1.1,
+                repeat_last_n=64,
+                n_batch=9,
+                # required kwargs for cli ux (incremental response)
+                streaming=True,
+            )
+            response = io.StringIO()
+            for token in response_generator:
+                print(token, end='', flush=True)
+                response.write(token)
+
+            # record assistant's response to messages
+            response_message = {'role': 'assistant', 'content': response.getvalue()}
+            response.close()
+            gpt4all_instance.current_chat_session.append(response_message)
+            MESSAGES.append(response_message)
+            print() # newline before next prompt
+
+
 @app.command()
 def version():
-    print("gpt4all-cli v0.1.0")
+    """The CLI version command."""
+    print(f"gpt4all-cli v{VERSION}")
 
 
 if __name__ == "__main__":

gpt4all-bindings/cli/developer_notes.md

@@ -0,0 +1,25 @@
+# Developing the CLI
+## Documentation
+Documentation can be found in three places:
+- `app.py` docstrings & comments
+- a Readme: `gpt4all-bindings/cli/README.md`
+- the actual CLI documentation: `gpt4all-bindings/python/docs/gpt4all_cli.md`
+
+The _docstrings_ are meant for programmatic use. Since the CLI is primarily geared towards users and
+not to build on top, they're kept terse.
+
+The _Readme_ is mostly meant for users and includes:
+- a link to the _CLI documentation_ (on the [website])
+- a Quickstart section with some guidance on how to get started with a sane setup
+
+The _CLI documentation_ and other documentation are located in the above mentioned `docs/` folder.
+They're in Markdown format and built for the [website]. Of the three, they should be the most
+detailed.
+
+[website]: https://docs.gpt4all.io/gpt4all_cli.html
+
+
+## Versioning
+The version number should now follow the `gpt4all` PyPI package, so compatibility is more clear.
+
+The one place to change it is the `namedtuple` called `VERSION_INFO`.

gpt4all-bindings/csharp/.editorconfig

@@ -1,346 +0,0 @@
-# EditorConfig is awesome: https://EditorConfig.org
-
-# top-most EditorConfig file
-root = true
-
-# Don't use tabs for indentation.
-[*]
-indent_style = space
-# (Please don't specify an indent_size here; that has too many unintended consequences.)
-
-# Code files
-[*.{cs,csx,vb,vbx}]
-indent_size = 4
-insert_final_newline = true
-charset = utf-8-bom
-
-# XML project files
-[*.{csproj,vbproj,vcxproj,vcxproj.filters,proj,projitems,shproj}]
-indent_size = 4
-
-# XML config files
-[*.{props,targets,ruleset,config,nuspec,resx,vsixmanifest,vsct}]
-indent_size = 2
-
-# JSON files
-[*.json]
-indent_size = 2
-
-# Powershell files
-[*.ps1]
-indent_size = 2
-
-# Shell script files
-[*.sh]
-end_of_line = lf
-indent_size = 2
-insert_final_newline = true
-
-# Dotnet code style settings:
-[*.{cs,vb}]
-
-# IDE0055: Fix formatting
-dotnet_diagnostic.IDE0055.severity = error
-
-# Sort using and Import directives with System.* appearing first
-dotnet_sort_system_directives_first = true
-dotnet_separate_import_directive_groups = false
-
-# Avoid "this." and "Me." if not necessary
-dotnet_style_qualification_for_field = false:suggestion
-dotnet_style_qualification_for_property = false:suggestion
-dotnet_style_qualification_for_method = false:suggestion
-dotnet_style_qualification_for_event = false:suggestion
-
-# Use language keywords instead of framework type names for type references
-dotnet_style_predefined_type_for_locals_parameters_members = true:warning
-dotnet_style_predefined_type_for_member_access = true:warning
-
-# Suggest more modern language features when available
-dotnet_style_object_initializer = true:suggestion
-dotnet_style_collection_initializer = true:suggestion
-dotnet_style_coalesce_expression = true:suggestion
-dotnet_style_null_propagation = true:suggestion
-dotnet_style_explicit_tuple_names = true:suggestion
-
-# Whitespace options
-dotnet_style_allow_multiple_blank_lines_experimental = false
-
-# Private fields are camelCase with '_' prefix
-dotnet_naming_rule.private_members_with_underscore.symbols  = private_fields
-dotnet_naming_rule.private_members_with_underscore.style    = prefix_underscore
-dotnet_naming_rule.private_members_with_underscore.severity = error
-dotnet_naming_symbols.private_fields.applicable_kinds           = field
-dotnet_naming_symbols.private_fields.applicable_accessibilities = private
-dotnet_naming_style.prefix_underscore.capitalization = camel_case
-dotnet_naming_style.prefix_underscore.required_prefix = _
-
-# Non-private static fields are PascalCase
-dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.severity = suggestion
-dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.symbols = non_private_static_fields
-dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.style = non_private_static_field_style
-
-dotnet_naming_symbols.non_private_static_fields.applicable_kinds = field
-dotnet_naming_symbols.non_private_static_fields.applicable_accessibilities = public, protected, internal, protected_internal, private_protected
-dotnet_naming_symbols.non_private_static_fields.required_modifiers = static
-
-dotnet_naming_style.non_private_static_field_style.capitalization = pascal_case
-
-# Non-private readonly fields are PascalCase
-dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.severity = suggestion
-dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.symbols = non_private_readonly_fields
-dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.style = non_private_static_field_style
-
-dotnet_naming_symbols.non_private_readonly_fields.applicable_kinds = field
-dotnet_naming_symbols.non_private_readonly_fields.applicable_accessibilities = public, protected, internal, protected_internal, private_protected
-dotnet_naming_symbols.non_private_readonly_fields.required_modifiers = readonly
-
-dotnet_naming_style.non_private_readonly_field_style.capitalization = pascal_case
-
-# Constants are PascalCase
-dotnet_naming_rule.constants_should_be_pascal_case.severity = suggestion
-dotnet_naming_rule.constants_should_be_pascal_case.symbols = constants
-dotnet_naming_rule.constants_should_be_pascal_case.style = non_private_static_field_style
-
-dotnet_naming_symbols.constants.applicable_kinds = field, local
-dotnet_naming_symbols.constants.required_modifiers = const
-
-dotnet_naming_style.constant_style.capitalization = pascal_case
-
-# Static fields are camelCase and start with s_
-dotnet_naming_rule.static_fields_should_be_camel_case.severity = none
-dotnet_naming_rule.static_fields_should_be_camel_case.symbols = static_fields
-dotnet_naming_rule.static_fields_should_be_camel_case.style = static_field_style
-
-dotnet_naming_symbols.static_fields.applicable_kinds = field
-dotnet_naming_symbols.static_fields.required_modifiers = static
-
-dotnet_naming_style.static_field_style.capitalization = camel_case
-dotnet_naming_style.static_field_style.required_prefix = s_
-
-# Instance fields are camelCase and start with _
-dotnet_naming_rule.instance_fields_should_be_camel_case.severity = none
-dotnet_naming_rule.instance_fields_should_be_camel_case.symbols = instance_fields
-dotnet_naming_rule.instance_fields_should_be_camel_case.style = instance_field_style
-
-dotnet_naming_symbols.instance_fields.applicable_kinds = field
-
-dotnet_naming_style.instance_field_style.capitalization = camel_case
-dotnet_naming_style.instance_field_style.required_prefix = _
-
-# Locals and parameters are camelCase
-dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion
-dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters
-dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style
-
-dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local
-
-dotnet_naming_style.camel_case_style.capitalization = camel_case
-
-# Local functions are PascalCase
-dotnet_naming_rule.local_functions_should_be_pascal_case.severity = suggestion
-dotnet_naming_rule.local_functions_should_be_pascal_case.symbols = local_functions
-dotnet_naming_rule.local_functions_should_be_pascal_case.style = non_private_static_field_style
-
-dotnet_naming_symbols.local_functions.applicable_kinds = local_function
-
-dotnet_naming_style.local_function_style.capitalization = pascal_case
-
-# By default, name items with PascalCase
-dotnet_naming_rule.members_should_be_pascal_case.severity = suggestion
-dotnet_naming_rule.members_should_be_pascal_case.symbols = all_members
-dotnet_naming_rule.members_should_be_pascal_case.style = non_private_static_field_style
-
-dotnet_naming_symbols.all_members.applicable_kinds = *
-
-dotnet_naming_style.pascal_case_style.capitalization = pascal_case
-
-# error RS2008: Enable analyzer release tracking for the analyzer project containing rule '{0}'
-dotnet_diagnostic.RS2008.severity = none
-
-# IDE0073: File header
-dotnet_diagnostic.IDE0073.severity = none
-#file_header_template = Licensed to the .NET Foundation under one or more agreements.\nThe .NET Foundation licenses this file to you under the MIT license.\nSee the LICENSE file in the project root for more information.
-
-# IDE0035: Remove unreachable code
-dotnet_diagnostic.IDE0035.severity = warning
-
-# IDE0036: Order modifiers
-dotnet_diagnostic.IDE0036.severity = warning
-
-# IDE0043: Format string contains invalid placeholder
-dotnet_diagnostic.IDE0043.severity = warning
-
-# IDE0044: Make field readonly
-dotnet_diagnostic.IDE0044.severity = warning
-
-# IDE1006: Naming rule violation
-#dotnet_diagnostic.IDE1006.severity = none
-
-# RS0016: Only enable if API files are present
-dotnet_public_api_analyzer.require_api_files = true
-dotnet_style_operator_placement_when_wrapping = beginning_of_line
-tab_width = 4
-end_of_line = crlf
-dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion
-dotnet_style_prefer_auto_properties = true:silent
-dotnet_style_prefer_simplified_boolean_expressions = true:suggestion
-dotnet_style_prefer_conditional_expression_over_assignment = true:silent
-dotnet_style_prefer_conditional_expression_over_return = true:silent
-dotnet_style_prefer_inferred_tuple_names = true:suggestion
-dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion
-dotnet_style_prefer_compound_assignment = true:suggestion
-dotnet_style_prefer_simplified_interpolation = true:suggestion
-dotnet_style_namespace_match_folder = true:suggestion
-
-# CSharp code style settings:
-[*.cs]
-# Newline settings
-csharp_new_line_before_open_brace = all
-csharp_new_line_before_else = true
-csharp_new_line_before_catch = true
-csharp_new_line_before_finally = true
-csharp_new_line_before_members_in_object_initializers = true
-csharp_new_line_before_members_in_anonymous_types = true
-csharp_new_line_between_query_expression_clauses = true
-
-# Indentation preferences
-csharp_indent_block_contents = true
-csharp_indent_braces = false
-csharp_indent_case_contents = true
-csharp_indent_case_contents_when_block = true
-csharp_indent_switch_labels = true
-csharp_indent_labels = flush_left
-
-# Whitespace options
-csharp_style_allow_embedded_statements_on_same_line_experimental = false
-csharp_style_allow_blank_lines_between_consecutive_braces_experimental = false
-csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = false
-
-# Prefer "var" everywhere
-csharp_style_var_for_built_in_types = true:suggestion
-csharp_style_var_when_type_is_apparent = true:suggestion
-csharp_style_var_elsewhere = true:suggestion
-
-# Prefer method-like constructs to have a block body
-csharp_style_expression_bodied_methods = false:none
-csharp_style_expression_bodied_constructors = false:none
-csharp_style_expression_bodied_operators = false:none
-
-# Prefer property-like constructs to have an expression-body
-csharp_style_expression_bodied_properties = true:none
-csharp_style_expression_bodied_indexers = true:none
-csharp_style_expression_bodied_accessors = true:none
-
-# Suggest more modern language features when available
-csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion
-csharp_style_pattern_matching_over_as_with_null_check = true:suggestion
-csharp_style_inlined_variable_declaration = true:suggestion
-csharp_style_throw_expression = true:suggestion
-csharp_style_conditional_delegate_call = true:suggestion
-
-# Space preferences
-csharp_space_after_cast = false
-csharp_space_after_colon_in_inheritance_clause = true
-csharp_space_after_comma = true
-csharp_space_after_dot = false
-csharp_space_after_keywords_in_control_flow_statements = true
-csharp_space_after_semicolon_in_for_statement = true
-csharp_space_around_binary_operators = before_and_after
-csharp_space_around_declaration_statements = do_not_ignore
-csharp_space_before_colon_in_inheritance_clause = true
-csharp_space_before_comma = false
-csharp_space_before_dot = false
-csharp_space_before_open_square_brackets = false
-csharp_space_before_semicolon_in_for_statement = false
-csharp_space_between_empty_square_brackets = false
-csharp_space_between_method_call_empty_parameter_list_parentheses = false
-csharp_space_between_method_call_name_and_opening_parenthesis = false
-csharp_space_between_method_call_parameter_list_parentheses = false
-csharp_space_between_method_declaration_empty_parameter_list_parentheses = false
-csharp_space_between_method_declaration_name_and_open_parenthesis = false
-csharp_space_between_method_declaration_parameter_list_parentheses = false
-csharp_space_between_parentheses = false
-csharp_space_between_square_brackets = false
-
-# Blocks are allowed
-csharp_prefer_braces = true:silent
-csharp_preserve_single_line_blocks = true
-csharp_preserve_single_line_statements = true
-
-# Target-type new expressio
-csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion
-
-# Currently only enabled for C# due to crash in VB analyzer.  VB can be enabled once
-# https://github.com/dotnet/roslyn/pull/54259 has been published.
-dotnet_style_allow_statement_immediately_after_block_experimental = false
-dotnet_diagnostic.RCS0003.severity=warning
-dotnet_diagnostic.RCS1036.severity=error
-dotnet_diagnostic.IDE0005.severity=warning
-dotnet_diagnostic.IDE0007.severity=error
-csharp_using_directive_placement = outside_namespace:silent
-csharp_prefer_simple_using_statement = true:suggestion
-csharp_style_namespace_declarations = block_scoped:silent
-csharp_style_expression_bodied_lambdas = true:silent
-csharp_style_expression_bodied_local_functions = false:silent
-csharp_style_prefer_null_check_over_type_check = true:suggestion
-dotnet_diagnostic.RCS1075.severity = suggestion
-
-[src/CodeStyle/**.{cs,vb}]
-# warning RS0005: Do not use generic CodeAction.Create to create CodeAction
-dotnet_diagnostic.RS0005.severity = none
-
-[src/{Analyzers,CodeStyle,Features,Workspaces,EditorFeatures,VisualStudio}/**/*.{cs,vb}]
-
-# IDE0011: Add braces
-csharp_prefer_braces = when_multiline:warning
-# NOTE: We need the below severity entry for Add Braces due to https://github.com/dotnet/roslyn/issues/44201
-dotnet_diagnostic.IDE0011.severity = warning
-
-# IDE0040: Add accessibility modifiers
-dotnet_diagnostic.IDE0040.severity = warning
-
-# CONSIDER: Are IDE0051 and IDE0052 too noisy to be warnings for IDE editing scenarios? Should they be made build-only warnings?
-# IDE0051: Remove unused private member
-dotnet_diagnostic.IDE0051.severity = warning
-
-# IDE0052: Remove unread private member
-dotnet_diagnostic.IDE0052.severity = warning
-
-# IDE0059: Unnecessary assignment to a value
-dotnet_diagnostic.IDE0059.severity = warning
-
-# IDE0060: Remove unused parameter
-dotnet_diagnostic.IDE0060.severity = warning
-
-# CA1012: Abstract types should not have public constructors
-dotnet_diagnostic.CA1012.severity = warning
-
-# CA1822: Make member static
-dotnet_diagnostic.CA1822.severity = warning
-
-# Prefer "var" everywhere
-dotnet_diagnostic.IDE0007.severity = warning
-csharp_style_var_for_built_in_types = true:warning
-csharp_style_var_when_type_is_apparent = true:warning
-csharp_style_var_elsewhere = true:warning
-
-# dotnet_style_allow_multiple_blank_lines_experimental
-dotnet_diagnostic.IDE2000.severity = warning
-
-# csharp_style_allow_embedded_statements_on_same_line_experimental
-dotnet_diagnostic.IDE2001.severity = warning
-
-# csharp_style_allow_blank_lines_between_consecutive_braces_experimental
-dotnet_diagnostic.IDE2002.severity = warning
-
-# dotnet_style_allow_statement_immediately_after_block_experimental
-dotnet_diagnostic.IDE2003.severity = warning
-
-# csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental
-dotnet_diagnostic.IDE2004.severity = warning
-
-[src/{VisualStudio}/**/*.{cs,vb}]
-# CA1822: Make member static
-# There is a risk of accidentally breaking an internal API that partners rely on though IVT.
-dotnet_code_quality.CA1822.api_surface = private

gpt4all-bindings/csharp/.gitignore

@@ -1,379 +0,0 @@
-## Ignore Visual Studio temporary files, build results, and
-## files generated by popular Visual Studio add-ons.
-##
-## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
-
-runtimes
-**/*nuget
-
-*.zip
-include/
-*.exp
-*.lib
-*.dll
-
-# User-specific files
-*.rsuser
-*.suo
-*.user
-*.userosscache
-*.sln.docstates
-
-# User-specific files (MonoDevelop/Xamarin Studio)
-*.userprefs
-
-# Mono auto generated files
-mono_crash.*
-Tests/**/launchSettings.json
-
-# Build results
-[Dd]ebug/
-[Dd]ebugPublic/
-[Rr]elease/
-[Rr]eleases/
-x64/
-x86/
-[Ww][Ii][Nn]32/
-[Aa][Rr][Mm]/
-[Aa][Rr][Mm]64/
-bld/
-[Bb]in/
-[Oo]bj/
-[Oo]ut/
-[Ll]og/
-[Ll]ogs/
-
-# Visual Studio 2015/2017 cache/options directory
-.vs/
-# Uncomment if you have tasks that create the project's static files in wwwroot
-#wwwroot/
-
-# Visual Studio 2017 auto generated files
-Generated\ Files/
-
-# MSTest test Results
-[Tt]est[Rr]esult*/
-[Bb]uild[Ll]og.*
-
-# NUnit
-*.VisualState.xml
-TestResult.xml
-nunit-*.xml
-
-# Build Results of an ATL Project
-[Dd]ebugPS/
-[Rr]eleasePS/
-dlldata.c
-
-# Benchmark Results
-BenchmarkDotNet.Artifacts/
-
-# .NET Core
-project.lock.json
-project.fragment.lock.json
-artifacts/
-
-# ASP.NET Scaffolding
-ScaffoldingReadMe.txt
-
-# StyleCop
-StyleCopReport.xml
-
-# Files built by Visual Studio
-*_i.c
-*_p.c
-*_h.h
-*.ilk
-*.meta
-*.obj
-*.iobj
-*.pch
-*.pdb
-*.ipdb
-*.pgc
-*.pgd
-*.rsp
-*.sbr
-*.tlb
-*.tli
-*.tlh
-*.tmp
-*.tmp_proj
-*_wpftmp.csproj
-*.log
-*.vspscc
-*.vssscc
-.builds
-*.pidb
-*.svclog
-*.scc
-
-# Chutzpah Test files
-_Chutzpah*
-
-# Visual C++ cache files
-ipch/
-*.aps
-*.ncb
-*.opendb
-*.opensdf
-*.sdf
-*.cachefile
-*.VC.db
-*.VC.VC.opendb
-
-# Visual Studio profiler
-*.psess
-*.vsp
-*.vspx
-*.sap
-
-# Visual Studio Trace Files
-*.e2e
-
-# TFS 2012 Local Workspace
-$tf/
-
-# Guidance Automation Toolkit
-*.gpState
-
-# ReSharper is a .NET coding add-in
-_ReSharper*/
-*.[Rr]e[Ss]harper
-*.DotSettings.user
-
-# TeamCity is a build add-in
-_TeamCity*
-
-# DotCover is a Code Coverage Tool
-*.dotCover
-
-# AxoCover is a Code Coverage Tool
-.axoCover/*
-!.axoCover/settings.json
-
-# Coverlet is a free, cross platform Code Coverage Tool
-coverage*.json
-coverage*.xml
-coverage*.info
-
-# Visual Studio code coverage results
-*.coverage
-*.coveragexml
-
-# NCrunch
-_NCrunch_*
-.*crunch*.local.xml
-nCrunchTemp_*
-
-# MightyMoose
-*.mm.*
-AutoTest.Net/
-
-# Web workbench (sass)
-.sass-cache/
-
-# Installshield output folder
-[Ee]xpress/
-
-# DocProject is a documentation generator add-in
-DocProject/buildhelp/
-DocProject/Help/*.HxT
-DocProject/Help/*.HxC
-DocProject/Help/*.hhc
-DocProject/Help/*.hhk
-DocProject/Help/*.hhp
-DocProject/Help/Html2
-DocProject/Help/html
-
-# Click-Once directory
-publish/
-
-# Publish Web Output
-*.[Pp]ublish.xml
-*.azurePubxml
-# Note: Comment the next line if you want to checkin your web deploy settings,
-# but database connection strings (with potential passwords) will be unencrypted
-*.pubxml
-*.publishproj
-
-# Microsoft Azure Web App publish settings. Comment the next line if you want to
-# checkin your Azure Web App publish settings, but sensitive information contained
-# in these scripts will be unencrypted
-PublishScripts/
-
-# NuGet Packages
-*.nupkg
-# NuGet Symbol Packages
-*.snupkg
-# The packages folder can be ignored because of Package Restore
-**/[Pp]ackages/*
-# except build/, which is used as an MSBuild target.
-!**/[Pp]ackages/build/
-# Uncomment if necessary however generally it will be regenerated when needed
-#!**/[Pp]ackages/repositories.config
-# NuGet v3's project.json files produces more ignorable files
-*.nuget.props
-*.nuget.targets
-
-# Microsoft Azure Build Output
-csx/
-*.build.csdef
-
-# Microsoft Azure Emulator
-ecf/
-rcf/
-
-# Windows Store app package directories and files
-AppPackages/
-BundleArtifacts/
-Package.StoreAssociation.xml
-_pkginfo.txt
-*.appx
-*.appxbundle
-*.appxupload
-
-# Visual Studio cache files
-# files ending in .cache can be ignored
-*.[Cc]ache
-# but keep track of directories ending in .cache
-!?*.[Cc]ache/
-
-# Others
-ClientBin/
-~$*
-*~
-*.dbmdl
-*.dbproj.schemaview
-*.jfm
-*.pfx
-*.publishsettings
-orleans.codegen.cs
-
-# Including strong name files can present a security risk
-# (https://github.com/github/gitignore/pull/2483#issue-259490424)
-#*.snk
-
-# Since there are multiple workflows, uncomment next line to ignore bower_components
-# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
-#bower_components/
-
-# RIA/Silverlight projects
-Generated_Code/
-
-# Backup & report files from converting an old project file
-# to a newer Visual Studio version. Backup files are not needed,
-# because we have git ;-)
-_UpgradeReport_Files/
-Backup*/
-UpgradeLog*.XML
-UpgradeLog*.htm
-ServiceFabricBackup/
-*.rptproj.bak
-
-# SQL Server files
-*.mdf
-*.ldf
-*.ndf
-
-# Business Intelligence projects
-*.rdl.data
-*.bim.layout
-*.bim_*.settings
-*.rptproj.rsuser
-*- [Bb]ackup.rdl
-*- [Bb]ackup ([0-9]).rdl
-*- [Bb]ackup ([0-9][0-9]).rdl
-
-# Microsoft Fakes
-FakesAssemblies/
-
-# GhostDoc plugin setting file
-*.GhostDoc.xml
-
-# Node.js Tools for Visual Studio
-.ntvs_analysis.dat
-node_modules/
-
-# Visual Studio 6 build log
-*.plg
-
-# Visual Studio 6 workspace options file
-*.opt
-
-# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
-*.vbw
-
-# Visual Studio LightSwitch build output
-**/*.HTMLClient/GeneratedArtifacts
-**/*.DesktopClient/GeneratedArtifacts
-**/*.DesktopClient/ModelManifest.xml
-**/*.Server/GeneratedArtifacts
-**/*.Server/ModelManifest.xml
-_Pvt_Extensions
-
-# Paket dependency manager
-.paket/paket.exe
-paket-files/
-
-# FAKE - F# Make
-.fake/
-
-# CodeRush personal settings
-.cr/personal
-
-# Python Tools for Visual Studio (PTVS)
-__pycache__/
-*.pyc
-
-# Cake - Uncomment if you are using it
-# tools/**
-# !tools/packages.config
-
-# Tabs Studio
-*.tss
-
-# Telerik's JustMock configuration file
-*.jmconfig
-
-# BizTalk build output
-*.btp.cs
-*.btm.cs
-*.odx.cs
-*.xsd.cs
-
-# OpenCover UI analysis results
-OpenCover/
-
-# Azure Stream Analytics local run output
-ASALocalRun/
-
-# MSBuild Binary and Structured Log
-*.binlog
-
-# NVidia Nsight GPU debugger configuration file
-*.nvuser
-
-# MFractors (Xamarin productivity tool) working folder
-.mfractor/
-
-# Local History for Visual Studio
-.localhistory/
-
-# BeatPulse healthcheck temp database
-healthchecksdb
-
-# Backup folder for Package Reference Convert tool in Visual Studio 2017
-MigrationBackup/
-
-# Ionide (cross platform F# VS Code tools) working folder
-.ionide/
-
-# Fody - auto-generated XML schema
-FodyWeavers.xsd
-
-# JetBrains Rider
-.idea
-
-# Visual Studio Code
-.vscode

gpt4all-bindings/csharp/Directory.Build.props

@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project>
-
-    <PropertyGroup>
-        <Company></Company>
-        <Copyright></Copyright>
-        <NeutralLanguage>en-US</NeutralLanguage>
-        <Version>0.6.1-alpha</Version>
-        <VersionSuffix>$(VersionSuffix)</VersionSuffix>
-        <Version Condition=" '$(VersionSuffix)' != '' ">$(Version)$(VersionSuffix)</Version>
-        <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
-        <RepositoryUrl></RepositoryUrl>
-        <RepositoryType>git</RepositoryType>
-        <IncludeSymbols>true</IncludeSymbols>
-        <IncludeSource>true</IncludeSource>
-        <AnalysisLevel>latest-minimum</AnalysisLevel>
-		<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
-    </PropertyGroup>
-
-    <ItemGroup>
-        <Using Include="System"/>
-    </ItemGroup>
-
-    <PropertyGroup>
-        <LangVersion>preview</LangVersion>
-        <Features>strict</Features>
-    </PropertyGroup>
-	
-	<ItemGroup>
-        <PackageReference Include="Roslynator.Analyzers" Version="4.2.0">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-        </PackageReference>
-        <PackageReference Include="Roslynator.CodeAnalysis.Analyzers" Version="4.2.0">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-        </PackageReference>
-        <PackageReference Include="Roslynator.Formatting.Analyzers" Version="4.2.0">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-        </PackageReference>
-    </ItemGroup>
-
-</Project>

gpt4all-bindings/csharp/Gpt4All.Samples/Gpt4All.Samples.csproj

@@ -1,18 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Folder Include="Properties\" />
-  </ItemGroup>
-
-</Project>

gpt4all-bindings/csharp/Gpt4All.Samples/Program.cs

@@ -1,22 +0,0 @@
-using Gpt4All;
-
-var modelFactory = new Gpt4AllModelFactory();
-if (args.Length < 2)
-{
-    Console.WriteLine($"Usage: Gpt4All.Samples <model-path> <prompt>");
-    return;
-}
-
-var modelPath = args[0];
-var prompt = args[1];
-
-using var model = modelFactory.LoadModel(modelPath);
-
-var result = await model.GetStreamingPredictionAsync(
-    prompt,
-    PredictRequestOptions.Defaults);
-
-await foreach (var token in result.GetPredictionStreamingAsync())
-{
-    Console.Write(token);
-}

gpt4all-bindings/csharp/Gpt4All.Tests/Constants.cs

@@ -1,10 +0,0 @@
-namespace Gpt4All.Tests
-{
-    public static class Constants
-    {
-        public const string MODELS_BASE_DIR = "../../../models";
-        public const string LLAMA_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-l13b-snoozy.bin";
-        public const string GPTJ_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-j-v1.3-groovy.bin";
-        public const string MPT_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-mpt-7b-chat.bin";
-    }
-}

gpt4all-bindings/csharp/Gpt4All.Tests/Gpt4All.Tests.csproj

@@ -1,27 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>net6.0</TargetFramework>
-        <Nullable>enable</Nullable>
-
-        <IsPackable>false</IsPackable>
-    </PropertyGroup>
-
-    <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
-        <PackageReference Include="xunit" Version="2.4.1" />
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-            <PrivateAssets>all</PrivateAssets>
-        </PackageReference>
-        <PackageReference Include="coverlet.collector" Version="3.1.0">
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-            <PrivateAssets>all</PrivateAssets>
-        </PackageReference>
-    </ItemGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
-    </ItemGroup>
-
-</Project>

gpt4all-bindings/csharp/Gpt4All.Tests/ModelFactoryTests.cs

@@ -1,31 +0,0 @@
-using Xunit;
-
-namespace Gpt4All.Tests;
-
-public class ModelFactoryTests
-{
-    private readonly Gpt4AllModelFactory _modelFactory;
-
-    public ModelFactoryTests()
-    {
-        _modelFactory = new Gpt4AllModelFactory();
-    }
-
-    [Fact]
-    public void CanLoadLlamaModel()
-    {
-        using var model = _modelFactory.LoadLlamaModel(Constants.LLAMA_MODEL_PATH);
-    }
-
-    [Fact]
-    public void CanLoadGptjModel()
-    {
-        using var model = _modelFactory.LoadGptjModel(Constants.GPTJ_MODEL_PATH);
-    }
-
-    [Fact]
-    public void CanLoadMptModel()
-    {
-        using var model = _modelFactory.LoadMptModel(Constants.MPT_MODEL_PATH);
-    }
-}

gpt4all-bindings/csharp/Gpt4All.sln

@@ -1,47 +0,0 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.5.33516.290
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Gpt4All.Samples", "Gpt4All.Samples\Gpt4All.Samples.csproj", "{59864AE8-E45D-42F7-A7C0-1308EF185F39}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{DA396C11-CEAD-4368-8234-FB12255A30D2}"
-	ProjectSection(SolutionItems) = preProject
-		.gitignore = .gitignore
-		build_linux.sh = build_linux.sh
-		build_win-mingw.ps1 = build_win-mingw.ps1
-		build_win-msvc.ps1 = build_win-msvc.ps1
-		docs\gpt4all_csharp.md = docs\gpt4all_csharp.md
-		README.md = README.md
-	EndProjectSection
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Gpt4All", "Gpt4All\Gpt4All.csproj", "{6015C62B-2008-426B-A334-740D6F1FE38B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gpt4All.Tests", "Gpt4All.Tests\Gpt4All.Tests.csproj", "{33A72341-52C1-4EAE-878B-A98BC77F686A}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Release|Any CPU = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{59864AE8-E45D-42F7-A7C0-1308EF185F39}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6015C62B-2008-426B-A334-740D6F1FE38B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6015C62B-2008-426B-A334-740D6F1FE38B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6015C62B-2008-426B-A334-740D6F1FE38B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6015C62B-2008-426B-A334-740D6F1FE38B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{33A72341-52C1-4EAE-878B-A98BC77F686A}.Release|Any CPU.Build.0 = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {17632027-F4C2-4903-B88F-310CE3DE386B}
-	EndGlobalSection
-EndGlobal

gpt4all-bindings/csharp/Gpt4All/Bindings/ILLModel.cs

@@ -1,31 +0,0 @@
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Represents the interface exposed by the universal wrapper for GPT4All language models built around llmodel C-API.
-/// </summary>
-public interface ILLModel : IDisposable
-{
-    ModelType ModelType { get; }
-
-    ulong GetStateSizeBytes();
-
-    int GetThreadCount();
-
-    void SetThreadCount(int threadCount);
-
-    bool IsLoaded();
-
-    bool Load(string modelPath);
-
-    void Prompt(
-        string text,
-        LLModelPromptContext context,
-        Func<ModelPromptEventArgs, bool>? promptCallback = null,
-        Func<ModelResponseEventArgs, bool>? responseCallback = null,
-        Func<ModelRecalculatingEventArgs, bool>? recalculateCallback = null,
-        CancellationToken cancellationToken = default);
-
-    unsafe ulong RestoreStateData(byte* destination);
-
-    unsafe ulong SaveStateData(byte* source);
-}

gpt4all-bindings/csharp/Gpt4All/Bindings/LLModel.cs

@@ -1,247 +0,0 @@
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
-
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Arguments for the response processing callback
-/// </summary>
-/// <param name="TokenId">The token id of the response</param>
-/// <param name="Response"> The response string. NOTE: a token_id of -1 indicates the string is an error string</param>
-/// <return>
-/// A bool indicating whether the model should keep generating
-/// </return>
-public record ModelResponseEventArgs(int TokenId, string Response)
-{
-    public bool IsError => TokenId == -1;
-}
-
-/// <summary>
-/// Arguments for the prompt processing callback
-/// </summary>
-/// <param name="TokenId">The token id of the prompt</param>
-/// <return>
-/// A bool indicating whether the model should keep processing
-/// </return>
-public record ModelPromptEventArgs(int TokenId)
-{
-}
-
-/// <summary>
-/// Arguments for the recalculating callback
-/// </summary>
-/// <param name="IsRecalculating"> whether the model is recalculating the context.</param>
-/// <return>
-/// A bool indicating whether the model should keep generating
-/// </return>
-public record ModelRecalculatingEventArgs(bool IsRecalculating);
-
-/// <summary>
-/// Base class and universal wrapper for GPT4All language models built around llmodel C-API.
-/// </summary>
-public class LLModel : ILLModel
-{
-    protected readonly IntPtr _handle;
-    private readonly ModelType _modelType;
-    private readonly ILogger _logger;
-    private bool _disposed;
-
-    public ModelType ModelType => _modelType;
-
-    internal LLModel(IntPtr handle, ModelType modelType, ILogger? logger = null)
-    {
-        _handle = handle;
-        _modelType = modelType;
-        _logger = logger ?? NullLogger.Instance;
-    }
-
-    /// <summary>
-    /// Create a new model from a pointer
-    /// </summary>
-    /// <param name="handle">Pointer to underlying model</param>
-    /// <param name="modelType">The model type</param>
-    public static LLModel Create(IntPtr handle, ModelType modelType, ILogger? logger = null)
-    {
-        return new LLModel(handle, modelType, logger: logger);
-    }
-
-    /// <summary>
-    /// Generate a response using the model
-    /// </summary>
-    /// <param name="text">The input promp</param>
-    /// <param name="context">The context</param>
-    /// <param name="promptCallback">A callback function for handling the processing of prompt</param>
-    /// <param name="responseCallback">A callback function for handling the generated response</param>
-    /// <param name="recalculateCallback">A callback function for handling recalculation requests</param>
-    /// <param name="cancellationToken"></param>
-    public void Prompt(
-        string text,
-        LLModelPromptContext context,
-        Func<ModelPromptEventArgs, bool>? promptCallback = null,
-        Func<ModelResponseEventArgs, bool>? responseCallback = null,
-        Func<ModelRecalculatingEventArgs, bool>? recalculateCallback = null,
-        CancellationToken cancellationToken = default)
-    {
-        GC.KeepAlive(promptCallback);
-        GC.KeepAlive(responseCallback);
-        GC.KeepAlive(recalculateCallback);
-        GC.KeepAlive(cancellationToken);
-
-        _logger.LogInformation("Prompt input='{Prompt}' ctx={Context}", text, context.Dump());
-
-        NativeMethods.llmodel_prompt(
-            _handle,
-            text,
-            (tokenId) =>
-            {
-                if (cancellationToken.IsCancellationRequested) return false;
-                if (promptCallback == null) return true;
-                var args = new ModelPromptEventArgs(tokenId);
-                return promptCallback(args);
-            },
-            (tokenId, response) =>
-            {
-                if (cancellationToken.IsCancellationRequested)
-                {
-                    _logger.LogDebug("ResponseCallback evt=CancellationRequested");
-                    return false;
-                }
-
-                if (responseCallback == null) return true;
-                var args = new ModelResponseEventArgs(tokenId, response);
-                return responseCallback(args);
-            },
-            (isRecalculating) =>
-            {
-                if (cancellationToken.IsCancellationRequested) return false;
-                if (recalculateCallback == null) return true;
-                var args = new ModelRecalculatingEventArgs(isRecalculating);
-                return recalculateCallback(args);
-            },
-            ref context.UnderlyingContext
-        );
-    }
-
-    /// <summary>
-    ///  Set the number of threads to be used by the model.
-    /// </summary>
-    /// <param name="threadCount">The new thread count</param>
-    public void SetThreadCount(int threadCount)
-    {
-        NativeMethods.llmodel_setThreadCount(_handle, threadCount);
-    }
-
-    /// <summary>
-    /// Get  the number of threads used by the model.
-    /// </summary>
-    /// <returns>the number of threads used by the model</returns>
-    public int GetThreadCount()
-    {
-        return NativeMethods.llmodel_threadCount(_handle);
-    }
-
-    /// <summary>
-    /// Get the size of the internal state of the model.
-    /// </summary>
-    /// <remarks>
-    /// This state data is specific to the type of model you have created.
-    /// </remarks>
-    /// <returns>the size in bytes of the internal state of the model</returns>
-    public ulong GetStateSizeBytes()
-    {
-        return NativeMethods.llmodel_get_state_size(_handle);
-    }
-
-    /// <summary>
-    /// Saves the internal state of the model to the specified destination address.
-    /// </summary>
-    /// <param name="source">A pointer to the src</param>
-    /// <returns>The number of bytes copied</returns>
-    public unsafe ulong SaveStateData(byte* source)
-    {
-        return NativeMethods.llmodel_save_state_data(_handle, source);
-    }
-
-    /// <summary>
-    /// Restores the internal state of the model using data from the specified address.
-    /// </summary>
-    /// <param name="destination">A pointer to destination</param>
-    /// <returns>the number of bytes read</returns>
-    public unsafe ulong RestoreStateData(byte* destination)
-    {
-        return NativeMethods.llmodel_restore_state_data(_handle, destination);
-    }
-
-    /// <summary>
-    /// Check if the model is loaded.
-    /// </summary>
-    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
-    public bool IsLoaded()
-    {
-        return NativeMethods.llmodel_isModelLoaded(_handle);
-    }
-
-    /// <summary>
-    /// Load the model from a file.
-    /// </summary>
-    /// <param name="modelPath">The path to the model file.</param>
-    /// <returns>true if the model was loaded successfully, false otherwise.</returns>
-    public bool Load(string modelPath)
-    {
-        return NativeMethods.llmodel_loadModel(_handle, modelPath);
-    }
-
-    protected void Destroy()
-    {
-        NativeMethods.llmodel_model_destroy(_handle);
-    }
-
-    protected void DestroyLLama()
-    {
-        NativeMethods.llmodel_llama_destroy(_handle);
-    }
-
-    protected void DestroyGptj()
-    {
-        NativeMethods.llmodel_gptj_destroy(_handle);
-    }
-
-    protected void DestroyMtp()
-    {
-        NativeMethods.llmodel_mpt_destroy(_handle);
-    }
-
-    protected virtual void Dispose(bool disposing)
-    {
-        if (_disposed) return;
-
-        if (disposing)
-        {
-            // dispose managed state
-        }
-
-        switch (_modelType)
-        {
-            case ModelType.LLAMA:
-                DestroyLLama();
-                break;
-            case ModelType.GPTJ:
-                DestroyGptj();
-                break;
-            case ModelType.MPT:
-                DestroyMtp();
-                break;
-            default:
-                Destroy();
-                break;
-        }
-
-        _disposed = true;
-    }
-
-    public void Dispose()
-    {
-        Dispose(disposing: true);
-        GC.SuppressFinalize(this);
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Bindings/LLPromptContext.cs

@@ -1,138 +0,0 @@
-namespace Gpt4All.Bindings;
-
-/// <summary>
-/// Wrapper around the llmodel_prompt_context structure for holding the prompt context.
-/// </summary>
-/// <remarks>
-/// The implementation takes care of all the memory handling of the raw logits pointer and the
-/// raw tokens pointer.Attempting to resize them or modify them in any way can lead to undefined behavior
-/// </remarks>
-public unsafe class LLModelPromptContext
-{
-    private llmodel_prompt_context _ctx;
-
-    internal ref llmodel_prompt_context UnderlyingContext => ref _ctx;
-
-    public LLModelPromptContext()
-    {
-        _ctx = new();
-    }
-
-    /// <summary>
-    /// logits of current context
-    /// </summary>
-    public Span<float> Logits => new(_ctx.logits, (int)_ctx.logits_size);
-
-    /// <summary>
-    /// the size of the raw logits vector
-    /// </summary>
-    public nuint LogitsSize
-    {
-        get => _ctx.logits_size;
-        set => _ctx.logits_size = value;
-    }
-
-    /// <summary>
-    /// current tokens in the context window
-    /// </summary>
-    public Span<int> Tokens => new(_ctx.tokens, (int)_ctx.tokens_size);
-
-    /// <summary>
-    /// the size of the raw tokens vector
-    /// </summary>
-    public nuint TokensSize
-    {
-        get => _ctx.tokens_size;
-        set => _ctx.tokens_size = value;
-    }
-
-    /// <summary>
-    /// top k logits to sample from
-    /// </summary>
-    public int TopK
-    {
-        get => _ctx.top_k;
-        set => _ctx.top_k = value;
-    }
-
-    /// <summary>
-    /// nucleus sampling probability threshold
-    /// </summary>
-    public float TopP
-    {
-        get => _ctx.top_p;
-        set => _ctx.top_p = value;
-    }
-
-    /// <summary>
-    /// temperature to adjust model's output distribution
-    /// </summary>
-    public float Temperature
-    {
-        get => _ctx.temp;
-        set => _ctx.temp = value;
-    }
-
-    /// <summary>
-    /// number of tokens in past conversation
-    /// </summary>
-    public int PastNum
-    {
-        get => _ctx.n_past;
-        set => _ctx.n_past = value;
-    }
-
-    /// <summary>
-    /// number of predictions to generate in parallel
-    /// </summary>
-    public int Batches
-    {
-        get => _ctx.n_batch;
-        set => _ctx.n_batch = value;
-    }
-
-    /// <summary>
-    /// number of tokens to predict
-    /// </summary>
-    public int TokensToPredict
-    {
-        get => _ctx.n_predict;
-        set => _ctx.n_predict = value;
-    }
-
-    /// <summary>
-    /// penalty factor for repeated tokens
-    /// </summary>
-    public float RepeatPenalty
-    {
-        get => _ctx.repeat_penalty;
-        set => _ctx.repeat_penalty = value;
-    }
-
-    /// <summary>
-    /// last n tokens to penalize
-    /// </summary>
-    public int RepeatLastN
-    {
-        get => _ctx.repeat_last_n;
-        set => _ctx.repeat_last_n = value;
-    }
-
-    /// <summary>
-    /// number of tokens possible in context window
-    /// </summary>
-    public int ContextSize
-    {
-        get => _ctx.n_ctx;
-        set => _ctx.n_ctx = value;
-    }
-
-    /// <summary>
-    /// percent of context to erase if we exceed the context window
-    /// </summary>
-    public float ContextErase
-    {
-        get => _ctx.context_erase;
-        set => _ctx.context_erase = value;
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeMethods.cs

@@ -1,126 +0,0 @@
-using System.Runtime.InteropServices;
-
-namespace Gpt4All.Bindings;
-
-public unsafe partial struct llmodel_prompt_context
-{
-    public float* logits;
-
-    [NativeTypeName("size_t")]
-    public nuint logits_size;
-
-    [NativeTypeName("int32_t *")]
-    public int* tokens;
-
-    [NativeTypeName("size_t")]
-    public nuint tokens_size;
-
-    [NativeTypeName("int32_t")]
-    public int n_past;
-
-    [NativeTypeName("int32_t")]
-    public int n_ctx;
-
-    [NativeTypeName("int32_t")]
-    public int n_predict;
-
-    [NativeTypeName("int32_t")]
-    public int top_k;
-
-    public float top_p;
-
-    public float temp;
-
-    [NativeTypeName("int32_t")]
-    public int n_batch;
-
-    public float repeat_penalty;
-
-    [NativeTypeName("int32_t")]
-    public int repeat_last_n;
-
-    public float context_erase;
-}
-
-internal static unsafe partial class NativeMethods
-{
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelResponseCallback(int token_id, [MarshalAs(UnmanagedType.LPUTF8Str)] string response);
-
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelPromptCallback(int token_id);
-
-    [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public delegate bool LlmodelRecalculateCallback(bool isRecalculating);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_gptj_create();
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_gptj_destroy([NativeTypeName("llmodel_model")] IntPtr gptj);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_mpt_create();
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_mpt_destroy([NativeTypeName("llmodel_model")] IntPtr mpt);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_llama_create();
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_llama_destroy([NativeTypeName("llmodel_model")] IntPtr llama);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    [return: NativeTypeName("llmodel_model")]
-    public static extern IntPtr llmodel_model_create(
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_model_destroy([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    [return: MarshalAs(UnmanagedType.I1)]
-    public static extern bool llmodel_loadModel(
-        [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-
-    [return: MarshalAs(UnmanagedType.I1)]
-    public static extern bool llmodel_isModelLoaded([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_get_state_size([NativeTypeName("llmodel_model")] IntPtr model);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_save_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("uint8_t *")] byte* dest);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("uint64_t")]
-    public static extern ulong llmodel_restore_state_data([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("const uint8_t *")] byte* src);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
-    public static extern void llmodel_prompt(
-        [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string prompt,
-        LlmodelPromptCallback prompt_callback,
-        LlmodelResponseCallback response_callback,
-        LlmodelRecalculateCallback recalculate_callback,
-        ref llmodel_prompt_context ctx);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    public static extern void llmodel_setThreadCount([NativeTypeName("llmodel_model")] IntPtr model, [NativeTypeName("int32_t")] int n_threads);
-
-    [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-    [return: NativeTypeName("int32_t")]
-    public static extern int llmodel_threadCount([NativeTypeName("llmodel_model")] IntPtr model);
-}

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeTypeNameAttribute.cs

@@ -1,21 +0,0 @@
-using System.Diagnostics;
-
-namespace Gpt4All.Bindings;
-
-/// <summary>Defines the type of a member as it was used in the native signature.</summary>
-[AttributeUsage(AttributeTargets.Struct | AttributeTargets.Enum | AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.ReturnValue, AllowMultiple = false, Inherited = true)]
-[Conditional("DEBUG")]
-internal sealed partial class NativeTypeNameAttribute : Attribute
-{
-    private readonly string _name;
-
-    /// <summary>Initializes a new instance of the <see cref="NativeTypeNameAttribute" /> class.</summary>
-    /// <param name="name">The name of the type that was used in the native signature.</param>
-    public NativeTypeNameAttribute(string name)
-    {
-        _name = name;
-    }
-
-    /// <summary>Gets the name of the type that was used in the native signature.</summary>
-    public string Name => _name;
-}

gpt4all-bindings/csharp/Gpt4All/Extensions/LLPromptContextExtensions.cs

@@ -1,26 +0,0 @@
-using Gpt4All.Bindings;
-
-namespace Gpt4All;
-
-internal static class LLPromptContextExtensions
-{
-    public static string Dump(this LLModelPromptContext context)
-    {
-        var ctx = context.UnderlyingContext;
-        return @$"
-        {{
-            logits_size = {ctx.logits_size}
-            tokens_size = {ctx.tokens_size}
-            n_past = {ctx.n_past}
-            n_ctx = {ctx.n_ctx}
-            n_predict = {ctx.n_predict}
-            top_k = {ctx.top_k}
-            top_p = {ctx.top_p}
-            temp = {ctx.temp}
-            n_batch = {ctx.n_batch}
-            repeat_penalty = {ctx.repeat_penalty}
-            repeat_last_n = {ctx.repeat_last_n}
-            context_erase = {ctx.context_erase}
-        }}";
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Extensions/PredictRequestOptionsExtensions.cs

@@ -1,25 +0,0 @@
-using Gpt4All.Bindings;
-
-namespace Gpt4All;
-
-public static class PredictRequestOptionsExtensions
-{
-    public static LLModelPromptContext ToPromptContext(this PredictRequestOptions opts)
-    {
-        return new LLModelPromptContext
-        {
-            LogitsSize = opts.LogitsSize,
-            TokensSize = opts.TokensSize,
-            TopK = opts.TopK,
-            TopP = opts.TopP,
-            PastNum = opts.PastConversationTokensNum,
-            RepeatPenalty = opts.RepeatPenalty,
-            Temperature = opts.Temperature,
-            RepeatLastN = opts.RepeatLastN,
-            Batches = opts.Batches,
-            ContextErase = opts.ContextErase,
-            ContextSize = opts.ContextSize,
-            TokensToPredict = opts.TokensToPredict
-        };
-    }
-}

gpt4all-bindings/csharp/Gpt4All/GenLLModelBindings.rsp

@@ -1,21 +0,0 @@
---config
-exclude-funcs-with-body
---with-access-specifier
-*=Public
---include-directory
-..\..\..\gpt4all-backend\
---file
-..\..\..\gpt4all-backend\llmodel_c.h
---libraryPath
-libllmodel
---remap
-sbyte*=IntPtr
-void*=IntPtr
---namespace
-Gpt4All.Bindings
---methodClassName
-NativeMethods
---output
-.\Bindings\NativeMethods.cs
---output-mode
-CSharp

gpt4all-bindings/csharp/Gpt4All/Gpt4All.cs

@@ -1,132 +0,0 @@
-using System.Diagnostics;
-using Gpt4All.Bindings;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
-
-namespace Gpt4All;
-
-public class Gpt4All : IGpt4AllModel
-{
-    private readonly ILLModel _model;
-    private readonly ILogger _logger;
-
-    private const string ResponseErrorMessage =
-        "The model reported an error during token generation error={ResponseError}";
-
-    /// <inheritdoc/>
-    public IPromptFormatter? PromptFormatter { get; set; }
-
-    internal Gpt4All(ILLModel model, ILogger? logger = null)
-    {
-        _model = model;
-        _logger = logger ?? NullLogger.Instance;
-        PromptFormatter = new DefaultPromptFormatter();
-    }
-
-    private string FormatPrompt(string prompt)
-    {
-        if (PromptFormatter == null) return prompt;
-
-        return PromptFormatter.FormatPrompt(prompt);
-    }
-
-    public Task<ITextPredictionResult> GetPredictionAsync(string text, PredictRequestOptions opts, CancellationToken cancellationToken = default)
-    {
-        ArgumentNullException.ThrowIfNull(text);
-
-        return Task.Run(() =>
-        {
-            _logger.LogInformation("Start prediction task");
-
-            var sw = Stopwatch.StartNew();
-            var result = new TextPredictionResult();
-            var context = opts.ToPromptContext();
-            var prompt = FormatPrompt(text);
-
-            try
-            {
-                _model.Prompt(prompt, context, responseCallback: e =>
-                {
-                    if (e.IsError)
-                    {
-                        _logger.LogWarning(ResponseErrorMessage, e.Response);
-                        result.Success = false;
-                        result.ErrorMessage = e.Response;
-                        return false;
-                    }
-                    result.Append(e.Response);
-                    return true;
-                }, cancellationToken: cancellationToken);
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, "Prompt error");
-                result.Success = false;
-            }
-
-            sw.Stop();
-            _logger.LogInformation("Prediction task completed elapsed={Elapsed}s", sw.Elapsed.TotalSeconds);
-
-            return (ITextPredictionResult)result;
-        }, CancellationToken.None);
-    }
-
-    public Task<ITextPredictionStreamingResult> GetStreamingPredictionAsync(string text, PredictRequestOptions opts, CancellationToken cancellationToken = default)
-    {
-        ArgumentNullException.ThrowIfNull(text);
-
-        var result = new TextPredictionStreamingResult();
-
-        _ = Task.Run(() =>
-        {
-            _logger.LogInformation("Start streaming prediction task");
-            var sw = Stopwatch.StartNew();
-
-            try
-            {
-                var context = opts.ToPromptContext();
-                var prompt = FormatPrompt(text);
-
-                _model.Prompt(prompt, context, responseCallback: e =>
-                {
-                    if (e.IsError)
-                    {
-                        _logger.LogWarning(ResponseErrorMessage, e.Response);
-                        result.Success = false;
-                        result.ErrorMessage = e.Response;
-                        return false;
-                    }
-                    result.Append(e.Response);
-                    return true;
-                }, cancellationToken: cancellationToken);
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, "Prompt error");
-                result.Success = false;
-            }
-            finally
-            {
-                result.Complete();
-                sw.Stop();
-                _logger.LogInformation("Prediction task completed elapsed={Elapsed}s", sw.Elapsed.TotalSeconds);
-            }
-        }, CancellationToken.None);
-
-        return Task.FromResult((ITextPredictionStreamingResult)result);
-    }
-
-    protected virtual void Dispose(bool disposing)
-    {
-        if (disposing)
-        {
-            _model.Dispose();
-        }
-    }
-
-    public void Dispose()
-    {
-        Dispose(true);
-        GC.SuppressFinalize(this);
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Gpt4All.csproj

@@ -1,27 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-	<TargetFrameworks>net6.0</TargetFrameworks>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-  </PropertyGroup>
-
-	<ItemGroup>
-		<!-- Windows -->
-		<None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
-		<!-- Linux -->
-		<None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
-	</ItemGroup>
-
-	<ItemGroup>
-		<!-- Windows -->
-		<None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-		<!-- Linux -->
-		<None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
-	</ItemGroup>
-
-	<ItemGroup>
-	  <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
-	</ItemGroup>
-</Project>

gpt4all-bindings/csharp/Gpt4All/Model/DefaultPromptFormatter.cs

@@ -1,16 +0,0 @@
-namespace Gpt4All;
-
-public class DefaultPromptFormatter : IPromptFormatter
-{
-    public string FormatPrompt(string prompt)
-    {
-        return $"""
-        ### Instruction: 
-        The prompt below is a question to answer, a task to complete, or a conversation
-        to respond to; decide which and write an appropriate response.
-        ### Prompt:
-        {prompt}
-        ### Response:
-        """;
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Model/Gpt4AllModelFactory.cs

@@ -1,61 +0,0 @@
-using System.Diagnostics;
-using Microsoft.Extensions.Logging;
-using Gpt4All.Bindings;
-using Microsoft.Extensions.Logging.Abstractions;
-
-namespace Gpt4All;
-
-public class Gpt4AllModelFactory : IGpt4AllModelFactory
-{
-    private readonly ILoggerFactory _loggerFactory;
-    private readonly ILogger _logger;
-
-    public Gpt4AllModelFactory(ILoggerFactory? loggerFactory = null)
-    {
-        _loggerFactory = loggerFactory ?? NullLoggerFactory.Instance;
-        _logger = _loggerFactory.CreateLogger<Gpt4AllModelFactory>();
-    }
-
-    private IGpt4AllModel CreateModel(string modelPath, ModelType? modelType = null)
-    {
-        var modelType_ = modelType ?? ModelFileUtils.GetModelTypeFromModelFileHeader(modelPath);
-
-        _logger.LogInformation("Creating model path={ModelPath} type={ModelType}", modelPath, modelType_);
-
-        var handle = modelType_ switch
-        {
-            ModelType.LLAMA => NativeMethods.llmodel_llama_create(),
-            ModelType.GPTJ => NativeMethods.llmodel_gptj_create(),
-            ModelType.MPT => NativeMethods.llmodel_mpt_create(),
-            _ => NativeMethods.llmodel_model_create(modelPath),
-        };
-
-        _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
-        _logger.LogInformation("Model loading started");
-
-        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath);
-
-        _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
-
-        if (loadedSuccessfully == false)
-        {
-            throw new Exception($"Failed to load model: '{modelPath}'");
-        }
-
-        var logger = _loggerFactory.CreateLogger<LLModel>();
-
-        var underlyingModel = LLModel.Create(handle, modelType_, logger: logger);
-
-        Debug.Assert(underlyingModel.IsLoaded());
-
-        return new Gpt4All(underlyingModel, logger: logger);
-    }
-
-    public IGpt4AllModel LoadModel(string modelPath) => CreateModel(modelPath, modelType: null);
-
-    public IGpt4AllModel LoadMptModel(string modelPath) => CreateModel(modelPath, ModelType.MPT);
-
-    public IGpt4AllModel LoadGptjModel(string modelPath) => CreateModel(modelPath, ModelType.GPTJ);
-
-    public IGpt4AllModel LoadLlamaModel(string modelPath) => CreateModel(modelPath, ModelType.LLAMA);
-}

gpt4all-bindings/csharp/Gpt4All/Model/IGpt4AllModel.cs

@@ -1,10 +0,0 @@
-namespace Gpt4All;
-
-public interface IGpt4AllModel : ITextPrediction, IDisposable
-{
-    /// <summary>
-    /// The prompt formatter used to format the prompt before
-    /// feeding it to the model, if null no transformation is applied
-    /// </summary>
-    IPromptFormatter? PromptFormatter { get; set; }
-}

gpt4all-bindings/csharp/Gpt4All/Model/IGpt4AllModelFactory.cs

@@ -1,12 +0,0 @@
-namespace Gpt4All;
-
-public interface IGpt4AllModelFactory
-{
-    IGpt4AllModel LoadGptjModel(string modelPath);
-
-    IGpt4AllModel LoadLlamaModel(string modelPath);
-
-    IGpt4AllModel LoadModel(string modelPath);
-
-    IGpt4AllModel LoadMptModel(string modelPath);
-}

gpt4all-bindings/csharp/Gpt4All/Model/IPromptFormatter.cs

@@ -1,14 +0,0 @@
-namespace Gpt4All;
-
-/// <summary>
-/// Formats a prompt
-/// </summary>
-public interface IPromptFormatter
-{
-    /// <summary>
-    /// Format the provided prompt
-    /// </summary>
-    /// <param name="prompt">the input prompt</param>
-    /// <returns>The formatted prompt</returns>
-    string FormatPrompt(string prompt);
-}

gpt4all-bindings/csharp/Gpt4All/Model/ModelFileUtils.cs

@@ -1,24 +0,0 @@
-namespace Gpt4All;
-
-public static class ModelFileUtils
-{
-    private const uint GPTJ_MAGIC = 0x67676d6c;
-    private const uint LLAMA_MAGIC = 0x67676a74;
-    private const uint MPT_MAGIC = 0x67676d6d;
-
-    public static ModelType GetModelTypeFromModelFileHeader(string modelPath)
-    {
-        using var fileStream = new FileStream(modelPath, FileMode.Open);
-        using var binReader = new BinaryReader(fileStream);
-
-        var magic = binReader.ReadUInt32();
-
-        return magic switch
-        {
-            GPTJ_MAGIC => ModelType.GPTJ,
-            LLAMA_MAGIC => ModelType.LLAMA,
-            MPT_MAGIC => ModelType.MPT,
-            _ => throw new ArgumentOutOfRangeException($"Invalid model file. magic=0x{magic:X8}"),
-        };
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Model/ModelOptions.cs

@@ -1,8 +0,0 @@
-namespace Gpt4All;
-
-public record ModelOptions
-{
-    public int Threads { get; init; } = 4;
-
-    public ModelType ModelType { get; init; } = ModelType.GPTJ;
-}

gpt4all-bindings/csharp/Gpt4All/Model/ModelType.cs

@@ -1,11 +0,0 @@
-namespace Gpt4All;
-
-/// <summary>
-/// The supported model types
-/// </summary>
-public enum ModelType
-{
-    LLAMA = 0,
-    GPTJ,
-    MPT
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/ITextPrediction.cs

@@ -1,31 +0,0 @@
-namespace Gpt4All;
-
-/// <summary>
-/// Interface for text prediction services
-/// </summary>
-public interface ITextPrediction
-{
-    /// <summary>
-    /// Get prediction results for the prompt and provided options.
-    /// </summary>
-    /// <param name="text">The text to complete</param>
-    /// <param name="opts">The prediction settings</param>
-    /// <param name="cancellationToken">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
-    /// <returns>The prediction result generated by the model</returns>
-    Task<ITextPredictionResult> GetPredictionAsync(
-        string text,
-        PredictRequestOptions opts,
-        CancellationToken cancellation = default);
-
-    /// <summary>
-    /// Get streaming prediction results for the prompt and provided options.
-    /// </summary>
-    /// <param name="text">The text to complete</param>
-    /// <param name="opts">The prediction settings</param>
-    /// <param name="cancellationToken">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
-    /// <returns>The prediction result generated by the model</returns>
-    Task<ITextPredictionStreamingResult> GetStreamingPredictionAsync(
-        string text,
-        PredictRequestOptions opts,
-        CancellationToken cancellationToken = default);
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/ITextPredictionResult.cs

@@ -1,10 +0,0 @@
-namespace Gpt4All;
-
-public interface ITextPredictionResult
-{
-    bool Success { get; }
-
-    string? ErrorMessage { get; }
-
-    Task<string> GetPredictionAsync(CancellationToken cancellationToken = default);
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/ITextPredictionStreamingResult.cs

@@ -1,6 +0,0 @@
-namespace Gpt4All;
-
-public interface ITextPredictionStreamingResult : ITextPredictionResult
-{
-    IAsyncEnumerable<string> GetPredictionStreamingAsync(CancellationToken cancellationToken = default);
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/PredictRequestOptions.cs

@@ -1,30 +0,0 @@
-namespace Gpt4All;
-
-public record PredictRequestOptions
-{
-    public nuint LogitsSize { get; init; } = 0;
-
-    public nuint TokensSize { get; init; } = 0;
-
-    public int PastConversationTokensNum { get; init; } = 0;
-
-    public int ContextSize { get; init; } = 1024;
-
-    public int TokensToPredict { get; init; } = 128;
-
-    public int TopK { get; init; } = 40;
-
-    public float TopP { get; init; } = 0.9f;
-
-    public float Temperature { get; init; } = 0.1f;
-
-    public int Batches { get; init; } = 8;
-
-    public float RepeatPenalty { get; init; } = 1.2f;
-
-    public int RepeatLastN { get; init; } = 10;
-
-    public float ContextErase { get; init; } = 0.5f;
-
-    public static readonly PredictRequestOptions Defaults = new();
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/TextPredictionResult.cs

@@ -1,27 +0,0 @@
-using System.Text;
-
-namespace Gpt4All;
-
-public record TextPredictionResult : ITextPredictionResult
-{
-    private readonly StringBuilder _result;
-
-    public bool Success { get; internal set; } = true;
-
-    public string? ErrorMessage { get; internal set; }
-
-    internal TextPredictionResult()
-    {
-        _result = new StringBuilder();
-    }
-
-    internal void Append(string token)
-    {
-        _result.Append(token);
-    }
-
-    public Task<string> GetPredictionAsync(CancellationToken cancellationToken = default)
-    {
-        return Task.FromResult(_result.ToString());
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/TextPredictionStreamingResult.cs

@@ -1,49 +0,0 @@
-using System.Text;
-using System.Threading.Channels;
-
-namespace Gpt4All;
-
-public record TextPredictionStreamingResult : ITextPredictionStreamingResult
-{
-    private readonly Channel<string> _channel;
-
-    public bool Success { get; internal set; } = true;
-
-    public string? ErrorMessage { get; internal set; }
-
-    public Task Completion => _channel.Reader.Completion;
-
-    internal TextPredictionStreamingResult()
-    {
-        _channel = Channel.CreateUnbounded<string>();
-    }
-
-    internal bool Append(string token)
-    {
-        return _channel.Writer.TryWrite(token);
-    }
-
-    internal void Complete()
-    {
-        _channel.Writer.Complete();
-    }
-
-    public async Task<string> GetPredictionAsync(CancellationToken cancellationToken = default)
-    {
-        var sb = new StringBuilder();
-
-        var tokens = GetPredictionStreamingAsync(cancellationToken).ConfigureAwait(false);
-
-        await foreach (var token in tokens)
-        {
-            sb.Append(token);
-        }
-
-        return sb.ToString();
-    }
-
-    public IAsyncEnumerable<string> GetPredictionStreamingAsync(CancellationToken cancellationToken = default)
-    {
-        return _channel.Reader.ReadAllAsync(cancellationToken);
-    }
-}

gpt4all-bindings/csharp/Gpt4All/gen_bindings.ps1

@@ -1 +0,0 @@
-ClangSharpPInvokeGenerator @(Get-Content .\GenLLModelBindings.rsp)

gpt4all-bindings/csharp/README.md

@@ -1,115 +0,0 @@
-# C# GPT4All
-
-This package contains a set of C# bindings around the `llmodel` C-API.
-
-## Documentation
-TBD
-
-## Installation
-TBD NuGet
-
-## Project Structure
-```
-gpt4all-bindings/
-└── csharp                
-    ├── Gpt4All               // .NET Bindigs
-    ├── Gpt4All.Samples       // Sample project
-    ├── build_win-msvc.ps1    // Native build scripts
-    ├── build_win-mingw.ps1   
-    ├── build_linux.sh        
-    └── runtimes              // [POST-BUILD] Platform-specific native libraries
-          ├── win-x64
-          ├── ...
-          └── linux-x64
-```
-
-## Local Build Instructions
-> **Note** 
-> Tested On:
->  - Windows 11 22H + VS2022 (CE) x64
->  - Linux Ubuntu x64
->  - Linux Ubuntu (WSL2) x64
-
-1. Setup the repository
-2. Build the native libraries for the platform of choice (see below)
-3. Build the C# Bindings (NET6+ SDK is required)
-```
-git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
-cd gpt4all/gpt4all-bindings/csharp
-```
-### Linux
-1. Setup build environment and install NET6+ SDK with the appropriate procedure for your distribution
-```
-sudo apt-get update
-sudo apt-get install -y cmake build-essential
-chmod +x ./build_linux.sh
-```
-2. `./build_linux.sh`
-3. The native libraries should be present at `.\native\linux-x64`
-
-### Windows - MinGW64
-#### Additional requirements
-  - [MinGW64](https://www.mingw-w64.org/) 
-  - CMAKE
-1. Setup
-```
-choco install mingw
-$env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
-choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
-```
-2. Run the `./build_win-mingw.ps1` build script
-3. The native libraries should be present at `.\native\win-x64`
-
-### Windows - MSVC
-#### Additional requirements
-  - Visual Studio 2022
-1. Open a terminal using the  `x64 Native Tools Command Prompt for VS 2022` (`vcvars64.bat`)
-2. Run the `./build_win-msvc.ps1` build script
-3. `libllmodel.dll` and `libllama.dll` should be present at `.\native\win-x64`
-
-> **Warning** 
-> If the build fails with: '**error C7555: use of designated initializers requires at least '/std:c++20'**'
->
-> Modify `cd gpt4all/gpt4all-backends/CMakeLists.txt` adding `CXX_STANDARD_20` to `llmodel` properties.
-> ```cmake
-> set_target_properties(llmodel PROPERTIES
->                              VERSION ${PROJECT_VERSION}
->                              CXX_STANDARD 20 # <---- ADD THIS -----------------------
->                              SOVERSION ${PROJECT_VERSION_MAJOR})
-> ```
-## C# Bindings Build Instructions
-Build the `Gpt4All` (or `Gpt4All.Samples`) projects from within VisualStudio.
-### Try the bindings
-```csharp
-using Gpt4All;
-
-// load the model
-var modelFactory = new ModelFactory();
-
-using var model = modelFactory.LoadModel("./path/to/ggml-gpt4all-j-v1.3-groovy.bin");
-
-var input = "Name 3 Colors";
-
-// request a prediction
-var result = await model.GetStreamingPredictionAsync(
-    input, 
-    PredictRequestOptions.Defaults);
-
-// asynchronously print the tokens as soon as they are produces by the model
-await foreach(var token in result.GetPredictionStreamingAsync())
-{
-    Console.Write(token);
-}
-```
-Output:
-```
-gptj_model_load: loading model from 'ggml-gpt4all-j-v1.3-groovy.bin' - please wait ...
-gptj_model_load: n_vocab = 50400
-[...TRUNCATED...]
-gptj_model_load: ggml ctx size = 5401.45 MB
-gptj_model_load: kv self size  =  896.00 MB
-gptj_model_load: ................................... done
-gptj_model_load: model size =  3609.38 MB / num tensors = 285
-
-Black, Blue and White
-```

gpt4all-bindings/csharp/build_linux.sh

@@ -1,8 +0,0 @@
-mkdir -p runtimes
-rm -rf runtimes/linux-x64
-mkdir -p runtimes/linux-x64/native
-mkdir runtimes/linux-x64/build
-cmake -S ../../gpt4all-backend -B runtimes/linux-x64/build
-cmake --build runtimes/linux-x64/build --parallel --config Release
-cp runtimes/linux-x64/build/libllmodel.so  runtimes/linux-x64/native/libllmodel.so
-cp runtimes/linux-x64/build/llama.cpp/libllama.so runtimes/linux-x64/native/libllama.so

gpt4all-bindings/csharp/build_win-msvc.ps1

@@ -1,5 +0,0 @@
-Remove-Item -Force -Recurse .\runtimes\win-x64\msvc -ErrorAction SilentlyContinue
-mkdir .\runtimes\win-x64\msvc\build | Out-Null
-cmake -G "Visual Studio 17 2022" -A X64 -S ..\..\gpt4all-backend -B .\runtimes\win-x64\msvc\build
-cmake --build .\runtimes\win-x64\msvc\build --parallel --config Release
-cp .\runtimes\win-x64\msvc\build\bin\Release\*.dll .\runtimes\win-x64

gpt4all-bindings/csharp/docs/gpt4all_csharp.md

@@ -1 +0,0 @@
-# GPT4All C# API

gpt4all-bindings/golang/Makefile

@@ -1,163 +0,0 @@
-INCLUDE_PATH := $(abspath ./)
-LIBRARY_PATH := $(abspath ./)
-CMAKEFLAGS=
-
-ifndef UNAME_S
-UNAME_S := $(shell uname -s)
-endif
-
-ifndef UNAME_P
-UNAME_P := $(shell uname -p)
-endif
-
-ifndef UNAME_M
-UNAME_M := $(shell uname -m)
-endif
-
-CCV := $(shell $(CC) --version | head -n 1)
-CXXV := $(shell $(CXX) --version | head -n 1)
-
-# Mac OS + Arm can report x86_64
-# ref: https://github.com/ggerganov/whisper.cpp/issues/66#issuecomment-1282546789
-ifeq ($(UNAME_S),Darwin)
-	ifneq ($(UNAME_P),arm)
-		SYSCTL_M := $(shell sysctl -n hw.optional.arm64 2>/dev/null)
-		ifeq ($(SYSCTL_M),1)
-			# UNAME_P := arm
-			# UNAME_M := arm64
-			warn := $(warning Your arch is announced as x86_64, but it seems to actually be ARM64. Not fixing that can lead to bad performance. For more info see: https://github.com/ggerganov/whisper.cpp/issues/66\#issuecomment-1282546789)
-		endif
-	endif
-endif
-
-#
-# Compile flags
-#
-
-# keep standard at C11 and C++11
-CFLAGS   = -I. -I../../gpt4all-backend/llama.cpp -I../../gpt4all-backend -I -O3 -DNDEBUG -std=c11 -fPIC
-CXXFLAGS = -I. -I../../gpt4all-backend/llama.cpp -I../../gpt4all-backend -O3 -DNDEBUG -std=c++17 -fPIC
-LDFLAGS  =
-
-# warnings
-CFLAGS   += -Wall -Wextra -Wpedantic -Wcast-qual -Wdouble-promotion -Wshadow -Wstrict-prototypes -Wpointer-arith -Wno-unused-function
-CXXFLAGS += -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wno-multichar
-
-# OS specific
-# TODO: support Windows
-ifeq ($(UNAME_S),Linux)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-ifeq ($(UNAME_S),Darwin)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-ifeq ($(UNAME_S),FreeBSD)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-ifeq ($(UNAME_S),NetBSD)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-ifeq ($(UNAME_S),OpenBSD)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-ifeq ($(UNAME_S),Haiku)
-	CFLAGS   += -pthread
-	CXXFLAGS += -pthread
-endif
-
-# Architecture specific
-# TODO: probably these flags need to be tweaked on some architectures
-#       feel free to update the Makefile for your architecture and send a pull request or issue
-ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686))
-	# Use all CPU extensions that are available:
-	CFLAGS += -march=native -mtune=native
-	CXXFLAGS += -march=native -mtune=native
-endif
-ifneq ($(filter ppc64%,$(UNAME_M)),)
-	POWER9_M := $(shell grep "POWER9" /proc/cpuinfo)
-	ifneq (,$(findstring POWER9,$(POWER9_M)))
-		CFLAGS += -mcpu=power9
-		CXXFLAGS += -mcpu=power9
-	endif
-	# Require c++23's std::byteswap for big-endian support.
-	ifeq ($(UNAME_M),ppc64)
-		CXXFLAGS += -std=c++23 -DGGML_BIG_ENDIAN
-	endif
-endif
-ifndef LLAMA_NO_ACCELERATE
-	# Mac M1 - include Accelerate framework.
-	# `-framework Accelerate` works on Mac Intel as well, with negliable performance boost (as of the predict time).
-	ifeq ($(UNAME_S),Darwin)
-		CFLAGS  += -DGGML_USE_ACCELERATE
-		LDFLAGS += -framework Accelerate
-	endif
-endif
-ifdef LLAMA_OPENBLAS
-	CFLAGS  += -DGGML_USE_OPENBLAS -I/usr/local/include/openblas
-	LDFLAGS += -lopenblas
-endif
-ifdef LLAMA_GPROF
-	CFLAGS   += -pg
-	CXXFLAGS += -pg
-endif
-ifneq ($(filter aarch64%,$(UNAME_M)),)
-	CFLAGS += -mcpu=native
-	CXXFLAGS += -mcpu=native
-endif
-ifneq ($(filter armv6%,$(UNAME_M)),)
-	# Raspberry Pi 1, 2, 3
-	CFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access
-endif
-ifneq ($(filter armv7%,$(UNAME_M)),)
-	# Raspberry Pi 4
-	CFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access -funsafe-math-optimizations
-endif
-ifneq ($(filter armv8%,$(UNAME_M)),)
-	# Raspberry Pi 4
-	CFLAGS += -mfp16-format=ieee -mno-unaligned-access
-endif
-
-#
-# Print build information
-#
-
-$(info I go-gpt4all build info: )
-$(info I UNAME_S:  $(UNAME_S))
-$(info I UNAME_P:  $(UNAME_P))
-$(info I UNAME_M:  $(UNAME_M))
-$(info I CFLAGS:   $(CFLAGS))
-$(info I CXXFLAGS: $(CXXFLAGS))
-$(info I LDFLAGS:  $(LDFLAGS))
-$(info I CMAKEFLAGS:  $(CMAKEFLAGS))
-$(info I CC:       $(CCV))
-$(info I CXX:      $(CXXV))
-$(info )
-
-llmodel.o:
-	mkdir buildllm
-	cd buildllm && cmake ../../../gpt4all-backend/ $(CMAKEFLAGS) && make
-	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel_c.cpp.o ../llmodel_c.o
-	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel.cpp.o ../llmodel.o
-
-clean:
-	rm -f *.o
-	rm -f *.a
-	rm -rf buildllm
-	rm -rf example/main
-
-binding.o: 
-	$(CXX) $(CXXFLAGS) binding.cpp -o binding.o -c $(LDFLAGS)
-
-libgpt4all.a: binding.o llmodel.o
-	ar src libgpt4all.a llmodel.o binding.o
-
-test: libgpt4all.a
-	@C_INCLUDE_PATH=${INCLUDE_PATH} LIBRARY_PATH=${LIBRARY_PATH} go test -v ./...
-
-example/main: libgpt4all.a
-	C_INCLUDE_PATH=$(INCLUDE_PATH) LIBRARY_PATH=$(INCLUDE_PATH) go build -o example/main ./example/

gpt4all-bindings/golang/README.md

@@ -1,59 +0,0 @@
-# GPT4All Golang bindings
-
-The golang bindings have been tested on:
-- MacOS
-- Linux
-
-### Usage
-
-```
-import (
-	"github.com/nomic-ai/gpt4all/gpt4all-bindings/golang"
-)
-
-func main() {
-	// Load the model
-	model, err := gpt4all.New("model.bin", gpt4all.SetModelType(gpt4all.GPTJType))
-	if err != nil {
-		panic(err)
-	}
-	defer model.Free()
-
-	model.SetTokenCallback(func(s string) bool {
-		fmt.Print(s)
-		return true
-	})
-
-	_, err = model.Predict("Here are 4 steps to create a website:", gpt4all.SetTemperature(0.1))
-	if err != nil {
-		panic(err)
-	}
-}
-```
-
-## Building
-
-In order to use the bindings you will need to build `libgpt4all.a`:
-
-```
-git clone https://github.com/nomic-ai/gpt4all
-cd gpt4all/gpt4all-bindings/golang
-make libgpt4all.a
-```
-
-To use the bindings in your own software:
-
-- Import `github.com/nomic-ai/gpt4all/gpt4all-bindings/golang`;
-- Compile `libgpt4all.a` (you can use `make libgpt4all.a` in the bindings/go directory);
-- Link your go binary against whisper by setting the environment variables `C_INCLUDE_PATH` and `LIBRARY_PATH` to point to the `binding.h` file directory and `libgpt4all.a` file directory respectively.
-- Note: you need to have *.so/*.dynlib/*.dll files of the implementation nearby the binary produced by the binding in order to make this to work
-
-## Testing
-
-To run tests, run `make test`:
-
-```
-git clone https://github.com/nomic-ai/gpt4all
-cd gpt4all/gpt4all-bindings/golang
-make test
-```

gpt4all-bindings/golang/binding.cpp

@@ -1,104 +0,0 @@
-#include "../../gpt4all-backend/llmodel_c.h"
-#include "../../gpt4all-backend/llmodel.h"
-#include "../../gpt4all-backend/llmodel_c.cpp"
-
-#include "binding.h"
-#include <cassert>
-#include <cmath>
-#include <cstddef>
-#include <cstdio>
-#include <cstring>
-#include <fstream>
-#include <map>
-#include <string>
-#include <vector>
-#include <iostream>
-#include <unistd.h>
-
-void* load_model(const char *fname, int n_threads) {
-    // load the model
-    llmodel_error new_error{};
-    auto model = llmodel_model_create2(fname, "auto", &new_error);
-    if (model == nullptr ){
-        fprintf(stderr, "%s: error '%s'\n",
-                __func__, new_error.message);
-        return nullptr;
-    }
-    llmodel_setThreadCount(model,  n_threads);
-    if (!llmodel_loadModel(model, fname)) {
-        return nullptr;
-    }
-
-    return model;
-}
-
-std::string res = "";
-void * mm;
-
-void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float temp, int n_batch,float ctx_erase)
-{
-    llmodel_model* model = (llmodel_model*) m;
-
-   // std::string res = "";
- 
-    auto lambda_prompt = [](int token_id)  {
-	        return true;
-    };
-
-    mm=model;
-    res="";
-
-    auto lambda_response = [](int token_id, const char *responsechars) {
-        res.append((char*)responsechars);
-        return !!getTokenCallback(mm, (char*)responsechars);
-	};
-	
-	auto lambda_recalculate = [](bool is_recalculating) {
-	        // You can handle recalculation requests here if needed
-	    return is_recalculating;
-	};
-
-    llmodel_prompt_context* prompt_context = new llmodel_prompt_context{
-        .logits = NULL,
-        .logits_size = 0,
-        .tokens = NULL,
-        .tokens_size = 0,
-        .n_past = 0,
-        .n_ctx = 1024,
-        .n_predict = 50,
-        .top_k = 10,
-        .top_p = 0.9,
-        .temp = 1.0,
-        .n_batch = 1,
-        .repeat_penalty = 1.2,
-        .repeat_last_n = 10,
-        .context_erase = 0.5
-    };
-
-    prompt_context->n_predict = tokens;
-    prompt_context->repeat_last_n = repeat_last_n;
-    prompt_context->repeat_penalty = repeat_penalty;
-    prompt_context->n_ctx = n_ctx;
-    prompt_context->top_k = top_k;
-    prompt_context->context_erase = ctx_erase;
-    prompt_context->top_p = top_p;
-    prompt_context->temp = temp;
-    prompt_context->n_batch = n_batch;    
-
-    llmodel_prompt(model, prompt,
-                        lambda_prompt,
-                        lambda_response,
-                    lambda_recalculate,
-                    prompt_context );
-
-    strcpy(result, res.c_str()); 
-
-    free(prompt_context);
-}
-
-void free_model(void *state_ptr) {
-    llmodel_model* ctx = (llmodel_model*) state_ptr;
-    llmodel_model_destroy(*ctx);
-}
-

gpt4all-bindings/golang/binding.h

@@ -1,18 +0,0 @@
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdbool.h>
-
-void* load_model(const char *fname, int n_threads);
-
-void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float temp, int n_batch,float ctx_erase);
-
-void free_model(void *state_ptr);
-
-extern unsigned char getTokenCallback(void *, char *);
-
-#ifdef __cplusplus
-}
-#endif

gpt4all-bindings/golang/example/main.go

@@ -1,82 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"flag"
-	"fmt"
-	"io"
-	"os"
-	"runtime"
-	"strings"
-
-	gpt4all "github.com/nomic-ai/gpt4all/gpt4all-bindings/golang"
-)
-
-var (
-	threads = 4
-	tokens  = 128
-)
-
-func main() {
-	var model string
-
-	flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
-	flags.StringVar(&model, "m", "./models/7B/ggml-model-q4_0.bin", "path to q4_0.bin model file to load")
-	flags.IntVar(&threads, "t", runtime.NumCPU(), "number of threads to use during computation")
-	flags.IntVar(&tokens, "n", 512, "number of tokens to predict")
-
-	err := flags.Parse(os.Args[1:])
-	if err != nil {
-		fmt.Printf("Parsing program arguments failed: %s", err)
-		os.Exit(1)
-	}
-	l, err := gpt4all.New(model, gpt4all.SetThreads(threads))
-	if err != nil {
-		fmt.Println("Loading the model failed:", err.Error())
-		os.Exit(1)
-	}
-	fmt.Printf("Model loaded successfully.\n")
-
-	l.SetTokenCallback(func(token string) bool {
-		fmt.Print(token)
-		return true
-	})
-
-	reader := bufio.NewReader(os.Stdin)
-
-	for {
-		text := readMultiLineInput(reader)
-
-		_, err := l.Predict(text, gpt4all.SetTokens(tokens), gpt4all.SetTopK(90), gpt4all.SetTopP(0.86))
-		if err != nil {
-			panic(err)
-		}
-		fmt.Printf("\n\n")
-	}
-}
-
-// readMultiLineInput reads input until an empty line is entered.
-func readMultiLineInput(reader *bufio.Reader) string {
-	var lines []string
-	fmt.Print(">>> ")
-
-	for {
-		line, err := reader.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				os.Exit(0)
-			}
-			fmt.Printf("Reading the prompt failed: %s", err)
-			os.Exit(1)
-		}
-
-		if len(strings.TrimSpace(line)) == 0 {
-			break
-		}
-
-		lines = append(lines, line)
-	}
-
-	text := strings.Join(lines, "")
-	return text
-}

gpt4all-bindings/golang/go.mod

@@ -1,20 +0,0 @@
-module github.com/nomic-ai/gpt4all/gpt4all-bindings/golang
-
-go 1.19
-
-require (
-	github.com/onsi/ginkgo/v2 v2.9.4
-	github.com/onsi/gomega v1.27.6
-)
-
-require (
-	github.com/go-logr/logr v1.2.4 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
-	golang.org/x/net v0.9.0 // indirect
-	golang.org/x/sys v0.7.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/tools v0.8.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-)

gpt4all-bindings/golang/go.sum

@@ -1,40 +0,0 @@
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
-github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
-github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
-github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
-github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
-golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
-golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=