cmake: fix LLAMA_DIR use before set

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/config.yml

@@ -11,8 +11,10 @@ workflows:
           base-revision: main
           config-path: .circleci/continue_config.yml
           mapping: |
+            .circleci/.* run-all-workflows true
+            gpt4all-backend/.* run-all-workflows true
             gpt4all-bindings/python/.* run-python-workflow true
+            gpt4all-bindings/typescript/.* run-ts-workflow true
             gpt4all-bindings/csharp/.* run-csharp-workflow true
-            gpt4all-backend/.* run-chat-workflow true
             gpt4all-chat/.* run-chat-workflow true
             .* run-default-workflow true

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
-ignore-words-list = blong, belong, afterall
+ignore-words-list = blong, afterall, som, assistent, crasher
 skip = .git,*.pdf,*.svg,*.lock

.github/ISSUE_TEMPLATE/bindings-bug.md

@@ -0,0 +1,35 @@
+---
+name: "\U0001F6E0 Bindings Bug Report"
+about: A bug report for the GPT4All Bindings
+labels: ["bindings", "bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. --!>
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Example Code
+
+<!-- Please provide a minimal code example that can be used to experience this issue. Delete this section if it does not apply. --!>
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. --!>
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? --!>
+
+### Your Environment
+
+- Bindings version (e.g. "Version" from `pip show gpt4all`):
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,70 +0,0 @@
-name: "\U0001F41B Bug Report"
-description: Submit a bug report to help us improve GPT4All
-labels: ["02 Bug Report"]
-body:
-  - type: markdown
-    attributes:
-      value: >
-        Thank you for taking the time to file a bug report. Before creating a new
-        issue, please make sure to take a few moments to check the issue tracker
-        for existing issues about the bug.
-
-  - type: textarea
-    id: system-info
-    attributes:
-      label: System Info
-      description: Please share your system info with us.
-      placeholder: GPT4All version, platform, python version, etc...
-    validations:
-      required: true
-
-  - type: checkboxes
-    id: information-scripts-examples
-    attributes:
-      label: Information
-      description: "The problem arises when using:"
-      options:
-        - label: "The official example notebooks/scripts"
-        - label: "My own modified scripts"
-
-  - type: checkboxes
-    id: related-components
-    attributes:
-      label: Related Components
-      description: "Select the components related to the issue (if applicable):"
-      options:
-        - label: "backend"
-        - label: "bindings"
-        - label: "python-bindings"
-        - label: "chat-ui"
-        - label: "models"
-        - label: "circleci"
-        - label: "docker"
-        - label: "api"
-
-  - type: textarea
-    id: reproduction
-    validations:
-      required: true
-    attributes:
-      label: Reproduction
-      description: |
-        Please provide a [code sample](https://stackoverflow.com/help/minimal-reproducible-example) that reproduces the problem you ran into. It can be a Colab link or just a code snippet.
-        If you have code snippets, error messages, stack traces please provide them here as well.
-        Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting
-        Avoid screenshots when possible, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code.
-
-      placeholder: |
-        Steps to reproduce the behavior:
-
-          1.
-          2.
-          3.
-
-  - type: textarea
-    id: expected-behavior
-    validations:
-      required: true
-    attributes:
-      label: Expected behavior
-      description: "A clear and concise description of what you would expect to happen."

.github/ISSUE_TEMPLATE/chat-bug.md

@@ -0,0 +1,31 @@
+---
+name: "\U0001F4AC Chat UI Bug Report"
+about: A bug report for the GPT4All Chat UI
+labels: ["chat", "bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. --!>
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. --!>
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? --!>
+
+### Your Environment
+
+- GPT4All version:
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -1,2 +1 @@
-blank_issues_enabled: false
-version: 2.1
+version: 2.1

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,9 @@
+---
+name: "\U0001F4C4 Documentation"
+about: An issue related to the GPT4All documentation
+labels: ["documentation"]
+---
+
+### Documentation
+
+<!-- Please describe the issue with the documentation as clearly as possible. --!>

.github/ISSUE_TEMPLATE/documentation.yml

@@ -1,19 +0,0 @@
-name: Documentation
-description: Report an issue related to the GPT4All documentation.
-title: "DOC: <Please write a comprehensive title after the 'DOC: ' prefix>"
-labels: [03 - Documentation]
-
-body:
-- type: textarea
-  attributes: 
-    label: "Issue with current documentation:"
-    description: >
-      Please make sure to leave a reference to the document/code you're
-      referring to.
-
-- type: textarea
-  attributes:
-    label: "Idea or request for content:"
-    description: >
-      Please describe as clearly as possible what topics you think are missing
-      from the current documentation.

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,9 @@
+---
+name: "\U0001F680 Feature Request"
+about: Submit a proposal/request for a new GPT4All feature
+labels: ["enhancement"]
+---
+
+### Feature Request
+
+<!-- A clear and concise description of the feature proposal. -->

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -1,30 +0,0 @@
-name: "\U0001F680 Feature Request"
-description: Submit a proposal/request for a new GPT4All feature
-labels: ["02 Feature Request"]
-body:
-  - type: textarea
-    id: feature-request
-    validations:
-      required: true
-    attributes:
-      label: Feature request
-      description: |
-        A clear and concise description of the feature proposal. Please provide links to any relevant GitHub repos, papers, or other resources if relevant.
-
-  - type: textarea
-    id: motivation
-    validations:
-      required: true
-    attributes:
-      label: Motivation
-      description: |
-        Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too.
-
-  - type: textarea
-    id: contribution
-    validations:
-      required: true
-    attributes:
-      label: Your contribution
-      description: |
-        Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/nomic-ai/gpt4all/blob/main/CONTRIBUTING.md)

.github/ISSUE_TEMPLATE/other-bug.md

@@ -0,0 +1,32 @@
+---
+name: "\U0001F41B Other Bug Report"
+about: A bug in another component of GPT4All
+labels: ["bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. --!>
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. If this bug involves original code, please provide a minimal version that can reproduce the issue. --!>
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? --!>
+
+### Your Environment
+
+- GPT4All version (if applicable):
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->
+

.github/ISSUE_TEMPLATE/other.yml

@@ -1,18 +0,0 @@
-name: Other Issue
-description: Raise an issue that wouldn't be covered by the other templates.
-title: "Issue: <Please write a comprehensive title after the 'Issue: ' prefix>"
-labels: [04 - Other]
-
-body:
-  - type: textarea
-    attributes:
-      label: "Issue you'd like to raise."
-      description: >
-        Please describe the issue you'd like to raise as clearly as possible.
-        Make sure to include any relevant links or references.
-
-  - type: textarea
-    attributes:
-      label: "Suggestion:"
-      description: >
-        Please outline a suggestion to improve the issue here.

.gitignore

@@ -183,4 +183,7 @@ build_*
 build-*
 
 # IntelliJ
-.idea/
+.idea/
+
+# LLM models
+*.gguf

.gitmodules

@@ -1,9 +1,4 @@
-[submodule "llama.cpp-230519"]
-	path = gpt4all-backend/llama.cpp-230519
-	url = https://github.com/ggerganov/llama.cpp.git
-[submodule "llama.cpp-230511"]
-	path = gpt4all-backend/llama.cpp-230511
-	url = https://github.com/nomic-ai/llama.cpp
 [submodule "llama.cpp-mainline"]
 	path = gpt4all-backend/llama.cpp-mainline
 	url = https://github.com/nomic-ai/llama.cpp.git
+	branch = gguf

LICENSE_SOM.txt

@@ -0,0 +1,30 @@
+Software for Open Models License (SOM)
+Version 1.0 dated August 30th, 2023
+
+This license governs use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
+
+This license is intended to encourage open release of models created, modified, processed, or otherwise used via the Software under open licensing terms, and should be interpreted in light of that intent.
+
+1. Definitions
+The “Licensor” is the person or entity who is making the Software available under this license. “Software” is the software made available by Licensor under this license.
+A “Model” is the output of a machine learning algorithm, and excludes the Software.
+“Model Source Materials” must include the Model and model weights, and may include any input data, input data descriptions, documentation or training descriptions for the Model.
+“Open Licensing Terms” means: (a) any open source license approved by the Open Source Initiative, or (b) any other terms that make the Model Source Materials publicly available free of charge, and allow recipients to use, modify and distribute the Model Source Materials. Terms described in (b) may include reasonable restrictions such as non-commercial or non-production limitations, or require use in compliance with law.
+
+2. Grant of Rights. Subject to the conditions and limitations in section 3:
+(A) Copyright Grant. Licensor grants you a non-exclusive, worldwide, royalty-free copyright license to copy, modify, and distribute the Software and any modifications of the Software you create under this license. The foregoing license includes without limitation the right to create, modify, and use Models using this Software.
+
+(B) Patent Grant. Licensor grants you a non-exclusive, worldwide, royalty-free license, under any patents owned or controlled by Licensor, to make, have made, use, sell, offer for sale, import, or otherwise exploit the Software.  No license is granted to patent rights that are not embodied in the operation of the Software in the form provided by Licensor.
+
+3. Conditions and Limitations
+(A) Model Licensing and Access. If you use the Software to create, modify, process, or otherwise use any Model, including usage to create inferences with a Model, whether or not you make the Model available to others, you must make that Model Source Materials publicly available under Open Licensing Terms. 
+
+(B) No Re-Licensing. If you redistribute the Software, or modifications to the Software made under the license granted above, you must make it available only under the terms of this license. You may offer additional terms such as warranties, maintenance and support, but You, and not Licensor, are responsible for performing such terms.
+
+(C) No Trademark License. This license does not grant you rights to use the Licensor’s name, logo, or trademarks.
+
+(D) If you assert in writing a claim against any person or entity alleging that the use of the Software infringes any patent, all of your licenses to the Software under Section 2 end automatically as of the date you asserted the claim.
+
+(E) If you distribute any portion of the Software, you must retain all copyright, patent, trademark, and attribution notices that are present in the Software, and you must include a copy of this license.
+
+(F) The Software is licensed “as-is.” You bear the entire risk of using it. Licensor gives You no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws that this license cannot change. To the extent permitted under your local laws, the Licensor disclaims and excludes the implied warranties of merchantability, fitness for a particular purpose and non-infringement. To the extent this disclaimer is unlawful, you, and not Licensor, are responsible for any liability.

README.md

@@ -1,8 +1,9 @@
 <h1 align="center">GPT4All</h1>
-<p align="center">Open-source assistant-style large language models that run locally on your CPU</p>
+
+<p align="center">Open-source large language models that run locally on your CPU and nearly any GPU</p>
 
 <p align="center">
-<a href="https://gpt4all.io">GPT4All Website</a>
+<a href="https://gpt4all.io">GPT4All Website and Models</a>
 </p>
 
 <p align="center">
@@ -29,13 +30,24 @@ Run on an M1 macOS Device (not sped up!)
 </p>
 
 ## GPT4All: An ecosystem of open-source on-edge large language models.
-GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs.
+
+> [!IMPORTANT]
+> GPT4All v2.5.0 and newer only supports models in GGUF format (.gguf). Models used with a previous version of GPT4All (.bin extension) will no longer work.
+
+GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and any GPU. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
 
 Learn more in the [documentation](https://docs.gpt4all.io).
 
-The goal is simple - be the best instruction tuned assistant-style language model that any person or enterprise can freely use, distribute and build on.
+A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models.
 
-A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models. 
+### What's New ([Issue Tracker](https://github.com/orgs/nomic-ai/projects/2))
+- **October 19th, 2023**: GGUF Support Launches with Support for:
+    - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
+    - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4_0, Q6 quantizations in GGUF.
+    - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
+- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on AMD, Intel, Samsung, Qualcomm and NVIDIA GPUs.
+- **August 15th, 2023**: GPT4All API launches allowing inference of local LLMs from docker containers.
+- **July 2023**: Stable support for LocalDocs, a GPT4All Plugin that allows you to privately and locally chat with your data.
 
 
 ### Chat Client
@@ -57,12 +69,15 @@ Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io
 
 ### Bindings
 
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a>
+* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
 * <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/java">:computer: Official Java Bindings</a>
 
+### Integrations
+
+* 🗃️ [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
 
 ## Contributing
 GPT4All welcomes contributions, involvement, and discussion from the open source community!

gpt4all-api/.isort.cfg

@@ -0,0 +1,7 @@
+[settings]
+known_third_party=geopy,nltk,np,numpy,pandas,pysbd,fire,torch
+
+line_length=120
+include_trailing_comma=True
+multi_line_output=3
+use_parentheses=True

gpt4all-api/README.md

@@ -4,9 +4,13 @@ for serving inference from GPT4All models. The API matches the OpenAI API spec.
 
 ## Tutorial
 
+The following tutorial assumes that you have checked out this repo and cd'd into it.
+
 ### Starting the app
 
-First build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
+First change your working directory to `gpt4all/gpt4all-api`.
+
+Now you can build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
 ```bash
 DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
 ```
@@ -17,6 +21,18 @@ Then, start the backend with:
 docker compose up --build
 ```
 
+This will run both the API and locally hosted GPU inference server. If you want to run the API without the GPU inference server, you can run:
+
+```bash
+docker compose up --build gpt4all_api
+```
+
+To run the API with the GPU inference server, you will need to include environment variables (like the `MODEL_ID`). Edit the `.env` file and run
+```bash
+docker compose --env-file .env up --build
+```
+
+
 #### Spinning up your app
 Run `docker compose up` to spin up the backend. Monitor the logs for errors in-case you forgot to set an environment variable above.
 
@@ -27,7 +43,7 @@ Run
 ```bash
 docker compose up --build
 ```
-and edit files in the `api` directory. The api will hot-reload on changes.
+and edit files in the `app` directory. The api will hot-reload on changes.
 
 You can run the unit tests with
 

gpt4all-api/docker-compose.gpu.yaml

@@ -0,0 +1,24 @@
+version: "3.8"
+
+services:
+  gpt4all_gpu:
+    image: ghcr.io/huggingface/text-generation-inference:0.9.3
+    container_name: gpt4all_gpu
+    restart: always #restart on error (usually code compilation from save during bad state)
+    environment:
+      - HUGGING_FACE_HUB_TOKEN=token
+      - USE_FLASH_ATTENTION=false
+      - MODEL_ID=''
+      - NUM_SHARD=1
+    command: --model-id $MODEL_ID --num-shard $NUM_SHARD
+    volumes:
+      - ./:/data
+    ports:
+      - "8080:80"
+    shm_size: 1g
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: [gpu]

gpt4all-api/docker-compose.yaml

@@ -1,4 +1,4 @@
-version: "3.5"
+version: "3.8"
 
 services:
   gpt4all_api:
@@ -7,12 +7,16 @@ services:
     restart: always #restart on error (usually code compilation from save during bad state)
     ports:
       - "4891:4891"
+    env_file:
+      - .env
     environment:
       - APP_ENVIRONMENT=dev
       - WEB_CONCURRENCY=2
       - LOGLEVEL=debug
       - PORT=4891
-      - model=ggml-mpt-7b-chat.bin
+      - model=${MODEL_BIN} # using variable from .env file
+      - inference_mode=cpu
     volumes:
       - './gpt4all_api/app:/app'
+      - './gpt4all_api/models:/models' # models are mounted in the container
     command: ["/start-reload.sh"]

gpt4all-api/gpt4all_api/Dockerfile.buildkit

@@ -1,8 +1,6 @@
 # syntax=docker/dockerfile:1.0.0-experimental
 FROM tiangolo/uvicorn-gunicorn:python3.11
 
-ARG MODEL_BIN=ggml-mpt-7b-chat.bin
-
 # Put first so anytime this file changes other cached layers are invalidated.
 COPY gpt4all_api/requirements.txt /requirements.txt
 
@@ -17,7 +15,3 @@ COPY gpt4all_api/app /app
 
 RUN mkdir -p /models
 
-# Include the following line to bake a model into the image and not have to download it on API start.
-RUN wget -q --show-progress=off https://gpt4all.io/models/${MODEL_BIN} -P /models \
-  && md5sum /models/${MODEL_BIN}
-

gpt4all-api/gpt4all_api/app/api_v1/api.py

@@ -1,4 +1,4 @@
-from api_v1.routes import chat, completions, engines
+from api_v1.routes import chat, completions, engines, health
 from fastapi import APIRouter
 
 router = APIRouter()
@@ -6,3 +6,4 @@ router = APIRouter()
 router.include_router(chat.router)
 router.include_router(completions.router)
 router.include_router(engines.router)
+router.include_router(health.router)

gpt4all-api/gpt4all_api/app/api_v1/events.py

@@ -1,8 +1,10 @@
 import logging
+
+from api_v1.settings import settings
 from fastapi import HTTPException
 from fastapi.responses import JSONResponse
 from starlette.requests import Request
-from api_v1.settings import settings
+
 log = logging.getLogger(__name__)
 
 
@@ -19,8 +21,9 @@ async def on_startup(app):
     startup_msg = startup_msg_fmt.format(settings=settings)
     log.info(startup_msg)
 
+
 def startup_event_handler(app):
     async def start_app() -> None:
         await on_startup(app)
 
-    return start_app
+    return start_app

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -1,29 +1,28 @@
-from fastapi import APIRouter, Depends, Response, Security, status
-from pydantic import BaseModel, Field
-from typing import List, Dict
 import logging
 import time
+from typing import List
+from uuid import uuid4
+from fastapi import APIRouter
+from pydantic import BaseModel, Field
 from api_v1.settings import settings
+from fastapi.responses import StreamingResponse
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
 ### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-
 class ChatCompletionMessage(BaseModel):
     role: str
     content: str
 
 class ChatCompletionRequest(BaseModel):
-    model: str = Field(..., description='The model to generate a completion from.')
-    messages: List[ChatCompletionMessage] = Field(..., description='The model to generate a completion from.')
-
+    model: str = Field(settings.model, description='The model to generate a completion from.')
+    messages: List[ChatCompletionMessage] = Field(..., description='Messages for the chat completion.')
 
 class ChatCompletionChoice(BaseModel):
     message: ChatCompletionMessage
     index: int
+    logprobs: float
     finish_reason: str
 
 class ChatCompletionUsage(BaseModel):
@@ -39,25 +38,38 @@ class ChatCompletionResponse(BaseModel):
     choices: List[ChatCompletionChoice]
     usage: ChatCompletionUsage
 
-
 router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
 
 @router.post("/completions", response_model=ChatCompletionResponse)
 async def chat_completion(request: ChatCompletionRequest):
     '''
-    Completes a GPT4All model response.
+    Completes a GPT4All model response based on the last message in the chat.
     '''
+    # Example: Echo the last message content with some modification
+    if request.messages:
+        last_message = request.messages[-1].content
+        response_content = f"Echo: {last_message}"
+    else:
+        response_content = "No messages received."
 
-    return ChatCompletionResponse(
-        id='asdf',
-        created=time.time(),
-        model=request.model,
-        choices=[{}],
-        usage={
-            'prompt_tokens': 0,
-            'completion_tokens': 0,
-            'total_tokens': 0
-        }
+    # Create a chat message for the response
+    response_message = ChatCompletionMessage(role="system", content=response_content)
+
+    # Create a choice object with the response message
+    response_choice = ChatCompletionChoice(
+        message=response_message,
+        index=0,
+        logprobs=-1.0,  # Placeholder value
+        finish_reason="length"  # Placeholder value
     )
 
+    # Create the response object
+    chat_response = ChatCompletionResponse(
+        id=str(uuid4()),
+        created=int(time.time()),
+        model=request.model,
+        choices=[response_choice],
+        usage=ChatCompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0),  # Placeholder values
+    )
 
+    return chat_response

gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py

@@ -1,14 +1,16 @@
 import json
-
-from fastapi import APIRouter, Depends, Response, Security, status
-from fastapi.responses import StreamingResponse
-from pydantic import BaseModel, Field
 from typing import List, Dict, Iterable, AsyncIterable
 import logging
-from uuid import uuid4
-from api_v1.settings import settings
-from gpt4all import GPT4All
 import time
+from typing import Dict, List, Union, Optional
+from uuid import uuid4
+import aiohttp
+import asyncio
+from api_v1.settings import settings
+from fastapi import APIRouter, Depends, Response, Security, status, HTTPException
+from fastapi.responses import StreamingResponse
+from gpt4all import GPT4All
+from pydantic import BaseModel, Field
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -16,14 +18,17 @@ logger.setLevel(logging.DEBUG)
 
 ### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
 
+
 class CompletionRequest(BaseModel):
-    model: str = Field(..., description='The model to generate a completion from.')
-    prompt: str = Field(..., description='The prompt to begin completing from.')
-    max_tokens: int = Field(7, description='Max tokens to generate')
-    temperature: float = Field(0, description='Model temperature')
-    top_p: float = Field(1.0, description='top_p')
-    n: int = Field(1, description='')
+    model: str = Field(settings.model, description='The model to generate a completion from.')
+    prompt: Union[List[str], str] = Field(..., description='The prompt to begin completing from.')
+    max_tokens: int = Field(None, description='Max tokens to generate')
+    temperature: float = Field(settings.temp, description='Model temperature')
+    top_p: Optional[float] = Field(settings.top_p, description='top_p')
+    top_k: Optional[int] = Field(settings.top_k, description='top_k')
+    n: int = Field(1, description='How many completions to generate for each prompt')
     stream: bool = Field(False, description='Stream responses')
+    repeat_penalty: float = Field(settings.repeat_penalty, description='Repeat penalty')
 
 
 class CompletionChoice(BaseModel):
@@ -58,7 +63,6 @@ class CompletionStreamResponse(BaseModel):
 
 router = APIRouter(prefix="/completions", tags=["Completion Endpoints"])
 
-
 def stream_completion(output: Iterable, base_response: CompletionStreamResponse):
     """
     Streams a GPT4All output to the client.
@@ -80,49 +84,132 @@ def stream_completion(output: Iterable, base_response: CompletionStreamResponse)
         ))]
         yield f"data: {json.dumps(dict(chunk))}\n\n"
 
+async def gpu_infer(payload, header):
+    async with aiohttp.ClientSession() as session:
+        try:
+            async with session.post(
+                settings.hf_inference_server_host, headers=header, data=json.dumps(payload)
+            ) as response:
+                resp = await response.json()
+            return resp
+
+        except aiohttp.ClientError as e:
+            # Handle client-side errors (e.g., connection error, invalid URL)
+            logger.error(f"Client error: {e}")
+        except aiohttp.ServerError as e:
+            # Handle server-side errors (e.g., internal server error)
+            logger.error(f"Server error: {e}")
+        except json.JSONDecodeError as e:
+            # Handle JSON decoding errors
+            logger.error(f"JSON decoding error: {e}")
+        except Exception as e:
+            # Handle other unexpected exceptions
+            logger.error(f"Unexpected error: {e}")
 
 @router.post("/", response_model=CompletionResponse)
 async def completions(request: CompletionRequest):
     '''
     Completes a GPT4All model response.
     '''
+    if settings.inference_mode == "gpu":
+        params = request.dict(exclude={'model', 'prompt', 'max_tokens', 'n'})
+        params["max_new_tokens"] = request.max_tokens
+        params["num_return_sequences"] = request.n
 
-    model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+        header = {"Content-Type": "application/json"}
+        if isinstance(request.prompt, list):
+            tasks = []
+            for prompt in request.prompt:
+                payload = {"parameters": params}
+                payload["inputs"] = prompt
+                task = gpu_infer(payload, header)
+                tasks.append(task)
+            results = await asyncio.gather(*tasks)
 
-    output = model.generate(prompt=request.prompt,
-                            n_predict=request.max_tokens,
-                            streaming=request.stream,
-                            top_k=20,
-                            top_p=request.top_p,
-                            temp=request.temperature,
-                            n_batch=1024,
-                            repeat_penalty=1.2,
-                            repeat_last_n=10)
+            choices = []
+            for response in results:
+                scores = response["scores"] if "scores" in response else -1.0
+                choices.append(
+                    dict(
+                        CompletionChoice(
+                            text=response["generated_text"], index=0, logprobs=scores, finish_reason='stop'
+                        )
+                    )
+                )
+
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=choices,
+                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+            )
+
+        else:
+            payload = {"parameters": params}
+            # If streaming, we need to return a StreamingResponse
+            payload["inputs"] = request.prompt
+
+            resp = await gpu_infer(payload, header)
+
+            output = resp["generated_text"]
+            # this returns all logprobs
+            scores = resp["scores"] if "scores" in resp else -1.0
+
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[dict(CompletionChoice(text=output, index=0, logprobs=scores, finish_reason='stop'))],
+                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+            )
 
-    # If streaming, we need to return a StreamingResponse
-    if request.stream:
-        base_chunk = CompletionStreamResponse(
-            id=str(uuid4()),
-            created=time.time(),
-            model=request.model,
-            choices=[]
-        )
-        return StreamingResponse((response for response in stream_completion(output, base_chunk)),
-                                 media_type="text/event-stream")
     else:
-        return CompletionResponse(
-            id=str(uuid4()),
-            created=time.time(),
-            model=request.model,
-            choices=[dict(CompletionChoice(
-                text=output,
-                index=0,
-                logprobs=-1,
-                finish_reason='stop'
-            ))],
-            usage={
-                'prompt_tokens': 0, #TODO how to compute this?
-                'completion_tokens': 0,
-                'total_tokens': 0
-            }
-        )
+
+        if request.model != settings.model:
+            raise HTTPException(status_code=400,
+                                detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
+
+        if isinstance(request.prompt, list):
+            if len(request.prompt) > 1:
+                raise HTTPException(status_code=400, detail="Can only infer one inference per request in CPU mode.")
+            else:
+                request.prompt = request.prompt[0]
+
+        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+        output = model.generate(prompt=request.prompt,
+                                max_tokens=request.max_tokens,
+                                streaming=request.stream,
+                                top_k=request.top_k,
+                                top_p=request.top_p,
+                                temp=request.temperature,
+                                )
+
+        # If streaming, we need to return a StreamingResponse
+        if request.stream:
+            base_chunk = CompletionStreamResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[]
+            )
+            return StreamingResponse((response for response in stream_completion(output, base_chunk)),
+                                     media_type="text/event-stream")
+        else:
+            return CompletionResponse(
+                id=str(uuid4()),
+                created=time.time(),
+                model=request.model,
+                choices=[dict(CompletionChoice(
+                    text=output,
+                    index=0,
+                    logprobs=-1,
+                    finish_reason='stop'
+                ))],
+                usage={
+                    'prompt_tokens': 0,  # TODO how to compute this?
+                    'completion_tokens': 0,
+                    'total_tokens': 0
+                }
+            )

gpt4all-api/gpt4all_api/app/api_v1/routes/embeddings.py

@@ -0,0 +1,65 @@
+from typing import List, Union
+from fastapi import APIRouter
+from api_v1.settings import settings
+from gpt4all import Embed4All
+from pydantic import BaseModel, Field
+
+### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+
+
+class EmbeddingRequest(BaseModel):
+    model: str = Field(
+        settings.model, description="The model to generate an embedding from."
+    )
+    input: Union[str, List[str], List[int], List[List[int]]] = Field(
+        ..., description="Input text to embed, encoded as a string or array of tokens."
+    )
+
+
+class EmbeddingUsage(BaseModel):
+    prompt_tokens: int = 0
+    total_tokens: int = 0
+
+
+class Embedding(BaseModel):
+    index: int = 0
+    object: str = "embedding"
+    embedding: List[float]
+
+
+class EmbeddingResponse(BaseModel):
+    object: str = "list"
+    model: str
+    data: List[Embedding]
+    usage: EmbeddingUsage
+
+
+router = APIRouter(prefix="/embeddings", tags=["Embedding Endpoints"])
+
+embedder = Embed4All()
+
+
+def get_embedding(data: EmbeddingRequest) -> EmbeddingResponse:
+    """
+    Calculates the embedding for the given input using a specified model.
+
+    Args:
+        data (EmbeddingRequest): An EmbeddingRequest object containing the input data
+        and model name.
+
+    Returns:
+        EmbeddingResponse: An EmbeddingResponse object encapsulating the calculated embedding,
+        usage info, and the model name.
+    """
+    embedding = embedder.embed(data.input)
+    return EmbeddingResponse(
+        data=[Embedding(embedding=embedding)], usage=EmbeddingUsage(), model=data.model
+    )
+
+
+@router.post("/", response_model=EmbeddingResponse)
+def embeddings(data: EmbeddingRequest):
+    """
+    Creates a GPT4All embedding
+    """
+    return get_embedding(data)

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -1,38 +1,39 @@
-from fastapi import APIRouter, Depends, Response, Security, status
+import requests
+from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel, Field
 from typing import List, Dict
-import logging
-from api_v1.settings import settings
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+# Define the router for the engines module
+router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
 
+# Define the models for the engines module
 class ListEnginesResponse(BaseModel):
     data: List[Dict] = Field(..., description="All available models.")
 
 class EngineResponse(BaseModel):
     data: List[Dict] = Field(..., description="All available models.")
 
-router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
 
+# Define the routes for the engines module
 @router.get("/", response_model=ListEnginesResponse)
 async def list_engines():
-    '''
-    List all available GPT4All models from
-    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json
-    '''
-    raise NotImplementedError()
-    return ListEnginesResponse(data=[])
-
+    try:
+        response = requests.get('https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json')
+        response.raise_for_status()  # This will raise an HTTPError if the HTTP request returned an unsuccessful status code
+        engines = response.json()
+        return ListEnginesResponse(data=engines)
+    except requests.RequestException as e:
+        logger.error(f"Error fetching engine list: {e}")
+        raise HTTPException(status_code=500, detail="Error fetching engine list")
 
+# Define the routes for the engines module
 @router.get("/{engine_id}", response_model=EngineResponse)
 async def retrieve_engine(engine_id: str):
-    '''
-
-    '''
-
-    raise NotImplementedError()
-    return EngineResponse()
-
+    try:
+        # Implement logic to fetch a specific engine's details
+        # This is a placeholder, replace with your actual data retrieval logic
+        engine_details = {"id": engine_id, "name": "Engine Name", "description": "Engine Description"}
+        return EngineResponse(data=[engine_details])
+    except Exception as e:
+        logger.error(f"Error fetching engine details: {e}")
+        raise HTTPException(status_code=500, detail=f"Error fetching details for engine {engine_id}")

gpt4all-api/gpt4all_api/app/api_v1/routes/health.py

@@ -1,6 +1,7 @@
 import logging
 from fastapi import APIRouter
 from fastapi.responses import JSONResponse
+
 log = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/health", tags=["Health"])

gpt4all-api/gpt4all_api/app/api_v1/settings.py

@@ -5,6 +5,15 @@ class Settings(BaseSettings):
     app_environment = 'dev'
     model: str = 'ggml-mpt-7b-chat.bin'
     gpt4all_path: str = '/models'
+    inference_mode: str = "cpu"
+    hf_inference_server_host: str = "http://gpt4all_gpu:80/generate"
+    sentry_dns: str = None
+
+    temp: float = 0.18
+    top_p: float = 1.0
+    top_k: int = 50
+    repeat_penalty: float = 1.18
+
 
 
 settings = Settings()

gpt4all-api/gpt4all_api/app/main.py

@@ -1,19 +1,19 @@
-import os
-import docs
 import logging
-from fastapi import FastAPI, HTTPException, Request
-from starlette.middleware.cors import CORSMiddleware
-from fastapi.logger import logger as fastapi_logger
-from api_v1.settings import settings
-from api_v1.api import router as v1_router
-from api_v1 import events
 import os
 
+import docs
+from api_v1 import events
+from api_v1.api import router as v1_router
+from api_v1.settings import settings
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.logger import logger as fastapi_logger
+from starlette.middleware.cors import CORSMiddleware
+
 logger = logging.getLogger(__name__)
 
 app = FastAPI(title='GPT4All API', description=docs.desc)
 
-#CORS Configuration (in-case you want to deploy)
+# CORS Configuration (in-case you want to deploy)
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -29,20 +29,41 @@ app.include_router(v1_router, prefix='/v1')
 app.add_event_handler('startup', events.startup_event_handler(app))
 app.add_exception_handler(HTTPException, events.on_http_error)
 
+
 @app.on_event("startup")
 async def startup():
     global model
-    logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
-    from gpt4all import GPT4All
-    model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+    if settings.inference_mode == "cpu":
+        logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
+        from gpt4all import GPT4All
+
+        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
+
+    else:
+        # is it possible to do this once the server is up?
+        ## TODO block until HF inference server is up.
+        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
+
 
-    logger.info("GPT4All API is ready.")
 
 @app.on_event("shutdown")
 async def shutdown():
     logger.info("Shutting down API")
 
 
+if settings.sentry_dns is not None:
+    import sentry_sdk
+
+    def traces_sampler(sampling_context):
+        if 'health' in sampling_context['transaction_context']['name']:
+            return False
+
+    sentry_sdk.init(
+        dsn=settings.sentry_dns, traces_sample_rate=0.1, traces_sampler=traces_sampler, send_default_pii=False
+    )
+
 # This is needed to get logs to show up in the app
 if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
     gunicorn_error_logger = logging.getLogger("gunicorn.error")
@@ -57,5 +78,7 @@ if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
     uvicorn_logger.handlers = gunicorn_error_logger.handlers
 else:
     # https://github.com/tiangolo/fastapi/issues/2019
-    LOG_FORMAT2 = "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
-    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)
+    LOG_FORMAT2 = (
+        "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
+    )
+    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -1,31 +1,35 @@
 """
 Use the OpenAI python API to test gpt4all models.
 """
-import openai
-openai.api_base = "http://localhost:4891/v1"
+from typing import List, get_args
+import os
+from dotenv import load_dotenv
 
+import openai
+
+openai.api_base = "http://localhost:4891/v1"
 openai.api_key = "not needed for a local LLM"
 
+# Load the .env file
+env_path = 'gpt4all-api/gpt4all_api/.env'
+load_dotenv(dotenv_path=env_path)
+
+# Fetch MODEL_ID from .env file
+model_id = os.getenv('MODEL_BIN', 'default_model_id')
+embedding = os.getenv('EMBEDDING', 'default_embedding_model_id')
+print (model_id)
+print (embedding)
 
 def test_completion():
-    model = "gpt4all-j-v1.3-groovy"
+    model = model_id
     prompt = "Who is Michael Jordan?"
     response = openai.Completion.create(
-        model=model,
-        prompt=prompt,
-        max_tokens=50,
-        temperature=0.28,
-        top_p=0.95,
-        n=1,
-        echo=True,
-        stream=False
+        model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
     )
     assert len(response['choices'][0]['text']) > len(prompt)
-    print(response)
-
 
 def test_streaming_completion():
-    model = "gpt4all-j-v1.3-groovy"
+    model = model_id
     prompt = "Who is Michael Jordan?"
     tokens = []
     for resp in openai.Completion.create(
@@ -42,10 +46,32 @@ def test_streaming_completion():
     assert (len(tokens) > 0)
     assert (len("".join(tokens)) > len(prompt))
 
-# def test_chat_completions():
-#     model = "gpt4all-j-v1.3-groovy"
-#     prompt = "Who is Michael Jordan?"
-#     response = openai.ChatCompletion.create(
-#         model=model,
-#         messages=[]
-#     )
+# Modified test batch, problems with keyerror in response
+def test_batched_completion():
+    model = model_id  # replace with your specific model ID
+    prompt = "Who is Michael Jordan?"
+    responses = []
+    
+    # Loop to create completions one at a time
+    for _ in range(3):
+        response = openai.Completion.create(
+            model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
+        )
+        responses.append(response)
+
+    # Assertions to check the responses
+    for response in responses:
+        assert len(response['choices'][0]['text']) > len(prompt)
+    
+    assert len(responses) == 3
+
+def test_embedding():
+    model = embedding
+    prompt = "Who is Michael Jordan?"
+    response = openai.Embedding.create(model=model, input=prompt)
+    output = response["data"][0]["embedding"]
+    args = get_args(List[float])
+
+    assert response["model"] == model
+    assert isinstance(output, list)
+    assert all(isinstance(x, args) for x in output)

gpt4all-api/gpt4all_api/env

@@ -0,0 +1,3 @@
+# Add your GGUF compatible model LLM here. ie: MODEL_BIN="mistral-7b-instruct-v0.1.Q4_0", rename file ".env"
+# Make sure this LLM matches the model you placed inside the models folder
+MODEL_BIN=""

gpt4all-api/gpt4all_api/models/README.md

@@ -0,0 +1 @@
+### Drop GGUF compatible models here, make sure it matches MODEL_BIN on your .env file

gpt4all-api/gpt4all_api/requirements.txt

@@ -1,10 +1,13 @@
 aiohttp>=3.6.2
 aiofiles
-pydantic>=1.4.0
+pydantic>=1.4.0,<2.0.0
 requests>=2.24.0
 ujson>=2.0.2
 fastapi>=0.95.0
 Jinja2>=3.0
-gpt4all==1.0.1
+gpt4all>=1.0.0
 pytest
-openai
+openai==0.28.0
+black
+isort
+python-dotenv

gpt4all-api/makefile

@@ -1,22 +1,26 @@
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 APP_NAME:=gpt4all_api
 PYTHON:=python3.8
+SHELL := /bin/bash
 
 all: dependencies
 
 fresh: clean dependencies
 
 testenv: clean_testenv test_build
-	docker compose up --build
+	docker compose -f docker-compose.yaml up --build
+
+testenv_gpu: clean_testenv test_build
+	docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up --build
 
 testenv_d: clean_testenv test_build
-	docker compose up --build -d
+	docker compose env up --build -d
 
 test:
-	docker compose exec gpt4all_api pytest -svv --disable-warnings -p no:cacheprovider /app/tests
+	docker compose exec $(APP_NAME) pytest -svv --disable-warnings -p no:cacheprovider /app/tests
 
 test_build:
-    DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
+    DOCKER_BUILDKIT=1 docker build -t $(APP_NAME) --progress plain -f $(APP_NAME)/Dockerfile.buildkit .
 
 clean_testenv:
 	docker compose down -v
@@ -24,14 +28,19 @@ clean_testenv:
 fresh_testenv: clean_testenv testenv
 
 venv:
-	if [ ! -d $(ROOT_DIR)/env ]; then $(PYTHON) -m venv $(ROOT_DIR)/env; fi
+	if [ ! -d $(ROOT_DIR)/venv ]; then $(PYTHON) -m venv $(ROOT_DIR)/venv; fi
 
 dependencies: venv
-	source $(ROOT_DIR)/env/bin/activate; yes w | python -m pip install -r $(ROOT_DIR)/gpt4all_api/requirements.txt
+	source $(ROOT_DIR)/venv/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
 
 clean: clean_testenv
 	# Remove existing environment
-	rm -rf $(ROOT_DIR)/env;
+	rm -rf $(ROOT_DIR)/venv;
 	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
 
 
+black:
+	source $(ROOT_DIR)/venv/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
+
+isort:
+	source $(ROOT_DIR)/venv/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)

gpt4all-backend/CMakeLists.txt

@@ -20,7 +20,7 @@ endif()
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 
 set(LLMODEL_VERSION_MAJOR 0)
-set(LLMODEL_VERSION_MINOR 2)
+set(LLMODEL_VERSION_MINOR 5)
 set(LLMODEL_VERSION_PATCH 0)
 set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
 project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
@@ -69,11 +69,6 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     # Include GGML
     set(LLAMA_K_QUANTS YES)
     include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
-    if (NOT LLAMA_METAL)
-        set(LLAMA_K_QUANTS NO)
-        include_ggml(llama.cpp-230511 -230511-${BUILD_VARIANT} ON)
-        include_ggml(llama.cpp-230519 -230519-${BUILD_VARIANT} ON)
-    endif()
 
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
@@ -98,33 +93,15 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    add_library(replit-mainline-${BUILD_VARIANT} SHARED
-    replit.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-    prepare_target(replit-mainline llama-mainline)
-
     if (NOT LLAMA_METAL)
-        add_library(llamamodel-230519-${BUILD_VARIANT} SHARED
-            llamamodel.cpp llmodel_shared.cpp)
-        target_compile_definitions(llamamodel-230519-${BUILD_VARIANT} PRIVATE
-            LLAMA_VERSIONS===2 LLAMA_DATE=230519)
-        prepare_target(llamamodel-230519 llama-230519)
-        add_library(llamamodel-230511-${BUILD_VARIANT} SHARED
-            llamamodel.cpp llmodel_shared.cpp)
-        target_compile_definitions(llamamodel-230511-${BUILD_VARIANT} PRIVATE
-            LLAMA_VERSIONS=<=1 LLAMA_DATE=230511)
-        prepare_target(llamamodel-230511 llama-230511)
-
         add_library(gptj-${BUILD_VARIANT} SHARED
             gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        prepare_target(gptj ggml-230511)
+        prepare_target(gptj llama-mainline)
 
-        add_library(falcon-${BUILD_VARIANT} SHARED
-            falcon.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        prepare_target(falcon llama-mainline)
-
-        add_library(mpt-${BUILD_VARIANT} SHARED
-            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        prepare_target(mpt ggml-230511)
+        add_library(bert-${BUILD_VARIANT} SHARED
+            bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
+        target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+        prepare_target(bert llama-mainline)
     endif()
 endforeach()
 

gpt4all-backend/bert.cpp

@@ -0,0 +1,908 @@
+#define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#include "bert_impl.h"
+#include "llmodel_shared.h"
+#include "ggml.h"
+
+#include <cassert>
+#include <cinttypes>
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+#include <map>
+#include <string>
+#include <vector>
+#include <iostream>
+#include <regex>
+#include <thread>
+#include <algorithm>
+#include <numeric>
+
+//#define DEBUG_BERT
+
+namespace {
+const char *modelType_ = "Bert";
+}
+
+typedef int32_t bert_vocab_id;
+
+// default hparams (all-MiniLM-L6-v2)
+struct bert_hparams
+{
+    int32_t n_vocab = 30522;
+    int32_t n_max_tokens = 512;
+    int32_t n_embd = 256;
+    int32_t n_intermediate = 1536;
+    int32_t n_head = 12;
+    int32_t n_layer = 6;
+};
+
+struct bert_layer
+{
+    // normalization
+    struct ggml_tensor *ln_att_w;
+    struct ggml_tensor *ln_att_b;
+
+    struct ggml_tensor *ln_out_w;
+    struct ggml_tensor *ln_out_b;
+
+    // attention
+    struct ggml_tensor *q_w;
+    struct ggml_tensor *q_b;
+    struct ggml_tensor *k_w;
+    struct ggml_tensor *k_b;
+    struct ggml_tensor *v_w;
+    struct ggml_tensor *v_b;
+
+    struct ggml_tensor *o_w;
+    struct ggml_tensor *o_b;
+
+    // ff
+    struct ggml_tensor *ff_i_w;
+    struct ggml_tensor *ff_i_b;
+
+    struct ggml_tensor *ff_o_w;
+    struct ggml_tensor *ff_o_b;
+};
+
+struct bert_vocab
+{
+    std::map<std::string, bert_vocab_id> token_to_id;
+    std::map<std::string, bert_vocab_id> subword_token_to_id;
+
+    std::map<bert_vocab_id, std::string> _id_to_token;
+    std::map<bert_vocab_id, std::string> _id_to_subword_token;
+};
+
+struct bert_model
+{
+    bert_hparams hparams;
+
+    // embeddings weights
+    struct ggml_tensor *word_embeddings;
+    struct ggml_tensor *token_type_embeddings;
+    struct ggml_tensor *position_embeddings;
+    struct ggml_tensor *ln_e_w;
+    struct ggml_tensor *ln_e_b;
+
+    std::vector<bert_layer> layers;
+
+    struct ggml_context *ctx;
+};
+
+// Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
+struct bert_ctx
+{
+    bert_model model;
+    bert_vocab vocab;
+
+    size_t mem_per_token;
+    int64_t mem_per_input;
+    int32_t max_batch_n;
+    llm_buffer buf_compute;
+    llm_buffer work_buf;
+};
+
+int32_t bert_n_embd(bert_ctx * ctx)
+{
+    return ctx->model.hparams.n_embd;
+}
+
+int32_t bert_n_max_tokens(bert_ctx * ctx)
+{
+    return ctx->model.hparams.n_max_tokens;
+}
+
+const char* bert_vocab_id_to_token(bert_ctx * ctx, bert_vocab_id id) {
+    bert_vocab & vocab = ctx->vocab;
+    auto it = vocab._id_to_token.find(id);
+    if (it != vocab._id_to_token.end())
+    {
+        return it->second.c_str();
+    }
+    it = vocab._id_to_subword_token.find(id);
+    if (it != vocab._id_to_subword_token.end())
+    {
+        return it->second.c_str();
+    }
+    return "[UNK TOKEN from bert_vocab]";
+}
+
+//
+// Tokenizing
+//
+
+static size_t utf8_len(char src)
+{
+    const size_t lookup[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4};
+    uint8_t highbits = static_cast<uint8_t>(src) >> 4;
+    return lookup[highbits];
+}
+
+std::string stripAccents(const std::string &inputString)
+{
+    std::string resultString;
+    std::map<std::string, char> accentMap = {{"À", 'A'},{"Á", 'A'},
+        {"Â", 'A'},{"Ã", 'A'},{"Ä", 'A'},{"Å", 'A'},{"à", 'a'},{"á", 'a'},
+        {"â", 'a'},{"ã", 'a'},{"ä", 'a'},{"å", 'a'},{"È", 'E'},{"É", 'E'},
+        {"Ê", 'E'},{"Ë", 'E'},{"è", 'e'},{"é", 'e'},{"ê", 'e'},{"ë", 'e'},
+        {"Ì", 'I'},{"Í", 'I'},{"Î", 'I'},{"Ï", 'I'},{"ì", 'i'},{"í", 'i'},
+        {"î", 'i'},{"ï", 'i'},{"Ò", 'O'},{"Ó", 'O'},{"Ô", 'O'},{"Õ", 'O'},
+        {"Ö", 'O'},{"ò", 'o'},{"ó", 'o'},{"ô", 'o'},{"õ", 'o'},{"ö", 'o'},
+        {"Ù", 'U'},{"Ú", 'U'},{"Û", 'U'},{"Ü", 'U'},{"ù", 'u'},{"ú", 'u'},
+        {"û", 'u'},{"ü", 'u'},{"Ý", 'Y'},{"ý", 'y'},{"Ç", 'C'},{"ç", 'c'},
+        {"Ñ", 'N'},{"ñ", 'n'},
+    };
+
+    for (size_t i = 0; i < inputString.length();)
+    {
+        int len = utf8_len(inputString[i]);
+        std::string curChar = inputString.substr(i, len);
+        auto iter = accentMap.find(curChar);
+        if (iter != accentMap.end())
+        {
+            resultString += iter->second;
+        }
+        else
+        {
+            resultString += curChar;
+        }
+        i += len;
+    }
+
+    return resultString;
+}
+
+std::string bert_normalize_prompt(const std::string &text)
+{
+    // TODO: handle chinese characters? https://github.com/huggingface/tokenizers/blob/ef5f50605ddf9f8caef1598c0e4853862b9707a7/tokenizers/src/normalizers/bert.rs#L98
+    std::string text2 = stripAccents(text);
+    for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i]))
+    {
+        char c = text2[i];
+        if (c >= 'A' && c <= 'Z')
+            text2[i] = c - 'A' + 'a';
+    }
+    return text2;
+}
+
+std::vector<bert_vocab_id> bert_tokenize(
+    struct bert_ctx * ctx,
+    const char * text)
+{
+    const bert_vocab &vocab = ctx->vocab;
+
+    std::string str = text;
+
+    std::vector<std::string> words;
+    // first split the text into words
+    {
+        str = bert_normalize_prompt(str);
+
+        std::string pat = R"([[:punct:]]|[[:alpha:]]+|[[:digit:]]+)";
+
+        std::regex re(pat);
+        std::smatch m;
+
+        while (std::regex_search(str, m, re))
+        {
+            for (std::string x : m)
+            {
+                words.push_back(x);
+            }
+            str = m.suffix();
+        }
+    }
+
+    // find the longest tokens that form the words:
+    std::vector<bert_vocab_id> tokens;
+    int cls_tok_id = 101;
+    tokens.push_back(cls_tok_id);
+    for (const auto &word : words)
+    {
+        if (word.size() == 0)
+            continue;
+
+        int i = 0;
+        int n = word.size();
+        auto *token_map = &vocab.token_to_id;
+        while (i < n)
+        {
+            int j = n;
+            while (j > i)
+            {
+                auto it = token_map->find(word.substr(i, j - i));
+                if (it != token_map->end())
+                {
+                    tokens.push_back(it->second);
+                    i = j;
+                    token_map = &vocab.subword_token_to_id;
+                }
+                --j;
+            }
+            if (j == i)
+            {
+                fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data());
+                token_map = &vocab.subword_token_to_id;
+                ++i;
+            }
+        }
+    }
+
+    return tokens;
+}
+
+void bert_resize_ctx(bert_ctx * ctx, int32_t new_size) {
+    int64_t buf_size_new = ctx->mem_per_input * new_size;
+
+    // TODO: Max memory should be a param? Now just 1 GB
+    int64_t GB = 1 << 30;
+#if defined(DEBUG_BERT)
+    printf("%s: requested_buf_size %lldMB\n", __func__, buf_size_new / (1 << 20));
+#endif
+    if (buf_size_new > GB) {
+        int32_t adjusted_new_size = GB / ctx->mem_per_input;
+        if (adjusted_new_size < 1) adjusted_new_size = 1;
+#if defined(DEBUG_BERT)
+        printf("%s: requested batch size %d, actual new batch size %d\n", __func__, new_size, adjusted_new_size);
+#endif
+        new_size = adjusted_new_size;
+        buf_size_new = ctx->mem_per_input * new_size;
+    }
+    if (new_size > ctx->max_batch_n) {
+        ctx->buf_compute.resize(buf_size_new);
+        ctx->max_batch_n = new_size;
+    }
+}
+
+void bert_eval(
+    struct bert_ctx *ctx,
+    int32_t n_threads,
+    const bert_vocab_id *raw_tokens,
+    int32_t n_tokens,
+    float *embeddings)
+{
+    const bert_model& model = ctx->model;
+    bool mem_req_mode = !embeddings;
+
+    // batch_embeddings is nullptr for the initial memory requirements run
+    if (!mem_req_mode && 1 > ctx->max_batch_n)
+        bert_resize_ctx(ctx, 1);
+
+    const int N = n_tokens;
+    const auto &tokens = raw_tokens;
+
+    const auto &hparams = model.hparams;
+
+    const int n_embd = hparams.n_embd;
+    const int n_layer = hparams.n_layer;
+    const int n_max_tokens = hparams.n_max_tokens;
+    const int n_head = hparams.n_head;
+
+    const int d_head = n_embd / n_head;
+
+    std::vector<float> result;
+    if (N > n_max_tokens)
+    {
+        fprintf(stderr, "Too many tokens, maximum is %d\n", n_max_tokens);
+        return;
+    }
+
+    auto & mem_per_token = ctx->mem_per_token;
+    auto & buf_compute   = ctx->buf_compute;
+
+    struct ggml_init_params params = {
+        .mem_size = buf_compute.size,
+        .mem_buffer = buf_compute.addr,
+        .no_alloc = false,
+    };
+
+    struct ggml_context *ctx0 = ggml_init(params);
+    struct ggml_cgraph *gf = ggml_new_graph(ctx0);
+
+    // Embeddings. word_embeddings + token_type_embeddings + position_embeddings
+    struct ggml_tensor *token_layer = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    memcpy(token_layer->data, tokens, N * ggml_element_size(token_layer));
+
+    struct ggml_tensor *token_types = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    ggml_set_zero(token_types);
+
+    struct ggml_tensor *positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    for (int i = 0; i < N; i++)
+    {
+        ggml_set_i32_1d(positions, i, i);
+    }
+
+    struct ggml_tensor *inpL = ggml_get_rows(ctx0, model.word_embeddings, token_layer);
+
+    inpL = ggml_add(ctx0,
+                    ggml_get_rows(ctx0, model.token_type_embeddings, token_types),
+                    inpL);
+    inpL = ggml_add(ctx0,
+                    ggml_get_rows(ctx0, model.position_embeddings, positions),
+                    inpL);
+
+    // embd norm
+    {
+        inpL = ggml_norm(ctx0, inpL, 1e-5f);
+
+        inpL = ggml_add(ctx0,
+                        ggml_mul(ctx0,
+                                 ggml_repeat(ctx0, model.ln_e_w, inpL),
+                                 inpL),
+                        ggml_repeat(ctx0, model.ln_e_b, inpL));
+    }
+    // layers
+    for (int il = 0; il < n_layer; il++)
+    {
+        struct ggml_tensor *cur = inpL;
+
+        // self-attention
+        {
+            struct ggml_tensor *Qcur = cur;
+            Qcur = ggml_reshape_3d(ctx0,
+                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].q_b, Qcur),
+                                            ggml_mul_mat(ctx0, model.layers[il].q_w, Qcur)),
+                                   d_head, n_head, N);
+            struct ggml_tensor *Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+
+            struct ggml_tensor *Kcur = cur;
+            Kcur = ggml_reshape_3d(ctx0,
+                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].k_b, Kcur),
+                                            ggml_mul_mat(ctx0, model.layers[il].k_w, Kcur)),
+                                   d_head, n_head, N);
+            struct ggml_tensor *K = ggml_permute(ctx0, Kcur, 0, 2, 1, 3);
+
+            struct ggml_tensor *Vcur = cur;
+            Vcur = ggml_reshape_3d(ctx0,
+                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].v_b, Vcur),
+                                            ggml_mul_mat(ctx0, model.layers[il].v_w, Vcur)),
+                                   d_head, n_head, N);
+            struct ggml_tensor *V = ggml_permute(ctx0, Vcur, 0, 2, 1, 3);
+
+            struct ggml_tensor *KQ = ggml_mul_mat(ctx0, K, Q);
+            // KQ = soft_max(KQ / sqrt(head width))
+            KQ = ggml_soft_max(
+                ctx0, ggml_scale(ctx0, KQ, 1.0f / sqrt((float)d_head))
+            );
+
+            V = ggml_cont(ctx0, ggml_transpose(ctx0, V));
+            struct ggml_tensor *KQV = ggml_mul_mat(ctx0, V, KQ);
+            KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
+
+            cur = ggml_cpy(ctx0,
+                           KQV,
+                           ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
+        }
+        // attention output
+        cur = ggml_add(ctx0,
+                       ggml_repeat(ctx0, model.layers[il].o_b, cur),
+                       ggml_mul_mat(ctx0, model.layers[il].o_w, cur));
+
+        // re-add the layer input
+        cur = ggml_add(ctx0, cur, inpL);
+
+        // attention norm
+        {
+            cur = ggml_norm(ctx0, cur, 1e-5f);
+
+            cur = ggml_add(ctx0,
+                           ggml_mul(ctx0,
+                                    ggml_repeat(ctx0, model.layers[il].ln_att_w, cur),
+                                    cur),
+                           ggml_repeat(ctx0, model.layers[il].ln_att_b, cur));
+        }
+        struct ggml_tensor *att_output = cur;
+        // intermediate_output = self.intermediate(attention_output)
+        cur = ggml_mul_mat(ctx0, model.layers[il].ff_i_w, cur);
+        cur = ggml_add(ctx0,
+                       ggml_repeat(ctx0, model.layers[il].ff_i_b, cur),
+                       cur);
+        cur = ggml_gelu(ctx0, cur);
+
+        // layer_output = self.output(intermediate_output, attention_output)
+        cur = ggml_mul_mat(ctx0, model.layers[il].ff_o_w, cur);
+        cur = ggml_add(ctx0,
+                       ggml_repeat(ctx0, model.layers[il].ff_o_b, cur),
+                       cur);
+        // attentions bypass the intermediate layer
+        cur = ggml_add(ctx0, att_output, cur);
+
+        // output norm
+        {
+            cur = ggml_norm(ctx0, cur, 1e-5f);
+
+            cur = ggml_add(ctx0,
+                           ggml_mul(ctx0,
+                                    ggml_repeat(ctx0, model.layers[il].ln_out_w, cur),
+                                    cur),
+                           ggml_repeat(ctx0, model.layers[il].ln_out_b, cur));
+        }
+        inpL = cur;
+    }
+    inpL = ggml_cont(ctx0, ggml_transpose(ctx0, inpL));
+    // pooler
+    struct ggml_tensor *sum = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, N, 1);
+    ggml_set_f32(sum, 1.0f / N);
+    inpL = ggml_mul_mat(ctx0, inpL, sum);
+
+    ggml_tensor *output = inpL;
+    // run the computation
+    ggml_build_forward_expand(gf, output);
+    //ggml_graph_compute_g4a()
+    ggml_graph_compute_g4a(ctx->work_buf, gf, n_threads);
+    //ggml_graph_compute(ctx0, gf);
+
+
+    // float *dat = ggml_get_data_f32(output);
+    // pretty_print_tensor(dat, output->ne, output->nb, output->n_dims - 1, "");
+
+    #ifdef GGML_PERF
+        // print timing information per ggml operation (for debugging purposes)
+        // requires GGML_PERF to be defined
+        ggml_graph_print(gf);
+    #endif
+
+    if (!mem_req_mode) {
+        memcpy(embeddings, (float *)ggml_get_data(output), sizeof(float) * n_embd);
+    } else {
+        mem_per_token = ggml_used_mem(ctx0) / N;
+    }
+
+    // printf("used_mem = %zu KB \n", ggml_used_mem(ctx0) / 1024);
+    // printf("mem_per_token = %zu KB \n", mem_per_token / 1024);
+
+    ggml_free(ctx0);
+}
+
+//
+// Loading and setup
+//
+
+void bert_free(bert_ctx * ctx) {
+    delete ctx;
+}
+
+struct bert_ctx * bert_load_from_file(const char *fname)
+{
+#if defined(DEBUG_BERT)
+    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname);
+#endif
+
+    bert_ctx * new_bert = new bert_ctx;
+
+    bert_model & model = new_bert->model;
+    bert_vocab & vocab = new_bert->vocab;
+
+    struct gguf_init_params params = {
+        /*.no_alloc = */ false,
+        /*.ctx      = */ &model.ctx,
+    };
+    gguf_context *ggufctx = gguf_init_from_file(fname, params);
+    if (!ggufctx) {
+        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
+        return nullptr;
+    }
+
+    printf("%s: gguf version     = %d\n", __func__, gguf_get_version(ggufctx));
+    printf("%s: gguf alignment   = %zu\n", __func__, gguf_get_alignment(ggufctx));
+    printf("%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx));
+
+    // print some standard metadata
+    {
+        int keyidx;
+
+        keyidx = gguf_find_key(ggufctx, "general.name");
+        if (keyidx != -1) { printf("%s: model name           = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.description");
+        if (keyidx != -1) { printf("%s: model description    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.author");
+        if (keyidx != -1) { printf("%s: model author         = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.license");
+        if (keyidx != -1) { printf("%s: model license        = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.architecture");
+        if (keyidx != -1) { printf("%s: model architecture   = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.file_type");
+        if (keyidx != -1) { printf("%s: model file type      = %" PRIu32 "\n", __func__, gguf_get_val_u32(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "gptneox.tensor_data_layout");
+        if (keyidx != -1) { printf("%s: model data layout    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+        keyidx = gguf_find_key(ggufctx, "general.source.huggingface.repository");
+        if (keyidx != -1) { printf("%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
+    }
+
+    // check required metadata
+    {
+        // check model architecture kv
+        int keyidx = gguf_find_key(ggufctx, "general.architecture");
+        if (keyidx == -1) {
+            fprintf(stderr, "%s: gguf model architecture not found!\n", __func__);
+            return nullptr;
+        }
+        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
+            fprintf(stderr, "%s: model architecture not supported!\n", __func__);
+            return nullptr;
+        }
+    }
+
+    // load hparams
+    {
+        auto &hparams = model.hparams;
+
+        bool ok = false;
+        int keyidx;
+
+        do {
+            keyidx = gguf_find_key(ggufctx, "bert.context_length");
+            if (keyidx == -1) { break; }
+            hparams.n_max_tokens = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.embedding_length");
+            if (keyidx == -1) { break; }
+            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.feed_forward_length");
+            if (keyidx == -1) { break; }
+            hparams.n_intermediate = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.attention.head_count");
+            if (keyidx == -1) { break; }
+            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "bert.block_count");
+            if (keyidx == -1) { break; }
+            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
+
+            ok = true;
+        } while (false);
+
+        if (!ok) {
+            fprintf(stderr, "%s: required hparam missing!\n", __func__);
+            return nullptr;
+        }
+
+#if defined(DEBUG_BERT)
+        printf("%s: n_max_tokens   = %d\n", __func__, hparams.n_max_tokens);
+        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
+        printf("%s: n_intermediate = %d\n", __func__, hparams.n_intermediate);
+        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
+        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
+#endif
+    }
+
+    // load vocab
+    {
+        auto & hparams = model.hparams;
+
+        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
+        if (keyidx == -1) {
+            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
+            return nullptr;
+        }
+        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
+            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
+            return nullptr;
+        }
+
+        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
+        if (tokens_keyidx == -1) {
+            fprintf(stderr, "%s: bert tokenizer vocab not found!\n", __func__);
+            return nullptr;
+        }
+
+        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
+        printf("%s: bert tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
+
+        for (int i = 0; i < hparams.n_vocab; i++) {
+            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
+
+            if (word[0] == '#' && word[1] == '#')
+            {
+                vocab.subword_token_to_id[word.substr(2)] = i;
+                vocab._id_to_subword_token[i] = word;
+            }
+
+            if (vocab.token_to_id.count(word) == 0)
+            {
+                vocab.token_to_id[word] = i;
+                vocab._id_to_token[i] = word;
+            }
+        }
+    }
+
+    auto &ctx = model.ctx;
+
+#if defined(DEBUG_BERT)
+    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ggml_get_mem_size(ctx) / (1024.0 * 1024.0));
+#endif
+
+    // prepare memory for the weights
+    {
+        const int n_layer = model.hparams.n_layer;
+        model.layers.resize(n_layer);
+
+        model.word_embeddings       = ggml_get_tensor(ctx, "token_embd.weight");
+        model.token_type_embeddings = ggml_get_tensor(ctx, "token_types.weight");
+        model.position_embeddings   = ggml_get_tensor(ctx, "position_embd.weight");
+        model.ln_e_w                = ggml_get_tensor(ctx, "output_norm.weight");
+        model.ln_e_b                = ggml_get_tensor(ctx, "output_norm.bias");
+
+        auto name = [](int i, std::string n) {
+            static std::string key;
+            key = "blk." + std::to_string(i) + "." + n;
+            return key.c_str();
+        };
+
+        for (int i = 0; i < n_layer; ++i)
+        {
+            auto &layer = model.layers[i];
+
+            layer.ln_att_w = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
+            layer.ln_att_b = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
+            layer.ln_out_w = ggml_get_tensor(ctx, name(i, "ffn_norm.weight"));
+            layer.ln_out_b = ggml_get_tensor(ctx, name(i, "ffn_norm.bias"));
+            layer.q_w      = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
+            layer.q_b      = ggml_get_tensor(ctx, name(i, "attn_q.bias"));
+            layer.k_w      = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
+            layer.k_b      = ggml_get_tensor(ctx, name(i, "attn_k.bias"));
+            layer.v_w      = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
+            layer.v_b      = ggml_get_tensor(ctx, name(i, "attn_v.bias"));
+            layer.o_w      = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
+            layer.o_b      = ggml_get_tensor(ctx, name(i, "attn_output.bias"));
+            layer.ff_i_w   = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
+            layer.ff_i_b   = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
+            layer.ff_o_w   = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
+            layer.ff_o_b   = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
+        }
+    }
+
+    // Calculate space requirements for setting up context buffers later
+    {
+        bert_vocab_id tokens[] = {0, 1, 2, 3};
+        // TODO: We set the initial buffer size to 16MB and hope it's enough. Maybe there is a better way to do this?
+        new_bert->buf_compute.resize(16 * 1024 * 1024);
+        bert_eval(new_bert, 1, tokens, 4, nullptr);
+        new_bert->max_batch_n = 0;
+
+        // TODO: Max tokens should be a param?
+        int32_t N = new_bert->model.hparams.n_max_tokens;
+        new_bert->mem_per_input = 2.2 * (new_bert->mem_per_token * N); // add 10% to account for ggml object overhead
+
+    }
+#if defined(DEBUG_BERT)
+    printf("%s: mem_per_token %ld KB, mem_per_input %ld MB\n", __func__, new_bert->mem_per_token / (1 << 10), new_bert->mem_per_input / (1 << 20));
+#endif
+
+    return new_bert;
+}
+
+struct BertPrivate {
+    const std::string modelPath;
+    bool modelLoaded;
+    bert_ctx *ctx = nullptr;
+    int64_t n_threads = 0;
+};
+
+Bert::Bert() : d_ptr(new BertPrivate) {
+    d_ptr->modelLoaded = false;
+}
+
+Bert::~Bert() {
+    bert_free(d_ptr->ctx);
+}
+
+bool Bert::loadModel(const std::string &modelPath, int n_ctx, int ngl)
+{
+    (void)n_ctx;
+    (void)ngl;
+    d_ptr->modelLoaded = false;
+
+    auto * ctx = bert_load_from_file(modelPath.c_str());
+    fflush(stdout);
+    if (!ctx)
+        return false;
+
+    d_ptr->ctx = ctx;
+    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    d_ptr->modelLoaded = true;
+    return true;
+}
+
+bool Bert::isModelLoaded() const
+{
+    return d_ptr->modelLoaded;
+}
+
+size_t Bert::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
+{
+    (void)modelPath;
+    (void)n_ctx;
+    (void)ngl;
+    return 0;
+}
+
+size_t Bert::stateSize() const
+{
+    return 0;
+}
+
+size_t Bert::saveState(uint8_t */*dest*/) const
+{
+    return 0;
+}
+
+size_t Bert::restoreState(const uint8_t */*src*/)
+{
+    return 0;
+}
+
+void Bert::setThreadCount(int32_t n_threads)
+{
+    d_ptr->n_threads = n_threads;
+}
+
+int32_t Bert::threadCount() const
+{
+    return d_ptr->n_threads;
+}
+
+std::vector<float> Bert::embedding(const std::string &text)
+{
+    const int overlap = 32;
+    const LLModel::Token clsToken = 101;
+    const size_t contextLength = bert_n_max_tokens(d_ptr->ctx);
+    typedef std::vector<LLModel::Token> TokenString;
+    TokenString tokens = ::bert_tokenize(d_ptr->ctx, text.c_str());
+#if defined(DEBUG_BERT)
+    std::cerr << "embedding: " << tokens.size()
+              << " contextLength " << contextLength
+              << "\n";
+#endif
+    std::vector<double> embeddingsSum(bert_n_embd(d_ptr->ctx), 0);
+    int embeddingsSumTotal = 0;
+    size_t start_pos = 0;
+    bool isFirstChunk = true;
+    while (start_pos < tokens.size()) {
+        TokenString chunk;
+        if (!isFirstChunk)
+            chunk.push_back(clsToken);
+        const size_t l = isFirstChunk ? contextLength : contextLength - 1;
+        if (tokens.size() - start_pos > l) {
+            chunk.insert(chunk.end(), tokens.begin() + start_pos, tokens.begin() + start_pos + l);
+            start_pos = start_pos + contextLength - overlap;
+        } else {
+            chunk.insert(chunk.end(), tokens.begin() + start_pos, tokens.end());
+            start_pos = tokens.size();
+        }
+#if defined(DEBUG_BERT)
+        std::cerr << "chunk length: " << chunk.size()
+            << " embeddingsSumTotal " << embeddingsSumTotal
+            << " contextLength " << contextLength
+            << " start_pos " << start_pos
+            << "\n";
+#endif
+        embeddingsSumTotal++;
+        std::vector<float> embeddings(bert_n_embd(d_ptr->ctx));
+        bert_eval(d_ptr->ctx, d_ptr->n_threads, chunk.data(), chunk.size(), embeddings.data());
+        std::transform(embeddingsSum.begin(), embeddingsSum.end(), embeddings.begin(), embeddingsSum.begin(), std::plus<float>());
+        isFirstChunk = false;
+    }
+
+    std::transform(embeddingsSum.begin(), embeddingsSum.end(), embeddingsSum.begin(), [embeddingsSumTotal](float num){ return num / embeddingsSumTotal; });
+    double magnitude = std::sqrt(std::inner_product(embeddingsSum.begin(), embeddingsSum.end(), embeddingsSum.begin(), 0.0));
+    for (auto &value : embeddingsSum)
+        value /= magnitude;
+    std::vector<float> finalEmbeddings(embeddingsSum.begin(), embeddingsSum.end());
+    return finalEmbeddings;
+}
+
+std::vector<LLModel::Token> Bert::tokenize(PromptContext &, const std::string &str) const
+{
+    return ::bert_tokenize(d_ptr->ctx, str.c_str());
+}
+
+LLModel::Token Bert::sampleToken(PromptContext &/*promptCtx*/) const
+{
+    return 999 /*!*/;
+}
+
+std::string Bert::tokenToString(Token id) const
+{
+    return bert_vocab_id_to_token(d_ptr->ctx, id);
+}
+
+bool Bert::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
+{
+    std::vector<float> embeddings(bert_n_embd(d_ptr->ctx));
+    int32_t cls = 101;
+    const bool useCLS = tokens.front() != cls;
+    if (useCLS) {
+        std::vector<int32_t> myTokens;
+        myTokens.push_back(cls);
+        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
+        bert_eval(d_ptr->ctx, d_ptr->n_threads, myTokens.data(), myTokens.size(), embeddings.data());
+    } else
+        bert_eval(d_ptr->ctx, d_ptr->n_threads, tokens.data(), tokens.size(), embeddings.data());
+    ctx.n_past = 0; // bert does not store any context
+    return true;
+}
+
+int32_t Bert::contextLength() const
+{
+    return bert_n_max_tokens(d_ptr->ctx);
+}
+
+const std::vector<LLModel::Token> &Bert::endTokens() const
+{
+    static const std::vector<LLModel::Token> out = { 102 /*sep*/};
+    return out;
+}
+
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != GGUF_TYPE_STRING) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
+#if defined(_WIN32)
+#define DLL_EXPORT __declspec(dllexport)
+#else
+#define DLL_EXPORT __attribute__ ((visibility ("default")))
+#endif
+
+extern "C" {
+DLL_EXPORT bool is_g4a_backend_model_implementation() {
+    return true;
+}
+
+DLL_EXPORT const char *get_model_type() {
+    return modelType_;
+}
+
+DLL_EXPORT const char *get_build_variant() {
+    return GGML_BUILD_VARIANT;
+}
+
+DLL_EXPORT bool magic_match(const char * fname) {
+    struct ggml_context * ctx_meta = NULL;
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ &ctx_meta,
+    };
+    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
+    if (!ctx_gguf)
+        return false;
+
+    bool isValid = gguf_get_version(ctx_gguf) <= 3;
+    isValid = isValid && get_arch_name(ctx_gguf) == "bert";
+
+    gguf_free(ctx_gguf);
+    return isValid;
+}
+
+DLL_EXPORT LLModel *construct() {
+    return new Bert;
+}
+}

gpt4all-backend/bert_impl.h

@@ -1,8 +1,8 @@
-#ifndef FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of falcon.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#ifndef BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
+#error This file is NOT meant to be included outside of bert.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
 #endif
-#ifndef FALCON_H
-#define FALCON_H
+#ifndef BERT_H
+#define BERT_H
 
 #include <string>
 #include <functional>
@@ -10,23 +10,27 @@
 #include <memory>
 #include "llmodel.h"
 
-struct FalconPrivate;
-class Falcon : public LLModel {
+struct BertPrivate;
+class Bert : public LLModel {
 public:
-    Falcon();
-    ~Falcon();
+    Bert();
+    ~Bert();
 
-    bool loadModel(const std::string &modelPath) override;
+    bool supportsEmbedding() const override { return true; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
 
+    std::vector<float> embedding(const std::string &text) override;
+
 private:
-    std::unique_ptr<FalconPrivate> d_ptr;
+    std::unique_ptr<BertPrivate> d_ptr;
 
 protected:
     std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
@@ -37,4 +41,4 @@ protected:
     const std::vector<Token>& endTokens() const override;
 };
 
-#endif // Falcon_H
+#endif // BERT_H

gpt4all-backend/dlhandle.h

@@ -53,6 +53,8 @@ public:
     }
 };
 #else
+#include <algorithm>
+#include <filesystem>
 #include <string>
 #include <exception>
 #include <stdexcept>
@@ -75,7 +77,9 @@ public:
 
     Dlhandle() : chandle(nullptr) {}
     Dlhandle(const std::string& fpath) {
-        chandle = LoadLibraryA(fpath.c_str());
+        std::string afpath = std::filesystem::absolute(fpath).string();
+        std::replace(afpath.begin(), afpath.end(), '/', '\\');
+        chandle = LoadLibraryExA(afpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
         if (!chandle) {
             throw Exception("dlopen(\""+fpath+"\"): Error");
         }

gpt4all-backend/falcon.cpp

@@ -1,983 +0,0 @@
-#define FALCON_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "falcon_impl.h"
-#include "llama.h"
-#include "llama-util.h"
-#include "utils.h"
-#include "llmodel_shared.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <iostream>
-#include <sstream>
-
-namespace {
-const char *modelType_ = "Falcon";
-}
-
-// commented out 40B support as it presently would require forking ggml/llama.cpp
-// can re-add once mainline ggml supports it
-
-#define FALCON_MAGIC 0x67676a74
-
-// default hparams (Falcon 7B)
-struct falcon_hparams {
-    int32_t n_vocab = 65024;
-    int32_t n_embd  = 4544;
-    int32_t n_head  = 71;
-    int32_t n_head_kv = 1;
-    int32_t n_layer = 32;
-    int32_t falcon_version = 7; // 7 for Falcon-7B, 40 for Falcon-40B
-    int32_t ftype   = 1;
-    int32_t n_ctx   = 2048;
-};
-
-struct falcon_layer {
-    // normalization
-    struct ggml_tensor* input_layernorm;
-    struct ggml_tensor* input_layernorm_b;
-    //struct ggml_tensor* attention_norm;    // Falcon-40B only
-    //struct ggml_tensor* attention_norm_b;  // Falcon-40B only
-
-    // attention
-    struct ggml_tensor* query_key_value;
-    struct ggml_tensor* wo;
-
-    // ff
-    struct ggml_tensor* ffn_up;
-    struct ggml_tensor* ffn_down;
-};
-
-struct falcon_model {
-    falcon_hparams hparams;
-
-    struct ggml_tensor* tok_embeddings;
-    struct ggml_tensor* output_norm;
-    struct ggml_tensor* output_norm_b;
-    struct ggml_tensor* lm_head;
-
-    std::vector<falcon_layer> layers;
-
-    // key + value memory
-    llm_kv_cache kv_self;
-
-    struct ggml_context* ctx;
-    std::map<std::string, struct ggml_tensor*> tensors;
-
-    llm_buffer eval_buf;
-    llm_buffer scr0_buf;
-    llm_buffer scr1_buf;
-};
-
-static bool kv_cache_init(
-        const struct falcon_hparams & hparams,
-              struct llm_kv_cache & cache,
-                         ggml_type   wtype,
-                               int   n_ctx) {
-    const int n_embd  = hparams.n_embd;
-    const int dim_head  = n_embd / hparams.n_head;
-    const int dim_kv = dim_head * hparams.n_head_kv;
-    const int n_layer = hparams.n_layer;
-
-    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
-    const int64_t n_elements = dim_kv * n_mem;
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
-    struct ggml_init_params params;
-    params.mem_size   = cache.buf.size;
-    params.mem_buffer = cache.buf.addr;
-    params.no_alloc   = false;
-
-    cache.ctx = ggml_init(params);
-    if (!cache.ctx) {
-        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
-        return false;
-    }
-
-    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    return true;
-}
-
-// load the model's weights from a file
-bool falcon_model_load(const std::string & fname, falcon_model & model, gpt_vocab & vocab, size_t *mem_req) {
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
-    if (mem_req) {
-        *mem_req = 0;
-    }
-
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin) {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
-        return false;
-    }
-
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *) &magic, sizeof(magic));
-        if (magic != FALCON_MAGIC) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
-            return false;
-        }
-    }
-
-    uint32_t format_version;
-    fin.read((char *) &format_version, sizeof(format_version));
-
-    // load hparams
-    {
-        auto & hparams = model.hparams;
-
-        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
-        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
-        fin.read((char *) &hparams.n_head_kv, sizeof(hparams.n_head_kv));
-        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *) &hparams.falcon_version, sizeof(hparams.falcon_version));
-        fin.read((char *) &hparams.ftype,   sizeof(hparams.ftype));
-
-        if (hparams.falcon_version != 7) { // && hparams.falcon_version != 40) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad Falcon version: %d)\n", __func__, fname.c_str(), hparams.falcon_version);
-            return false;
-        }
-
-        const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
-
-        printf("%s: n_vocab   = %d\n", __func__, hparams.n_vocab);
-        printf("%s: n_embd    = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_head    = %d\n", __func__, hparams.n_head);
-        printf("%s: n_head_kv = %d\n", __func__, hparams.n_head_kv);
-        printf("%s: n_layer   = %d\n", __func__, hparams.n_layer);
-        printf("%s: ftype     = %d\n", __func__, hparams.ftype);
-        printf("%s: qntvr     = %d\n", __func__, qntvr);
-
-        hparams.ftype %= GGML_QNT_VERSION_FACTOR;
-    }
-
-    // load vocab
-    {
-        const int32_t n_vocab = model.hparams.n_vocab;
-
-        std::string word;
-        std::vector<char> buf(128);
-
-        for (int i = 0; i < n_vocab; i++) {
-            uint32_t len;
-            fin.read((char *) &len, sizeof(len));
-
-            buf.resize(len);
-            fin.read((char *) buf.data(), len);
-            word.assign(buf.data(), len);
-
-            uint32_t dummy;
-            fin.read((char *) &dummy, sizeof(dummy));
-
-            vocab.token_to_id[word] = i;
-            vocab.id_to_token[i] = word;
-        }
-    }
-
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
-    if (wtype == GGML_TYPE_COUNT) {
-        fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
-                __func__, fname.c_str(), model.hparams.ftype);
-        return false;
-    }
-
-    auto & ctx = model.ctx;
-
-    size_t ctx_size = 0;
-
-    {
-        const auto& hparams = model.hparams;
-
-        const int n_embd = hparams.n_embd;
-        const int n_head = hparams.n_head;
-        const int n_head_kv = hparams.n_head_kv;
-        const int n_layer = hparams.n_layer;
-        const int n_ctx = hparams.n_ctx;
-        const int n_ff = 4 * model.hparams.n_embd;
-        const int n_vocab = hparams.n_vocab;
-        const int head_dim = hparams.n_embd / hparams.n_head;
-
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // tok_embeddings
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd;  // output_norm_b
-        ctx_size += ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_vocab;  // lm_head
-
-        // if (hparams.version == 40) { // Falcon-40B
-        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm
-        //     ctx_size += n_layer * ggml_sizeof_tensor_1d(GGML_TYPE_F32, n_embd);  // attention_norm_b
-        // }
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(GGML_TYPE_F32) * n_embd);  // input_layernorm_b
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * (n_head_kv * 2 + n_head) * head_dim);  // query_key_value
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_embd);  // wo
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_embd * n_ff);  // ffn_up
-        ctx_size += n_layer * (ggml_tensor_overhead() + ggml_type_sizef(wtype) * n_ff * n_embd);  // ffn_down
- 
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
-    }
-
-    if (mem_req) {
-        const int n_embd  = model.hparams.n_embd;
-        const int dim_head  = n_embd / model.hparams.n_head;
-        const int dim_kv = dim_head * model.hparams.n_head_kv;
-        const int n_layer = model.hparams.n_layer;
-
-        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
-        const int64_t n_elements = dim_kv * n_mem;
-        size_t kv_cache_size = 2u*n_elements*ggml_type_size(wtype) + 2_MiB;
-        *mem_req = ctx_size + kv_cache_size;
-        return false;
-    }
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size   = ctx_size,
-            .mem_buffer = NULL,
-            .no_alloc   = false,
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx) {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            return false;
-        }
-    }
-
-    // prepare memory for the weights
-    {
-        const auto& hparams = model.hparams;
-
-        const int n_embd = hparams.n_embd;
-        const int n_head = hparams.n_head;
-        const int n_head_kv = hparams.n_head_kv;
-        const int n_layer = hparams.n_layer;
-        const int n_ff = 4 * model.hparams.n_embd;
-        const int n_vocab = hparams.n_vocab;
-        const int head_dim = hparams.n_embd / hparams.n_head;
-
-        model.layers.resize(n_layer);
-
-        model.tok_embeddings = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
-
-        model.output_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.output_norm_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
-
-        // map by name
-        model.tensors["transformer.word_embeddings.weight"] =
-            model.tok_embeddings;
-
-        model.tensors["transformer.ln_f.weight"] = model.output_norm;
-        model.tensors["transformer.ln_f.bias"] = model.output_norm_b;
-        model.tensors["lm_head.weight"] = model.lm_head;
-
-        for (int i = 0; i < n_layer; ++i) {
-            auto& layer = model.layers[i];
-
-            layer.input_layernorm =
-                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.input_layernorm_b =
-                ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            // if (hparams.version == 40) { // for Falcon-40B only
-            //     layer.attention_norm =
-            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            //     layer.attention_norm_b =
-            //         ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            // }
-
-            // query_key_value shape for config.multi_query == True:
-            layer.query_key_value = ggml_new_tensor_2d(
-                ctx, wtype, n_embd, (n_head_kv * 2 + n_head) * head_dim);
-            layer.wo = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
-
-            layer.ffn_up = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);
-            layer.ffn_down = ggml_new_tensor_2d(ctx, wtype, n_ff, n_embd);
-
-            // map by name
-            // if (hparams.version == 40) {
-            //     // Falcon-40B:
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_mlp.weight"] = layer.input_layernorm;
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_mlp.bias"] = layer.input_layernorm_b;
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_attn.weight"] = layer.attention_norm;
-            //     model.tensors["transformer.h." + std::to_string(i) +
-            //                   ".ln_attn.bias"] = layer.attention_norm_b;
-            // } else {
-            // Falcon-7B:
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".input_layernorm.weight"] = layer.input_layernorm;
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".input_layernorm.bias"] = layer.input_layernorm_b;
-            //}
-
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".self_attention.query_key_value.weight"] =
-                layer.query_key_value;
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".self_attention.dense.weight"] = layer.wo;
-
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".mlp.dense_h_to_4h.weight"] = layer.ffn_up;
-            model.tensors["transformer.h." + std::to_string(i) +
-                          ".mlp.dense_4h_to_h.weight"] = layer.ffn_down;
-        }
-    }
-
-    // key + value memory
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_layer = hparams.n_layer;
-        const int n_ctx   = hparams.n_ctx;
-        const int n_head_kv = hparams.n_head_kv;
-        const int head_dim = hparams.n_embd / hparams.n_head;
-
-        const int64_t n_mem      = n_layer*n_ctx;
-        const int64_t n_elements = head_dim*n_mem;
-
-        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F32, model.hparams.n_ctx)) {
-            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
-            ggml_free(ctx);
-            return false;
-        }
-        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
-
-        printf("%s: memory_size = %8.2f MB, n_mem = %" PRId64 "\n", __func__, memory_size/1024.0/1024.0, n_mem);
-    }
-
-    // load weights
-    {
-        int n_tensors = 0;
-        size_t total_size = 0;
-
-        printf("%s: ", __func__);
-
-        while (true) {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ttype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
-
-            if (fin.eof()) {
-                break;
-            }
-
-            int32_t nelements = 1;
-            int32_t ne[2] = { 1, 1 };
-            for (int i = 0; i < n_dims; ++i) {
-                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-            fin.seekg(-static_cast<ptrdiff_t>(fin.tellg()) & 31, std::ios_base::cur);
-
-            if (model.tensors.find(name.data()) == model.tensors.end()) {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                return false;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                return false;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%5d, %5d], expected [%5d, %5d]\n",
-                        __func__, name.data(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
-                return false;
-            }
-
-            // for debugging
-            if (0) {
-                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
-            }
-
-            const size_t bpe = ggml_type_size(ggml_type(ttype));
-
-            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
-                return false;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-            total_size += ggml_nbytes(tensor);
-            if (++n_tensors % 8 == 0) {
-                printf(".");
-                fflush(stdout);
-            }
-        }
-
-        printf(" done\n");
-
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
-    }
-
-    fin.close();
-
-    model.eval_buf.resize(1280u * 1024 * 1024);
-    model.scr0_buf.resize(256u * 1024 * 1024);
-    model.scr1_buf.resize(256u * 1024 * 1024);
-    return true;
-}
-
-
-// evaluate the transformer
-//
-//   - model:     the model
-//   - n_threads: number of threads to use
-//   - n_past:    the context size so far
-//   - embd_inp:  the embeddings of the tokens in the context
-//   - embd_w:    the predicted logits for the next token
-//
-bool falcon_eval(
-        const falcon_model & model,
-        const int n_threads,
-        const int n_past,
-        const std::vector<gpt_vocab::id> & embd_inp,
-              std::vector<float>         & embd_w,
-              size_t                     & mem_per_token) {
-    const int N = embd_inp.size();
-
-    const auto & hparams = model.hparams;
-
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_ctx   = hparams.n_ctx;
-    const int n_head  = hparams.n_head;
-    const int n_head_kv = hparams.n_head_kv;
-    const int n_vocab = hparams.n_vocab;
-    const int version = hparams.falcon_version;
-    const size_t head_dim = n_embd / n_head;
-
-   struct ggml_init_params eval_ctx_params = {
-        .mem_size = model.eval_buf.size,
-        .mem_buffer = model.eval_buf.addr,
-        .no_alloc = false,
-    };
-
-    struct ggml_context * ctx0 = ggml_init(eval_ctx_params);
-    struct ggml_cgraph gf = {};
-    gf.n_threads = n_threads;
-
-    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
-
-    // wte
-    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.tok_embeddings, embd);
-    struct ggml_tensor* repeat_dummy = ggml_new_tensor_3d(ctx0, inpL->type, head_dim, N + n_past, n_head);
-
-    ggml_type wtype = GGML_TYPE_F32;
-    const int sizeof_wtype = ggml_type_sizef(wtype);
-
-    for (int il = 0; il < n_layer; ++il) {
-        struct ggml_tensor * cur;
-        struct ggml_tensor * layernorm_output;
-
-        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-        // self-attention
-        {
-            layernorm_output = ggml_norm(ctx0, inpL);
-
-            layernorm_output = ggml_add(ctx0,
-                    ggml_mul(ctx0,
-                        ggml_repeat(ctx0, model.layers[il].input_layernorm, layernorm_output),
-                        layernorm_output),
-                    ggml_repeat(ctx0, model.layers[il].input_layernorm_b, layernorm_output));
-
-            // if (version == 40) { // Falcon-40B only
-            //     cur = ggml_norm(ctx0, inpL);
-
-            //     cur = ggml_add(ctx0,
-            //             ggml_mul(ctx0,
-            //                 ggml_repeat(ctx0, model.layers[il].attention_norm, cur),
-            //                 cur),
-            //             ggml_repeat(ctx0, model.layers[il].attention_norm_b, cur));
-            // }
-            // else {
-            cur = layernorm_output;
-            // }
-
-            // compute QKV
-
-            cur = ggml_mul_mat(ctx0, model.layers[il].query_key_value, cur);
-
-            // Note that the strides for Kcur, Vcur are set up so that the
-            // resulting views are misaligned with the tensor's storage
-            // (by applying the K/V offset we shift the tensor's original
-            // view to stick out behind the viewed QKV tensor's allocated
-            // memory, so to say). This is ok because no actual accesses
-            // happen to that out-of-range memory, but it can require some
-            // trickery when trying to accurately dump these views for
-            // debugging.
-
-            struct ggml_tensor * Qcur = ggml_view_3d(
-                ctx0, cur, head_dim, n_head, N,
-                head_dim * sizeof_wtype,
-                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
-                0);
-
-            struct ggml_tensor * Kcur = ggml_view_3d(
-                ctx0, cur, head_dim, n_head_kv, N,
-                head_dim * sizeof_wtype,
-                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
-                head_dim * n_head * sizeof_wtype);
-
-            struct ggml_tensor * Vcur = ggml_view_3d(
-                ctx0, cur, head_dim, n_head_kv, N,
-                head_dim * sizeof_wtype,
-                head_dim * (n_head + 2 * n_head_kv) * sizeof_wtype,
-                head_dim * (n_head + n_head_kv) * sizeof_wtype);
-
-            // using mode = 2 for neox mode
-            Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, head_dim, 2);
-            Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, head_dim, 2);
-
-            // store key and value to memory
-            {
-                struct ggml_tensor* k = ggml_view_1d(
-                    ctx0, model.kv_self.k, N * n_head_kv * head_dim,
-                    (ggml_element_size(model.kv_self.k) * n_head_kv * head_dim) *
-                        (il * n_ctx + n_past));
-                struct ggml_tensor* v = ggml_view_1d(
-                    ctx0, model.kv_self.v, N * n_head_kv * head_dim,
-                    (ggml_element_size(model.kv_self.v) * n_head_kv * head_dim) *
-                        (il * n_ctx + n_past));
-
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
-            }
-
-            struct ggml_tensor * K = ggml_permute(
-                ctx0,
-                ggml_view_3d(
-                    ctx0,
-                    model.kv_self.k,
-                    head_dim, n_head_kv, n_past + N,
-                    head_dim * sizeof_wtype,
-                    head_dim * n_head_kv * sizeof_wtype,
-                    il * n_ctx * ggml_element_size(model.kv_self.k) * n_head_kv * head_dim),
-                0, 2, 1, 3);
-
-            // K * Q
-
-            // changed from repeat2 back to repeat, will not support 40B!
-            K = ggml_cont(ctx0, ggml_repeat(ctx0, K, repeat_dummy));
-
-            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
-            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
-
-            // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled =
-                ggml_scale_inplace(ctx0,
-                        KQ,
-                        ggml_new_f32(ctx0, 1.0f/sqrt(float(head_dim)))
-                        );
-
-            // KQ_masked = mask_past(KQ_scaled)
-            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
-
-            // KQ = soft_max(KQ_masked)
-            struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
-
-            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor* V = ggml_permute(
-                ctx0,
-                ggml_view_3d(
-                    ctx0,
-                    model.kv_self.v,
-                    head_dim, n_head_kv, n_past + N,
-                    head_dim * sizeof_wtype,
-                    head_dim * n_head_kv * sizeof_wtype,
-                    il * n_ctx * ggml_element_size(model.kv_self.v) * n_head_kv * head_dim),
-                0, 2, 1, 3);
-
-            // changed from repeat2 back to repeat, will not support 40B!
-            V = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_repeat(ctx0, V, repeat_dummy)));
-
-            // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
-
-            // KQV_merged = KQV.permute(0, 2, 1, 3)
-            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            // cur = KQV_merged.contiguous().view(n_embd, N)
-            cur = ggml_cpy(ctx0,
-                    KQV_merged,
-                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-            // projection
-            {
-                cur = ggml_mul_mat(ctx0,
-                        model.layers[il].wo,
-                        cur);
-            }
-        }
-
-        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
-
-        struct ggml_tensor* inpFF = layernorm_output;
-        struct ggml_tensor* attn_out = ggml_cpy(
-            ctx0, cur, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-        {
-            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_up, inpFF);
-            cur = ggml_gelu(ctx0, cur);
-            cur = ggml_mul_mat(ctx0, model.layers[il].ffn_down, cur);
-        }
-
-        cur = ggml_add(ctx0, cur, attn_out);
-        cur = ggml_add(ctx0, cur, inpL);
-        // input for next layer
-        inpL = cur;
-    }
-
-    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-    // norm
-    {
-        inpL = ggml_norm(ctx0, inpL);
-
-        // inpL = ln_f_g*inpL + ln_f_b
-        inpL = ggml_add(ctx0,
-                ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.output_norm, inpL),
-                    inpL),
-                ggml_repeat(ctx0, model.output_norm_b, inpL));
-    }
-
-    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
-
-    // lm_head
-    {
-        inpL = ggml_mul_mat(ctx0, model.lm_head, inpL);
-
-        //inpL = ggml_add(ctx0,
-        //        ggml_repeat(ctx0, model.lmh_b, inpL),
-        //        inpL);
-    }
-
-    // logits -> probs
-    //inpL = ggml_soft_max_inplace(ctx0, inpL);
-
-    // run the computation
-    ggml_build_forward_expand(&gf, inpL);
-    ggml_graph_compute       (ctx0, &gf);
-
-    //if (n_past%100 == 0) {
-    //    ggml_graph_print   (&gf);
-    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
-    //}
-
-    //embd_w.resize(n_vocab*N);
-    //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
-
-    // return result for just the last token
-    embd_w.resize(n_vocab);
-    memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
-
-    if (mem_per_token == 0) {
-        mem_per_token = ggml_used_mem(ctx0)/N;
-    }
-    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
-
-    ggml_free(ctx0);
-
-    return true;
-}
-
-
-#define MAX_RNG_STATE 64*1024
-size_t falcon_get_state_size(const falcon_model &model) {
-    const size_t s_rng_size        = sizeof(size_t);
-    const size_t s_rng             = MAX_RNG_STATE;
-    const size_t s_kv_size         = sizeof(size_t);
-    const size_t s_kv_ntok         = sizeof(int);
-    const size_t s_kv              = model.kv_self.buf.size;
-    const size_t s_total = (
-        + s_rng_size
-        + s_rng
-        + s_kv_size
-        + s_kv_ntok
-        + s_kv
-    );
-    return s_total;
-}
-
-size_t falcon_copy_state_data(const falcon_model &model, const std::mt19937 &rng, uint8_t *dest)
-{
-    uint8_t * out = dest;
-    // copy rng
-    {
-        std::stringstream rng_ss;
-        rng_ss << rng;
-
-        const size_t rng_size = rng_ss.str().size();
-        char rng_buf[MAX_RNG_STATE];
-
-        memset(&rng_buf[0], 0, MAX_RNG_STATE);
-        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
-
-        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
-        memcpy(out, &rng_buf[0], MAX_RNG_STATE); out += MAX_RNG_STATE;
-    }
-
-    // copy kv cache
-    {
-        const size_t kv_size = model.kv_self.buf.size;
-        const int    kv_ntok = model.kv_self.n;
-
-        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
-        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
-
-        if (kv_size) {
-            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
-        }
-    }
-
-    const size_t written  = out - dest;
-    assert(written == falcon_get_state_size(model));
-    fflush(stdout);
-    return written;
-}
-
-size_t falcon_set_state_data(falcon_model *model, std::mt19937 *rng, const uint8_t *src)
-{
-    const uint8_t * in = src;
-
-    // set rng
-    {
-        size_t rng_size;
-        char   rng_buf[MAX_RNG_STATE];
-
-        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
-        memcpy(&rng_buf[0], in, MAX_RNG_STATE); in += MAX_RNG_STATE;
-
-        std::stringstream rng_ss;
-        rng_ss.str(std::string(&rng_buf[0], rng_size));
-        rng_ss >> *rng;
-
-        assert(rng_ss.fail() == false);
-    }
-
-    // set kv cache
-    {
-        size_t kv_size;
-        int kv_ntok;
-
-        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
-        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
-
-        if (kv_size) {
-            assert(model->kv_self.buf.size == kv_size);
-
-            void * k_data = model->kv_self.k->data; // remember data pointers
-            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
-
-            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
-
-            model->kv_self.k->data = k_data; // restore correct data pointers
-            model->kv_self.v->data = v_data;
-
-        }
-
-        model->kv_self.n = kv_ntok;
-    }
-
-    const size_t nread    = in - src;
-    assert(nread == falcon_get_state_size(*model));
-    fflush(stdout);
-    return nread;
-}
-
-struct FalconPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    gpt_vocab vocab;
-    falcon_model *model = nullptr;
-    int64_t n_threads = 0;
-    size_t mem_per_token = 0;
-    std::mt19937 rng;
-};
-
-Falcon::Falcon() : d_ptr(new FalconPrivate) {
-    d_ptr->model = new falcon_model;
-    d_ptr->model->ctx = nullptr;
-    d_ptr->modelLoaded = false;
-}
-
-Falcon::~Falcon() {
-    if(d_ptr->model->ctx) {
-        ggml_free(d_ptr->model->ctx);
-        d_ptr->model->ctx = nullptr;
-    }
-    delete d_ptr->model;
-}
-
-bool Falcon::loadModel(const std::string &modelPath)
-{
-    std::mt19937 rng(time(NULL));
-    d_ptr->rng = rng;
-
-    // load the model
-    if (!falcon_model_load(modelPath, *d_ptr->model, d_ptr->vocab, nullptr)) {
-        std::cerr << "FALCON ERROR: failed to load model from " <<  modelPath;
-        return false;
-    }
-
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    fflush(stdout);
-    return true;
-}
-
-bool Falcon::isModelLoaded() const
-{
-    return d_ptr -> modelLoaded;
-}
-
-size_t Falcon::requiredMem(const std::string &modelPath)
-{
-    falcon_model dummy_model;
-    gpt_vocab dummy_vocab;
-    size_t mem_req;
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-    falcon_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
-    return mem_req;
-}
-
-size_t Falcon::stateSize() const
-{
-    return falcon_get_state_size(*d_ptr->model);
-}
-
-size_t Falcon::saveState(uint8_t *dest) const
-{
-    return falcon_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
-}
-
-size_t Falcon::restoreState(const uint8_t *src)
-{
-    return falcon_set_state_data(d_ptr->model, &d_ptr->rng, src);
-}
-
-void Falcon::setThreadCount(int32_t n_threads)
-{
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t Falcon::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-std::vector<LLModel::Token> Falcon::tokenize(PromptContext &, const std::string &str) const
-{
-    return ::gpt_tokenize(d_ptr->vocab, str);
-}
-
-LLModel::Token Falcon::sampleToken(PromptContext &promptCtx) const
-{
-    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
-    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
-        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks,
-        promptCtx.logits,
-        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty,
-        d_ptr->rng);
-}
-
-std::string Falcon::tokenToString(Token id) const
-{
-    return d_ptr->vocab.id_to_token[id];
-}
-
-bool Falcon::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    // determine the required inference memory per token:
-    static bool initialized = false;
-    if (!initialized) {
-        falcon_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
-            d_ptr->mem_per_token);
-        initialized = true;
-    }
-
-    return falcon_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
-}
-
-int32_t Falcon::contextLength() const
-{
-    return d_ptr->model->hparams.n_ctx;
-}
-
-const std::vector<LLModel::Token> &Falcon::endTokens() const
-{
-    static const std::vector<LLModel::Token> out = { 11 };
-    return out;
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    uint32_t version = 0;
-    f.read(reinterpret_cast<char*>(&version), sizeof(version));
-    if (magic != FALCON_MAGIC) {
-         return false; 
-    }
-    falcon_hparams hparams;
-    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
-    // we're matching the file format of existing pre-converted models
-    // compatible with ctransformers llama.cpp based format, which also
-    // unfortunately shares its magic number what llama uses, so we now
-    // differentiate by n_vocab
-    // give some wiggle room over the max to allow for finetunes that expand the
-    // vocabulary
-    if (!(hparams.n_vocab >= 65024 && hparams.n_vocab <= 65100)) {
-        return false;
-    }
-    if (hparams.falcon_version != 7) {
-        return false;
-    }
-    return true;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new Falcon;
-}
-}

gpt4all-backend/gptj.cpp

@@ -9,7 +9,6 @@
 #include <cmath>
 #include <cstdio>
 #include <cstring>
-#include <fstream>
 #include <map>
 #include <string>
 #include <vector>
@@ -42,7 +41,7 @@ struct gptj_hparams {
     int32_t n_head  = 16;
     int32_t n_layer = 28;
     int32_t n_rot   = 64;
-    int32_t f16     = 1;
+    float norm_eps  = 1e-5;
 };
 
 struct gptj_layer {
@@ -128,216 +127,149 @@ static bool kv_cache_init(
     return true;
 }
 
-// load the model's weights from a stream
-bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
+// load the model's weights from a file path
+bool gptj_model_load(const std::string &fname, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr) {
     printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
     if(mem_req != nullptr) {
         *mem_req = 0;
     }
 
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *) &magic, sizeof(magic));
-        if (magic != 0x67676d6c) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
-            return false;
-        }
+    // create the ggml context
+    struct gguf_init_params params = {
+        /*.no_alloc = */ false,
+        /*.ctx      = */ &model.ctx,
+    };
+
+    gguf_context *ggufctx = gguf_init_from_file(fname.c_str(), params);
+    if (!ggufctx) {
+        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
+        return false;
     }
 
     // load hparams
     {
         auto & hparams = model.hparams;
 
-        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
-        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
-        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
-        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *) &hparams.n_rot,   sizeof(hparams.n_rot));
-        fin.read((char *) &hparams.f16,     sizeof(hparams.f16));
+        bool ok = false;
+        int keyidx;
+
+        do {
+            keyidx = gguf_find_key(ggufctx, "gptj.context_length");
+            if (keyidx == -1) { break; }
+            hparams.n_ctx = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.embedding_length");
+            if (keyidx == -1) { break; }
+            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.attention.head_count");
+            if (keyidx == -1) { break; }
+            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.block_count");
+            if (keyidx == -1) { break; }
+            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.rope.dimension_count");
+            if (keyidx == -1) { break; }
+            hparams.n_rot = gguf_get_val_u32(ggufctx, keyidx);
+
+            keyidx = gguf_find_key(ggufctx, "gptj.attention.layer_norm_epsilon");
+            if (keyidx == -1) { break; }
+            hparams.norm_eps = gguf_get_val_f32(ggufctx, keyidx);
+
+            ok = true;
+        } while (false);
+
+        if (!ok) {
+            fprintf(stderr, "%s: required hparam missing!\n", __func__);
+            return false;
+        }
 
-        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
         printf("%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
         printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
         printf("%s: n_head  = %d\n", __func__, hparams.n_head);
         printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
         printf("%s: n_rot   = %d\n", __func__, hparams.n_rot);
-        printf("%s: f16     = %d\n", __func__, hparams.f16);
     }
 
     // load vocab
     {
-        int32_t n_vocab = 0;
-        fin.read((char *) &n_vocab, sizeof(n_vocab));
+        auto & hparams = model.hparams;
 
-        if (n_vocab != model.hparams.n_vocab) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
-                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
+        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
+        if (keyidx == -1) {
+            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
+            return false;
+        }
+        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "gpt2") != 0) {
+            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
             return false;
         }
 
-        std::string word;
-        for (int i = 0; i < n_vocab; i++) {
-            uint32_t len;
-            fin.read((char *) &len, sizeof(len));
+        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
+        if (tokens_keyidx == -1) {
+            fprintf(stderr, "%s: gpt2 tokenizer vocab not found!\n", __func__);
+            return false;
+        }
 
-            word.resize(len);
-            fin.read((char *) word.data(), len);
+        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
+        printf("%s: gpt2 tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
 
+        for (int i = 0; i < hparams.n_vocab; i++) {
+            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
             vocab.token_to_id[word] = i;
             vocab.id_to_token[i] = word;
         }
     }
 
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = GGML_TYPE_COUNT;
-    switch (model.hparams.f16) {
-        case 0: wtype = GGML_TYPE_F32;  break;
-        case 1: wtype = GGML_TYPE_F16;  break;
-        case 2: wtype = GGML_TYPE_Q4_0; break;
-        case 3: wtype = GGML_TYPE_Q4_1; break;
-        case 5: wtype = GGML_TYPE_Q4_2; break;
-        default:
-                {
-                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
-                            __func__, fname.c_str(), model.hparams.f16);
-                    return false;
-                }
-    }
-
     auto & ctx = model.ctx;
 
-    size_t ctx_size = 0;
-
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_ctx   = hparams.n_ctx;
-        const int n_vocab = hparams.n_vocab;
-
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_g
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_b
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype); // wte
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(wtype);         // lmh_g
-        ctx_size +=        n_vocab*ggml_type_sizef(GGML_TYPE_F32); // lmh_b
-
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_g
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // ln_1_b
-
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_q_proj_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_k_proj_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_v_proj_w
-
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // c_attn_proj_w
-
-        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_fc_w
-        ctx_size += n_layer*(       4*n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_fc_b
-
-        ctx_size += n_layer*(4*n_embd*n_embd*ggml_type_sizef(wtype));         // c_mlp_proj_w
-        ctx_size += n_layer*(         n_embd*ggml_type_sizef(GGML_TYPE_F32)); // c_mlp_proj_b
-
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_k
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_v
-
-        ctx_size += (5 + 10*n_layer)*256; // object overhead
-
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
-    }
+    size_t ctx_size = ggml_get_mem_size(ctx);
+    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size / (1024.0 * 1024.0));
 
     if (mem_req != nullptr) {
-        *mem_req += ctx_size;
-        const int n_embd  = model.hparams.n_embd;
-        const int n_layer = model.hparams.n_layer;
-
-        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
-        const int64_t n_elements = n_embd*n_mem;
-
-        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
+        *mem_req = ctx_size;
+        gguf_free(ggufctx);
         return false;
     }
 
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size   = ctx_size,
-            .mem_buffer = NULL,
-            .no_alloc = false
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx) {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            return false;
-        }
-    }
-
     // prepare memory for the weights
     {
         const auto & hparams = model.hparams;
+        model.layers.resize(hparams.n_layer);
 
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_vocab = hparams.n_vocab;
+        model.wte    = ggml_get_tensor(ctx, "token_embd.weight");
 
-        model.layers.resize(n_layer);
+        model.ln_f_g = ggml_get_tensor(ctx, "output_norm.weight");
+        model.ln_f_b = ggml_get_tensor(ctx, "output_norm.bias");
 
-        model.wte    = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
+        model.lmh_g  = ggml_get_tensor(ctx, "output.weight");
+        model.lmh_b  = ggml_get_tensor(ctx, "output.bias");
 
-        model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-        model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
+        auto name = [](int i, std::string n) {
+            static std::string key;
+            key = "blk." + std::to_string(i) + "." + n;
+            return key.c_str();
+        };
 
-        model.lmh_g  = ggml_new_tensor_2d(ctx, wtype,         n_embd, n_vocab);
-        model.lmh_b  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab);
-
-        // map by name
-        model.tensors["transformer.wte.weight"] = model.wte;
-
-        model.tensors["transformer.ln_f.weight"] = model.ln_f_g;
-        model.tensors["transformer.ln_f.bias"]   = model.ln_f_b;
-
-        model.tensors["lm_head.weight"] = model.lmh_g;
-        model.tensors["lm_head.bias"]   = model.lmh_b;
-
-        for (int i = 0; i < n_layer; ++i) {
+        for (int i = 0; i < hparams.n_layer; ++i) {
             auto & layer = model.layers[i];
 
-            layer.ln_1_g          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
-            layer.ln_1_b          = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
+            layer.ln_1_g          = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
+            layer.ln_1_b          = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
 
-            layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
-            layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
-            layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_q_proj_w = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
+            layer.c_attn_k_proj_w = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
+            layer.c_attn_v_proj_w = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
 
-            layer.c_attn_proj_w   = ggml_new_tensor_2d(ctx, wtype,           n_embd,   n_embd);
+            layer.c_attn_proj_w   = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
 
-            layer.c_mlp_fc_w      = ggml_new_tensor_2d(ctx, wtype,           n_embd, 4*n_embd);
-            layer.c_mlp_fc_b      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
+            layer.c_mlp_fc_w      = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
+            layer.c_mlp_fc_b      = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
 
-            layer.c_mlp_proj_w    = ggml_new_tensor_2d(ctx, wtype,         4*n_embd,   n_embd);
-            layer.c_mlp_proj_b    = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,   n_embd);
-
-            // map by name
-            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.weight"]          = layer.ln_1_g;
-            model.tensors["transformer.h." + std::to_string(i) + ".ln_1.bias"]            = layer.ln_1_b;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.q_proj.weight"]   = layer.c_attn_q_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.k_proj.weight"]   = layer.c_attn_k_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.v_proj.weight"]   = layer.c_attn_v_proj_w;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".attn.out_proj.weight"] = layer.c_attn_proj_w;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.weight"]     = layer.c_mlp_fc_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.bias"]       = layer.c_mlp_fc_b;
-
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.weight"]    = layer.c_mlp_proj_w;
-            model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.bias"]      = layer.c_mlp_proj_b;
+            layer.c_mlp_proj_w    = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
+            layer.c_mlp_proj_b    = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
         }
     }
 
@@ -354,113 +286,12 @@ bool gptj_model_load(const std::string &fname, std::istream &fin, gptj_model & m
         printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
     }
 
-    // load weights
-    {
-        int n_tensors = 0;
-        size_t total_size = 0;
-
-        printf("%s: ", __func__);
-
-        while (true) {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ftype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ftype),  sizeof(ftype));
-
-            if (fin.eof()) {
-                break;
-            }
-
-            int32_t nelements = 1;
-            int32_t ne[2] = { 1, 1 };
-            for (int i = 0; i < n_dims; ++i) {
-                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-
-            if (model.tensors.find(name.data()) == model.tensors.end()) {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                return false;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                return false;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
-                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
-                return false;
-            }
-
-            if (0) {
-                static const char * ftype_str[] = { "f32", "f16", "q4_0", "q4_1", };
-                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ftype_str[ftype], ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
-            }
-
-            size_t bpe = 0;
-
-            switch (ftype) {
-                case 0: bpe = ggml_type_size(GGML_TYPE_F32);  break;
-                case 1: bpe = ggml_type_size(GGML_TYPE_F16);  break;
-                case 2: bpe = ggml_type_size(GGML_TYPE_Q4_0); assert(ne[0] % 64 == 0); break;
-                case 3: bpe = ggml_type_size(GGML_TYPE_Q4_1); assert(ne[0] % 64 == 0); break;
-                default:
-                        {
-                            fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype);
-                            return false;
-                        }
-            };
-
-            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
-                return false;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
-            total_size += ggml_nbytes(tensor);
-            if (++n_tensors % 8 == 0) {
-                printf(".");
-                fflush(stdout);
-            }
-        }
-
-        printf(" done\n");
-
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
-    }
-
     model.scr0_buf.resize(256u * 1024 * 1024);
     model.scr1_buf.resize(256u * 1024 * 1024);
 
     return true;
 }
 
-// load the model's weights from a file path
-bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) {
-
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin) {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
-        return false;
-    }
-
-    bool loaded = gptj_model_load(fname, fin, model, vocab);
-    fin.close();
-    return loaded;
-}
-
 // evaluate the transformer
 //
 //   - model:     the model
@@ -512,8 +343,14 @@ bool gptj_eval(
     };
 
     struct ggml_context * ctx0 = ggml_init(params);
-    struct ggml_cgraph gf = {};
-    gf.n_threads = n_threads;
+    struct ggml_cgraph * gf = ggml_new_graph(ctx0);
+
+    // KQ_pos - contains the positions
+    struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    int * data = (int *) KQ_pos->data;
+    for (int i = 0; i < N; ++i) {
+        data[i] = n_past + i;
+    }
 
     struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
     memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -526,7 +363,7 @@ bool gptj_eval(
         ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
         // norm
         {
-            cur = ggml_norm(ctx0, inpL);
+            cur = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
 
             // cur = ln_1_g*cur + ln_1_b
             cur = ggml_add(ctx0,
@@ -540,48 +377,44 @@ bool gptj_eval(
 
         // self-attention
         {
-            struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur);
-            struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur);
-            struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur);
+            struct ggml_tensor * Qcur = ggml_rope(
+                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N),
+                KQ_pos, n_rot, 0, 0
+            );
+            struct ggml_tensor * Kcur = ggml_rope(
+                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N),
+                KQ_pos, n_rot, 0, 0
+            );
 
             // store key and value to memory
             {
-                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_1d(ctx0, model.kv_self.v, N*n_embd, (ggml_element_size(model.kv_self.v)*n_embd)*(il*n_ctx + n_past));
+                struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur));
 
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
+                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
+                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
+                        (   n_ctx)*ggml_element_size(model.kv_self.v),
+                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
+
+                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
+                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
             }
 
             // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q =
-                ggml_permute(ctx0,
-                        ggml_rope(ctx0,
-                            ggml_cpy(ctx0,
-                                Qcur,
-                                ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),
-                            n_past, n_rot, 0),
-                        0, 2, 1, 3);
+            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
 
             // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
             struct ggml_tensor * K =
                 ggml_permute(ctx0,
-                        ggml_rope(ctx0,
-                            ggml_reshape_3d(ctx0,
-                                ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                                n_embd/n_head, n_head, n_past + N),
-                            n_past, n_rot, 1),
+                        ggml_reshape_3d(ctx0,
+                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
+                            n_embd/n_head, n_head, n_past + N),
                         0, 2, 1, 3);
 
             // K * Q
             struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
 
             // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled =
-                ggml_scale(ctx0,
-                        KQ,
-                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
-                        );
+            struct ggml_tensor * KQ_scaled = ggml_scale(ctx0, KQ, 1.0f/sqrt(float(n_embd)/n_head));
 
             // KQ_masked = mask_past(KQ_scaled)
             struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past);
@@ -590,17 +423,15 @@ bool gptj_eval(
             struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
 
             // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V_trans =
-                ggml_cpy(ctx0,
-                        ggml_permute(ctx0,
-                            ggml_reshape_3d(ctx0,
-                                ggml_view_1d(ctx0, model.kv_self.v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd),
-                                n_embd/n_head, n_head, n_past + N),
-                            1, 2, 0, 3),
-                        ggml_new_tensor_3d(ctx0, model.kv_self.v->type, n_past + N, n_embd/n_head, n_head));
+            struct ggml_tensor * V =
+                ggml_view_3d(ctx0, model.kv_self.v,
+                        n_past + N, n_embd/n_head, n_head,
+                        n_ctx*ggml_element_size(model.kv_self.v),
+                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
+                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
 
             // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);
+            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
 
             // KQV_merged = KQV.permute(0, 2, 1, 3)
             struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
@@ -656,7 +487,7 @@ bool gptj_eval(
 
     // norm
     {
-        inpL = ggml_norm(ctx0, inpL);
+        inpL = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
 
         // inpL = ln_f_g*inpL + ln_f_b
         inpL = ggml_add(ctx0,
@@ -680,13 +511,22 @@ bool gptj_eval(
     // logits -> probs
     //inpL = ggml_soft_max(ctx0, inpL);
 
+    ggml_build_forward_expand(gf, inpL);
+
     // run the computation
-    ggml_build_forward_expand(&gf, inpL);
-    ggml_graph_compute       (ctx0, &gf);
+    {
+        std::unique_ptr<uint8_t []> data;
+        auto plan = ggml_graph_plan(gf, n_threads);
+        if (plan.work_size > 0) {
+            data.reset(new uint8_t[plan.work_size]);
+            plan.work_data = data.get();
+        }
+        ggml_graph_compute(gf, &plan);
+    }
 
     //if (n_past%100 == 0) {
-    //    ggml_graph_print   (&gf);
-    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
+    //    ggml_graph_print   (gf);
+    //    ggml_graph_dump_dot(gf, NULL, "gpt-2.dot");
     //}
 
     //embd_w.resize(n_vocab*N);
@@ -832,30 +672,34 @@ GPTJ::GPTJ()
     d_ptr->modelLoaded = false;
 }
 
-size_t GPTJ::requiredMem(const std::string &modelPath) {
+size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx, int ngl) {
+    (void)n_ctx;
+    (void)ngl;
     gptj_model dummy_model;
     gpt_vocab dummy_vocab;
     size_t mem_req;
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-    gptj_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
+    gptj_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
     return mem_req;
 }
 
-bool GPTJ::loadModel(const std::string &modelPath) {
+bool GPTJ::loadModel(const std::string &modelPath, int n_ctx, int ngl) {
+    (void)n_ctx;
+    (void)ngl;
+    d_ptr->modelLoaded = false;
+
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
 
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-
     // load the model
-    if (!gptj_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab)) {
+    bool ok = gptj_model_load(modelPath, *d_ptr->model, d_ptr->vocab);
+    fflush(stdout);
+    if (!ok) {
         std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
         return false;
     }
 
     d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
     d_ptr->modelLoaded = true;
-    fflush(stdout);
     return true;
 }
 
@@ -939,6 +783,16 @@ const std::vector<LLModel::Token> &GPTJ::endTokens() const
     return fres;
 }
 
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != GGUF_TYPE_STRING) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -958,10 +812,21 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    return magic == 0x67676d6c;
+DLL_EXPORT bool magic_match(const char * fname) {
+    struct ggml_context * ctx_meta = NULL;
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ &ctx_meta,
+    };
+    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
+    if (!ctx_gguf)
+        return false;
+
+    bool isValid = gguf_get_version(ctx_gguf) <= 3;
+    isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
+
+    gguf_free(ctx_gguf);
+    return isValid;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/gptj_impl.h

@@ -15,9 +15,11 @@ public:
     GPTJ();
     ~GPTJ();
 
-    bool loadModel(const std::string &modelPath) override;
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;

gpt4all-backend/llama.cpp.cmake

@@ -1,3 +1,11 @@
+#
+# Copyright (c) 2023 Nomic, Inc. All rights reserved.
+#
+# This software is licensed under the terms of the Software for Open Models License (SOM),
+# version 1.0, as detailed in the LICENSE_SOM.txt file. A copy of this license should accompany
+# this software. Except as expressly granted in the SOM license, all rights are reserved by Nomic, Inc.
+#
+
 cmake_minimum_required(VERSION 3.12) # Don't bump this version for no reason
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@@ -30,6 +38,12 @@ else()
     endif()
 endif()
 
+if (APPLE)
+    set(LLAMA_KOMPUTE_DEFAULT OFF)
+else()
+    set(LLAMA_KOMPUTE_DEFAULT ON)
+endif()
+
 
 #
 # Option list
@@ -69,7 +83,7 @@ option(LLAMA_OPENBLAS               "llama: use OpenBLAS"
 #option(LLAMA_CUBLAS                 "llama: use cuBLAS"                                     OFF)
 #option(LLAMA_CLBLAST                "llama: use CLBlast"                                    OFF)
 #option(LLAMA_METAL                  "llama: use Metal"                                      OFF)
-#option(LLAMA_K_QUANTS               "llama: use k-quants"                                   ON)
+option(LLAMA_KOMPUTE                "llama: use Kompute"                                    ${LLAMA_KOMPUTE_DEFAULT})
 set(LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor")
 set(LLAMA_CUDA_DMMV_X "32" CACHE STRING "llama: x stride for dmmv CUDA kernels")
 set(LLAMA_CUDA_DMMV_Y "1" CACHE STRING  "llama: y block size for dmmv CUDA kernels")
@@ -145,6 +159,160 @@ if (LLAMA_OPENBLAS)
     endif()
 endif()
 
+if (LLAMA_KOMPUTE)
+    set(LLAMA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp-mainline)
+    if (NOT EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
+        message(FATAL_ERROR "Kompute not found")
+    endif()
+    message(STATUS "Kompute found")
+
+    add_compile_definitions(VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
+    find_package(Vulkan COMPONENTS glslc REQUIRED)
+    find_program(glslc_executable NAMES glslc HINTS Vulkan::glslc)
+    if (NOT glslc_executable)
+        message(FATAL_ERROR "glslc not found")
+    endif()
+
+    function(compile_shader)
+      set(options)
+      set(oneValueArgs)
+      set(multiValueArgs SOURCES)
+      cmake_parse_arguments(compile_shader "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+      foreach(source ${compile_shader_SOURCES})
+        get_filename_component(OP_FILE ${source} NAME)
+        set(spv_file ${CMAKE_CURRENT_BINARY_DIR}/${OP_FILE}.spv)
+        add_custom_command(
+            OUTPUT ${spv_file}
+            DEPENDS ${LLAMA_DIR}/${source}
+              ${LLAMA_DIR}/kompute-shaders/common.comp
+              ${LLAMA_DIR}/kompute-shaders/op_getrows.comp
+              ${LLAMA_DIR}/kompute-shaders/op_mul_mv_q_n_pre.comp
+              ${LLAMA_DIR}/kompute-shaders/op_mul_mv_q_n.comp
+            COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
+            COMMENT "Compiling ${source} to ${source}.spv"
+        )
+
+        get_filename_component(RAW_FILE_NAME ${spv_file} NAME)
+        set(FILE_NAME "shader${RAW_FILE_NAME}")
+        string(REPLACE ".comp.spv" ".h" HEADER_FILE ${FILE_NAME})
+        string(TOUPPER ${HEADER_FILE} HEADER_FILE_DEFINE)
+        string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}")
+        set(OUTPUT_HEADER_FILE "${HEADER_FILE}")
+        message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}")
+        if(CMAKE_GENERATOR MATCHES "Visual Studio")
+            add_custom_command(
+              OUTPUT ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              DEPENDS ${spv_file} xxd
+              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd"
+            )
+        else()
+            add_custom_command(
+              OUTPUT ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
+              DEPENDS ${spv_file} xxd
+              COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
+            )
+        endif()
+      endforeach()
+    endfunction()
+
+    set(KOMPUTE_OPT_LOG_LEVEL Critical CACHE STRING "Kompute log level")
+    add_subdirectory(${LLAMA_DIR}/kompute)
+
+    # Compile our shaders
+    compile_shader(SOURCES
+      kompute-shaders/op_scale.comp
+      kompute-shaders/op_scale_8.comp
+      kompute-shaders/op_add.comp
+      kompute-shaders/op_addrow.comp
+      kompute-shaders/op_mul.comp
+      kompute-shaders/op_silu.comp
+      kompute-shaders/op_relu.comp
+      kompute-shaders/op_gelu.comp
+      kompute-shaders/op_softmax.comp
+      kompute-shaders/op_norm.comp
+      kompute-shaders/op_rmsnorm.comp
+      kompute-shaders/op_diagmask.comp
+      kompute-shaders/op_mul_mat_mat_f32.comp
+      kompute-shaders/op_mul_mat_f16.comp
+      kompute-shaders/op_mul_mat_q8_0.comp
+      kompute-shaders/op_mul_mat_q4_0.comp
+      kompute-shaders/op_mul_mat_q4_1.comp
+      kompute-shaders/op_mul_mat_q6_k.comp
+      kompute-shaders/op_getrows_f16.comp
+      kompute-shaders/op_getrows_q4_0.comp
+      kompute-shaders/op_getrows_q4_1.comp
+      kompute-shaders/op_getrows_q6_k.comp
+      kompute-shaders/op_rope_f16.comp
+      kompute-shaders/op_rope_f32.comp
+      kompute-shaders/op_cpy_f16_f16.comp
+      kompute-shaders/op_cpy_f16_f32.comp
+      kompute-shaders/op_cpy_f32_f16.comp
+      kompute-shaders/op_cpy_f32_f32.comp
+    )
+
+    # Create a custom target for our generated shaders
+    add_custom_target(generated_shaders DEPENDS
+      shaderop_scale.h
+      shaderop_scale_8.h
+      shaderop_add.h
+      shaderop_addrow.h
+      shaderop_mul.h
+      shaderop_silu.h
+      shaderop_relu.h
+      shaderop_gelu.h
+      shaderop_softmax.h
+      shaderop_norm.h
+      shaderop_rmsnorm.h
+      shaderop_diagmask.h
+      shaderop_mul_mat_mat_f32.h
+      shaderop_mul_mat_f16.h
+      shaderop_mul_mat_q8_0.h
+      shaderop_mul_mat_q4_0.h
+      shaderop_mul_mat_q4_1.h
+      shaderop_mul_mat_q6_k.h
+      shaderop_getrows_f16.h
+      shaderop_getrows_q4_0.h
+      shaderop_getrows_q4_1.h
+      shaderop_getrows_q6_k.h
+      shaderop_rope_f16.h
+      shaderop_rope_f32.h
+      shaderop_cpy_f16_f16.h
+      shaderop_cpy_f16_f32.h
+      shaderop_cpy_f32_f16.h
+      shaderop_cpy_f32_f32.h
+    )
+
+    # Create a custom command that depends on the generated_shaders
+    add_custom_command(
+        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp
+        COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp
+        DEPENDS generated_shaders
+        COMMENT "Ensuring shaders are generated before compiling ggml-kompute.cpp"
+    )
+
+    # Add the stamp to the main sources to ensure dependency tracking
+    set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml-kompute.cpp ${LLAMA_DIR}/ggml-kompute.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp)
+    add_compile_definitions(GGML_USE_KOMPUTE)
+    set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute)
+    set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CMAKE_BINARY_DIR})
+endif()
+
 if (LLAMA_ALL_WARNINGS)
     if (NOT MSVC)
         set(c_flags
@@ -198,6 +366,13 @@ endif()
 # TODO: probably these flags need to be tweaked on some architectures
 #       feel free to update the Makefile for your architecture and send a pull request or issue
 message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
+if (MSVC)
+  string(TOLOWER "${CMAKE_GENERATOR_PLATFORM}" CMAKE_GENERATOR_PLATFORM_LWR)
+  message(STATUS "CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}")
+else ()
+  set(CMAKE_GENERATOR_PLATFORM_LWR "")
+endif ()
+
 if (NOT MSVC)
     if (LLAMA_STATIC)
         add_link_options(-static)
@@ -213,6 +388,139 @@ if (NOT MSVC)
     endif()
 endif()
 
+if ((${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm") OR (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") OR ("${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "arm64"))
+    message(STATUS "ARM detected")
+    if (MSVC)
+        add_compile_definitions(__ARM_NEON)
+        add_compile_definitions(__ARM_FEATURE_FMA)
+        add_compile_definitions(__ARM_FEATURE_DOTPROD)
+        # add_compile_definitions(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) # MSVC doesn't support vdupq_n_f16, vld1q_f16, vst1q_f16
+        add_compile_definitions(__aarch64__) # MSVC defines _M_ARM64 instead
+    else()
+        include(CheckCXXCompilerFlag)
+        check_cxx_compiler_flag(-mfp16-format=ieee COMPILER_SUPPORTS_FP16_FORMAT_I3E)
+        if (NOT "${COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
+            add_compile_options(-mfp16-format=ieee)
+        endif()
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv6")
+            # Raspberry Pi 1, Zero
+            add_compile_options(-mfpu=neon-fp-armv8 -mno-unaligned-access)
+        endif()
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7")
+            # Raspberry Pi 2
+            add_compile_options(-mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations)
+        endif()
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv8")
+            # Raspberry Pi 3, 4, Zero 2 (32-bit)
+            add_compile_options(-mno-unaligned-access)
+        endif()
+    endif()
+elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$" OR "${CMAKE_GENERATOR_PLATFORM_LWR}" MATCHES "^(x86_64|i686|amd64|x64)$" )
+    message(STATUS "x86 detected")
+    if (MSVC)
+        if (LLAMA_AVX512)
+            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX512>)
+            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX512>)
+            # MSVC has no compile-time flags enabling specific
+            # AVX512 extensions, neither it defines the
+            # macros corresponding to the extensions.
+            # Do it manually.
+            if (LLAMA_AVX512_VBMI)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
+            endif()
+            if (LLAMA_AVX512_VNNI)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
+            endif()
+        elseif (LLAMA_AVX2)
+            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX2>)
+            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX2>)
+        elseif (LLAMA_AVX)
+            add_compile_options($<$<COMPILE_LANGUAGE:C>:/arch:AVX>)
+            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/arch:AVX>)
+        endif()
+    else()
+        if (LLAMA_F16C)
+            add_compile_options(-mf16c)
+        endif()
+        if (LLAMA_FMA)
+            add_compile_options(-mfma)
+        endif()
+        if (LLAMA_AVX)
+            add_compile_options(-mavx)
+        endif()
+        if (LLAMA_AVX2)
+            add_compile_options(-mavx2)
+        endif()
+        if (LLAMA_AVX512)
+            add_compile_options(-mavx512f)
+            add_compile_options(-mavx512bw)
+        endif()
+        if (LLAMA_AVX512_VBMI)
+            add_compile_options(-mavx512vbmi)
+        endif()
+        if (LLAMA_AVX512_VNNI)
+            add_compile_options(-mavx512vnni)
+        endif()
+    endif()
+elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
+    message(STATUS "PowerPC detected")
+    add_compile_options(-mcpu=native -mtune=native)
+    #TODO: Add  targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be)
+else()
+    message(STATUS "Unknown architecture")
+endif()
+
+#
+# POSIX conformance
+#
+
+# clock_gettime came in POSIX.1b (1993)
+# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional
+# posix_memalign came in POSIX.1-2001 / SUSv3
+# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985)
+add_compile_definitions(_XOPEN_SOURCE=600)
+
+# Somehow in OpenBSD whenever POSIX conformance is specified
+# some string functions rely on locale_t availability,
+# which was introduced in POSIX.1-2008, forcing us to go higher
+if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
+    remove_definitions(-D_XOPEN_SOURCE=600)
+    add_compile_definitions(_XOPEN_SOURCE=700)
+endif()
+
+# Data types, macros and functions related to controlling CPU affinity and
+# some memory allocation are available on Linux through GNU extensions in libc
+if (CMAKE_SYSTEM_NAME MATCHES "Linux")
+    add_compile_definitions(_GNU_SOURCE)
+endif()
+
+# RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1,
+# and on macOS its availability depends on enabling Darwin extensions
+# similarly on DragonFly, enabling BSD extensions is necessary
+if (
+    CMAKE_SYSTEM_NAME MATCHES "Darwin" OR
+    CMAKE_SYSTEM_NAME MATCHES "iOS" OR
+    CMAKE_SYSTEM_NAME MATCHES "tvOS" OR
+    CMAKE_SYSTEM_NAME MATCHES "DragonFly"
+)
+    add_compile_definitions(_DARWIN_C_SOURCE)
+endif()
+
+# alloca is a non-standard interface that is not visible on BSDs when
+# POSIX conformance is specified, but not all of them provide a clean way
+# to enable it in such cases
+if (CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+    add_compile_definitions(__BSD_VISIBLE)
+endif()
+if (CMAKE_SYSTEM_NAME MATCHES "NetBSD")
+    add_compile_definitions(_NETBSD_SOURCE)
+endif()
+if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
+    add_compile_definitions(_BSD_SOURCE)
+endif()
+
 function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
     message(STATUS "Configuring ggml implementation target llama${SUFFIX} in ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}")
 
@@ -264,46 +572,41 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
         endif()
     endif()
 
-    set(GGML_SOURCES_QUANT_K )
-    set(GGML_METAL_SOURCES )
-    if (LLAMA_K_QUANTS)
-        set(GGML_SOURCES_QUANT_K
-            ${DIRECTORY}/k_quants.h
-            ${DIRECTORY}/k_quants.c)
+    set(GGML_METAL_SOURCES)
+    if (LLAMA_METAL)
+        find_library(FOUNDATION_LIBRARY         Foundation              REQUIRED)
+        find_library(METAL_FRAMEWORK            Metal                   REQUIRED)
+        find_library(METALKIT_FRAMEWORK         MetalKit                REQUIRED)
+        find_library(METALPERFORMANCE_FRAMEWORK MetalPerformanceShaders REQUIRED)
 
-        if (LLAMA_METAL)
-            find_library(FOUNDATION_LIBRARY         Foundation              REQUIRED)
-            find_library(METAL_FRAMEWORK            Metal                   REQUIRED)
-            find_library(METALKIT_FRAMEWORK         MetalKit                REQUIRED)
-            find_library(METALPERFORMANCE_FRAMEWORK MetalPerformanceShaders REQUIRED)
+        set(GGML_METAL_SOURCES ${DIRECTORY}/ggml-metal.m ${DIRECTORY}/ggml-metal.h)
+        # get full path to the file
+        #add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
 
-            set(GGML_METAL_SOURCES ${DIRECTORY}/ggml-metal.m ${DIRECTORY}/ggml-metal.h)
-            # get full path to the file
-            #add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
+        # copy ggml-metal.metal to bin directory
+        configure_file(${DIRECTORY}/ggml-metal.metal bin/ggml-metal.metal COPYONLY)
 
-            # copy ggml-metal.metal to bin directory
-            configure_file(${DIRECTORY}/ggml-metal.metal bin/ggml-metal.metal COPYONLY)
-
-            set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS}
-                ${FOUNDATION_LIBRARY}
-                ${METAL_FRAMEWORK}
-                ${METALKIT_FRAMEWORK}
-                ${METALPERFORMANCE_FRAMEWORK}
-            )
-        endif()
+        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS}
+            ${FOUNDATION_LIBRARY}
+            ${METAL_FRAMEWORK}
+            ${METALKIT_FRAMEWORK}
+            ${METALPERFORMANCE_FRAMEWORK}
+        )
     endif()
 
     add_library(ggml${SUFFIX} OBJECT
                 ${DIRECTORY}/ggml.c
                 ${DIRECTORY}/ggml.h
-                ${GGML_SOURCES_QUANT_K}
+                ${DIRECTORY}/ggml-alloc.c
+                ${DIRECTORY}/ggml-alloc.h
+                ${DIRECTORY}/ggml-backend.c
+                ${DIRECTORY}/ggml-backend.h
+                ${DIRECTORY}/ggml-quants.h
+                ${DIRECTORY}/ggml-quants.c
                 ${GGML_SOURCES_CUDA}
                 ${GGML_METAL_SOURCES}
-                ${GGML_OPENCL_SOURCES})
-
-    if (LLAMA_K_QUANTS)
-        target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_K_QUANTS)
-    endif()
+                ${GGML_OPENCL_SOURCES}
+                ${GGML_SOURCES_KOMPUTE})
 
     if (LLAMA_METAL AND GGML_METAL_SOURCES)
         target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)
@@ -317,15 +620,14 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
 
     if (WITH_LLAMA)
         # Backwards compatibility with old llama.cpp versions
-        set(LLAMA_UTIL_SOURCE_FILE llama-util.h)
+#        set(LLAMA_UTIL_SOURCE_FILE llama-util.h)
         if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
             set(LLAMA_UTIL_SOURCE_FILE llama_util.h)
         endif()
 
         add_library(llama${SUFFIX} STATIC
                     ${DIRECTORY}/llama.cpp
-                    ${DIRECTORY}/llama.h
-                    ${DIRECTORY}/${LLAMA_UTIL_SOURCE_FILE})
+                    ${DIRECTORY}/llama.h)
 
         if (LLAMA_METAL AND GGML_METAL_SOURCES)
             target_compile_definitions(llama${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)

gpt4all-backend/llamamodel.cpp

@@ -28,33 +28,45 @@
 #include <llama.h>
 #include <ggml.h>
 
+#ifdef GGML_USE_KOMPUTE
+#include "ggml-kompute.h"
+#endif
+
+// Maximum supported GGUF version
+static constexpr int GGUF_VER_MAX = 3;
 
 namespace {
 const char *modelType_ = "LLaMA";
 }
 
+static bool llama_verbose() {
+    const char* var = getenv("GPT4ALL_VERBOSE_LLAMACPP");
+    return var && *var;
+}
+
+static void llama_log_callback(enum ggml_log_level level, const char *text, void *userdata) {
+    (void)userdata;
+    if (llama_verbose() || level <= GGML_LOG_LEVEL_ERROR) {
+        fputs(text, stderr);
+    }
+}
+
 struct gpt_params {
     int32_t seed          = -1;   // RNG seed
     int32_t n_keep        = 0;    // number of tokens to keep from initial prompt
-#if LLAMA_DATE <= 230511
-    int32_t n_parts       = -1;   // amount of model parts (-1 = determine from model dimensions)
-#endif
 
-#if LLAMA_DATE >= 230519
     // sampling parameters
     float   tfs_z         = 1.0f; // 1.0 = disabled
     float   typical_p     = 1.0f; // 1.0 = disabled
-#endif
 
     std::string prompt = "";
 
-    bool memory_f16        = true;  // use f16 instead of f32 for memory kv
+    enum ggml_type kv_type = GGML_TYPE_F16; // use f16 instead of f32 for memory kv
 
     bool use_mmap          = true;  // use mmap for faster loads
     bool use_mlock         = false; // use mlock to keep model in memory
 };
 
-#if LLAMA_DATE >= 230519
 static int llama_sample_top_p_top_k(
         llama_context *ctx,
         const llama_token *last_n_tokens_data,
@@ -62,9 +74,10 @@ static int llama_sample_top_p_top_k(
         int top_k,
         float top_p,
         float temp,
-        float repeat_penalty) {
-    auto logits = llama_get_logits(ctx);
-    auto n_vocab = llama_n_vocab(ctx);
+        float repeat_penalty,
+        int32_t pos) {
+    auto logits = llama_get_logits_ith(ctx, pos);
+    auto n_vocab = llama_n_vocab(llama_get_model(ctx));
     // Populate initial list of all candidates
     std::vector<llama_token_data> candidates;
     candidates.reserve(n_vocab);
@@ -73,23 +86,26 @@ static int llama_sample_top_p_top_k(
     }
     llama_token_data_array candidates_p = {candidates.data(), candidates.size(), false};
     // Sample repeat penalty
-    llama_sample_repetition_penalty(nullptr, &candidates_p, last_n_tokens_data, last_n_tokens_size, repeat_penalty);
+    llama_sample_repetition_penalties(nullptr, &candidates_p, last_n_tokens_data, last_n_tokens_size, repeat_penalty, 0.0f, 0.0f);
     // Temperature sampling
     llama_sample_top_k(ctx, &candidates_p, top_k, 1);
     llama_sample_tail_free(ctx, &candidates_p, 1.0f, 1);
     llama_sample_typical(ctx, &candidates_p, 1.0f, 1);
     llama_sample_top_p(ctx, &candidates_p, top_p, 1);
-    llama_sample_temperature(ctx, &candidates_p, temp);
+    llama_sample_temp(ctx, &candidates_p, temp);
     return llama_sample_token(ctx, &candidates_p);
 }
-#endif
 
 struct LLamaPrivate {
     const std::string modelPath;
     bool modelLoaded;
+    int device = -1;
+    llama_model *model = nullptr;
     llama_context *ctx = nullptr;
-    llama_context_params params;
+    llama_model_params model_params;
+    llama_context_params ctx_params;
     int64_t n_threads = 0;
+    std::vector<LLModel::Token> end_tokens;
 };
 
 LLamaModel::LLamaModel()
@@ -108,7 +124,9 @@ struct llama_file_hparams {
     enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16;
 };
 
-size_t LLamaModel::requiredMem(const std::string &modelPath) {
+size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx, int ngl) {
+    // TODO(cebtenzzre): update to GGUF
+    (void)ngl; // FIXME(cetenzzre): use this value
     auto fin = std::ifstream(modelPath, std::ios::binary);
     fin.seekg(0, std::ios_base::end);
     size_t filesize = fin.tellg();
@@ -125,51 +143,115 @@ size_t LLamaModel::requiredMem(const std::string &modelPath) {
     fin.read(reinterpret_cast<char*>(&hparams.n_layer), sizeof(hparams.n_layer));
     fin.read(reinterpret_cast<char*>(&hparams.n_rot), sizeof(hparams.n_rot));
     fin.read(reinterpret_cast<char*>(&hparams.ftype), sizeof(hparams.ftype));
-    const size_t n_ctx = 2048;
     const size_t kvcache_element_size = 2; // fp16
     const size_t est_kvcache_size = hparams.n_embd * hparams.n_layer * 2u * n_ctx * kvcache_element_size;
     return filesize + est_kvcache_size;
 }
 
-bool LLamaModel::loadModel(const std::string &modelPath)
+bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
 {
-    // load the model
-    d_ptr->params = llama_context_default_params();
+    d_ptr->modelLoaded = false;
+
+    // clean up after previous loadModel()
+    if (d_ptr->model) {
+        llama_free_model(d_ptr->model);
+        d_ptr->model = nullptr;
+    }
+    if (d_ptr->ctx) {
+        llama_free(d_ptr->ctx);
+        d_ptr->ctx = nullptr;
+    }
+
+    if (n_ctx < 8) {
+        std::cerr << "warning: minimum context size is 8, using minimum size.\n";
+        n_ctx = 8;
+    }
+
+    // -- load the model --
 
     gpt_params params;
-    d_ptr->params.n_ctx      = 2048;
-    d_ptr->params.seed       = params.seed;
-    d_ptr->params.f16_kv     = params.memory_f16;
-    d_ptr->params.use_mmap   = params.use_mmap;
+
+    d_ptr->model_params = llama_model_default_params();
+
+    d_ptr->model_params.use_mmap  = params.use_mmap;
 #if defined (__APPLE__)
-    d_ptr->params.use_mlock  = true;
+    d_ptr->model_params.use_mlock = true;
 #else
-    d_ptr->params.use_mlock  = params.use_mlock;
-#endif
-#if LLAMA_DATE <= 230511
-    d_ptr->params.n_parts  = params.n_parts;
-#endif
-#ifdef GGML_USE_METAL
-    std::cerr << "llama.cpp: using Metal" << std::endl;
-    // metal always runs the whole model if n_gpu_layers is not 0, at least
-    // currently
-    d_ptr->params.n_gpu_layers = 1;
+    d_ptr->model_params.use_mlock = params.use_mlock;
 #endif
 
-    d_ptr->ctx = llama_init_from_file(modelPath.c_str(), d_ptr->params);
-    if (!d_ptr->ctx) {
+#ifdef GGML_USE_METAL
+    if (llama_verbose()) {
+        std::cerr << "llama.cpp: using Metal" << std::endl;
+    }
+
+    // always fully offload on Metal
+    // TODO(cebtenzzre): use this parameter to allow using more than 53% of system RAM to load a model
+    d_ptr->model_params.n_gpu_layers = 100;
+#elif defined(GGML_USE_KOMPUTE)
+    if (d_ptr->device != -1) {
+        d_ptr->model_params.main_gpu = d_ptr->device;
+        d_ptr->model_params.n_gpu_layers = ngl;
+    }
+#endif
+
+    d_ptr->model = llama_load_model_from_file_gpt4all(modelPath.c_str(), &d_ptr->model_params);
+    if (!d_ptr->model) {
+        fflush(stdout);
+        d_ptr->device = -1;
         std::cerr << "LLAMA ERROR: failed to load model from " <<  modelPath << std::endl;
         return false;
     }
 
+    const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
+    if (n_ctx > n_ctx_train) {
+        std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
+                  << n_ctx << " specified)\n";
+    }
+
+    // -- initialize the context --
+
+    d_ptr->ctx_params = llama_context_default_params();
+
+    d_ptr->ctx_params.n_ctx   = n_ctx;
+    d_ptr->ctx_params.seed    = params.seed;
+    d_ptr->ctx_params.type_k  = params.kv_type;
+    d_ptr->ctx_params.type_v  = params.kv_type;
+
+    // The new batch API provides space for n_vocab*n_tokens logits. Tell llama.cpp early
+    // that we want this many logits so the state serializes consistently.
+    d_ptr->ctx_params.logits_all = true;
+
     d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    d_ptr->ctx_params.n_threads       = d_ptr->n_threads;
+    d_ptr->ctx_params.n_threads_batch = d_ptr->n_threads;
+
+    d_ptr->ctx = llama_new_context_with_model(d_ptr->model, d_ptr->ctx_params);
+    if (!d_ptr->ctx) {
+        fflush(stdout);
+        std::cerr << "LLAMA ERROR: failed to init context for model " <<  modelPath << std::endl;
+        llama_free_model(d_ptr->model);
+        d_ptr->model = nullptr;
+        d_ptr->device = -1;
+        return false;
+    }
+
+    d_ptr->end_tokens = {llama_token_eos(d_ptr->model)};
+
+#ifdef GGML_USE_KOMPUTE
+    if (usingGPUDevice() && ggml_vk_has_device()) {
+        std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
+    }
+#endif
+
+    fflush(stdout);
     d_ptr->modelLoaded = true;
-    fflush(stderr);
     return true;
 }
 
 void LLamaModel::setThreadCount(int32_t n_threads) {
     d_ptr->n_threads = n_threads;
+    llama_set_n_threads(d_ptr->ctx, n_threads, n_threads);
 }
 
 int32_t LLamaModel::threadCount() const {
@@ -178,9 +260,10 @@ int32_t LLamaModel::threadCount() const {
 
 LLamaModel::~LLamaModel()
 {
-    if(d_ptr->ctx) {
+    if (d_ptr->ctx) {
         llama_free(d_ptr->ctx);
     }
+    llama_free_model(d_ptr->model);
 }
 
 bool LLamaModel::isModelLoaded() const
@@ -206,16 +289,17 @@ size_t LLamaModel::restoreState(const uint8_t *src)
 
 std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str) const
 {
-    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
+    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos(d_ptr->model));
     std::vector<LLModel::Token> fres(str.size()+4);
-    auto fres_len = llama_tokenize(d_ptr->ctx, str.c_str(), fres.data(), fres.size(), useBOS);
+    // TODO(cebtenzzre): we may want to use special=true here to process special tokens
+    auto fres_len = llama_tokenize(d_ptr->model, str.c_str(), str.length(), fres.data(), fres.size(), useBOS, false);
     fres.resize(fres_len);
     return fres;
 }
 
 std::string LLamaModel::tokenToString(Token id) const
 {
-    return llama_token_to_str(d_ptr->ctx, id);
+    return llama_token_to_piece(d_ptr->ctx, id);
 }
 
 LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
@@ -224,21 +308,32 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
     return llama_sample_top_p_top_k(d_ptr->ctx,
         promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
         n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty);
+        promptCtx.repeat_penalty, promptCtx.n_last_batch_tokens - 1);
 }
 
 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
 {
-    // When we recalculate context we could have erased the original BOS token... we need to replace it
-    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
-    if (useBOS) {
-        std::vector<int32_t> myTokens;
-        myTokens.push_back(llama_token_bos());
-        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
-        ctx.n_past += 1;
-        return llama_eval(d_ptr->ctx, myTokens.data(), myTokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
-    } else
-        return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+    llama_kv_cache_seq_rm(d_ptr->ctx, 0, ctx.n_past, -1);
+
+    llama_batch batch = llama_batch_init(tokens.size(), 0, 1);
+
+    batch.n_tokens = tokens.size();
+    ctx.n_last_batch_tokens = tokens.size();
+
+    for (int32_t i = 0; i < batch.n_tokens; i++) {
+        batch.token   [i] = tokens[i];
+        batch.pos     [i] = ctx.n_past + i;
+        batch.n_seq_id[i] = 1;
+        batch.seq_id  [i][0] = 0;
+        batch.logits  [i] = false;
+    }
+
+    // llama_decode will output logits only for the last token of the prompt
+    batch.logits[batch.n_tokens - 1] = true;
+
+    int res = llama_decode(d_ptr->ctx, batch);
+    llama_batch_free(batch);
+    return res == 0;
 }
 
 int32_t LLamaModel::contextLength() const
@@ -248,8 +343,151 @@ int32_t LLamaModel::contextLength() const
 
 const std::vector<LLModel::Token> &LLamaModel::endTokens() const
 {
-    static const std::vector<LLModel::Token> fres = {llama_token_eos()};
-    return fres;
+    return d_ptr->end_tokens;
+}
+
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != (GGUF_TYPE_STRING)) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
+static gguf_context *load_gguf(const char *fname, std::string &arch) {
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ nullptr,
+    };
+    gguf_context *ctx = gguf_init_from_file(fname, params);
+    if (!ctx) {
+        std::cerr << __func__ << ": gguf_init_from_file failed\n";
+        return nullptr;
+    }
+
+    int gguf_ver = gguf_get_version(ctx);
+    if (gguf_ver > GGUF_VER_MAX) {
+        std::cerr << __func__ << ": unsupported gguf version: " << gguf_ver << "\n";
+        gguf_free(ctx);
+        return nullptr;
+    }
+
+    arch = get_arch_name(ctx);
+    return ctx;
+}
+
+static int32_t get_arch_key_u32(std::string const &modelPath, std::string const &archKey) {
+    std::string arch;
+    auto * ctx = load_gguf(modelPath.c_str(), arch);
+
+    int32_t value = -1;
+    if (ctx) {
+        auto key = arch + "." + archKey;
+        int keyidx = gguf_find_key(ctx, key.c_str());
+        if (keyidx != -1) {
+            value = gguf_get_val_u32(ctx, keyidx);
+        } else {
+            std::cerr << __func__ << ": " << key << "not found in " << modelPath << "\n";
+        }
+    }
+
+    gguf_free(ctx);
+    return value;
+}
+
+int32_t LLamaModel::maxContextLength(std::string const &modelPath) const
+{
+    return get_arch_key_u32(modelPath, "context_length");
+}
+
+int32_t LLamaModel::layerCount(std::string const &modelPath) const
+{
+    return get_arch_key_u32(modelPath, "block_count");
+}
+
+std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryRequired) const
+{
+#ifdef GGML_USE_KOMPUTE
+    size_t count = 0;
+    auto * vkDevices = ggml_vk_available_devices(memoryRequired, &count);
+
+    if (vkDevices) {
+        std::vector<LLModel::GPUDevice> devices;
+        devices.reserve(count);
+
+        for (size_t i = 0; i < count; ++i) {
+            auto & dev = vkDevices[i];
+            devices.emplace_back(
+                /* index    = */ dev.index,
+                /* type     = */ dev.type,
+                /* heapSize = */ dev.heapSize,
+                /* name     = */ dev.name,
+                /* vendor   = */ dev.vendor
+            );
+            ggml_vk_device_destroy(&dev);
+        }
+
+        free(vkDevices);
+        return devices;
+    }
+#else
+    std::cerr << __func__ << ": built without Kompute\n";
+#endif
+
+    return {};
+}
+
+bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string &name) const
+{
+#if defined(GGML_USE_KOMPUTE)
+    ggml_vk_device device;
+    bool ok = ggml_vk_get_device(&device, memoryRequired, name.c_str());
+    if (ok) {
+        d_ptr->device = device.index;
+        return true;
+    }
+#else
+    (void)memoryRequired;
+    (void)name;
+#endif
+    return false;
+}
+
+bool LLamaModel::initializeGPUDevice(int device, std::string *unavail_reason) const
+{
+#if defined(GGML_USE_KOMPUTE)
+    (void)unavail_reason;
+    d_ptr->device = device;
+    return true;
+#else
+    (void)device;
+    if (unavail_reason) {
+        *unavail_reason = "built without Kompute";
+    }
+    return false;
+#endif
+}
+
+bool LLamaModel::hasGPUDevice()
+{
+#if defined(GGML_USE_KOMPUTE)
+    return d_ptr->device != -1;
+#else
+    return false;
+#endif
+}
+
+bool LLamaModel::usingGPUDevice()
+{
+#if defined(GGML_USE_KOMPUTE)
+    return hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
+#elif defined(GGML_USE_METAL)
+    return true;
+#else
+    return false;
+#endif
 }
 
 #if defined(_WIN32)
@@ -271,42 +509,31 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(std::istream& f) {
-    // Check magic
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    if (magic != 0x67676a74) return false;
-    // Check version
-    uint32_t version = 0;
-    f.read(reinterpret_cast<char*>(&version), sizeof(version));
-    if (!(version LLAMA_VERSIONS)) {
-        return false;
+DLL_EXPORT bool magic_match(const char *fname) {
+    std::string arch;
+    auto * ctx = load_gguf(fname, arch);
+
+    bool valid = true;
+
+    static const std::vector<const char *> known_arches {
+        "baichuan", "bloom", "codeshell", "falcon", "gpt2", "llama", "mpt", "orion", "persimmon", "phi2", "plamo",
+        "qwen", "qwen2", "refact", "stablelm", "starcoder"
+    };
+
+    if (std::find(known_arches.begin(), known_arches.end(), arch) == known_arches.end()) {
+        // not supported by this version of llama.cpp
+        if (!(arch == "gptj" || arch == "bert")) { // we support these via other modules
+            std::cerr << __func__ << ": unsupported model architecture: " << arch << "\n";
+        }
+        valid = false;
     }
-    llama_file_hparams hparams;
-    f.read(reinterpret_cast<char*>(&hparams), sizeof(hparams));
-    if (!(hparams.n_vocab >= 32000 && hparams.n_vocab <= 32100)) {
-        return false; // not a llama.
-    }
-#ifdef GGML_USE_METAL
-    // Check quant supported on metal
-    // skip fields
-    switch(hparams.ftype) {
-        // currently supported on Metal https://github.com/ggerganov/llama.cpp/blob/ae9663f1887513e152839e91f61c513075a19422/ggml-metal.m#L51-L55
-        case LLAMA_FTYPE_MOSTLY_F16:
-        case LLAMA_FTYPE_MOSTLY_Q2_K:
-        case LLAMA_FTYPE_MOSTLY_Q4_0:
-        case LLAMA_FTYPE_MOSTLY_Q6_K:
-        case LLAMA_FTYPE_MOSTLY_Q4_K_S:
-        case LLAMA_FTYPE_MOSTLY_Q4_K_M:
-            return true;
-        default: // unsupported quant-type for Metal
-            return false;
-    }
-#endif
-    return true;
+
+    gguf_free(ctx);
+    return valid;
 }
 
 DLL_EXPORT LLModel *construct() {
+    llama_log_set(llama_log_callback, nullptr);
     return new LLamaModel;
 }
 }

gpt4all-backend/llamamodel_impl.h

@@ -4,8 +4,9 @@
 #ifndef LLAMAMODEL_H
 #define LLAMAMODEL_H
 
-#include <string>
 #include <functional>
+#include <memory>
+#include <string>
 #include <vector>
 #include "llmodel.h"
 
@@ -15,17 +16,24 @@ public:
     LLamaModel();
     ~LLamaModel();
 
-    bool loadModel(const std::string &modelPath) override;
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
+    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const override;
+    bool initializeGPUDevice(size_t memoryRequired, const std::string& name) const override;
+    bool initializeGPUDevice(int device, std::string *unavail_reason) const override;
+    bool hasGPUDevice() override;
+    bool usingGPUDevice() override;
 
 private:
-    LLamaPrivate *d_ptr;
+    std::unique_ptr<LLamaPrivate> d_ptr;
 
 protected:
     std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
@@ -34,6 +42,9 @@ protected:
     bool evalTokens(PromptContext& ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
     const std::vector<Token>& endTokens() const override;
+
+    int32_t maxContextLength(std::string const &modelPath) const override;
+    int32_t layerCount(std::string const &modelPath) const override;
 };
 
 #endif // LLAMAMODEL_H

gpt4all-backend/llmodel.cpp

@@ -2,27 +2,31 @@
 #include "dlhandle.h"
 #include "sysinfo.h"
 
-#include <iostream>
-#include <string>
-#include <vector>
-#include <fstream>
-#include <filesystem>
 #include <cassert>
 #include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <regex>
 #include <sstream>
+#include <string>
+#include <vector>
+
 #ifdef _MSC_VER
-#include <windows.h>
-#include <processthreadsapi.h>
+#include <intrin.h>
 #endif
 
 std::string s_implementations_search_path = ".";
 
 static bool has_at_least_minimal_hardware() {
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(_M_X64)
     #ifndef _MSC_VER
         return __builtin_cpu_supports("avx");
     #else
-        return IsProcessorFeaturePresent(PF_AVX_INSTRUCTIONS_AVAILABLE);
+        int cpuInfo[4];
+        __cpuid(cpuInfo, 1);
+        return cpuInfo[2] & (1 << 28);
     #endif
 #else
     return true; // Don't know how to handle non-x86_64
@@ -30,53 +34,63 @@ static bool has_at_least_minimal_hardware() {
 }
 
 static bool requires_avxonly() {
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(_M_X64)
     #ifndef _MSC_VER
         return !__builtin_cpu_supports("avx2");
     #else
-        return !IsProcessorFeaturePresent(PF_AVX2_INSTRUCTIONS_AVAILABLE);
+        int cpuInfo[4];
+        __cpuidex(cpuInfo, 7, 0);
+        return !(cpuInfo[1] & (1 << 5));
     #endif
 #else
     return false; // Don't know how to handle non-x86_64
 #endif
 }
 
-LLModel::Implementation::Implementation(Dlhandle &&dlhandle_) : dlhandle(new Dlhandle(std::move(dlhandle_))) {
-    auto get_model_type = dlhandle->get<const char *()>("get_model_type");
+LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
+    : m_dlhandle(new Dlhandle(std::move(dlhandle_))) {
+    auto get_model_type = m_dlhandle->get<const char *()>("get_model_type");
     assert(get_model_type);
-    modelType = get_model_type();
-    auto get_build_variant = dlhandle->get<const char *()>("get_build_variant");
+    m_modelType = get_model_type();
+    auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
-    buildVariant = get_build_variant();
-    magicMatch = dlhandle->get<bool(std::ifstream&)>("magic_match");
-    assert(magicMatch);
-    construct_ = dlhandle->get<LLModel *()>("construct");
-    assert(construct_);
+    m_buildVariant = get_build_variant();
+    m_magicMatch = m_dlhandle->get<bool(const char*)>("magic_match");
+    assert(m_magicMatch);
+    m_construct = m_dlhandle->get<LLModel *()>("construct");
+    assert(m_construct);
 }
 
 LLModel::Implementation::Implementation(Implementation &&o)
-    : construct_(o.construct_)
-    , modelType(o.modelType)
-    , buildVariant(o.buildVariant)
-    , magicMatch(o.magicMatch)
-    , dlhandle(o.dlhandle) {
-    o.dlhandle = nullptr;
+    : m_magicMatch(o.m_magicMatch)
+    , m_construct(o.m_construct)
+    , m_modelType(o.m_modelType)
+    , m_buildVariant(o.m_buildVariant)
+    , m_dlhandle(o.m_dlhandle) {
+    o.m_dlhandle = nullptr;
 }
 
 LLModel::Implementation::~Implementation() {
-    if (dlhandle) delete dlhandle;
+    if (m_dlhandle) delete m_dlhandle;
 }
 
 bool LLModel::Implementation::isImplementation(const Dlhandle &dl) {
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
-const std::vector<LLModel::Implementation> &LLModel::implementationList() {
+const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
     // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
     // individual models without the cleanup of the static list interfering
-    static auto* libs = new std::vector<LLModel::Implementation>([] () {
-        std::vector<LLModel::Implementation> fres;
+    static auto* libs = new std::vector<Implementation>([] () {
+        std::vector<Implementation> fres;
 
+        std::string impl_name_re = "(bert|gptj|llamamodel-mainline)";
+        if (requires_avxonly()) {
+            impl_name_re += "-avxonly";
+        } else {
+            impl_name_re += "-(default|metal)";
+        }
+        std::regex re(impl_name_re);
         auto search_in_directory = [&](const std::string& paths) {
             std::stringstream ss(paths);
             std::string path;
@@ -86,7 +100,10 @@ const std::vector<LLModel::Implementation> &LLModel::implementationList() {
                 // Iterate over all libraries
                 for (const auto& f : std::filesystem::directory_iterator(fs_path)) {
                     const std::filesystem::path& p = f.path();
+
                     if (p.extension() != LIB_FILE_EXT) continue;
+                    if (!std::regex_search(p.stem().string(), re)) continue;
+
                     // Add to list if model implementation
                     try {
                         Dlhandle dl(p.string());
@@ -107,34 +124,43 @@ const std::vector<LLModel::Implementation> &LLModel::implementationList() {
     return *libs;
 }
 
-const LLModel::Implementation* LLModel::implementation(std::ifstream& f, const std::string& buildVariant) {
+const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
+    bool buildVariantMatched = false;
     for (const auto& i : implementationList()) {
-        f.seekg(0);
-        if (!i.magicMatch(f)) continue;
-        if (buildVariant != i.buildVariant) continue;
+        if (buildVariant != i.m_buildVariant) continue;
+        buildVariantMatched = true;
+
+        if (!i.m_magicMatch(fname)) continue;
         return &i;
     }
+
+    if (!buildVariantMatched) {
+        std::cerr << "LLModel ERROR: Could not find any implementations for build variant: " << buildVariant << "\n";
+    }
     return nullptr;
 }
 
-LLModel *LLModel::construct(const std::string &modelPath, std::string buildVariant) {
-
-    if (!has_at_least_minimal_hardware())
+LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant, int n_ctx) {
+    if (!has_at_least_minimal_hardware()) {
+        std::cerr << "LLModel ERROR: CPU does not support AVX\n";
         return nullptr;
+    }
 
-    // Read magic
-    std::ifstream f(modelPath, std::ios::binary);
-    if (!f) return nullptr;
     // Get correct implementation
-    const LLModel::Implementation* impl = nullptr;
+    const Implementation* impl = nullptr;
 
     #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
         if (buildVariant == "auto") {
             size_t total_mem = getSystemTotalRAMInBytes();
-            impl = implementation(f, "metal");
+            impl = implementation(modelPath.c_str(), "metal");
             if(impl) {
-                LLModel* metalimpl = impl->construct();
-                size_t req_mem = metalimpl->requiredMem(modelPath);
+                LLModel* metalimpl = impl->m_construct();
+                metalimpl->m_implementation = impl;
+                /* TODO(cebtenzzre): after we fix requiredMem, we should change this to happen at
+                 * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
+                 * most (all?) places where this is called, causing underestimation of required
+                 * memory. */
+                size_t req_mem = metalimpl->requiredMem(modelPath, n_ctx, 100);
                 float req_to_total = (float) req_mem / (float) total_mem;
                 // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
                 if (req_to_total >= 0.53) {
@@ -145,6 +171,8 @@ LLModel *LLModel::construct(const std::string &modelPath, std::string buildVaria
                 }
             }
         }
+    #else
+        (void)n_ctx;
     #endif
 
     if (!impl) {
@@ -156,18 +184,54 @@ LLModel *LLModel::construct(const std::string &modelPath, std::string buildVaria
                 buildVariant = "default";
             }
         }
-        impl = implementation(f, buildVariant);
+        impl = implementation(modelPath.c_str(), buildVariant);
         if (!impl) return nullptr;
     }
-    f.close();
+
     // Construct and return llmodel implementation
-    return impl->construct();
+    auto fres = impl->m_construct();
+    fres->m_implementation = impl;
+    return fres;
 }
 
-void LLModel::setImplementationsSearchPath(const std::string& path) {
+LLModel *LLModel::Implementation::constructDefaultLlama() {
+    static std::unique_ptr<LLModel> llama([]() -> LLModel * {
+        const LLModel::Implementation *impl = nullptr;
+        for (const auto &i : implementationList()) {
+            if (i.m_buildVariant == "metal" || i.m_modelType != "LLaMA") continue;
+            impl = &i;
+        }
+        if (!impl) {
+            std::cerr << "LLModel ERROR: Could not find CPU LLaMA implementation\n";
+            return nullptr;
+        }
+        auto fres = impl->m_construct();
+        fres->m_implementation = impl;
+        return fres;
+    }());
+    return llama.get();
+}
+
+std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices() {
+    auto * llama = constructDefaultLlama();
+    if (llama) { return llama->availableGPUDevices(0); }
+    return {};
+}
+
+int32_t LLModel::Implementation::maxContextLength(const std::string &modelPath) {
+    auto * llama = constructDefaultLlama();
+    return llama ? llama->maxContextLength(modelPath) : -1;
+}
+
+int32_t LLModel::Implementation::layerCount(const std::string &modelPath) {
+    auto * llama = constructDefaultLlama();
+    return llama ? llama->layerCount(modelPath) : -1;
+}
+
+void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
     s_implementations_search_path = path;
 }
 
-const std::string& LLModel::implementationsSearchPath() {
+const std::string& LLModel::Implementation::implementationsSearchPath() {
     return s_implementations_search_path;
 }

gpt4all-backend/llmodel.h

@@ -12,32 +12,50 @@
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
 class Dlhandle;
-
 class LLModel {
 public:
     using Token = int32_t;
 
-    class Implementation {
-        LLModel *(*construct_)();
+    struct GPUDevice {
+        int index;
+        int type;
+        size_t heapSize;
+        std::string name;
+        std::string vendor;
 
+        GPUDevice(int index, int type, size_t heapSize, std::string name, std::string vendor):
+            index(index), type(type), heapSize(heapSize), name(std::move(name)), vendor(std::move(vendor)) {}
+    };
+
+    class Implementation {
     public:
         Implementation(Dlhandle&&);
         Implementation(const Implementation&) = delete;
         Implementation(Implementation&&);
         ~Implementation();
 
+        std::string_view modelType() const { return m_modelType; }
+        std::string_view buildVariant() const { return m_buildVariant; }
+
         static bool isImplementation(const Dlhandle&);
+        static const std::vector<Implementation>& implementationList();
+        static const Implementation *implementation(const char *fname, const std::string& buildVariant);
+        static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto", int n_ctx = 2048);
+        static std::vector<GPUDevice> availableGPUDevices();
+        static int32_t maxContextLength(const std::string &modelPath);
+        static int32_t layerCount(const std::string &modelPath);
+        static void setImplementationsSearchPath(const std::string& path);
+        static const std::string& implementationsSearchPath();
 
-        std::string_view modelType, buildVariant;
-        bool (*magicMatch)(std::ifstream& f);
-        Dlhandle *dlhandle;
+    private:
+        static LLModel *constructDefaultLlama();
 
-        // The only way an implementation should be constructed
-        LLModel *construct() const {
-            auto fres = construct_();
-            fres->m_implementation = this;
-            return fres;
-        }
+        bool (*m_magicMatch)(const char *fname);
+        LLModel *(*m_construct)();
+
+        std::string_view m_modelType;
+        std::string_view m_buildVariant;
+        Dlhandle *m_dlhandle;
     };
 
     struct PromptContext {
@@ -52,25 +70,32 @@ public:
         int32_t n_batch = 9;
         float   repeat_penalty = 1.10f;
         int32_t repeat_last_n = 64;     // last n tokens to penalize
-        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context
-            // window
+        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context window
+        int32_t n_last_batch_tokens = 0;
     };
 
     explicit LLModel() {}
     virtual ~LLModel() {}
 
-    virtual bool loadModel(const std::string &modelPath) = 0;
+    virtual bool supportsEmbedding() const = 0;
+    virtual bool supportsCompletion() const = 0;
+    virtual bool loadModel(const std::string &modelPath, int n_ctx, int ngl) = 0;
     virtual bool isModelLoaded() const = 0;
-    virtual size_t requiredMem(const std::string &modelPath) = 0;
+    virtual size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) = 0;
     virtual size_t stateSize() const { return 0; }
     virtual size_t saveState(uint8_t */*dest*/) const { return 0; }
     virtual size_t restoreState(const uint8_t */*src*/) { return 0; }
+
+    // This method requires the model to return true from supportsCompletion otherwise it will throw
+    // an error
     virtual void prompt(const std::string &prompt,
                         std::function<bool(int32_t)> promptCallback,
                         std::function<bool(int32_t, const std::string&)> responseCallback,
                         std::function<bool(bool)> recalculateCallback,
                         PromptContext &ctx);
 
+    virtual std::vector<float> embedding(const std::string &text);
+
     virtual void setThreadCount(int32_t /*n_threads*/) {}
     virtual int32_t threadCount() const { return 1; }
 
@@ -78,12 +103,27 @@ public:
         return *m_implementation;
     }
 
-    static const std::vector<Implementation>& implementationList();
-    static const Implementation *implementation(std::ifstream& f, const std::string& buildVariant);
-    static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
+    virtual std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const {
+        (void)memoryRequired;
+        return {};
+    }
 
-    static void setImplementationsSearchPath(const std::string& path);
-    static const std::string& implementationsSearchPath();
+    virtual bool initializeGPUDevice(size_t memoryRequired, const std::string& name) const {
+        (void)memoryRequired;
+        (void)name;
+        return false;
+    }
+
+    virtual bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const {
+        (void)device;
+        if (unavail_reason) {
+            *unavail_reason = "model has no GPU support";
+        }
+        return false;
+    }
+
+    virtual bool hasGPUDevice() { return false; }
+    virtual bool usingGPUDevice() { return false; }
 
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
@@ -95,10 +135,26 @@ protected:
     virtual int32_t contextLength() const = 0;
     virtual const std::vector<Token>& endTokens() const = 0;
 
+    virtual int32_t maxContextLength(std::string const &modelPath) const
+    {
+        (void)modelPath;
+        return -1;
+    }
+
+    virtual int32_t layerCount(std::string const &modelPath) const
+    {
+        (void)modelPath;
+        return -1;
+    }
+
     // This is a helper function called from the default implementation of 'prompt' but it can be
     // shared by all base classes so it isn't virtual
     void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate);
 
     const Implementation *m_implementation = nullptr;
+
+private:
+    friend class LLMImplementation;
 };
+
 #endif // LLMODEL_H

gpt4all-backend/llmodel_c.cpp

@@ -5,52 +5,39 @@
 #include <cerrno>
 #include <utility>
 
-
 struct LLModelWrapper {
     LLModel *llModel = nullptr;
     LLModel::PromptContext promptContext;
     ~LLModelWrapper() { delete llModel; }
 };
 
-
 thread_local static std::string last_error_message;
 
-
 llmodel_model llmodel_model_create(const char *model_path) {
-    auto fres = llmodel_model_create2(model_path, "auto", nullptr);
+    const char *error;
+    auto fres = llmodel_model_create2(model_path, "auto", &error);
     if (!fres) {
-        fprintf(stderr, "Invalid model file\n");
+        fprintf(stderr, "Unable to instantiate model: %s\n", error);
     }
     return fres;
 }
 
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error) {
+llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error) {
     auto wrapper = new LLModelWrapper;
-    int error_code = 0;
 
     try {
-        wrapper->llModel = LLModel::construct(model_path, build_variant);
+        wrapper->llModel = LLModel::Implementation::construct(model_path, build_variant);
+        if (!wrapper->llModel) {
+            last_error_message = "Model format not supported (no matching implementation found)";
+        }
     } catch (const std::exception& e) {
-        error_code = EINVAL;
         last_error_message = e.what();
     }
 
     if (!wrapper->llModel) {
         delete std::exchange(wrapper, nullptr);
-        // Get errno and error message if none
-        if (error_code == 0) {
-            if (errno != 0) {
-                error_code = errno;
-                last_error_message = std::strerror(error_code);
-            } else {
-                error_code = ENOTSUP;
-                last_error_message = "Model format not supported (no matching implementation found)";
-            }
-        }
-        // Set error argument
         if (error) {
-            error->message = last_error_message.c_str();
-            error->code = error_code;
+            *error = last_error_message.c_str();
         }
     }
     return reinterpret_cast<llmodel_model*>(wrapper);
@@ -60,16 +47,16 @@ void llmodel_model_destroy(llmodel_model model) {
     delete reinterpret_cast<LLModelWrapper*>(model);
 }
 
-size_t llmodel_required_mem(llmodel_model model, const char *model_path)
+size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl)
 {
     LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->requiredMem(model_path);
+    return wrapper->llModel->requiredMem(model_path, n_ctx, ngl);
 }
 
-bool llmodel_loadModel(llmodel_model model, const char *model_path)
+bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl)
 {
     LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->loadModel(model_path);
+    return wrapper->llModel->loadModel(model_path, n_ctx, ngl);
 }
 
 bool llmodel_isModelLoaded(llmodel_model model)
@@ -166,6 +153,29 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->context_erase = wrapper->promptContext.contextErase;
 }
 
+float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size)
+{
+    if (model == nullptr || text == nullptr || !strlen(text)) {
+        *embedding_size = 0;
+        return nullptr;
+    }
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    std::vector<float> embeddingVector = wrapper->llModel->embedding(text);
+    float *embedding = (float *)malloc(embeddingVector.size() * sizeof(float));
+    if (embedding == nullptr) {
+        *embedding_size = 0;
+        return nullptr;
+    }
+    std::copy(embeddingVector.begin(), embeddingVector.end(), embedding);
+    *embedding_size = embeddingVector.size();
+    return embedding;
+}
+
+void llmodel_free_embedding(float *ptr)
+{
+    free(ptr);
+}
+
 void llmodel_setThreadCount(llmodel_model model, int32_t n_threads)
 {
     LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
@@ -180,10 +190,58 @@ int32_t llmodel_threadCount(llmodel_model model)
 
 void llmodel_set_implementation_search_path(const char *path)
 {
-    LLModel::setImplementationsSearchPath(path);
+    LLModel::Implementation::setImplementationsSearchPath(path);
 }
 
 const char *llmodel_get_implementation_search_path()
 {
-    return LLModel::implementationsSearchPath().c_str();
+    return LLModel::Implementation::implementationsSearchPath().c_str();
+}
+
+struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    std::vector<LLModel::GPUDevice> devices = wrapper->llModel->availableGPUDevices(memoryRequired);
+
+    // Set the num_devices
+    *num_devices = devices.size();
+
+    if (*num_devices == 0) return nullptr;  // Return nullptr if no devices are found
+
+    // Allocate memory for the output array
+    struct llmodel_gpu_device* output = (struct llmodel_gpu_device*) malloc(*num_devices * sizeof(struct llmodel_gpu_device));
+
+    for (int i = 0; i < *num_devices; i++) {
+        output[i].index = devices[i].index;
+        output[i].type = devices[i].type;
+        output[i].heapSize = devices[i].heapSize;
+        output[i].name = strdup(devices[i].name.c_str());  // Convert std::string to char* and allocate memory
+        output[i].vendor = strdup(devices[i].vendor.c_str());  // Convert std::string to char* and allocate memory
+    }
+
+    return output;
+}
+
+bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(memoryRequired, std::string(device));
+}
+
+bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(device->index);
+}
+
+bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->initializeGPUDevice(device);
+}
+
+bool llmodel_has_gpu_device(llmodel_model model)
+{
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    return wrapper->llModel->hasGPUDevice();
 }

gpt4all-backend/llmodel_c.h

@@ -23,17 +23,6 @@ extern "C" {
  */
 typedef void *llmodel_model;
 
-/**
- * Structure containing any errors that may eventually occur
- */
-struct llmodel_error {
-    const char *message;  // Human readable error description; Thread-local; guaranteed to survive until next llmodel C API call
-    int code;             // errno; 0 if none
-};
-#ifndef __cplusplus
-typedef struct llmodel_error llmodel_error;
-#endif
-
 /**
  * llmodel_prompt_context structure for holding the prompt context.
  * NOTE: The implementation takes care of all the memory handling of the raw logits pointer and the
@@ -56,8 +45,18 @@ struct llmodel_prompt_context {
     int32_t repeat_last_n;  // last n tokens to penalize
     float context_erase;    // percent of context to erase if we exceed the context window
 };
+
+struct llmodel_gpu_device {
+    int index = 0;
+    int type = 0;           // same as VkPhysicalDeviceType
+    size_t heapSize = 0;
+    const char * name;
+    const char * vendor;
+};
+
 #ifndef __cplusplus
 typedef struct llmodel_prompt_context llmodel_prompt_context;
+typedef struct llmodel_gpu_device llmodel_gpu_device;
 #endif
 
 /**
@@ -95,10 +94,10 @@ DEPRECATED llmodel_model llmodel_model_create(const char *model_path);
  * Recognises correct model type from file at model_path
  * @param model_path A string representing the path to the model file; will only be used to detect model type.
  * @param build_variant A string representing the implementation to use (auto, default, avxonly, ...),
- * @param error A pointer to a llmodel_error; will only be set on error.
+ * @param error A pointer to a string; will only be set on error.
  * @return A pointer to the llmodel_model instance; NULL on error.
  */
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error);
+llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error);
 
 /**
  * Destroy a llmodel instance.
@@ -111,17 +110,21 @@ void llmodel_model_destroy(llmodel_model model);
  * Estimate RAM requirement for a model file
  * @param model A pointer to the llmodel_model instance.
  * @param model_path A string representing the path to the model file.
+ * @param n_ctx Maximum size of context window
+ * @param ngl Number of GPU layers to use (Vulkan)
  * @return size greater than 0 if the model was parsed successfully, 0 if file could not be parsed.
  */
-size_t llmodel_required_mem(llmodel_model model, const char *model_path);
+size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl);
 
 /**
  * Load a model from a file.
  * @param model A pointer to the llmodel_model instance.
  * @param model_path A string representing the path to the model file.
+ * @param n_ctx Maximum size of context window
+ * @param ngl Number of GPU layers to use (Vulkan)
  * @return true if the model was loaded successfully, false otherwise.
  */
-bool llmodel_loadModel(llmodel_model model, const char *model_path);
+bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl);
 
 /**
  * Check if a model is loaded.
@@ -171,6 +174,25 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
                     llmodel_recalculate_callback recalculate_callback,
                     llmodel_prompt_context *ctx);
 
+/**
+ * Generate an embedding using the model.
+ * NOTE: If given NULL pointers for the model or text, or an empty text, a NULL pointer will be
+ * returned. Bindings should signal an error when NULL is the return value.
+ * @param model A pointer to the llmodel_model instance.
+ * @param text A string representing the text to generate an embedding for.
+ * @param embedding_size A pointer to a size_t type that will be set by the call indicating the length
+ * of the returned floating point array.
+ * @return A pointer to an array of floating point values passed to the calling method which then will
+ * be responsible for lifetime of this memory.
+ */
+float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size);
+
+/**
+ * Frees the memory allocated by the llmodel_embedding function.
+ * @param ptr A pointer to the embedding as returned from llmodel_embedding.
+ */
+void llmodel_free_embedding(float *ptr);
+
 /**
  * Set the number of threads to be used by the model.
  * @param model A pointer to the llmodel_model instance.
@@ -199,6 +221,50 @@ void llmodel_set_implementation_search_path(const char *path);
  */
 const char *llmodel_get_implementation_search_path();
 
+/**
+ * Get a list of available GPU devices given the memory required.
+ * @return A pointer to an array of llmodel_gpu_device's whose number is given by num_devices.
+ */
+struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices);
+
+/**
+ * Initializes a GPU device based on a specified string criterion.
+ *
+ * This function initializes a GPU device based on a string identifier provided. The function
+ * allows initialization based on general device type ("gpu"), vendor name ("amd", "nvidia", "intel"),
+ * or any specific device name.
+ *
+ * @param memoryRequired The amount of memory (in bytes) required by the application or task
+ *                       that will utilize the GPU device.
+ * @param device A string specifying the desired criterion for GPU device selection. It can be:
+ *               - "gpu": To initialize the best available GPU.
+ *               - "amd", "nvidia", or "intel": To initialize the best available GPU from that vendor.
+ *               - A specific GPU device name: To initialize a GPU with that exact name.
+ *
+ * @return True if the GPU device is successfully initialized based on the provided string
+ *         criterion. Returns false if the desired GPU device could not be initialized.
+ */
+bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device);
+
+/**
+ * Initializes a GPU device by specifying a valid gpu device pointer.
+ * @param device A gpu device pointer.
+ * @return True if the GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device);
+
+/**
+ * Initializes a GPU device by its index.
+ * @param device An integer representing the index of the GPU device to be initialized.
+ * @return True if the GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
+
+/**
+ * @return True if a GPU device is successfully initialized, false otherwise.
+ */
+bool llmodel_has_gpu_device(llmodel_model model);
+
 #ifdef __cplusplus
 }
 #endif

gpt4all-backend/llmodel_shared.cpp

@@ -33,7 +33,14 @@ void LLModel::prompt(const std::string &prompt,
                      PromptContext &promptCtx)
 {
     if (!isModelLoaded()) {
-        std::cerr << implementation().modelType << " ERROR: prompt won't work with an unloaded model!\n";
+        std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
+        return;
+    }
+
+    if (!supportsCompletion()) {
+        std::string errorMessage = "ERROR: this model does not support text completion or chat!\n";
+        responseCallback(-1, errorMessage);
+        std::cerr << implementation().modelType() << errorMessage;
         return;
     }
 
@@ -45,8 +52,8 @@ void LLModel::prompt(const std::string &prompt,
 
     if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
         responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed.");
-        std::cerr << implementation().modelType << " ERROR: The prompt is" << embd_inp.size() <<
-            "tokens and the context window is" << promptCtx.n_ctx << "!\n";
+        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
+            " tokens and the context window is " << promptCtx.n_ctx << "!\n";
         return;
     }
 
@@ -64,7 +71,7 @@ void LLModel::prompt(const std::string &prompt,
         if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
             const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
             // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n";
+            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
             promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
             promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
@@ -72,7 +79,7 @@ void LLModel::prompt(const std::string &prompt,
         }
 
         if (!evalTokens(promptCtx, batch)) {
-            std::cerr << implementation().modelType << " ERROR: Failed to process prompt\n";
+            std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
             return;
         }
 
@@ -81,10 +88,10 @@ void LLModel::prompt(const std::string &prompt,
             if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
                 promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(batch.at(t));
+            promptCtx.n_past += 1;
             if (!promptCallback(batch.at(t)))
                 return;
         }
-        promptCtx.n_past += batch.size();
         i = batch_end;
     }
 
@@ -103,7 +110,7 @@ void LLModel::prompt(const std::string &prompt,
         if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
             const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
             // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n";
+            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
             promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
             promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
@@ -111,12 +118,10 @@ void LLModel::prompt(const std::string &prompt,
         }
 
         if (!evalTokens(promptCtx, { id })) {
-            std::cerr << implementation().modelType << " ERROR: Failed to predict next token\n";
+            std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
             return;
         }
 
-        promptCtx.n_past += 1;
-
         // display text
         for (const auto token : endTokens()) {
             if (id == token) return;
@@ -151,6 +156,7 @@ void LLModel::prompt(const std::string &prompt,
             if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
                 promptCtx.tokens.erase(promptCtx.tokens.begin());
             promptCtx.tokens.push_back(t);
+            promptCtx.n_past += 1;
             //TODO: Conversion to std::string can be avoided here...
             if (!responseCallback(t, std::string(tokenToString(t))))
                 return;
@@ -158,3 +164,12 @@ void LLModel::prompt(const std::string &prompt,
         cachedTokens.clear();
     }
 }
+
+std::vector<float> LLModel::embedding(const std::string &/*text*/)
+{
+    if (!supportsCompletion()) {
+        std::string errorMessage = "ERROR: this model does not support generating embeddings!\n";
+        std::cerr << implementation().modelType() << errorMessage;
+    }
+    return std::vector<float>();
+}

gpt4all-backend/llmodel_shared.h

@@ -1,6 +1,7 @@
 #pragma once
 #include <cstdint>
 #include <cstddef>
+#include <vector>
 #include <ggml.h>
 
 struct llm_buffer {
@@ -34,3 +35,12 @@ struct llm_kv_cache {
         }
     }
 };
+
+inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads) {
+    struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
+    if (plan.work_size > 0) {
+        buf.resize(plan.work_size);
+        plan.work_data = buf.addr;
+    }
+    ggml_graph_compute(graph, &plan);
+}

gpt4all-backend/mpt.cpp

@@ -1,893 +0,0 @@
-#define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "mpt_impl.h"
-
-#include "utils.h"
-#include "llmodel_shared.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <fstream>
-#include <map>
-#include <random>
-#include <string>
-#include <vector>
-#include <iostream>
-#if defined(_WIN32) && defined(_MSC_VER)
-    #define WIN32_LEAN_AND_MEAN
-    #ifndef NOMINMAX
-        #define NOMINMAX
-    #endif
-    #include <windows.h>
-    #include <io.h>
-    #include <stdio.h>
-#else
-    #include <unistd.h>
-#endif
-#include <sstream>
-#include <thread>
-#include <unordered_set>
-#include <regex>
-#include <ggml.h>
-
-
-namespace {
-const char *modelType_ = "MPT";
-}
-
-// default hparams (MPT 7B)
-struct mpt_hparams {
-    int32_t n_vocab      = 50432;
-    int32_t n_ctx        = 2048;
-    int32_t n_embd       = 4096;
-    int32_t n_head       = 32;
-    int32_t n_layer      = 32;
-    float alibi_bias_max = 8;
-    float clip_qkv       = 0;
-    int32_t expand       = 4;
-    int32_t f16          = 1;
-};
-
-struct mpt_layer {
-    // normalization
-    struct ggml_tensor * norm_1_w;
-    struct ggml_tensor * norm_2_w;
-
-    // attention
-    struct ggml_tensor * attn_Wqkv_w;
-    struct ggml_tensor * attn_out_proj_w;
-
-    // ff
-    struct ggml_tensor * ffn_up_proj_w;
-    struct ggml_tensor * ffn_down_proj_w;
-};
-
-struct mpt_model {
-    mpt_hparams hparams;
-
-    // normalization
-    struct ggml_tensor * norm_f_w;
-
-    struct ggml_tensor * wte; // position embedding
-
-    // mpt does weight tying
-
-    std::vector<mpt_layer> layers;
-
-    struct llm_kv_cache kv_self;
-    struct ggml_context * ctx;
-    std::map<std::string, struct ggml_tensor *> tensors;
-
-
-    llm_buffer eval_buf;
-    llm_buffer scr0_buf;
-    llm_buffer scr1_buf;
-
-    ~mpt_model() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
-static bool kv_cache_init(
-        const struct mpt_hparams & hparams,
-             struct llm_kv_cache & cache,
-                         ggml_type   wtype,
-                               int   n_ctx) {
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-
-    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
-    const int64_t n_elements = n_embd*n_mem;
-
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
-
-    struct ggml_init_params params;
-    params.mem_size   = cache.buf.size;
-    params.mem_buffer = cache.buf.addr;
-    params.no_alloc   = false;
-
-    cache.ctx = ggml_init(params);
-
-    if (!cache.ctx) {
-        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
-        return false;
-    }
-
-    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-
-    return true;
-}
-
-// load the model's weights from a stream. if mem_req ptr is passed the model is
-// only partially parsed to estimate required memory
-bool mpt_model_load(const std::string &fname, std::istream &fin, mpt_model & model, gpt_vocab & vocab, size_t * mem_req) {
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
-    if (mem_req != nullptr) {
-        *mem_req = 0;
-    }
-
-    // verify magic
-    {
-        uint32_t magic;
-        fin.read((char *) &magic, sizeof(magic));
-        if (magic != 0x67676d6d) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
-            return false;
-        }
-    }
-
-    // load hparams
-    {
-        auto & hparams = model.hparams;
-
-        fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
-        fin.read((char *) &hparams.n_ctx,   sizeof(hparams.n_ctx));
-        fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
-        fin.read((char *) &hparams.n_head,  sizeof(hparams.n_head));
-        fin.read((char *) &hparams.n_embd,  sizeof(hparams.n_embd));
-        fin.read((char *) &hparams.alibi_bias_max,  sizeof(hparams.alibi_bias_max));
-        fin.read((char *) &hparams.clip_qkv,  sizeof(hparams.clip_qkv));
-        fin.read((char *) &hparams.f16,   sizeof(hparams.f16));
-
-        printf("%s: n_vocab        = %d\n", __func__, hparams.n_vocab);
-        printf("%s: n_ctx          = %d\n", __func__, hparams.n_ctx);
-        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
-        printf("%s: alibi_bias_max = %f\n", __func__, hparams.alibi_bias_max);
-        printf("%s: clip_qkv       = %f\n", __func__, hparams.clip_qkv);
-        printf("%s: ftype          = %d\n", __func__, hparams.f16);
-    }
-
-    // load vocab
-    {
-        int32_t n_vocab = model.hparams.n_vocab;
-        fin.read((char *) &n_vocab, sizeof(n_vocab));
-
-        if (n_vocab != model.hparams.n_vocab) {
-            fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
-                    __func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
-            return false;
-        }
-
-        std::string word;
-        for (int i = 0; i < n_vocab; i++) {
-            uint32_t len;
-            fin.read((char *) &len, sizeof(len));
-            bool special = false;
-            if (len & (1<<31)) {
-                len = len &~ (1<<31);
-                special = true;
-            }
-
-            if (len > 0) {
-                word.resize(len);
-                fin.read((char *) word.data(), len);
-                vocab.token_to_id[word] = i;
-                vocab.id_to_token[i] = word;
-            }
-
-            if(special) {
-                vocab.add_special_token(word);
-            }
-        }
-    }
-
-    // for the big tensors, we have the option to store the data in 16-bit floats or quantized
-    // in order to save memory and also to speed up the computation
-    ggml_type wtype = GGML_TYPE_COUNT;
-    switch (model.hparams.f16) {
-        case 0: wtype = GGML_TYPE_F32;  break;
-        case 1: wtype = GGML_TYPE_F16;  break;
-        case 2: wtype = GGML_TYPE_Q4_0; break;
-        case 3: wtype = GGML_TYPE_Q4_1; break;
-        case 5: wtype = GGML_TYPE_Q4_2; break;
-        default:
-                {
-                    fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
-                            __func__, fname.c_str(), model.hparams.f16);
-                    return false;
-                }
-    }
-
-    auto & ctx = model.ctx;
-
-    size_t ctx_size = 0;
-
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_ctx   = hparams.n_ctx;
-        const int n_vocab = hparams.n_vocab;
-        const int expand  = hparams.expand;
-
-
-        ctx_size += n_embd*ggml_type_sizef(GGML_TYPE_F32); // ln_f_w
-
-        ctx_size += n_embd*n_vocab*ggml_type_sizef(GGML_TYPE_F32); // wte
-
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_1_w
-        ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // norm_2_w
-
-        ctx_size += n_layer*(3*n_embd*n_embd*ggml_type_sizef(wtype)); // attn_Wqkv_w
-        ctx_size += n_layer*(n_embd*n_embd*ggml_type_sizef(wtype)); // attn_out_proj_w
-
-        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype));  // ffn_up_proj_w
-        ctx_size += n_layer*(expand*n_embd*n_embd*ggml_type_sizef(wtype)); // ffn_down_proj_w
-
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_k
-        ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F16); // memory_v
-
-        // TODO probably less now?
-        ctx_size += (5 + 10*n_layer)*256; // object overhead
-
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
-    }
-
-    if (mem_req != nullptr) {
-        *mem_req += ctx_size;
-        const int n_embd  = model.hparams.n_embd;
-        const int n_layer = model.hparams.n_layer;
-
-        const int64_t n_mem      = (int64_t)n_layer*model.hparams.n_ctx;
-        const int64_t n_elements = n_embd*n_mem;
-
-        *mem_req += (2u*n_elements*ggml_type_size(wtype) + 2_MiB);
-        return false;
-    }
-
-    // create the ggml context
-    {
-        struct ggml_init_params params = {
-            .mem_size   = ctx_size,
-            .mem_buffer = NULL,
-            .no_alloc   = false,
-        };
-
-        model.ctx = ggml_init(params);
-        if (!model.ctx) {
-            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
-            return false;
-        }
-    }
-
-    // prepare memory for the weights
-    {
-        const auto & hparams = model.hparams;
-
-        const int n_embd  = hparams.n_embd;
-        const int n_layer = hparams.n_layer;
-        const int n_vocab = hparams.n_vocab;
-        const int expand  = hparams.expand;
-
-        model.layers.resize(n_layer);
-
-        model.wte    = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_vocab);
-        model.norm_f_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-        // map by name
-        model.tensors["transformer.wte.weight"] = model.wte;
-        model.tensors["transformer.norm_f.weight"] = model.norm_f_w;
-
-        for (int i = 0; i < n_layer; ++i) {
-            auto & layer = model.layers[i];
-
-            layer.norm_1_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-            layer.norm_2_w        = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
-
-            layer.attn_Wqkv_w     = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd * 3);
-            layer.attn_out_proj_w = ggml_new_tensor_2d(ctx, wtype,        n_embd, n_embd);
-            layer.ffn_up_proj_w   = ggml_new_tensor_2d(ctx, wtype,        n_embd, expand*n_embd);
-            layer.ffn_down_proj_w = ggml_new_tensor_2d(ctx, wtype, expand*n_embd, n_embd);
-
-            // map by name
-            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_1.weight"]        = layer.norm_1_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".norm_2.weight"]        = layer.norm_2_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.Wqkv.weight"]     = layer.attn_Wqkv_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".attn.out_proj.weight"] = layer.attn_out_proj_w;
-
-            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.up_proj.weight"]   = layer.ffn_up_proj_w;
-            model.tensors["transformer.blocks." + std::to_string(i) + ".ffn.down_proj.weight"] = layer.ffn_down_proj_w;
-        }
-    }
-
-    // key + value memory
-    {
-        const auto & hparams = model.hparams;
-        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
-            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
-            ggml_free(ctx);
-            return false;
-        }
-
-        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
-        printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
-    }
-
-    // load weights
-    {
-        int n_tensors = 0;
-        size_t total_size = 0;
-
-        printf("%s: ", __func__);
-
-        while (true) {
-            int32_t n_dims;
-            int32_t length;
-            int32_t ttype;
-
-            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
-            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
-            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));
-
-            if (fin.eof()) {
-                break;
-            }
-
-            int32_t nelements = 1;
-            int32_t ne[2] = { 1, 1 };
-            for (int i = 0; i < n_dims; ++i) {
-                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
-                nelements *= ne[i];
-            }
-
-            std::string name(length, 0);
-            fin.read(&name[0], length);
-
-            if (model.tensors.find(name.data()) == model.tensors.end()) {
-                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data());
-                return false;
-            }
-
-            auto tensor = model.tensors[name.data()];
-            if (ggml_nelements(tensor) != nelements) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data());
-                return false;
-            }
-
-            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
-                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%" PRId64 ", %" PRId64 "], expected [%d, %d]\n",
-                        __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
-                return false;
-            }
-
-            // for debugging
-            if (0) {
-                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
-            }
-
-            const size_t bpe = ggml_type_size(ggml_type(ttype));
-
-            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
-                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
-                        __func__, name.data(), ggml_nbytes(tensor), nelements*bpe);
-                return false;
-            }
-
-            fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
-
-            //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ttype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
-            total_size += ggml_nbytes(tensor);
-            if (++n_tensors % 8 == 0) {
-                printf(".");
-                fflush(stdout);
-            }
-        }
-
-        printf(" done\n");
-
-        printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
-    }
-
-    model.scr0_buf.resize(256u * 1024 * 1024);
-    model.scr1_buf.resize(256u * 1024 * 1024);
-
-    return true;
-}
-
-// load the model's weights from a file path
-bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vocab) {
-
-    auto fin = std::ifstream(fname, std::ios::binary);
-    if (!fin) {
-        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
-        return false;
-    }
-
-    bool loaded = mpt_model_load(fname, fin, model, vocab, nullptr);
-    fin.close();
-    return loaded;
-}
-
-bool mpt_eval(
-        mpt_model & model,
-        const int n_threads,
-        const int n_past,
-        const std::vector<int>           & embd_inp,
-              std::vector<float>         & embd_w,
-              size_t                     & mem_per_token) {
-    const int N = embd_inp.size();
-
-    const auto & hparams = model.hparams;
-
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_ctx   = hparams.n_ctx;
-    const int n_head  = hparams.n_head;
-    const int n_vocab = hparams.n_vocab;
-
-    const size_t init_buf_size = 1024_MiB;
-    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
-        model.eval_buf.resize(init_buf_size);
-
-    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
-        const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
-        // printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
-
-        // reallocate
-        model.eval_buf.resize(buf_size_new);
-        if (model.eval_buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
-            return false;
-        }
-    }
-
-    struct ggml_init_params params = {
-        .mem_size   = model.eval_buf.size,
-        .mem_buffer = model.eval_buf.addr,
-        .no_alloc = false
-    };
-
-    struct ggml_context * ctx0 = ggml_init(params);
-    struct ggml_cgraph gf = {};
-    gf.n_threads = n_threads;
-
-    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
-
-    // wte
-    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
-
-    for (int il = 0; il < n_layer; ++il) {
-        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-        struct ggml_tensor * inpSA = inpL;
-        struct ggml_tensor * cur = inpSA;
-        // self-attention
-        {
-
-            // norm1
-            cur = ggml_norm(ctx0, cur);
-            cur = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].norm_1_w, cur),
-                    cur);
-            // compute QKV
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].attn_Wqkv_w,
-                    cur);
-
-            // TODO: clip_qkv
-            struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 0*ggml_element_size(cur)*n_embd));
-            struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 1*ggml_element_size(cur)*n_embd));
-            struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 2*ggml_element_size(cur)*n_embd));
-
-            // TODO: qk_ln? (seems to be False in MPT-7B configs)
-            {
-                Vcur = ggml_transpose(ctx0, Vcur);
-
-                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
-                                        (   n_ctx)*ggml_element_size(model.kv_self.v),
-                                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
-
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
-            }
-            // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q =
-                ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0, Qcur, n_embd/n_head, n_head, N),
-                        0, 2, 1, 3);
-
-            struct ggml_tensor * K =
-                ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0,
-                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                            n_embd/n_head, n_head, n_past + N),
-                        0, 2, 1, 3);
-
-            // K * Q
-            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
-
-            // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled =
-                ggml_scale(ctx0,
-                        KQ,
-                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
-                        );
-
-
-            // Alibi
-            struct ggml_tensor * KQ_scaled_biased = ggml_alibi(ctx0, ggml_cont(ctx0, KQ_scaled), n_past, n_head);
-
-            // KQ_masked = mask_past(KQ_scaled)
-            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled_biased, n_past);
-
-            // KQ = soft_max(KQ_masked)
-            struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
-
-            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V =
-                ggml_view_3d(ctx0, model.kv_self.v,
-                        n_past + N, n_embd/n_head, n_head,
-                        n_ctx*ggml_element_size(model.kv_self.v),
-                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
-                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
-
-            // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
-
-            // KQV_merged = KQV.permute(0, 2, 1, 3)
-            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            // cur = KQV_merged.contiguous().view(n_embd, N)
-            cur = ggml_cpy(ctx0,
-                    KQV_merged,
-                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-            // projection (no bias)
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].attn_out_proj_w,
-                    cur);
-        }
-
-        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
-        // residual
-        struct ggml_tensor * resSA = ggml_add(ctx0, cur, inpSA);
-        // feed-forward network
-        {
-            cur = resSA;
-            // norm2
-            cur = ggml_norm(ctx0, cur);
-            cur = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].norm_2_w, cur),
-                    cur);
-            // ffn
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].ffn_up_proj_w,
-                    cur);
-            cur = ggml_gelu(ctx0, cur);
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].ffn_down_proj_w,
-                    cur);
-
-        }
-
-        // self-attention + FF
-        inpL = ggml_add(ctx0, cur, resSA);
-    }
-    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-    struct ggml_tensor * out = inpL;
-    // -> logits
-    {
-        out = ggml_norm(ctx0, out);
-        out = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.norm_f_w, out),
-                    out);
-        ggml_set_scratch(ctx0, { 0, 0, nullptr, });
-        out = ggml_mul_mat(ctx0, model.wte, out);
-    }
-
-
-    // run the computation
-    ggml_build_forward_expand(&gf, out);
-    ggml_graph_compute       (ctx0, &gf);
-
-
-    // return result for just the last token
-    embd_w.resize(n_vocab);
-    memcpy(embd_w.data(), (float *) ggml_get_data(out) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
-
-    if (mem_per_token == 0) {
-        mem_per_token = ggml_used_mem(ctx0)/N;
-    }
-    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
-
-    ggml_free(ctx0);
-
-    return true;
-}
-
-
-#define MPT_MAX_RNG_STATE 64*1024
-
-size_t mpt_get_state_size(const mpt_model &model)
-{
-    // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state.
-    // for reference, std::mt19937(1337) serializes to 6701 bytes.
-    const size_t s_rng_size        = sizeof(size_t);
-    const size_t s_rng             = MPT_MAX_RNG_STATE;
-    const size_t s_kv_size         = sizeof(size_t);
-    const size_t s_kv_ntok         = sizeof(int);
-    const size_t s_kv              = model.kv_self.buf.size;
-    const size_t s_total = (
-        + s_rng_size
-        + s_rng
-        + s_kv_size
-        + s_kv_ntok
-        + s_kv
-    );
-    fflush(stdout);
-    return s_total;
-}
-
-size_t mpt_copy_state_data(const mpt_model &model, const std::mt19937 &rng, uint8_t *dest)
-{
-    uint8_t * out = dest;
-    fflush(stdout);
-    // copy rng
-    {
-        std::stringstream rng_ss;
-        rng_ss << rng;
-
-        const size_t rng_size = rng_ss.str().size();
-        char rng_buf[MPT_MAX_RNG_STATE];
-
-        memset(&rng_buf[0], 0, MPT_MAX_RNG_STATE);
-        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
-
-        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
-        memcpy(out, &rng_buf[0], MPT_MAX_RNG_STATE); out += MPT_MAX_RNG_STATE;
-    }
-
-    // copy kv cache
-    {
-        const size_t kv_size = model.kv_self.buf.size;
-        const int    kv_ntok = model.kv_self.n;
-
-        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
-        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
-
-        if (kv_size) {
-            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
-        }
-    }
-
-    const size_t written  = out - dest;
-    assert(written == mpt_get_state_size(model));
-    fflush(stdout);
-    return written;
-}
-
-size_t mpt_set_state_data(mpt_model *model, std::mt19937 *rng, const uint8_t *src)
-{
-    const uint8_t * in = src;
-
-    // set rng
-    {
-        size_t rng_size;
-        char   rng_buf[MPT_MAX_RNG_STATE];
-
-        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
-        memcpy(&rng_buf[0], in, MPT_MAX_RNG_STATE); in += MPT_MAX_RNG_STATE;
-
-        std::stringstream rng_ss;
-        rng_ss.str(std::string(&rng_buf[0], rng_size));
-        rng_ss >> *rng;
-
-        assert(rng_ss.fail() == false);
-    }
-
-    // set kv cache
-    {
-        size_t kv_size;
-        int kv_ntok;
-
-        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
-        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
-
-        if (kv_size) {
-            assert(model->kv_self.buf.size == kv_size);
-
-            void * k_data = model->kv_self.k->data; // remember data pointers
-            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
-
-            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
-
-            model->kv_self.k->data = k_data; // restore correct data pointers
-            model->kv_self.v->data = v_data;
-
-        }
-
-        model->kv_self.n = kv_ntok;
-    }
-
-    const size_t nread    = in - src;
-    assert(nread == mpt_get_state_size(*model));
-    fflush(stdout);
-    return nread;
-}
-
-struct MPTPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    gpt_vocab vocab;
-    mpt_model *model = nullptr;
-    int64_t n_threads = 0;
-    size_t mem_per_token = 0;
-    std::mt19937 rng;
-    bool has_im_end = false;
-};
-
-MPT::MPT()
-    : d_ptr(new MPTPrivate) {
-    d_ptr->model = new mpt_model;
-    d_ptr->model->ctx = nullptr;
-    d_ptr->modelLoaded = false;
-}
-
-size_t MPT::requiredMem(const std::string &modelPath) {
-    mpt_model dummy_model;
-    gpt_vocab dummy_vocab;
-    size_t mem_req;
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-    mpt_model_load(modelPath, fin, dummy_model, dummy_vocab, &mem_req);
-    return mem_req;
-}
-
-bool MPT::loadModel(const std::string &modelPath) {
-    std::mt19937 rng(time(NULL));
-    d_ptr->rng = rng;
-
-    auto fin = std::ifstream(modelPath, std::ios::binary);
-
-    // load the model
-    if (!mpt_model_load(modelPath, fin, *d_ptr->model, d_ptr->vocab, nullptr)) {
-        std::cerr << "MPT ERROR: failed to load model from " <<  modelPath;
-        return false;
-    }
-
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    d_ptr->has_im_end = d_ptr->vocab.token_to_id.find("<|im_end|>") != d_ptr->vocab.token_to_id.end();
-    fflush(stdout);
-    return true;
-}
-
-void MPT::setThreadCount(int32_t n_threads) {
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t MPT::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-MPT::~MPT()
-{
-    delete d_ptr->model;
-}
-
-bool MPT::isModelLoaded() const
-{
-    return d_ptr->modelLoaded;
-}
-
-size_t MPT::stateSize() const
-{
-    return mpt_get_state_size(*d_ptr->model);
-}
-
-size_t MPT::saveState(uint8_t *dest) const
-{
-    return mpt_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
-}
-
-size_t MPT::restoreState(const uint8_t *src)
-{
-    return mpt_set_state_data(d_ptr->model, &d_ptr->rng, src);
-}
-
-std::vector<LLModel::Token> MPT::tokenize(PromptContext &, const std::string &str) const
-{
-    return ::gpt_tokenize(d_ptr->vocab, str);
-}
-
-std::string MPT::tokenToString(Token id) const
-{
-    return d_ptr->vocab.id_to_token[id];
-}
-
-LLModel::Token MPT::sampleToken(PromptContext &promptCtx) const
-{
-    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
-    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
-        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks,
-        promptCtx.logits,
-        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty,
-        d_ptr->rng);
-}
-
-bool MPT::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    // determine the required inference memory per token:
-    static bool initialized = false;
-    if (!initialized) {
-        mpt_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
-            d_ptr->mem_per_token);
-        initialized = true;
-    }
-
-    return mpt_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
-}
-
-int32_t MPT::contextLength() const
-{
-    return d_ptr->model->hparams.n_ctx;
-}
-
-const std::vector<LLModel::Token> &MPT::endTokens() const
-{
-    static const std::vector<LLModel::Token> fres = {0, d_ptr->vocab.token_to_id["<|im_end|>"]};
-    return fres;
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(std::istream& f) {
-    uint32_t magic = 0;
-    f.read(reinterpret_cast<char*>(&magic), sizeof(magic));
-    return magic == 0x67676d6d;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new MPT;
-}
-}

gpt4all-backend/mpt_impl.h

@@ -1,39 +0,0 @@
-#ifndef MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of mpt.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef MPT_H
-#define MPT_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include "llmodel.h"
-
-struct MPTPrivate;
-class MPT : public LLModel {
-public:
-    MPT();
-    ~MPT();
-
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    MPTPrivate *d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string tokenToString(Token) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // MPT_H

gpt4all-backend/replit_impl.h

@@ -1,41 +0,0 @@
-#ifndef REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of replit.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define REPLIT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef REPLIT_H
-#define REPLIT_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include "llmodel.h"
-
-#define GGML_QNT_VERSION_FACTOR 1000 // do not change this
-
-struct ReplitPrivate;
-class Replit : public LLModel {
-public:
-    Replit();
-    ~Replit();
-
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string & modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    ReplitPrivate *d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string tokenToString(Token) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // REPLIT_H

gpt4all-backend/scripts/convert_bert_hf_to_gguf.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import struct
+import sys
+from pathlib import Path
+
+import gguf
+import numpy as np
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+
+
+if not 2 <= len(sys.argv) < 4:
+    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = Path(sys.argv[1])
+
+with open(dir_model / "vocab.txt", encoding="utf-8") as f:
+    vocab = f.readlines()
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+
+fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
+
+
+ARCH = gguf.MODEL_ARCH.BERT
+gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
+
+print("gguf: get model metadata")
+
+config = AutoConfig.from_pretrained(dir_model)
+
+block_count = config.num_hidden_layers
+gguf_writer.add_name("BERT")
+gguf_writer.add_context_length(config.max_position_embeddings)
+gguf_writer.add_embedding_length(config.hidden_size)
+gguf_writer.add_feed_forward_length(config.intermediate_size)
+gguf_writer.add_block_count(block_count)
+gguf_writer.add_head_count(config.num_attention_heads)
+gguf_writer.add_file_type(ftype)
+
+print("gguf: get tokenizer metadata")
+
+try:
+    with open(dir_model / "tokenizer.json", encoding="utf-8") as f:
+        tokenizer_json = json.load(f)
+except FileNotFoundError as e:
+    print(f'Error: Missing {e.filename!r}', file=sys.stderr)
+    sys.exit(1)
+
+print("gguf: get wordpiece tokenizer vocab")
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+print(tokenizer.encode('I believe the meaning of life is'))
+
+tokens: list[bytearray] = []
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+
+# The number of tokens in tokenizer.json can differ from the expected vocab size.
+# This causes downstream issues with mismatched tensor sizes when running the inference
+for i in range(config.vocab_size):
+    try:
+        text = reverse_vocab[i]
+    except KeyError:
+        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
+        pad_token = f"[PAD{i}]".encode("utf8")
+        text = bytearray(pad_token)
+
+    tokens.append(text)
+
+gguf_writer.add_tokenizer_model("bert")  # wordpiece
+gguf_writer.add_token_list(tokens)
+
+special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
+special_vocab.add_to_gguf(gguf_writer)
+
+print("gguf: get tensor metadata")
+
+model = AutoModel.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
+print(model)
+
+tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
+
+list_vars = model.state_dict()
+for name in list_vars.keys():
+    print(name, list_vars[name].shape, list_vars[name].dtype)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
+        continue
+    print("Processing variable:", name, "with shape:", data.shape)
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+        print("  Converting to float16")
+        data = data.astype(np.float16)
+        l_type = 1
+    else:
+        l_type = 0
+
+    # map tensor names
+    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
+    if new_name is None:
+        print("Can not map tensor '" + name + "'")
+        sys.exit()
+
+    gguf_writer.add_tensor(new_name, data)
+
+
+print("gguf: write header")
+gguf_writer.write_header_to_file()
+print("gguf: write metadata")
+gguf_writer.write_kv_data_to_file()
+print("gguf: write tensors")
+gguf_writer.write_tensors_to_file()
+
+gguf_writer.close()
+
+print(f"gguf: model successfully exported to '{fname_out}'")
+print()

gpt4all-backend/scripts/convert_falcon_hf_to_ggml.py

@@ -1,143 +0,0 @@
-# Based on: https://github.com/KerfuffleV2/ggml-falcon/blob/feat-improve-falcon-convert-hf/examples/falcon/convert-hf-to-ggml.py
-# Convert Hugging Face fine-tuned bloom-like models to ggml format
-#
-# Usage:
-#
-#   python3 convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]
-#
-# This script is similar to "convert-pt-to-ggml.py"
-#
-
-import io
-import os
-import sys
-import struct
-import json
-import code
-import torch
-import numpy as np
-import gc
-
-from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
-
-# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
-def bytes_to_unicode():
-    """
-    Returns list of utf-8 byte and a corresponding list of unicode strings.
-    The reversible bpe codes work on unicode strings.
-    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
-    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
-    This is a significant percentage of your normal, say, 32K bpe vocab.
-    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
-    And avoids mapping to whitespace/control characters the bpe code barfs on.
-    """
-    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
-    cs = bs[:]
-    n = 0
-    for b in range(2**8):
-        if b not in bs:
-            bs.append(b)
-            cs.append(2**8+n)
-            n += 1
-    cs = [chr(n) for n in cs]
-    return dict(zip(bs, cs))
-
-if len(sys.argv) < 3:
-    print("INFO: GGML V1 files produced are meant to be finalized through examples/falcon_quantize which will bring them to latest version and precision of choice");
-    print("Usage: python convert_falcon_hf_to_ggml.py model_directory output_directory [use-f32]")
-    print("  model_directory: name of the directory and model you convert (it should be a subdirectory)")
-    print("  output-directory: directory where the output file will be written")
-    print("  use-f32:    if present, use float32 instead of float16 (f32 is recommended)")
-    sys.exit(1)
-
-# num_parts = int(sys.argv[1])
-dir_model = sys.argv[1] # name and dir of model
-dir_out = sys.argv[2]   # output directory
-
-# make sure the output directory exists
-os.makedirs(dir_out, exist_ok=True)
-
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-ftype = 1
-if len(sys.argv) > 3:
-    ftype = 0
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-# print(tokenizer)
-config = AutoConfig.from_pretrained(dir_model, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(dir_model, trust_remote_code=True, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)
-hparams = config.to_dict()
-
-n_head = hparams["n_head"]
-n_head_kv = hparams["n_head_kv"] if "n_head_kv" in hparams else 1
-head_dim = hparams["hidden_size"] // n_head
-print("* Loading model from: ", dir_model)
-
-fname_out = dir_out + f"/ggml-model-{dir_model.split('/')[-1]}-{ftype_str[ftype]}.bin"
-fout = open(fname_out, "wb")
-fout.write(struct.pack("i", 0x67676a74)) # magic: ggmf in hex (version 1) - possibly change to ggfc ?
-fout.write(struct.pack("i", 1)) # version
-fout.write(struct.pack("i", hparams["vocab_size"]))
-fout.write(struct.pack("i", hparams["hidden_size"]))
-fout.write(struct.pack("i", n_head))
-fout.write(struct.pack("i", n_head_kv))
-fout.write(struct.pack("i", hparams["n_layer"]))
-fout.write(struct.pack("i", 40 if "n_head_kv" in hparams else 7)) # obsolete field that breaks ggml compatibility - todo again remove one day
-fout.write(struct.pack("i", ftype))
-
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-byte_encoder = bytes_to_unicode()
-byte_decoder = {v:k for k, v in byte_encoder.items()}
-
-for i in range(hparams["vocab_size"]):
-    text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
-    fout.write(struct.pack("i", len(text)))
-    fout.write(text)
-    fout.write(struct.pack("f", 0.0)) # falcon uses bpe on RefinedWeb - no probability scores used
-
-model = model.state_dict()
-for name in model.keys():
-    src = name
-    # The original query_key_value tensor contains n_head_kv "kv groups",
-    # each consisting of n_head/n_head_kv query weights followed by one key
-    # and one value weight (shared by all query heads in the kv group).
-    # This layout makes it a big pain to work with in GGML.
-    # So we rearrange them here,, so that we have n_head query weights
-    # followed by n_head_kv key weights followed by n_head_kv value weights,
-    # in contiguous fashion.
-
-    if "query_key_value" in src:
-        qkv = model[src].view(
-            n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)
-
-        q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head)
-        k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)
-        v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)
-
-        model[src] = torch.cat((q,k,v)).reshape_as(model[src])
-    data = model[src].squeeze()
-    n_dims = len(data.shape)
-    # default type is fp32
-    ftype_cur = 1 if ftype == 1 and n_dims > 1 else 0
-    data = data.to(dtype = torch.float16 if ftype_cur == 1 else torch.float32).numpy()
-    print(f'  |', name, data.shape, '->', data.dtype)
-    # header
-    str = name.encode('utf-8')
-    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str)
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-backend/scripts/convert_gptj_to_gguf.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+# Convert GPT-J-6B h5 transformer model to ggml format
+#
+# Load the model using GPTJForCausalLM.
+# Iterate over all variables and write them to a binary file.
+#
+# For each variable, write the following:
+#   - Number of dimensions (int)
+#   - Name length (int)
+#   - Dimensions (int[n_dims])
+#   - Name (char[name_length])
+#   - Data (float[n_dims])
+#
+# By default, the bigger matrices are converted to 16-bit floats.
+# This can be disabled by adding the "ftype" CLI argument.
+#
+# At the start of the ggml file we write the model parameters
+# and vocabulary.
+#
+
+from __future__ import annotations
+
+import sys
+import struct
+import json
+from pathlib import Path
+
+import gguf
+import numpy as np
+from transformers import AutoConfig, AutoTokenizer, GPTJForCausalLM
+from transformers.models.gpt2 import tokenization_gpt2
+
+
+if not 2 <= len(sys.argv) < 4:
+    print("Usage: python {} dir-model [ftype]\n".format(Path(__file__).name))
+    print("  ftype == 0 -> float32")
+    print("  ftype == 1 -> float16")
+    sys.exit(1)
+
+# output in the same directory as the model
+dir_model = Path(sys.argv[1])
+fname_out = dir_model / "ggml-model.gguf"
+
+# possible data types
+#   ftype == 0 -> float32
+#   ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if len(sys.argv) > 2:
+    ftype = int(sys.argv[2])
+    if ftype < 0 or ftype > 1:
+        print("Invalid ftype: " + str(ftype))
+        sys.exit(1)
+
+fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
+
+
+ARCH = gguf.MODEL_ARCH.GPTJ
+gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
+
+print("gguf: get model metadata")
+
+config = AutoConfig.from_pretrained(dir_model)
+
+block_count = config.n_layer
+gguf_writer.add_name("GPT-J")
+gguf_writer.add_context_length(config.n_positions)
+gguf_writer.add_embedding_length(config.n_embd)
+gguf_writer.add_block_count(block_count)
+gguf_writer.add_feed_forward_length(4 * config.n_embd)
+gguf_writer.add_head_count(config.n_head)
+gguf_writer.add_rope_dimension_count(config.rotary_dim)
+gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
+gguf_writer.add_file_type(ftype)
+
+print("gguf: get gpt2 tokenizer vocab")
+
+tokenizer = AutoTokenizer.from_pretrained(dir_model)
+
+reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+byte_encoder = tokenization_gpt2.bytes_to_unicode()
+byte_decoder = {v: k for k, v in byte_encoder.items()}
+
+tokens: list[bytearray] = []
+
+for i in range(config.vocab_size):
+    if i in reverse_vocab:
+        try:
+            text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+        except KeyError:
+            text = bytearray()
+            for c in reverse_vocab[i]:
+                if ord(c) < 256:  # single byte character
+                    text.append(byte_decoder[c])
+                else:  # multibyte special token character
+                    text.extend(c.encode('utf-8'))
+    else:
+        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
+        pad_token = f"[PAD{i}]".encode("utf8")
+        text = bytearray(pad_token)
+
+    tokens.append(text)
+
+
+gguf_writer.add_tokenizer_model("gpt2")
+gguf_writer.add_token_list(tokens)
+
+special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
+special_vocab.add_to_gguf(gguf_writer)
+
+print("gguf: get tensor metadata")
+
+model = GPTJForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
+#print (model)
+
+tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
+
+list_vars = model.state_dict()
+#print (list_vars)
+
+for name in list_vars.keys():
+    data = list_vars[name].squeeze().numpy()
+    print("Processing variable:", name, "with shape:", data.shape)
+
+    # we don't need these
+    if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
+        print("  Skipping variable:", name)
+        continue
+
+    n_dims = len(data.shape)
+
+    # ftype == 0 -> float32, ftype == 1 -> float16
+    ftype_cur = 0
+    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
+        print("  Converting to float16")
+        data = data.astype(np.float16)
+        ftype_cur = 1
+    elif ftype == 1 or data.dtype != np.float32:
+        print("  Converting to float32")
+        data = data.astype(np.float32)
+        ftype_cur = 0
+
+    # map tensor names
+    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
+    if new_name is None:
+        print("Can not map tensor '" + name + "'")
+        sys.exit()
+
+    gguf_writer.add_tensor(new_name, data)
+
+
+print("gguf: write header")
+gguf_writer.write_header_to_file()
+print("gguf: write metadata")
+gguf_writer.write_kv_data_to_file()
+print("gguf: write tensors")
+gguf_writer.write_tensors_to_file()
+
+gguf_writer.close()
+
+print(f"gguf: model successfully exported to '{fname_out}'")
+print()

gpt4all-backend/scripts/convert_mpt_hf_to_ggml.py

@@ -1,145 +0,0 @@
-# Convert Hugging Face fine-tuned bloom-like models to ggml format
-#
-# Usage:
-#
-#   python3 models/convert-h5-to-ggml.py 
-#
-# This script is similar to "convert-pt-to-ggml.py"
-#
-
-import io
-import os
-import sys
-import struct
-import json
-import code
-import torch
-import numpy as np
-
-from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BloomForCausalLM
-
-# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
-def bytes_to_unicode():
-    """
-    Returns list of utf-8 byte and a corresponding list of unicode strings.
-    The reversible bpe codes work on unicode strings.
-    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
-    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
-    This is a significant percentage of your normal, say, 32K bpe vocab.
-    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
-    And avoids mapping to whitespace/control characters the bpe code barfs on.
-    """
-    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
-    cs = bs[:]
-    n = 0
-    for b in range(2**8):
-        if b not in bs:
-            bs.append(b)
-            cs.append(2**8+n)
-            n += 1
-    cs = [chr(n) for n in cs]
-    return dict(zip(bs, cs))
-
-if len(sys.argv) < 3:
-    print("Usage: python convert-hf-to-ggml.py model_name dir-output [use-f32]")
-    print("  model_name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
-    print("  dir-output: directory where the output file will be written")
-    print("  use-f32:    if present, use float32 instead of float16")
-    sys.exit(1)
-
-model_name = sys.argv[1]
-dir_out = sys.argv[2]
-
-# make sure the output directory exists
-os.makedirs(dir_out, exist_ok=True)
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-ftype = 1
-if len(sys.argv) > 3:
-    ftype = 0
-
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
-hparams = config.to_dict()
-print("Loading model: ", model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32, low_cpu_mem_usage=True)
-print("Model loaded: ", model_name)
-
-
-fname_out = dir_out + f"/ggml-model-{model_name.split('/')[-1]}-{ftype_str[ftype]}.bin"
-fout = open(fname_out, "wb")
-vocab = tokenizer.vocab
-
-hparams["multiple_of"] = 1
-fout.write(struct.pack("I", 0x67676d6d)) # magic: ggml in hex
-fout.write(struct.pack("I", model.config.vocab_size))
-fout.write(struct.pack("I", model.config.max_seq_len))
-fout.write(struct.pack("I", model.config.n_layers))
-fout.write(struct.pack("I", model.config.n_heads))
-fout.write(struct.pack("I", model.config.d_model))
-fout.write(struct.pack("f", model.config.attn_config['alibi_bias_max']))
-clip_qkv = model.config.attn_config['clip_qkv']
-fout.write(struct.pack("f",  clip_qkv if clip_qkv is not None else 0))
-fout.write(struct.pack("I", ftype))
-
-# # Is this correct??
-# dot_token = tokenizer.encode(".")[0]
-# write tokens to ggml file 
-dot_token = tokenizer.encode('.')[0]
-fout.write(struct.pack("I", model.config.vocab_size))
-
-for i in range(model.config.vocab_size):
-    text = tokenizer.decode([dot_token, i]).encode('utf-8')
-    # remove the first byte (it's always '.')
-    text = text[1:]
-    enclen = len(text)
-    if i in tokenizer.all_special_ids:
-        print(f"special token: {text}")
-        enclen = enclen | 1<<31
-    fout.write(struct.pack("I", enclen))
-    fout.write(text)
-    
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable: " + name + " with shape: ", data.shape)
-
-    n_dims = len(data.shape);
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0;
-    if ftype != 0:
-        # Keep token embeddings in fp32
-        if name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
-            print("  Converting to float16")
-            data = data.astype(np.float16)
-            ftype_cur = 1
-        else:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-    else:
-        if data.dtype != np.float32:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-
-    # header
-    str = name.encode('utf-8')
-    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str);
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-backend/scripts/convert_replit_hf_to_ggml.py

@@ -1,113 +0,0 @@
-from pathlib import Path
-import sys
-import struct
-import json
-import numpy as np
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import sentencepiece.sentencepiece_model_pb2 as model
-
-if len(sys.argv) < 3:
-    print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-
-# output in the same directory as the model
-dir_model = sys.argv[1]
-fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b.bin"
-
-
-with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
-    hparams = json.load(f)
-
-sp_proto = model.ModelProto()
-sp_proto.ParseFromString(open(Path(sys.argv[1]) / "spiece.model", "rb").read())
-
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-    fname_out = sys.argv[1] + "/ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".bin"
-
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(
-    dir_model, low_cpu_mem_usage=True, trust_remote_code=True
-)
-# print (model)
-
-# print(tokenizer.encode('I believe the meaning of life is'))
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-fout = open(fname_out, "wb")
-
-print(hparams)
-
-fout.write(struct.pack("i", 0x7265706c))  # magic: repl in hex
-fout.write(struct.pack("i", hparams["vocab_size"]))
-fout.write(struct.pack("i", hparams["max_seq_len"]))
-fout.write(struct.pack("i", hparams["d_model"]))
-fout.write(struct.pack("i", hparams["n_heads"]))
-fout.write(struct.pack("i", hparams["n_layers"]))
-fout.write(struct.pack("i", ftype))
-
-
-# TODO: temporary hack to not deal with implementing the tokenizer
-for piece in sp_proto.pieces:
-    encoded_piece = piece.piece.encode("utf-8")
-    fout.write(struct.pack("i", len(encoded_piece)))
-    fout.write(encoded_piece)
-    fout.write(struct.pack("f", piece.score))
-
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable: " + name + " with shape: ", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    if ftype != 0:
-        if name[-7:] == ".weight" and n_dims == 2:
-            print("  Converting to float16")
-            data = data.astype(np.float16)
-            ftype_cur = 1
-        else:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-    else:
-        if data.dtype != np.float32:
-            print("  Converting to float32")
-            data = data.astype(np.float32)
-            ftype_cur = 0
-
-    # header
-    str = name.encode("utf-8")
-    fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
-    for i in range(n_dims):
-        fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
-    fout.write(str)
-
-    # data
-    data.tofile(fout)
-
-fout.close()
-
-print("Done. Output file: " + fname_out)
-print("")

gpt4all-bindings/cli/README.md

@@ -40,5 +40,5 @@ directory, if necessary.
 If you have already saved a model beforehand, specify its path with the `-m`/`--model` argument,
 for example:
 ```shell
-python app.py repl --model /home/user/my-gpt4all-models/GPT4All-13B-snoozy.ggmlv3.q4_0.bin
+python app.py repl --model /home/user/my-gpt4all-models/gpt4all-13b-snoozy-q4_0.gguf
 ```

gpt4all-bindings/cli/app.py

@@ -1,16 +1,17 @@
+#!/usr/bin/env python3
 """GPT4All CLI
 
 The GPT4All CLI is a self-contained script based on the `gpt4all` and `typer` packages. It offers a
 REPL to communicate with a language model similar to the chat GUI application, but more basic.
 """
 
+import importlib.metadata
 import io
-import pkg_resources  # should be present as a dependency of gpt4all
 import sys
-import typer
-
 from collections import namedtuple
 from typing_extensions import Annotated
+
+import typer
 from gpt4all import GPT4All
 
 
@@ -53,14 +54,18 @@ def repl(
     model: Annotated[
         str,
         typer.Option("--model", "-m", help="Model to use for chatbot"),
-    ] = "ggml-gpt4all-j-v1.3-groovy",
+    ] = "mistral-7b-instruct-v0.1.Q4_0.gguf",
     n_threads: Annotated[
         int,
         typer.Option("--n-threads", "-t", help="Number of threads to use for chatbot"),
     ] = None,
+    device: Annotated[
+        str,
+        typer.Option("--device", "-d", help="Device to use for chatbot, e.g. gpu, amd, nvidia, intel. Defaults to CPU."),
+    ] = None,
 ):
     """The CLI read-eval-print loop."""
-    gpt4all_instance = GPT4All(model)
+    gpt4all_instance = GPT4All(model, device=device)
 
     # if threads are passed, set them
     if n_threads is not None:
@@ -79,7 +84,7 @@ def repl(
 
     use_new_loop = False
     try:
-        version = pkg_resources.Environment()['gpt4all'][0].version
+        version = importlib.metadata.version('gpt4all')
         version_major = int(version.split('.')[0])
         if version_major >= 1:
             use_new_loop = True

gpt4all-bindings/csharp/.editorconfig

@@ -41,6 +41,8 @@ insert_final_newline = true
 
 # IDE0055: Fix formatting
 dotnet_diagnostic.IDE0055.severity = error
+dotnet_diagnostic.CS1573.severity = suggestion
+dotnet_diagnostic.CS1591.severity = suggestion
 
 # Sort using and Import directives with System.* appearing first
 dotnet_sort_system_directives_first = true
@@ -343,4 +345,4 @@ dotnet_diagnostic.IDE2004.severity = warning
 [src/{VisualStudio}/**/*.{cs,vb}]
 # CA1822: Make member static
 # There is a risk of accidentally breaking an internal API that partners rely on though IVT.
-dotnet_code_quality.CA1822.api_surface = private
+dotnet_code_quality.CA1822.api_surface = private

gpt4all-bindings/csharp/Directory.Build.props

@@ -5,7 +5,7 @@
         <Company></Company>
         <Copyright></Copyright>
         <NeutralLanguage>en-US</NeutralLanguage>
-        <Version>0.6.3-alpha</Version>
+        <Version>0.6.4-alpha</Version>
         <VersionSuffix>$(VersionSuffix)</VersionSuffix>
         <Version Condition=" '$(VersionSuffix)' != '' ">$(Version)$(VersionSuffix)</Version>
         <TreatWarningsAsErrors>true</TreatWarningsAsErrors>

gpt4all-bindings/csharp/Gpt4All.Samples/Gpt4All.Samples.csproj

@@ -2,9 +2,10 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net7.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
+        <GenerateDocumentationFile>true</GenerateDocumentationFile>
     </PropertyGroup>
 
     <ItemGroup>

gpt4all-bindings/csharp/Gpt4All.Tests/Gpt4All.Tests.csproj

@@ -1,10 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net7.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <Nullable>enable</Nullable>
 
         <IsPackable>false</IsPackable>
+        <GenerateDocumentationFile>true</GenerateDocumentationFile>
     </PropertyGroup>
 
     <ItemGroup>

gpt4all-bindings/csharp/Gpt4All/Bindings/ILLModel.cs

@@ -5,8 +5,6 @@
 /// </summary>
 public interface ILLModel : IDisposable
 {
-    ModelType ModelType { get; }
-
     ulong GetStateSizeBytes();
 
     int GetThreadCount();

gpt4all-bindings/csharp/Gpt4All/Bindings/LLModel.cs

@@ -42,16 +42,12 @@ public record ModelRecalculatingEventArgs(bool IsRecalculating);
 public class LLModel : ILLModel
 {
     protected readonly IntPtr _handle;
-    private readonly ModelType _modelType;
     private readonly ILogger _logger;
     private bool _disposed;
 
-    public ModelType ModelType => _modelType;
-
-    internal LLModel(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    internal LLModel(IntPtr handle, ILogger? logger = null)
     {
         _handle = handle;
-        _modelType = modelType;
         _logger = logger ?? NullLogger.Instance;
     }
 
@@ -59,10 +55,9 @@ public class LLModel : ILLModel
     /// Create a new model from a pointer
     /// </summary>
     /// <param name="handle">Pointer to underlying model</param>
-    /// <param name="modelType">The model type</param>
-    public static LLModel Create(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    public static LLModel Create(IntPtr handle, ILogger? logger = null)
     {
-        return new LLModel(handle, modelType, logger: logger);
+        return new LLModel(handle, logger: logger);
     }
 
     /// <summary>
@@ -188,7 +183,7 @@ public class LLModel : ILLModel
     /// <returns>true if the model was loaded successfully, false otherwise.</returns>
     public bool Load(string modelPath)
     {
-        return NativeMethods.llmodel_loadModel(_handle, modelPath);
+        return NativeMethods.llmodel_loadModel(_handle, modelPath, 2048, 100);
     }
 
     protected void Destroy()
@@ -204,12 +199,7 @@ public class LLModel : ILLModel
             // dispose managed state
         }
 
-        switch (_modelType)
-        {
-            default:
-                Destroy();
-                break;
-        }
+        Destroy();
 
         _disposed = true;
     }

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeMethods.cs

@@ -70,7 +70,9 @@ internal static unsafe partial class NativeMethods
     [return: MarshalAs(UnmanagedType.I1)]
     public static extern bool llmodel_loadModel(
         [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path,
+        [NativeTypeName("int32_t")] int n_ctx,
+        [NativeTypeName("int32_t")] int ngl);
 
     [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
 

gpt4all-bindings/csharp/Gpt4All/Gpt4All.csproj

@@ -1,9 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk">
     <PropertyGroup>
-        <TargetFramework>net6.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+        <GenerateDocumentationFile>true</GenerateDocumentationFile>
+        <TargetFramework>net8.0</TargetFramework>
     </PropertyGroup>
     <ItemGroup>
         <!-- Windows -->

gpt4all-bindings/csharp/Gpt4All/Model/Gpt4AllModelFactory.cs

@@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Logging;
 using Gpt4All.Bindings;
 using Gpt4All.LibraryLoader;
+using System.Runtime.InteropServices;
 
 namespace Gpt4All;
 
@@ -31,15 +32,18 @@ public class Gpt4AllModelFactory : IGpt4AllModelFactory
         }
     }
 
-    private IGpt4AllModel CreateModel(string modelPath)
+    private Gpt4All CreateModel(string modelPath)
     {
-        var modelType_ = ModelFileUtils.GetModelTypeFromModelFileHeader(modelPath);
-        _logger.LogInformation("Creating model path={ModelPath} type={ModelType}", modelPath, modelType_);
+        _logger.LogInformation("Creating model path={ModelPath}", modelPath);
         IntPtr error;
         var handle = NativeMethods.llmodel_model_create2(modelPath, "auto", out error);
+        if (error != IntPtr.Zero)
+        {
+            throw new Exception(Marshal.PtrToStringAnsi(error));
+        }
         _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
         _logger.LogInformation("Model loading started");
-        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath);
+        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath, 2048, 100);
         _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
         if (!loadedSuccessfully)
         {
@@ -47,7 +51,7 @@ public class Gpt4AllModelFactory : IGpt4AllModelFactory
         }
 
         var logger = _loggerFactory.CreateLogger<LLModel>();
-        var underlyingModel = LLModel.Create(handle, modelType_, logger: logger);
+        var underlyingModel = LLModel.Create(handle, logger: logger);
 
         Debug.Assert(underlyingModel.IsLoaded());
 

gpt4all-bindings/csharp/Gpt4All/Model/ModelFileUtils.cs

@@ -1,24 +0,0 @@
-namespace Gpt4All;
-
-public static class ModelFileUtils
-{
-    private const uint GPTJ_MAGIC = 0x67676d6c;
-    private const uint LLAMA_MAGIC = 0x67676a74;
-    private const uint MPT_MAGIC = 0x67676d6d;
-
-    public static ModelType GetModelTypeFromModelFileHeader(string modelPath)
-    {
-        using var fileStream = new FileStream(modelPath, FileMode.Open);
-        using var binReader = new BinaryReader(fileStream);
-
-        var magic = binReader.ReadUInt32();
-
-        return magic switch
-        {
-            GPTJ_MAGIC => ModelType.GPTJ,
-            LLAMA_MAGIC => ModelType.LLAMA,
-            MPT_MAGIC => ModelType.MPT,
-            _ => throw new ArgumentOutOfRangeException($"Invalid model file. magic=0x{magic:X8}"),
-        };
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Model/ModelOptions.cs

@@ -3,6 +3,4 @@
 public record ModelOptions
 {
     public int Threads { get; init; } = 4;
-
-    public ModelType ModelType { get; init; } = ModelType.GPTJ;
 }

gpt4all-bindings/csharp/Gpt4All/Model/ModelType.cs

@@ -1,11 +0,0 @@
-namespace Gpt4All;
-
-/// <summary>
-/// The supported model types
-/// </summary>
-public enum ModelType
-{
-    LLAMA = 0,
-    GPTJ,
-    MPT
-}

gpt4all-bindings/csharp/README.md

@@ -6,7 +6,10 @@ This package contains a set of C# bindings around the `llmodel` C-API.
 TBD
 
 ## Installation
-TBD NuGet
+
+Windows and Linux builds are available on NuGet: https://www.nuget.org/packages/Gpt4All
+
+macOS is WIP due to code signing issues, contributions are welcome.
 
 ## Project Structure
 ```
@@ -23,6 +26,12 @@ gpt4all-bindings/
           └── linux-x64
 ```
 
+## Prerequisites
+
+On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+
+macOS users do not need Vulkan, as GPT4All will use Metal instead.
+
 ## Local Build Instructions
 > **Note** 
 > Tested On:
@@ -54,7 +63,7 @@ chmod +x ./build_linux.sh
 1. Setup
 ```
 choco install mingw
-$env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
+$env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
 choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
 ```
 2. Run the `./build_win-mingw.ps1` build script

gpt4all-bindings/csharp/build_linux.sh

@@ -1,3 +1,4 @@
+#!/bin/sh
 mkdir -p runtimes
 rm -rf runtimes/linux-x64
 mkdir -p runtimes/linux-x64/native
@@ -7,4 +8,3 @@ cmake --build runtimes/linux-x64/build --parallel --config Release
 cp runtimes/linux-x64/build/libllmodel.so  runtimes/linux-x64/native/libllmodel.so
 cp runtimes/linux-x64/build/libgptj*.so  runtimes/linux-x64/native/
 cp runtimes/linux-x64/build/libllama*.so  runtimes/linux-x64/native/
-cp runtimes/linux-x64/build/libmpt*.so  runtimes/linux-x64/native/

gpt4all-bindings/csharp/build_win-mingw.ps1

@@ -12,5 +12,5 @@ cmake -G "MinGW Makefiles" -S ..\..\gpt4all-backend -B $BUILD_DIR
 cmake --build $BUILD_DIR --parallel --config Release
 
 # copy native dlls
-cp "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll" $LIBS_DIR
-cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR
+cp "C:\ProgramData\mingw64\mingw64\bin\*dll" $LIBS_DIR
+cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR

gpt4all-bindings/golang/Makefile

@@ -139,7 +139,7 @@ $(info I CXX:      $(CXXV))
 $(info )
 
 llmodel.o:
-	mkdir buildllm
+	[ -e buildllm ] || mkdir buildllm
 	cd buildllm && cmake ../../../gpt4all-backend/ $(CMAKEFLAGS) && make
 	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel_c.cpp.o ../llmodel_c.o
 	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel.cpp.o ../llmodel.o
@@ -150,7 +150,7 @@ clean:
 	rm -rf buildllm
 	rm -rf example/main
 
-binding.o: 
+binding.o: binding.cpp binding.h
 	$(CXX) $(CXXFLAGS) binding.cpp -o binding.o -c $(LDFLAGS)
 
 libgpt4all.a: binding.o llmodel.o

gpt4all-bindings/golang/README.md

@@ -36,7 +36,7 @@ func main() {
 In order to use the bindings you will need to build `libgpt4all.a`:
 
 ```
-git clone https://github.com/nomic-ai/gpt4all
+git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
 cd gpt4all/gpt4all-bindings/golang
 make libgpt4all.a
 ```

gpt4all-bindings/golang/binding.cpp

@@ -17,14 +17,13 @@
 
 void* load_model(const char *fname, int n_threads) {
     // load the model
-    llmodel_error new_error{};
+    const char *new_error;
     auto model = llmodel_model_create2(fname, "auto", &new_error);
-    if (model == nullptr ){
-        fprintf(stderr, "%s: error '%s'\n",
-                __func__, new_error.message);
+    if (model == nullptr) {
+        fprintf(stderr, "%s: error '%s'\n", __func__, new_error);
         return nullptr;
     }
-    if (!llmodel_loadModel(model, fname)) {
+    if (!llmodel_loadModel(model, fname, 2048, 100)) {
         llmodel_model_destroy(model);
         return nullptr;
     }

gpt4all-bindings/java/README.md

@@ -12,17 +12,17 @@ You can add Java bindings into your Java project by adding the following depende
 <dependency>
     <groupId>com.hexadevlabs</groupId>
     <artifactId>gpt4all-java-binding</artifactId>
-    <version>1.1.3</version>
+    <version>1.1.5</version>
 </dependency>
 ```
 **Gradle**
 ```
-implementation 'com.hexadevlabs:gpt4all-java-binding:1.1.3'
+implementation 'com.hexadevlabs:gpt4all-java-binding:1.1.5'
 ```
 
 To add the library dependency for another build system see [Maven Central Java bindings](https://central.sonatype.com/artifact/com.hexadevlabs/gpt4all-java-binding/).
 
-To download model binary weights file use a URL such as [`https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin`](https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin).
+To download model binary weights file use a URL such as [`https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf`](https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf).
 
 For information about other models available see the [model file list](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-chat#manual-download-of-models).
 
@@ -121,4 +121,6 @@ If this is the case you can easily download and install the latest x64 Microsoft
 3. Version **1.1.4**:
    - Java bindings is compatible with gpt4all version 2.4.11
    - Falcon model support included.
-   
+4. Version **1.1.5**:
+   - Add a check for model file readability before loading model.
+   

gpt4all-bindings/java/TODO.md

@@ -1,2 +1,6 @@
+## Needed
+1. Integrate with circleci build pipeline like the C# binding.
+
+## These are just ideas
 1. Better Chat completions function.
 2. Chat completion that returns result in OpenAI compatible format.

gpt4all-bindings/java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.hexadevlabs</groupId>
     <artifactId>gpt4all-java-binding</artifactId>
-    <version>1.1.4</version>
+    <version>1.1.5</version>
     <packaging>jar</packaging>
 
     <properties>

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java

@@ -1,6 +1,7 @@
 package com.hexadevlabs.gpt4all;
 
 import jnr.ffi.Pointer;
+import jnr.ffi.byref.PointerByReference;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -8,9 +9,8 @@ import java.io.ByteArrayOutputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+import java.util.stream.Collectors;
 
 public  class LLModel implements AutoCloseable {
 
@@ -177,20 +177,25 @@ public  class LLModel implements AutoCloseable {
         modelName = modelPath.getFileName().toString();
         String modelPathAbs = modelPath.toAbsolutePath().toString();
 
-        LLModelLibrary.LLModelError error = new LLModelLibrary.LLModelError(jnr.ffi.Runtime.getSystemRuntime());
+        PointerByReference error = new PointerByReference();
 
         // Check if model file exists
         if(!Files.exists(modelPath)){
             throw new IllegalStateException("Model file does not exist: " + modelPathAbs);
         }
 
+        // Check if file is Readable
+        if(!Files.isReadable(modelPath)){
+            throw new IllegalStateException("Model file cannot be read: " + modelPathAbs);
+        }
+
         // Create Model Struct. Will load dynamically the correct backend based on model type
         model = library.llmodel_model_create2(modelPathAbs, "auto", error);
 
         if(model == null) {
-            throw new IllegalStateException("Could not load gpt4all backend :" + error.message);
+            throw new IllegalStateException("Could not load, gpt4all backend returned error: " + error.getValue().getString(0));
         }
-        library.llmodel_loadModel(model, modelPathAbs);
+        library.llmodel_loadModel(model, modelPathAbs, 2048, 100);
 
         if(!library.llmodel_isModelLoaded(model)){
             throw new IllegalStateException("The model " + modelName + " could not be loaded");
@@ -301,6 +306,197 @@ public  class LLModel implements AutoCloseable {
         };
     }
 
+    /**
+     * The array of messages for the conversation.
+     */
+    public static class Messages {
+
+        private final List<PromptMessage> messages = new ArrayList<>();
+
+        public Messages(PromptMessage...messages) {
+            this.messages.addAll(Arrays.asList(messages));
+        }
+
+        public Messages(List<PromptMessage> messages) {
+            this.messages.addAll(messages);
+        }
+
+        public Messages addPromptMessage(PromptMessage promptMessage) {
+            this.messages.add(promptMessage);
+            return this;
+        }
+
+        List<PromptMessage> toList() {
+            return Collections.unmodifiableList(this.messages);
+        }
+
+        List<Map<String, String>> toListMap() {
+            return messages.stream()
+                    .map(PromptMessage::toMap).collect(Collectors.toList());
+        }
+
+    }
+
+    /**
+     * A message in the conversation, identical to OpenAI's chat message.
+     */
+    public static class PromptMessage {
+
+        private static final String ROLE = "role";
+        private static final String CONTENT = "content";
+
+        private final Map<String, String> message = new HashMap<>();
+
+        public PromptMessage() {
+        }
+
+        public PromptMessage(Role role, String content) {
+            addRole(role);
+            addContent(content);
+        }
+
+        public PromptMessage addRole(Role role) {
+            return this.addParameter(ROLE, role.type());
+        }
+
+        public PromptMessage addContent(String content) {
+            return this.addParameter(CONTENT, content);
+        }
+
+        public PromptMessage addParameter(String key, String value) {
+            this.message.put(key, value);
+            return this;
+        }
+
+        public String content() {
+            return this.parameter(CONTENT);
+        }
+
+        public Role role() {
+            String role = this.parameter(ROLE);
+            return Role.from(role);
+        }
+
+        public String parameter(String key) {
+            return this.message.get(key);
+        }
+
+        Map<String, String> toMap() {
+            return Collections.unmodifiableMap(this.message);
+        }
+
+    }
+
+    public enum Role {
+
+        SYSTEM("system"), ASSISTANT("assistant"), USER("user");
+
+        private final String type;
+
+        String type() {
+            return this.type;
+        }
+
+        static Role from(String type) {
+
+            if (type == null) {
+                return null;
+            }
+
+            switch (type) {
+                case "system": return SYSTEM;
+                case "assistant": return ASSISTANT;
+                case "user": return USER;
+                default: throw new IllegalArgumentException(
+                        String.format("You passed %s type but only %s are supported",
+                                type, Arrays.toString(Role.values())
+                        )
+                );
+            }
+        }
+
+        Role(String type) {
+            this.type = type;
+        }
+
+        @Override
+        public String toString() {
+            return type();
+        }
+    }
+
+    /**
+     * The result of the completion, similar to OpenAI's format.
+     */
+    public static class CompletionReturn {
+        private String model;
+        private Usage usage;
+        private Choices choices;
+
+        public CompletionReturn(String model, Usage usage, Choices choices) {
+            this.model = model;
+            this.usage = usage;
+            this.choices = choices;
+        }
+
+        public Choices choices() {
+            return choices;
+        }
+
+        public String model() {
+            return model;
+        }
+
+        public Usage usage() {
+            return usage;
+        }
+    }
+
+    /**
+     * The generated completions.
+     */
+    public static class Choices {
+
+        private final List<CompletionChoice> choices = new ArrayList<>();
+
+        public Choices(List<CompletionChoice> choices) {
+            this.choices.addAll(choices);
+        }
+
+        public Choices(CompletionChoice...completionChoices){
+            this.choices.addAll(Arrays.asList(completionChoices));
+        }
+
+        public Choices addCompletionChoice(CompletionChoice completionChoice) {
+            this.choices.add(completionChoice);
+            return this;
+        }
+
+        public CompletionChoice first() {
+            return this.choices.get(0);
+        }
+
+        public int totalChoices() {
+            return this.choices.size();
+        }
+
+        public CompletionChoice get(int index) {
+            return this.choices.get(index);
+        }
+
+        public List<CompletionChoice> choices() {
+            return Collections.unmodifiableList(choices);
+        }
+    }
+
+    /**
+     * A completion choice, similar to OpenAI's format.
+     */
+    public static class CompletionChoice extends PromptMessage {
+        public CompletionChoice(Role role, String content) {
+            super(role, content);
+        }
+    }
 
     public static class ChatCompletionResponse {
         public String model;
@@ -318,6 +514,41 @@ public  class LLModel implements AutoCloseable {
         // Getters and setters
     }
 
+    public CompletionReturn chatCompletionResponse(Messages messages,
+                                                                GenerationConfig generationConfig) {
+        return chatCompletion(messages, generationConfig, false, false);
+    }
+
+    /**
+     * chatCompletion formats the existing chat conversation into a template to be
+     * easier to process for chat UIs. It is not absolutely necessary as generate method
+     * may be directly used to make generations with gpt models.
+     *
+     * @param messages object to create theMessages to send to GPT model
+     * @param generationConfig How to decode/process the generation.
+     * @param streamToStdOut Send tokens as they are calculated Standard output.
+     * @param outputFullPromptToStdOut Should full prompt built out of messages be sent to Standard output.
+     * @return CompletionReturn contains stats and generated Text.
+     */
+    public CompletionReturn chatCompletion(Messages messages,
+                                                  GenerationConfig generationConfig, boolean streamToStdOut,
+                                                  boolean outputFullPromptToStdOut) {
+
+        String fullPrompt = buildPrompt(messages.toListMap());
+
+        if(outputFullPromptToStdOut)
+            System.out.print(fullPrompt);
+
+        String generatedText = generate(fullPrompt, generationConfig, streamToStdOut);
+
+        final CompletionChoice promptMessage = new CompletionChoice(Role.ASSISTANT, generatedText);
+        final Choices choices = new Choices(promptMessage);
+
+        final Usage usage = getUsage(fullPrompt, generatedText);
+        return new CompletionReturn(this.modelName, usage, choices);
+
+    }
+
     public ChatCompletionResponse chatCompletion(List<Map<String, String>> messages,
                                                               GenerationConfig generationConfig) {
         return chatCompletion(messages, generationConfig, false, false);
@@ -347,19 +578,23 @@ public  class LLModel implements AutoCloseable {
         ChatCompletionResponse  response = new ChatCompletionResponse();
         response.model = this.modelName;
 
-        Usage usage = new Usage();
-        usage.promptTokens = fullPrompt.length();
-        usage.completionTokens = generatedText.length();
-        usage.totalTokens = fullPrompt.length() + generatedText.length();
-        response.usage =  usage;
+        response.usage = getUsage(fullPrompt, generatedText);
 
         Map<String, String> message = new HashMap<>();
         message.put("role", "assistant");
         message.put("content", generatedText);
 
         response.choices = List.of(message);
-
         return response;
+
+    }
+
+    private Usage getUsage(String fullPrompt, String generatedText) {
+        Usage usage = new Usage();
+        usage.promptTokens = fullPrompt.length();
+        usage.completionTokens = generatedText.length();
+        usage.totalTokens = fullPrompt.length() + generatedText.length();
+        return usage;
     }
 
     protected static String buildPrompt(List<Map<String, String>> messages) {
@@ -397,4 +632,4 @@ public  class LLModel implements AutoCloseable {
         library.llmodel_model_destroy(model);
     }
 
-}
+}

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModelLibrary.java

@@ -1,6 +1,7 @@
 package com.hexadevlabs.gpt4all;
 
 import jnr.ffi.Pointer;
+import jnr.ffi.byref.PointerByReference;
 import jnr.ffi.Struct;
 import jnr.ffi.annotations.Delegate;
 import jnr.ffi.annotations.Encoding;
@@ -58,9 +59,9 @@ public interface LLModelLibrary {
         }
     }
 
-    Pointer llmodel_model_create2(String model_path, String build_variant, @Out LLModelError llmodel_error);
+    Pointer llmodel_model_create2(String model_path, String build_variant, PointerByReference error);
     void llmodel_model_destroy(Pointer model);
-    boolean llmodel_loadModel(Pointer model, String model_path);
+    boolean llmodel_loadModel(Pointer model, String model_path, int n_ctx, int ngl);
     boolean llmodel_isModelLoaded(Pointer model);
     @u_int64_t long llmodel_get_state_size(Pointer model);
     @u_int64_t long llmodel_save_state_data(Pointer model, Pointer dest);

gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/BasicTests.java

@@ -28,6 +28,33 @@ import static org.mockito.Mockito.*;
 @ExtendWith(MockitoExtension.class)
 public class BasicTests {
 
+    @Test
+    public void simplePromptWithObject(){
+
+        LLModel model = Mockito.spy(new LLModel());
+
+        LLModel.GenerationConfig config =
+                LLModel.config()
+                        .withNPredict(20)
+                        .build();
+
+        // The generate method will return "4"
+        doReturn("4").when( model ).generate(anyString(), eq(config), eq(true));
+
+        LLModel.PromptMessage promptMessage1 = new LLModel.PromptMessage(LLModel.Role.SYSTEM, "You are a helpful assistant");
+        LLModel.PromptMessage promptMessage2 = new LLModel.PromptMessage(LLModel.Role.USER, "Add 2+2");
+
+        LLModel.Messages messages = new LLModel.Messages(promptMessage1, promptMessage2);
+
+        LLModel.CompletionReturn response = model.chatCompletion(
+                messages, config, true, true);
+
+        assertTrue( response.choices().first().content().contains("4") );
+
+        // Verifies the prompt and response are certain length.
+        assertEquals( 224 , response.usage().totalTokens );
+    }
+
     @Test
     public void simplePrompt(){
 

gpt4all-bindings/python/README.md

@@ -9,31 +9,52 @@ https://docs.gpt4all.io/gpt4all_python.html
 
 ## Installation
 
+The easiest way to install the Python bindings for GPT4All is to use pip:
+
 ```
 pip install gpt4all
 ```
 
-## Local Build Instructions
+This will download the latest version of the `gpt4all` package from PyPI.
 
-**NOTE**: If you are doing this on a Windows machine, you must build the GPT4All backend using [MinGW64](https://www.mingw-w64.org/) compiler.
+## Local Build
 
-1. Setup `llmodel`
+As an alternative to downloading via pip, you may build the Python bindings from source.
 
+### Prerequisites
+
+On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+
+macOS users do not need Vulkan, as GPT4All will use Metal instead.
+
+### Building the python bindings
+
+1. Clone GPT4All and change directory:
 ```
-git clone --recurse-submodules git@github.com:nomic-ai/gpt4all.git
-cd gpt4all/gpt4all-backend/
-mkdir build
-cd build
-cmake ..
-cmake --build . --parallel
+git clone --recurse-submodules https://github.com/nomic-ai/gpt4all.git
+cd gpt4all/gpt4all-backend
 ```
-Confirm that `libllmodel.*` exists in `gpt4all-backend/build`.
 
-2. Setup Python package
+2. Build the backend.
 
+If you are using Windows and have Visual Studio installed:
+```
+cmake -B build
+cmake --build build --parallel --config RelWithDebInfo
+```
+
+For all other platforms:
+```
+cmake -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo
+cmake --build build --parallel
+```
+
+`RelWithDebInfo` is a good default, but you can also use `Release` or `Debug` depending on the situation.
+
+2. Install the Python package:
 ```
 cd ../../gpt4all-bindings/python
-pip3 install -e .
+pip install -e .
 ```
 
 ## Usage
@@ -42,7 +63,16 @@ Test it out! In a Python script or console:
 
 ```python
 from gpt4all import GPT4All
-model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin")
+model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
+output = model.generate("The capital of France is ", max_tokens=3)
+print(output)
+```
+
+
+GPU Usage
+```python
+from gpt4all import GPT4All
+model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf", device='gpu') # device='amd', device='intel'
 output = model.generate("The capital of France is ", max_tokens=3)
 print(output)
 ```

gpt4all-bindings/python/docs/gpt4all_chat.md

@@ -5,48 +5,46 @@ The [GPT4All Chat Client](https://gpt4all.io) lets you easily interact with any
 It is optimized to run 7-13B parameter LLMs on the CPU's of any computer running OSX/Windows/Linux.
 
 ## Running LLMs on CPU
-The GPT4All Chat UI supports models from all newer versions of `GGML`, `llama.cpp` including the `LLaMA`, `MPT`, `replit`,  `GPT-J` and `falcon` architectures
+The GPT4All Chat UI supports models from all newer versions of `llama.cpp` with `GGUF` models including the `Mistral`, `LLaMA2`, `LLaMA`, `OpenLLaMa`, `Falcon`, `MPT`, `Replit`, `Starcoder`, and `Bert` architectures
 
-GPT4All maintains an official list of recommended models located in [models.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
+GPT4All maintains an official list of recommended models located in [models2.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
 
-#### Sideloading any GGML model
+#### Sideloading any GGUF model
 If a model is compatible with the gpt4all-backend, you can sideload it into GPT4All Chat by:
 
-1. Downloading your model in GGML format. It should be a 3-8 GB file similar to the ones [here](https://huggingface.co/TheBloke/Samantha-7B-GGML/tree/main).
-2. Identifying your GPT4All model downloads folder. This is the path listed at the bottom of the downloads dialog(Three lines in top left>Downloads).
-3. Placing your downloaded model inside the GPT4All's model downloads folder.
+1. Downloading your model in GGUF format. It should be a 3-8 GB file similar to the ones [here](https://huggingface.co/TheBloke/Orca-2-7B-GGUF/tree/main).
+2. Identifying your GPT4All model downloads folder. This is the path listed at the bottom of the downloads dialog.
+3. Placing your downloaded model inside GPT4All's model downloads folder.
 4. Restarting your GPT4ALL app. Your model should appear in the model selection list.
 
 ## Plugins
 GPT4All Chat Plugins allow you to expand the capabilities of Local LLMs.
 
-### LocalDocs Beta Plugin (Chat With Your Data)
-LocalDocs is a GPT4All plugin that allows you to chat with your local files and data.
+### LocalDocs Plugin (Chat With Your Data)
+LocalDocs is a GPT4All feature that allows you to chat with your local files and data.
 It allows you to utilize powerful local LLMs to chat with private data without any data leaving your computer or server.
-When using LocalDocs, your LLM will cite the sources that most likely contributed to a given output. Note, even an LLM equipped with LocalDocs can hallucinate. If the LocalDocs plugin decides to utilize your documents to help answer a prompt, you will see references appear below the response.
+When using LocalDocs, your LLM will cite the sources that most likely contributed to a given output. Note, even an LLM equipped with LocalDocs can hallucinate. The LocalDocs plugin will utilize your documents to help answer prompts and you will see references appear below the response.
 
 <p align="center">
-  <img width="70%" src="https://github.com/nomic-ai/gpt4all/assets/13879686/f70f40b4-9684-46d8-b388-ca186f63d13e">
-</p>
-<p align="center">
-GPT4All-Snoozy with LocalDocs. Try GPT4All-Groovy for a faster experience!
+  <img width="70%" src="https://github.com/nomic-ai/gpt4all/assets/10168/fe5dd3c0-b3cc-4701-98d3-0280dfbcf26f">
 </p>
 
 #### Enabling LocalDocs
 1. Install the latest version of GPT4All Chat from [GPT4All Website](https://gpt4all.io).
 2. Go to `Settings > LocalDocs tab`.
-3. Configure a collection (folder) on your computer that contains the files your LLM should have access to. You can alter the contents of the folder/directory at anytime. As you
+3. Download the SBert model
+4. Configure a collection (folder) on your computer that contains the files your LLM should have access to. You can alter the contents of the folder/directory at anytime. As you
 add more files to your collection, your LLM will dynamically be able to access them.
-4. Spin up a chat session with any LLM (including external ones like ChatGPT but warning data will leave your machine!)
-5. At the top right, click the database icon and select which collection you want your LLM to know about during your chat session.
+5. Spin up a chat session with any LLM (including external ones like ChatGPT but warning data will leave your machine!)
+6. At the top right, click the database icon and select which collection you want your LLM to know about during your chat session.
+7. You can begin searching with your localdocs even before the collection has completed indexing, but note the search will not include those parts of the collection yet to be indexed.
 
 #### LocalDocs Capabilities
-LocalDocs allows your LLM to have context about the contents of your documentation collection. Not all prompts/question will utilize your document
-collection for context. If LocalDocs was used in your LLMs response, you will see references to the document snippets that LocalDocs used.
+LocalDocs allows your LLM to have context about the contents of your documentation collection.
 
 LocalDocs **can**:
 
-- Query your documents based upon your prompt / question. If your documents contain answers that may help answer your question/prompt LocalDocs will try to utilize snippets of your documents to provide context.
+- Query your documents based upon your prompt / question. Your documents will be searched for snippets that can be used to provide context for an answer. The most relevant snippets will be inserted into your prompts context, but it will be up to the underlying model to decide how best to use the provided context.
 
 LocalDocs **cannot**:
 
@@ -62,32 +60,17 @@ The general technique this plugin uses is called [Retrieval Augmented Generation
 
 These document chunks help your LLM respond to queries with knowledge about the contents of your data.
 The number of chunks and the size of each chunk can be configured in the LocalDocs plugin settings tab.
-For indexing speed purposes, LocalDocs uses pre-deep-learning n-gram and TF-IDF based retrieval when deciding
-what document chunks your LLM should use as context. You'll find its of comparable quality
-with embedding based retrieval approaches but magnitudes faster to ingest data.
 
-LocalDocs supports the following file types:
-```json
-["txt", "doc", "docx", "pdf", "rtf", "odt", "html", "htm", "xls", "xlsx", "csv", "ods", "ppt", "pptx", "odp", "xml", "json", "log", "md", "org", "tex", "asc", "wks",
-"wpd", "wps", "wri", "xhtml", "xht", "xslt", "yaml", "yml", "dtd", "sgml", "tsv", "strings", "resx",
-"plist", "properties", "ini", "config", "bat", "sh", "ps1", "cmd", "awk", "sed", "vbs", "ics", "mht",
-"mhtml", "epub", "djvu", "azw", "azw3", "mobi", "fb2", "prc", "lit", "lrf", "tcr", "pdb", "oxps",
-"xps", "pages", "numbers", "key", "keynote", "abw", "zabw", "123", "wk1", "wk3", "wk4", "wk5", "wq1",
-"wq2", "xlw", "xlr", "dif", "slk", "sylk", "wb1", "wb2", "wb3", "qpw", "wdb", "wks", "wku", "wr1",
-"wrk", "xlk", "xlt", "xltm", "xltx", "xlsm", "xla", "xlam", "xll", "xld", "xlv", "xlw", "xlc", "xlm",
-"xlt", "xln"]
-```
+LocalDocs currently supports plain text files (`.txt`, `.md`, and `.rst`) and PDF files (`.pdf`).
 
 #### Troubleshooting and FAQ
 *My LocalDocs plugin isn't using my documents*
 
 - Make sure LocalDocs is enabled for your chat session (the DB icon on the top-right should have a border)
-- Try to modify your prompt to be more specific and use terminology that is in your document. This will increase the likelihood that LocalDocs matches document snippets for your question.
 - If your document collection is large, wait 1-2 minutes for it to finish indexing.
 
 
 #### LocalDocs Roadmap
-- Embedding based semantic search for retrieval. 
 - Customize model fine-tuned with retrieval in the loop.
 - Plugin compatibility with chat client server mode.
 

gpt4all-bindings/python/docs/gpt4all_cli.md

@@ -166,7 +166,7 @@ If you want to use a different model, you can do so with the `-m`/`--model` para
 model file name is provided, it will again check in `.cache/gpt4all/` and might start downloading.
 If instead given a path to an existing model, the command could for example look like this:
 ```shell
-python app.py repl --model /home/user/my-gpt4all-models/GPT4All-13B-snoozy.ggmlv3.q4_0.bin
+python app.py repl --model /home/user/my-gpt4all-models/gpt4all-13b-snoozy-q4_0.gguf
 ```
 
 When you're done and want to end a session, simply type `/exit`.

gpt4all-bindings/python/docs/gpt4all_faq.md

@@ -2,31 +2,42 @@
 
 ## What models are supported by the GPT4All ecosystem?
 
-Currently, there are five different model architectures that are supported:
+Currently, there are six different model architectures that are supported:
 
 1. GPT-J - Based off of the GPT-J architecture with examples found [here](https://huggingface.co/EleutherAI/gpt-j-6b)
 2. LLaMA - Based off of the LLaMA architecture with examples found [here](https://huggingface.co/models?sort=downloads&search=llama)
 3. MPT - Based off of Mosaic ML's MPT architecture with examples found [here](https://huggingface.co/mosaicml/mpt-7b)
 4. Replit - Based off of Replit Inc.'s Replit architecture with examples found [here](https://huggingface.co/replit/replit-code-v1-3b)
 5. Falcon - Based off of TII's Falcon architecture with examples found [here](https://huggingface.co/tiiuae/falcon-40b)
+6. StarCoder - Based off of BigCode's StarCoder architecture with examples found [here](https://huggingface.co/bigcode/starcoder)
 
 ## Why so many different architectures? What differentiates them?
 
-One of the major differences is license. Currently, the LLAMA based models are subject to a non-commercial license, whereas the GPTJ and MPT base models allow commercial usage. In the early advent of the recent explosion of activity in open source local models, the llama models have generally been seen as performing better, but that is changing quickly. Every week - even every day! - new models are released with some of the GPTJ and MPT models competitive in performance/quality with LLAMA. What's more, there are some very nice architectural innovations with the MPT models that could lead to new performance/quality gains.
+One of the major differences is license. Currently, the LLaMA based models are subject to a non-commercial license, whereas the GPTJ and MPT base
+models allow commercial usage. However, its successor [Llama 2 is commercially licensable](https://ai.meta.com/llama/license/), too. In the early
+advent of the recent explosion of activity in open source local models, the LLaMA models have generally been seen as performing better, but that is
+changing quickly. Every week - even every day! - new models are released with some of the GPTJ and MPT models competitive in performance/quality with
+LLaMA. What's more, there are some very nice architectural innovations with the MPT models that could lead to new performance/quality gains.
 
 ## How does GPT4All make these models available for CPU inference?
 
-By leveraging the ggml library written by Georgi Gerganov and a growing community of developers. There are currently multiple different versions of this library. The original github repo can be found [here](https://github.com/ggerganov/ggml), but the developer of the library has also created a LLAMA based version [here](https://github.com/ggerganov/llama.cpp). Currently, this backend is using the latter as a submodule.
+By leveraging the ggml library written by Georgi Gerganov and a growing community of developers. There are currently multiple different versions of
+this library. The original GitHub repo can be found [here](https://github.com/ggerganov/ggml), but the developer of the library has also created a
+LLaMA based version [here](https://github.com/ggerganov/llama.cpp). Currently, this backend is using the latter as a submodule.
 
 ## Does that mean GPT4All is compatible with all llama.cpp models and vice versa?
 
 Yes!
 
-The upstream [llama.cpp](https://github.com/ggerganov/llama.cpp) project has introduced several [compatibility breaking](https://github.com/ggerganov/llama.cpp/commit/b9fd7eee57df101d4a3e3eabc9fd6c2cb13c9ca1) quantization methods recently. This is a breaking change that renders all previous models (including the ones that GPT4All uses) inoperative with newer versions of llama.cpp since that change.
+The upstream [llama.cpp](https://github.com/ggerganov/llama.cpp) project has introduced several [compatibility breaking] quantization methods recently.
+This is a breaking change that renders all previous models (including the ones that GPT4All uses) inoperative with newer versions of llama.cpp since
+that change.
 
 Fortunately, we have engineered a submoduling system allowing us to dynamically load different versions of the underlying library so that
 GPT4All just works.
 
+[compatibility breaking]: https://github.com/ggerganov/llama.cpp/commit/b9fd7eee57df101d4a3e3eabc9fd6c2cb13c9ca1
+
 ## What are the system requirements?
 
 Your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) and you need enough RAM to load a model into memory.
@@ -35,8 +46,55 @@ Your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wi
 
 In newer versions of llama.cpp, there has been some added support for NVIDIA GPU's for inference. We're investigating how to incorporate this into our downloadable installers.
 
-## Ok, so bottom line... how do I make my model on huggingface compatible with GPT4All ecosystem right now?
+## Ok, so bottom line... how do I make my model on Hugging Face compatible with GPT4All ecosystem right now?
 
-1. Check to make sure the huggingface model is available in one of our three supported architectures
-2. If it is, then you can use the conversion script inside of our pinned llama.cpp submodule for GPTJ and LLAMA based models
+1. Check to make sure the Hugging Face model is available in one of our three supported architectures
+2. If it is, then you can use the conversion script inside of our pinned llama.cpp submodule for GPTJ and LLaMA based models
 3. Or if your model is an MPT model you can use the conversion script located directly in this backend directory under the scripts subdirectory 
+
+## Language Bindings
+
+#### There's a problem with the download
+
+Some bindings can download a model, if allowed to do so. For example, in Python or TypeScript if `allow_download=True`
+or `allowDownload=true` (default), a model is automatically downloaded into `.cache/gpt4all/` in the user's home folder,
+unless it already exists.
+
+In case of connection issues or errors during the download, you might want to manually verify the model file's MD5
+checksum by comparing it with the one listed in [models2.json].
+
+As an alternative to the basic downloader built into the bindings, you can choose to download from the 
+<https://gpt4all.io/> website instead. Scroll down to 'Model Explorer' and pick your preferred model.
+
+[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
+
+#### I need the chat GUI and bindings to behave the same
+
+The chat GUI and bindings are based on the same backend. You can make them behave the same way by following these steps:
+
+- First of all, ensure that all parameters in the chat GUI settings match those passed to the generating API, e.g.:
+
+    === "Python"
+        ``` py
+        from gpt4all import GPT4All
+        model = GPT4All(...)
+        model.generate("prompt text", temp=0, ...)  # adjust parameters
+        ```
+    === "TypeScript"
+        ``` ts
+        import { createCompletion, loadModel } from '../src/gpt4all.js'
+        const ll = await loadModel(...);
+        const messages = ...
+        const re = await createCompletion(ll, messages, { temp: 0, ... });  // adjust parameters
+        ```
+
+- To make comparing the output easier, set _Temperature_ in both to 0 for now. This will make the output deterministic.
+
+- Next you'll have to compare the templates, adjusting them as necessary, based on how you're using the bindings.
+    - Specifically, in Python:
+        - With simple `generate()` calls, the input has to be surrounded with system and prompt templates.
+        - When using a chat session, it depends on whether the bindings are allowed to download [models2.json]. If yes,
+          and in the chat GUI the default templates are used, it'll be handled automatically. If no, use
+          `chat_session()` template parameters to customize them.
+
+- Once you're done, remember to reset _Temperature_ to its previous value in both chat GUI and your custom code.

gpt4all-bindings/python/docs/gpt4all_modal.md

@@ -8,7 +8,7 @@ import modal
 
 def download_model():
     import gpt4all
-    #you can use any model from https://gpt4all.io/models/models.json
+    #you can use any model from https://gpt4all.io/models/models2.json
     return gpt4all.GPT4All("ggml-gpt4all-j-v1.3-groovy.bin")
 
 image=modal.Image.debian_slim().pip_install("gpt4all").run_function(download_model)
@@ -31,4 +31,4 @@ def main():
     model = GPT4All()
     for i in range(10):
         model.generate.call()
-```
+```

gpt4all-bindings/python/docs/gpt4all_nodejs.md

@@ -0,0 +1,805 @@
+# GPT4All Node.js API
+
+Native Node.js LLM bindings for all.
+
+```sh
+yarn add gpt4all@latest
+
+npm install gpt4all@latest
+
+pnpm install gpt4all@latest
+
+```
+
+The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
+
+*   New bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
+*   The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
+*   Everything should work out the box.
+*   See [API Reference](#api-reference)
+
+### Chat Completion
+
+```js
+import { createCompletion, loadModel } from '../src/gpt4all.js'
+
+const model = await loadModel('mistral-7b-openorca.Q4_0.gguf', { verbose: true });
+
+const response = await createCompletion(model, [
+    { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
+    { role : 'user', content: 'What is 1 + 1?'  } 
+]);
+
+```
+
+### Embedding
+
+```js
+import { createEmbedding, loadModel } from '../src/gpt4all.js'
+
+const model = await loadModel('ggml-all-MiniLM-L6-v2-f16', { verbose: true });
+
+const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional");
+```
+
+### Build Instructions
+
+*   binding.gyp is compile config
+*   Tested on Ubuntu. Everything seems to work fine
+*   Tested on Windows. Everything works fine.
+*   Sparse testing on mac os.
+*   MingW works as well to build the gpt4all-backend. **HOWEVER**, this package works only with MSVC built dlls.
+
+### Requirements
+
+*   git
+*   [node.js >= 18.0.0](https://nodejs.org/en)
+*   [yarn](https://yarnpkg.com/)
+*   [node-gyp](https://github.com/nodejs/node-gyp)
+    *   all of its requirements.
+*   (unix) gcc version 12
+*   (win) msvc version 143
+    *   Can be obtained with visual studio 2022 build tools
+*   python 3
+*   On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+*   macOS users do not need Vulkan, as GPT4All will use Metal instead.
+
+### Build (from source)
+
+```sh
+git clone https://github.com/nomic-ai/gpt4all.git
+cd gpt4all-bindings/typescript
+```
+
+*   The below shell commands assume the current working directory is `typescript`.
+
+*   To Build and Rebuild:
+
+```sh
+yarn
+```
+
+*   llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
+
+```sh
+git submodule update --init --depth 1 --recursive
+```
+
+```sh
+yarn build:backend
+```
+
+This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native The only current way to use them is to put them in the current working directory of your application. That is, **WHEREVER YOU RUN YOUR NODE APPLICATION**
+
+*   llama-xxxx.dll is required.
+*   According to whatever model you are using, you'll need to select the proper model loader.
+    *   For example, if you running an Mosaic MPT model, you will need to select the mpt-(buildvariant).(dynamiclibrary)
+
+### Test
+
+```sh
+yarn test
+```
+
+### Source Overview
+
+#### src/
+
+*   Extra functions to help aid devex
+*   Typings for the native node addon
+*   the javascript interface
+
+#### test/
+
+*   simple unit testings for some functions exported.
+*   more advanced ai testing is not handled
+
+#### spec/
+
+*   Average look and feel of the api
+*   Should work assuming a model and libraries are installed locally in working directory
+
+#### index.cc
+
+*   The bridge between nodejs and c. Where the bindings are.
+
+#### prompt.cc
+
+*   Handling prompting and inference of models in a threadsafe, asynchronous way.
+
+### Known Issues
+
+*   why your model may be spewing bull 💩
+    *   The downloaded model is broken (just reinstall or download from official site)
+    *   That's it so far
+
+### Roadmap
+
+This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
+
+*   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
+*   \[ ] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
+*   \[x] proper unit testing (integrate with circle ci)
+*   \[x] publish to npm under alpha tag `gpt4all@alpha`
+*   \[x] have more people test on other platforms (mac tester needed)
+*   \[x] switch to new pluggable backend
+*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
+    *   Should include prebuilds to avoid painful node-gyp errors
+*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
+
+### API Reference
+
+<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
+
+##### Table of Contents
+
+*   [ModelFile](#modelfile)
+    *   [gptj](#gptj)
+    *   [llama](#llama)
+    *   [mpt](#mpt)
+    *   [replit](#replit)
+*   [type](#type)
+*   [InferenceModel](#inferencemodel)
+    *   [dispose](#dispose)
+*   [EmbeddingModel](#embeddingmodel)
+    *   [dispose](#dispose-1)
+*   [LLModel](#llmodel)
+    *   [constructor](#constructor)
+        *   [Parameters](#parameters)
+    *   [type](#type-1)
+    *   [name](#name)
+    *   [stateSize](#statesize)
+    *   [threadCount](#threadcount)
+    *   [setThreadCount](#setthreadcount)
+        *   [Parameters](#parameters-1)
+    *   [raw\_prompt](#raw_prompt)
+        *   [Parameters](#parameters-2)
+    *   [embed](#embed)
+        *   [Parameters](#parameters-3)
+    *   [isModelLoaded](#ismodelloaded)
+    *   [setLibraryPath](#setlibrarypath)
+        *   [Parameters](#parameters-4)
+    *   [getLibraryPath](#getlibrarypath)
+    *   [initGpuByString](#initgpubystring)
+        *   [Parameters](#parameters-5)
+    *   [hasGpuDevice](#hasgpudevice)
+    *   [listGpu](#listgpu)
+    *   [dispose](#dispose-2)
+*   [GpuDevice](#gpudevice)
+    *   [type](#type-2)
+*   [LoadModelOptions](#loadmodeloptions)
+*   [loadModel](#loadmodel)
+    *   [Parameters](#parameters-6)
+*   [createCompletion](#createcompletion)
+    *   [Parameters](#parameters-7)
+*   [createEmbedding](#createembedding)
+    *   [Parameters](#parameters-8)
+*   [CompletionOptions](#completionoptions)
+    *   [verbose](#verbose)
+    *   [systemPromptTemplate](#systemprompttemplate)
+    *   [promptTemplate](#prompttemplate)
+    *   [promptHeader](#promptheader)
+    *   [promptFooter](#promptfooter)
+*   [PromptMessage](#promptmessage)
+    *   [role](#role)
+    *   [content](#content)
+*   [prompt\_tokens](#prompt_tokens)
+*   [completion\_tokens](#completion_tokens)
+*   [total\_tokens](#total_tokens)
+*   [CompletionReturn](#completionreturn)
+    *   [model](#model)
+    *   [usage](#usage)
+    *   [choices](#choices)
+*   [CompletionChoice](#completionchoice)
+    *   [message](#message)
+*   [LLModelPromptContext](#llmodelpromptcontext)
+    *   [logitsSize](#logitssize)
+    *   [tokensSize](#tokenssize)
+    *   [nPast](#npast)
+    *   [nCtx](#nctx)
+    *   [nPredict](#npredict)
+    *   [topK](#topk)
+    *   [topP](#topp)
+    *   [temp](#temp)
+    *   [nBatch](#nbatch)
+    *   [repeatPenalty](#repeatpenalty)
+    *   [repeatLastN](#repeatlastn)
+    *   [contextErase](#contexterase)
+*   [createTokenStream](#createtokenstream)
+    *   [Parameters](#parameters-9)
+*   [DEFAULT\_DIRECTORY](#default_directory)
+*   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
+*   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
+*   [DEFAULT\_PROMPT\_CONTEXT](#default_prompt_context)
+*   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
+*   [downloadModel](#downloadmodel)
+    *   [Parameters](#parameters-10)
+    *   [Examples](#examples)
+*   [DownloadModelOptions](#downloadmodeloptions)
+    *   [modelPath](#modelpath)
+    *   [verbose](#verbose-1)
+    *   [url](#url)
+    *   [md5sum](#md5sum)
+*   [DownloadController](#downloadcontroller)
+    *   [cancel](#cancel)
+    *   [promise](#promise)
+
+#### ModelFile
+
+Full list of models available
+DEPRECATED!! These model names are outdated and this type will not be maintained, please use a string literal instead
+
+##### gptj
+
+List of GPT-J Models
+
+Type: (`"ggml-gpt4all-j-v1.3-groovy.bin"` | `"ggml-gpt4all-j-v1.2-jazzy.bin"` | `"ggml-gpt4all-j-v1.1-breezy.bin"` | `"ggml-gpt4all-j.bin"`)
+
+##### llama
+
+List Llama Models
+
+Type: (`"ggml-gpt4all-l13b-snoozy.bin"` | `"ggml-vicuna-7b-1.1-q4_2.bin"` | `"ggml-vicuna-13b-1.1-q4_2.bin"` | `"ggml-wizardLM-7B.q4_2.bin"` | `"ggml-stable-vicuna-13B.q4_2.bin"` | `"ggml-nous-gpt4-vicuna-13b.bin"` | `"ggml-v3-13b-hermes-q5_1.bin"`)
+
+##### mpt
+
+List of MPT Models
+
+Type: (`"ggml-mpt-7b-base.bin"` | `"ggml-mpt-7b-chat.bin"` | `"ggml-mpt-7b-instruct.bin"`)
+
+##### replit
+
+List of Replit Models
+
+Type: `"ggml-replit-code-v1-3b.bin"`
+
+#### type
+
+Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
+
+Type: ModelType
+
+#### InferenceModel
+
+InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
+
+##### dispose
+
+delete and cleanup the native model
+
+Returns **void**&#x20;
+
+#### EmbeddingModel
+
+EmbeddingModel represents an LLM which can create embeddings, which are float arrays
+
+##### dispose
+
+delete and cleanup the native model
+
+Returns **void**&#x20;
+
+#### LLModel
+
+LLModel class representing a language model.
+This is a base class that provides common functionality for different types of language models.
+
+##### constructor
+
+Initialize a new LLModel.
+
+###### Parameters
+
+*   `path` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Absolute path to the model file.
+
+<!---->
+
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model file does not exist.
+
+##### type
+
+either 'gpt', mpt', or 'llama' or undefined
+
+Returns **(ModelType | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
+
+##### name
+
+The name of the model.
+
+Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+
+##### stateSize
+
+Get the size of the internal state of the model.
+NOTE: This state data is specific to the type of model you have created.
+
+Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the size in bytes of the internal state of the model
+
+##### threadCount
+
+Get the number of threads used for model inference.
+The default is the number of physical cores your computer has.
+
+Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The number of threads used for model inference.
+
+##### setThreadCount
+
+Set the number of threads used for model inference.
+
+###### Parameters
+
+*   `newNumber` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The new number of threads.
+
+Returns **void**&#x20;
+
+##### raw\_prompt
+
+Prompt the model with a given input and optional parameters.
+This is the raw output from model.
+Use the prompt function exported for a value
+
+###### Parameters
+
+*   `q` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
+*   `params` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
+*   `callback` **function (res: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): void**&#x20;
+
+Returns **void** The result of the model prompt.
+
+##### embed
+
+Embed text with the model. Keep in mind that
+not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
+Use the prompt function exported for a value
+
+###### Parameters
+
+*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+*   `q`  The prompt input.
+*   `params`  Optional parameters for the prompt context.
+
+Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The result of the model prompt.
+
+##### isModelLoaded
+
+Whether the model is loaded or not.
+
+Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)**&#x20;
+
+##### setLibraryPath
+
+Where to search for the pluggable backend libraries
+
+###### Parameters
+
+*   `s` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+
+Returns **void**&#x20;
+
+##### getLibraryPath
+
+Where to get the pluggable backend libraries
+
+Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+
+##### initGpuByString
+
+Initiate a GPU by a string identifier.
+
+###### Parameters
+
+*   `memory_required` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Should be in the range size\_t or will throw
+*   `device_name` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 'amd' | 'nvidia' | 'intel' | 'gpu' | gpu name.
+    read LoadModelOptions.device for more information
+
+Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)**&#x20;
+
+##### hasGpuDevice
+
+From C documentation
+
+Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** True if a GPU device is successfully initialized, false otherwise.
+
+##### listGpu
+
+GPUs that are usable for this LLModel
+
+*   Throws **any** if hasGpuDevice returns false (i think)
+
+Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[GpuDevice](#gpudevice)>**&#x20;
+
+##### dispose
+
+delete and cleanup the native model
+
+Returns **void**&#x20;
+
+#### GpuDevice
+
+an object that contains gpu data on this machine.
+
+##### type
+
+same as VkPhysicalDeviceType
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### LoadModelOptions
+
+Options that configure a model's behavior.
+
+#### loadModel
+
+Loads a machine learning model with the specified name. The defacto way to create a model.
+By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
+
+##### Parameters
+
+*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The name of the model to load.
+*   `options` **([LoadModelOptions](#loadmodeloptions) | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))?** (Optional) Additional options for loading the model.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<([InferenceModel](#inferencemodel) | [EmbeddingModel](#embeddingmodel))>** A promise that resolves to an instance of the loaded LLModel.
+
+#### createCompletion
+
+The nodejs equivalent to python binding's chat\_completion
+
+##### Parameters
+
+*   `model` **[InferenceModel](#inferencemodel)** The language model object.
+*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
+*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
+
+Returns **[CompletionReturn](#completionreturn)** The completion result.
+
+#### createEmbedding
+
+The nodejs moral equivalent to python binding's Embed4All().embed()
+meow
+
+##### Parameters
+
+*   `model` **[EmbeddingModel](#embeddingmodel)** The language model object.
+*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** text to embed
+
+Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The completion result.
+
+#### CompletionOptions
+
+**Extends Partial\<LLModelPromptContext>**
+
+The options for creating the completion.
+
+##### verbose
+
+Indicates if verbose logging is enabled.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### systemPromptTemplate
+
+Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
+Note that if this is not defined, system messages will not be included in the prompt.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### promptTemplate
+
+Template for user messages, with %1 being replaced by the message.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### promptHeader
+
+The initial instruction for the model, on top of the prompt
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### promptFooter
+
+The last instruction for the model, appended to the end of the prompt.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### PromptMessage
+
+A message in the conversation, identical to OpenAI's chat message.
+
+##### role
+
+The role of the message.
+
+Type: (`"system"` | `"assistant"` | `"user"`)
+
+##### content
+
+The message content.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### prompt\_tokens
+
+The number of tokens used in the prompt.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### completion\_tokens
+
+The number of tokens used in the completion.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### total\_tokens
+
+The total number of tokens used.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### CompletionReturn
+
+The result of the completion, similar to OpenAI's format.
+
+##### model
+
+The model used for the completion.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### usage
+
+Token usage report.
+
+Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
+
+##### choices
+
+The generated completions.
+
+Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[CompletionChoice](#completionchoice)>
+
+#### CompletionChoice
+
+A completion choice, similar to OpenAI's format.
+
+##### message
+
+Response message
+
+Type: [PromptMessage](#promptmessage)
+
+#### LLModelPromptContext
+
+Model inference arguments for generating completions.
+
+##### logitsSize
+
+The size of the raw logits vector.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### tokensSize
+
+The size of the raw tokens vector.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nPast
+
+The number of tokens in the past conversation.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nCtx
+
+The number of tokens possible in the context window.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nPredict
+
+The number of tokens to predict.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### topK
+
+The top-k logits to sample from.
+Top-K sampling selects the next token only from the top K most likely tokens predicted by the model.
+It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit
+the diversity of the output. A higher value for top-K (eg., 100) will consider more tokens and lead
+to more diverse text, while a lower value (eg., 10) will focus on the most probable tokens and generate
+more conservative text. 30 - 60 is a good range for most tasks.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### topP
+
+The nucleus sampling probability threshold.
+Top-P limits the selection of the next token to a subset of tokens with a cumulative probability
+above a threshold P. This method, also known as nucleus sampling, finds a balance between diversity
+and quality by considering both token probabilities and the number of tokens available for sampling.
+When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
+On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
+The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
+for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### temp
+
+The temperature to adjust the model's output distribution.
+Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
+(eg., 1.2) increase randomness, resulting in more imaginative and diverse text. Lower temperatures (eg., 0.5)
+make the output more focused, predictable, and conservative. When the temperature is set to 0, the output
+becomes completely deterministic, always selecting the most probable next token and producing identical results
+each time. A safe range would be around 0.6 - 0.85, but you are free to search what value fits best for you.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### nBatch
+
+The number of predictions to generate in parallel.
+By splitting the prompt every N tokens, prompt-batch-size reduces RAM usage during processing. However,
+this can increase the processing time as a trade-off. If the N value is set too low (e.g., 10), long prompts
+with 500+ tokens will be most affected, requiring numerous processing runs to complete the prompt processing.
+To ensure optimal performance, setting the prompt-batch-size to 2048 allows processing of all tokens in a single run.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### repeatPenalty
+
+The penalty factor for repeated tokens.
+Repeat-penalty can help penalize tokens based on how frequently they occur in the text, including the input prompt.
+A token that has already appeared five times is penalized more heavily than a token that has appeared only one time.
+A value of 1 means that there is no penalty and values larger than 1 discourage repeated tokens.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### repeatLastN
+
+The number of last tokens to penalize.
+The repeat-penalty-tokens N option controls the number of tokens in the history to consider for penalizing repetition.
+A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only
+consider recent tokens.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### contextErase
+
+The percentage of context to erase if the context window is exceeded.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### createTokenStream
+
+TODO: Help wanted to implement this
+
+##### Parameters
+
+*   `llmodel` **[LLModel](#llmodel)**&#x20;
+*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>**&#x20;
+*   `options` **[CompletionOptions](#completionoptions)**&#x20;
+
+Returns **function (ll: [LLModel](#llmodel)): AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>**&#x20;
+
+#### DEFAULT\_DIRECTORY
+
+From python api:
+models will be stored in (homedir)/.cache/gpt4all/\`
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### DEFAULT\_LIBRARIES\_DIRECTORY
+
+From python api:
+The default path for dynamic libraries to be stored.
+You may separate paths by a semicolon to search in multiple areas.
+This searches DEFAULT\_DIRECTORY/libraries, cwd/libraries, and finally cwd.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### DEFAULT\_MODEL\_CONFIG
+
+Default model configuration.
+
+Type: ModelConfig
+
+#### DEFAULT\_PROMPT\_CONTEXT
+
+Default prompt context.
+
+Type: [LLModelPromptContext](#llmodelpromptcontext)
+
+#### DEFAULT\_MODEL\_LIST\_URL
+
+Default model list url.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### downloadModel
+
+Initiates the download of a model file.
+By default this downloads without waiting. use the controller returned to alter this behavior.
+
+##### Parameters
+
+*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The model to be downloaded.
+*   `options` **DownloadOptions** to pass into the downloader. Default is { location: (cwd), verbose: false }.
+
+##### Examples
+
+```javascript
+const download = downloadModel('ggml-gpt4all-j-v1.3-groovy.bin')
+download.promise.then(() => console.log('Downloaded!'))
+```
+
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model already exists in the specified location.
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model cannot be found at the specified url.
+
+Returns **[DownloadController](#downloadcontroller)** object that allows controlling the download process.
+
+#### DownloadModelOptions
+
+Options for the model download process.
+
+##### modelPath
+
+location to download the model.
+Default is process.cwd(), or the current working directory
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### verbose
+
+Debug mode -- check how long it took to download in seconds
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### url
+
+Remote download url. Defaults to `https://gpt4all.io/models/gguf/<modelName>`
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### md5sum
+
+MD5 sum of the model file. If this is provided, the downloaded file will be checked against this sum.
+If the sums do not match, an error will be thrown and the file will be deleted.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### DownloadController
+
+Model download controller.
+
+##### cancel
+
+Cancel the request to download if this is called.
+
+Type: function (): void
+
+##### promise
+
+A promise resolving to the downloaded models config once the download is done
+
+Type: [Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<ModelConfig>