network: fix use of GNU asm statement with MSVC (#2267)

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/continue_config.yml

@@ -343,19 +343,18 @@ jobs:
     steps:
       - checkout
       - node/install:
-          install-yarn: true
           node-version: "18.16"
       - run: node --version
       - run: corepack enable
       - node/install-packages:
-          pkg-manager: yarn
+          pkg-manager: npm
           app-dir: gpt4all-bindings/typescript
-          override-ci-command: yarn install
+          override-ci-command: npm install --ignore-scripts
       - run:
           name: build docs ts yo
           command: |
             cd gpt4all-bindings/typescript
-            yarn docs:build
+            npm run docs:build
   build-py-docs:
     docker:
       - image: circleci/python:3.8
@@ -371,13 +370,13 @@ jobs:
       - run:
           name: Make Documentation
           command: |
-            cd gpt4all-bindings/python/
+            cd gpt4all-bindings/python
             mkdocs build
       - run:
           name: Deploy Documentation
           command: |
-            cd gpt4all-bindings/python/
-            aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat
+            cd gpt4all-bindings/python
+            aws s3 sync --delete site/ s3://docs.gpt4all.io/
       - run:
           name: Invalidate docs.gpt4all.io cloudfront
           command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"

README.md

@@ -3,16 +3,9 @@
 <p align="center">Open-source large language models that run locally on your CPU and nearly any GPU</p>
 
 <p align="center">
-<a href="https://gpt4all.io">GPT4All Website and Models</a>
+<a href="https://gpt4all.io">GPT4All Website and Models</a> • <a href="https://docs.gpt4all.io">GPT4All Documentation</a> • <a href="https://discord.gg/mGZE39AS3e">Discord</a>
 </p>
 
-<p align="center">
-<a href="https://docs.gpt4all.io">GPT4All Documentation</a>
-</p>
-
-<p align="center">
-<a href="https://discord.gg/mGZE39AS3e">Discord</a>
-</p>
 
 <p align="center">
 <a href="https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html">🦜️🔗 Official Langchain Backend</a> 
@@ -22,6 +15,10 @@
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
 </p>
 
+<p align="center">
+ <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg?&logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNSIgaGVpZ2h0PSI0IiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxwYXRoIGQ9Ik00LjQzIDEuODgyYTEuNDQgMS40NCAwIDAgMS0uMDk4LjQyNmMtLjA1LjEyMy0uMTE1LjIzLS4xOTIuMzIyLS4wNzUuMDktLjE2LjE2NS0uMjU1LjIyNmExLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxMmMtLjA5OS4wMTItLjE5Mi4wMTQtLjI3OS4wMDZsLTEuNTkzLS4xNHYtLjQwNmgxLjY1OGMuMDkuMDAxLjE3LS4xNjkuMjQ2LS4xOTFhLjYwMy42MDMgMCAwIDAgLjItLjEwNi41MjkuNTI5IDAgMCAwIC4xMzgtLjE3LjY1NC42NTQgMCAwIDAgLjA2NS0uMjRsLjAyOC0uMzJhLjkzLjkzIDAgMCAwLS4wMzYtLjI0OS41NjcuNTY3IDAgMCAwLS4xMDMtLjIuNTAyLjUwMiAwIDAgMC0uMTY4LS4xMzguNjA4LjYwOCAwIDAgMC0uMjQtLjA2N0wyLjQzNy43MjkgMS42MjUuNjcxYS4zMjIuMzIyIDAgMCAwLS4yMzIuMDU4LjM3NS4zNzUgMCAwIDAtLjExNi4yMzJsLS4xMTYgMS40NS0uMDU4LjY5Ny0uMDU4Ljc1NEwuNzA1IDRsLS4zNTctLjA3OUwuNjAyLjkwNkMuNjE3LjcyNi42NjMuNTc0LjczOS40NTRhLjk1OC45NTggMCAwIDEgLjI3NC0uMjg1Ljk3MS45NzEgMCAwIDEgLjMzNy0uMTRjLjExOS0uMDI2LjIyNy0uMDM0LjMyNS0uMDI2TDMuMjMyLjE2Yy4xNTkuMDE0LjMzNi4wMy40NTkuMDgyYTEuMTczIDEuMTczIDAgMCAxIC41NDUuNDQ3Yy4wNi4wOTQuMTA5LjE5Mi4xNDQuMjkzYTEuMzkyIDEuMzkyIDAgMCAxIC4wNzguNThsLS4wMjkuMzJaIiBmaWxsPSIjRjI3NzdBIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+Cjwvc3ZnPgo=" alt="phorm.ai"></a>
+</p>
+
 <p align="center">
   <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
 </p>
@@ -31,9 +28,6 @@ Run on an M1 macOS Device (not sped up!)
 
 ## GPT4All: An ecosystem of open-source on-edge large language models.
 
-> [!IMPORTANT]
-> GPT4All v2.5.0 and newer only supports models in GGUF format (.gguf). Models used with a previous version of GPT4All (.bin extension) will no longer work.
-
 GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and any GPU. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
 
 Learn more in the [documentation](https://docs.gpt4all.io).
@@ -41,9 +35,10 @@ Learn more in the [documentation](https://docs.gpt4all.io).
 A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models.
 
 ### What's New ([Issue Tracker](https://github.com/orgs/nomic-ai/projects/2))
+- [Latest Release ](https://github.com/nomic-ai/gpt4all/releases)
 - **October 19th, 2023**: GGUF Support Launches with Support for:
     - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
-    - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4_0, Q6 quantizations in GGUF.
+    - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4\_0 and Q4\_1 quantizations in GGUF.
     - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
 - **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on AMD, Intel, Samsung, Qualcomm and NVIDIA GPUs.
 - **August 15th, 2023**: GPT4All API launches allowing inference of local LLMs from docker containers.
@@ -87,6 +82,58 @@ Check project discord, with project owners, or through existing issues/PRs to av
 Please make sure to tag all of the above with relevant project identifiers or your contribution could potentially get lost.
 Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 
+
+## GPT4All 2024 Roadmap
+To contribute to the development of any of the below roadmap items, make or find the corresponding issue and cross-reference the [in-progress task](https://github.com/orgs/nomic-ai/projects/2/views/1).
+
+Each item should have an issue link below.
+
+- Chat UI Language Localization (localize UI into the native languages of users)
+    - [ ] Chinese
+    - [ ] German
+    - [ ] French
+    - [ ] Portuguese
+    - [ ] Your native language here. 
+- UI Redesign: an internal effort at Nomic to improve the UI/UX of gpt4all for all users.
+    - [ ] Design new user interface and gather community feedback
+    - [ ] Implement the new user interface and experience.
+- Installer and Update Improvements
+    - [ ] Seamless native installation and update process on OSX
+    - [ ] Seamless native installation and update process on Windows
+    - [ ] Seamless native installation and update process on Linux
+- Model discoverability improvements:
+    - [x] Support huggingface model discoverability
+    - [ ] Support Nomic hosted model discoverability
+- LocalDocs (towards a local perplexity)
+    - Multilingual LocalDocs Support
+        - [ ] Create a multilingual experience
+        - [ ] Incorporate a multilingual embedding model
+        - [ ] Specify a preferred multilingual LLM for localdocs
+    - Improved RAG techniques
+        - [ ] Query augmentation and re-writing
+        - [ ] Improved chunking and text extraction from arbitrary modalities
+            - [ ] Custom PDF extractor past the QT default (charts, tables, text)
+        - [ ] Faster indexing and local exact search with v1.5 hamming embeddings and reranking (skip ANN index construction!)
+    - Support queries like 'summarize X document'
+    - Multimodal LocalDocs support with Nomic Embed
+    - Nomic Dataset Integration with real-time LocalDocs
+        - [ ] Include an option to allow the export of private LocalDocs collections to Nomic Atlas for debugging data/chat quality
+        - [ ] Allow optional sharing of LocalDocs collections between users.
+        - [ ] Allow the import of a LocalDocs collection from an Atlas Datasets
+            - Chat with live version of Wikipedia, Chat with Pubmed, chat with the latest snapshot of world news.
+- First class Multilingual LLM Support
+    - [ ] Recommend and set a default LLM for German
+    - [ ] Recommend and set a default LLM for English
+    - [ ] Recommend and set a default LLM for Chinese
+    - [ ] Recommend and set a default LLM for Spanish
+
+- Server Mode improvements
+    - Improved UI and new requested features:
+        - [ ] Fix outstanding bugs and feature requests around networking configurations.
+        - [ ] Support Nomic Embed inferencing
+        - [ ] First class documentation
+        - [ ] Improving developer use and quality of server mode (e.g. support larger batches)
+
 ## Technical Reports
 
 <p align="center">

gpt4all-api/README.md

@@ -1,4 +1,7 @@
 # GPT4All REST API
+
+NOTICE: We are considering to deprecate this API as it has become challenging to maintain and test. If you have any interest in maintaining this or would like to takeover and adopt or discuss the future of this API please speak up in the discord channel.
+
 This directory contains the source code to run and build docker images that run a FastAPI app
 for serving inference from GPT4All models. The API matches the OpenAI API spec.
 

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -2,7 +2,8 @@ import logging
 import time
 from typing import List
 from uuid import uuid4
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException
+from gpt4all import GPT4All
 from pydantic import BaseModel, Field
 from api_v1.settings import settings
 from fastapi.responses import StreamingResponse
@@ -18,6 +19,7 @@ class ChatCompletionMessage(BaseModel):
 class ChatCompletionRequest(BaseModel):
     model: str = Field(settings.model, description='The model to generate a completion from.')
     messages: List[ChatCompletionMessage] = Field(..., description='Messages for the chat completion.')
+    temperature: float = Field(settings.temp, description='Model temperature')
 
 class ChatCompletionChoice(BaseModel):
     message: ChatCompletionMessage
@@ -45,15 +47,41 @@ async def chat_completion(request: ChatCompletionRequest):
     '''
     Completes a GPT4All model response based on the last message in the chat.
     '''
-    # Example: Echo the last message content with some modification
+    # GPU is not implemented yet
+    if settings.inference_mode == "gpu":
+        raise HTTPException(status_code=400,
+              detail=f"Not implemented yet: Can only infer in CPU mode.")
+
+    # we only support the configured model
+    if request.model != settings.model:
+        raise HTTPException(status_code=400,
+              detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
+
+    # run only of we have a message
     if request.messages:
-        last_message = request.messages[-1].content
-        response_content = f"Echo: {last_message}"
+        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+        # format system message and conversation history correctly
+        formatted_messages = ""
+        for message in request.messages:
+            formatted_messages += f"<|im_start|>{message.role}\n{message.content}<|im_end|>\n"
+
+        # the LLM will complete the response of the assistant
+        formatted_messages += "<|im_start|>assistant\n"
+        response = model.generate(
+            prompt=formatted_messages,
+            temp=request.temperature
+            )
+
+        # the LLM may continue to hallucinate the conversation, but we want only the first response
+        # so, cut off everything after first <|im_end|>
+        index = response.find("<|im_end|>")
+        response_content = response[:index].strip()
     else:
         response_content = "No messages received."
 
     # Create a chat message for the response
-    response_message = ChatCompletionMessage(role="system", content=response_content)
+    response_message = ChatCompletionMessage(role="assistant", content=response_content)
 
     # Create a choice object with the response message
     response_choice = ChatCompletionChoice(

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -51,7 +51,7 @@ def test_batched_completion():
     model = model_id  # replace with your specific model ID
     prompt = "Who is Michael Jordan?"
     responses = []
-    
+
     # Loop to create completions one at a time
     for _ in range(3):
         response = openai.Completion.create(
@@ -62,7 +62,7 @@ def test_batched_completion():
     # Assertions to check the responses
     for response in responses:
         assert len(response['choices'][0]['text']) > len(prompt)
-    
+
     assert len(responses) == 3
 
 def test_embedding():
@@ -74,4 +74,20 @@ def test_embedding():
 
     assert response["model"] == model
     assert isinstance(output, list)
-    assert all(isinstance(x, args) for x in output)
+    assert all(isinstance(x, args) for x in output)
+
+def test_chat_completion():
+    model = model_id
+
+    response = openai.ChatCompletion.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Knock knock."},
+            {"role": "assistant", "content": "Who's there?"},
+            {"role": "user", "content": "Orange."},
+            ]
+    )
+
+    assert response.choices[0].message.role == "assistant"
+    assert len(response.choices[0].message.content) > 0

gpt4all-backend/CMakeLists.txt

@@ -97,11 +97,6 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         add_library(gptj-${BUILD_VARIANT} SHARED
             gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
         prepare_target(gptj llama-mainline)
-
-        add_library(bert-${BUILD_VARIANT} SHARED
-            bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
-        prepare_target(bert llama-mainline)
     endif()
 endforeach()
 

gpt4all-backend/bert.cpp

@@ -1,910 +0,0 @@
-#define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "bert_impl.h"
-#include "llmodel_shared.h"
-#include "ggml.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <map>
-#include <string>
-#include <vector>
-#include <iostream>
-#include <regex>
-#include <thread>
-#include <algorithm>
-#include <numeric>
-
-//#define DEBUG_BERT
-
-namespace {
-const char *modelType_ = "Bert";
-}
-
-typedef int32_t bert_vocab_id;
-
-// default hparams (all-MiniLM-L6-v2)
-struct bert_hparams
-{
-    int32_t n_vocab = 30522;
-    int32_t n_max_tokens = 512;
-    int32_t n_embd = 256;
-    int32_t n_intermediate = 1536;
-    int32_t n_head = 12;
-    int32_t n_layer = 6;
-};
-
-struct bert_layer
-{
-    // normalization
-    struct ggml_tensor *ln_att_w;
-    struct ggml_tensor *ln_att_b;
-
-    struct ggml_tensor *ln_out_w;
-    struct ggml_tensor *ln_out_b;
-
-    // attention
-    struct ggml_tensor *q_w;
-    struct ggml_tensor *q_b;
-    struct ggml_tensor *k_w;
-    struct ggml_tensor *k_b;
-    struct ggml_tensor *v_w;
-    struct ggml_tensor *v_b;
-
-    struct ggml_tensor *o_w;
-    struct ggml_tensor *o_b;
-
-    // ff
-    struct ggml_tensor *ff_i_w;
-    struct ggml_tensor *ff_i_b;
-
-    struct ggml_tensor *ff_o_w;
-    struct ggml_tensor *ff_o_b;
-};
-
-struct bert_vocab
-{
-    std::map<std::string, bert_vocab_id> token_to_id;
-    std::map<std::string, bert_vocab_id> subword_token_to_id;
-
-    std::map<bert_vocab_id, std::string> _id_to_token;
-    std::map<bert_vocab_id, std::string> _id_to_subword_token;
-};
-
-struct bert_model
-{
-    bert_hparams hparams;
-
-    // embeddings weights
-    struct ggml_tensor *word_embeddings;
-    struct ggml_tensor *token_type_embeddings;
-    struct ggml_tensor *position_embeddings;
-    struct ggml_tensor *ln_e_w;
-    struct ggml_tensor *ln_e_b;
-
-    std::vector<bert_layer> layers;
-
-    struct ggml_context *ctx;
-};
-
-// Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
-struct bert_ctx
-{
-    bert_model model;
-    bert_vocab vocab;
-
-    size_t mem_per_token;
-    int64_t mem_per_input;
-    int32_t max_batch_n;
-    llm_buffer buf_compute;
-    llm_buffer work_buf;
-};
-
-int32_t bert_n_embd(bert_ctx * ctx)
-{
-    return ctx->model.hparams.n_embd;
-}
-
-int32_t bert_n_max_tokens(bert_ctx * ctx)
-{
-    return ctx->model.hparams.n_max_tokens;
-}
-
-const char* bert_vocab_id_to_token(bert_ctx * ctx, bert_vocab_id id) {
-    bert_vocab & vocab = ctx->vocab;
-    auto it = vocab._id_to_token.find(id);
-    if (it != vocab._id_to_token.end())
-    {
-        return it->second.c_str();
-    }
-    it = vocab._id_to_subword_token.find(id);
-    if (it != vocab._id_to_subword_token.end())
-    {
-        return it->second.c_str();
-    }
-    return "[UNK TOKEN from bert_vocab]";
-}
-
-//
-// Tokenizing
-//
-
-static size_t utf8_len(char src)
-{
-    const size_t lookup[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4};
-    uint8_t highbits = static_cast<uint8_t>(src) >> 4;
-    return lookup[highbits];
-}
-
-std::string stripAccents(const std::string &inputString)
-{
-    std::string resultString;
-    std::map<std::string, char> accentMap = {{"À", 'A'},{"Á", 'A'},
-        {"Â", 'A'},{"Ã", 'A'},{"Ä", 'A'},{"Å", 'A'},{"à", 'a'},{"á", 'a'},
-        {"â", 'a'},{"ã", 'a'},{"ä", 'a'},{"å", 'a'},{"È", 'E'},{"É", 'E'},
-        {"Ê", 'E'},{"Ë", 'E'},{"è", 'e'},{"é", 'e'},{"ê", 'e'},{"ë", 'e'},
-        {"Ì", 'I'},{"Í", 'I'},{"Î", 'I'},{"Ï", 'I'},{"ì", 'i'},{"í", 'i'},
-        {"î", 'i'},{"ï", 'i'},{"Ò", 'O'},{"Ó", 'O'},{"Ô", 'O'},{"Õ", 'O'},
-        {"Ö", 'O'},{"ò", 'o'},{"ó", 'o'},{"ô", 'o'},{"õ", 'o'},{"ö", 'o'},
-        {"Ù", 'U'},{"Ú", 'U'},{"Û", 'U'},{"Ü", 'U'},{"ù", 'u'},{"ú", 'u'},
-        {"û", 'u'},{"ü", 'u'},{"Ý", 'Y'},{"ý", 'y'},{"Ç", 'C'},{"ç", 'c'},
-        {"Ñ", 'N'},{"ñ", 'n'},
-    };
-
-    for (size_t i = 0; i < inputString.length();)
-    {
-        int len = utf8_len(inputString[i]);
-        std::string curChar = inputString.substr(i, len);
-        auto iter = accentMap.find(curChar);
-        if (iter != accentMap.end())
-        {
-            resultString += iter->second;
-        }
-        else
-        {
-            resultString += curChar;
-        }
-        i += len;
-    }
-
-    return resultString;
-}
-
-std::string bert_normalize_prompt(const std::string &text)
-{
-    // TODO: handle chinese characters? https://github.com/huggingface/tokenizers/blob/ef5f50605ddf9f8caef1598c0e4853862b9707a7/tokenizers/src/normalizers/bert.rs#L98
-    std::string text2 = stripAccents(text);
-    for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i]))
-    {
-        char c = text2[i];
-        if (c >= 'A' && c <= 'Z')
-            text2[i] = c - 'A' + 'a';
-    }
-    return text2;
-}
-
-std::vector<bert_vocab_id> bert_tokenize(
-    struct bert_ctx * ctx,
-    const char * text)
-{
-    const bert_vocab &vocab = ctx->vocab;
-
-    std::string str = text;
-
-    std::vector<std::string> words;
-    // first split the text into words
-    {
-        str = bert_normalize_prompt(str);
-
-        std::string pat = R"([[:punct:]]|[[:alpha:]]+|[[:digit:]]+)";
-
-        std::regex re(pat);
-        std::smatch m;
-
-        while (std::regex_search(str, m, re))
-        {
-            for (std::string x : m)
-            {
-                words.push_back(x);
-            }
-            str = m.suffix();
-        }
-    }
-
-    // find the longest tokens that form the words:
-    std::vector<bert_vocab_id> tokens;
-    int cls_tok_id = 101;
-    tokens.push_back(cls_tok_id);
-    for (const auto &word : words)
-    {
-        if (word.size() == 0)
-            continue;
-
-        int i = 0;
-        int n = word.size();
-        auto *token_map = &vocab.token_to_id;
-        while (i < n)
-        {
-            int j = n;
-            while (j > i)
-            {
-                auto it = token_map->find(word.substr(i, j - i));
-                if (it != token_map->end())
-                {
-                    tokens.push_back(it->second);
-                    i = j;
-                    token_map = &vocab.subword_token_to_id;
-                }
-                --j;
-            }
-            if (j == i)
-            {
-                fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data());
-                token_map = &vocab.subword_token_to_id;
-                ++i;
-            }
-        }
-    }
-
-    return tokens;
-}
-
-void bert_resize_ctx(bert_ctx * ctx, int32_t new_size) {
-    int64_t buf_size_new = ctx->mem_per_input * new_size;
-
-    // TODO: Max memory should be a param? Now just 1 GB
-    int64_t GB = 1 << 30;
-#if defined(DEBUG_BERT)
-    printf("%s: requested_buf_size %lldMB\n", __func__, buf_size_new / (1 << 20));
-#endif
-    if (buf_size_new > GB) {
-        int32_t adjusted_new_size = GB / ctx->mem_per_input;
-        if (adjusted_new_size < 1) adjusted_new_size = 1;
-#if defined(DEBUG_BERT)
-        printf("%s: requested batch size %d, actual new batch size %d\n", __func__, new_size, adjusted_new_size);
-#endif
-        new_size = adjusted_new_size;
-        buf_size_new = ctx->mem_per_input * new_size;
-    }
-    if (new_size > ctx->max_batch_n) {
-        ctx->buf_compute.resize(buf_size_new);
-        ctx->max_batch_n = new_size;
-    }
-}
-
-void bert_eval(
-    struct bert_ctx *ctx,
-    int32_t n_threads,
-    const bert_vocab_id *raw_tokens,
-    int32_t n_tokens,
-    float *embeddings)
-{
-    const bert_model& model = ctx->model;
-    bool mem_req_mode = !embeddings;
-
-    // batch_embeddings is nullptr for the initial memory requirements run
-    if (!mem_req_mode && 1 > ctx->max_batch_n)
-        bert_resize_ctx(ctx, 1);
-
-    const int N = n_tokens;
-    const auto &tokens = raw_tokens;
-
-    const auto &hparams = model.hparams;
-
-    const int n_embd = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_max_tokens = hparams.n_max_tokens;
-    const int n_head = hparams.n_head;
-
-    const int d_head = n_embd / n_head;
-
-    std::vector<float> result;
-    if (N > n_max_tokens)
-    {
-        fprintf(stderr, "Too many tokens, maximum is %d\n", n_max_tokens);
-        return;
-    }
-
-    auto & mem_per_token = ctx->mem_per_token;
-    auto & buf_compute   = ctx->buf_compute;
-
-    struct ggml_init_params params = {
-        .mem_size = buf_compute.size,
-        .mem_buffer = buf_compute.addr,
-        .no_alloc = false,
-    };
-
-    struct ggml_context *ctx0 = ggml_init(params);
-    struct ggml_cgraph *gf = ggml_new_graph(ctx0);
-
-    // Embeddings. word_embeddings + token_type_embeddings + position_embeddings
-    struct ggml_tensor *token_layer = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(token_layer->data, tokens, N * ggml_element_size(token_layer));
-
-    struct ggml_tensor *token_types = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    ggml_set_zero(token_types);
-
-    struct ggml_tensor *positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    for (int i = 0; i < N; i++)
-    {
-        ggml_set_i32_1d(positions, i, i);
-    }
-
-    struct ggml_tensor *inpL = ggml_get_rows(ctx0, model.word_embeddings, token_layer);
-
-    inpL = ggml_add(ctx0,
-                    ggml_get_rows(ctx0, model.token_type_embeddings, token_types),
-                    inpL);
-    inpL = ggml_add(ctx0,
-                    ggml_get_rows(ctx0, model.position_embeddings, positions),
-                    inpL);
-
-    // embd norm
-    {
-        inpL = ggml_norm(ctx0, inpL, 1e-12f);
-
-        inpL = ggml_add(ctx0,
-                        ggml_mul(ctx0,
-                                 ggml_repeat(ctx0, model.ln_e_w, inpL),
-                                 inpL),
-                        ggml_repeat(ctx0, model.ln_e_b, inpL));
-    }
-    // layers
-    for (int il = 0; il < n_layer; il++)
-    {
-        struct ggml_tensor *cur = inpL;
-
-        // self-attention
-        {
-            struct ggml_tensor *Qcur = cur;
-            Qcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].q_b, Qcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].q_w, Qcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *Kcur = cur;
-            Kcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].k_b, Kcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].k_w, Kcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *K = ggml_permute(ctx0, Kcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *Vcur = cur;
-            Vcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].v_b, Vcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].v_w, Vcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *V = ggml_permute(ctx0, Vcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *KQ = ggml_mul_mat(ctx0, K, Q);
-            // KQ = soft_max(KQ / sqrt(head width))
-            KQ = ggml_soft_max(
-                ctx0, ggml_scale(ctx0, KQ, 1.0f / sqrt((float)d_head))
-            );
-
-            V = ggml_cont(ctx0, ggml_transpose(ctx0, V));
-            struct ggml_tensor *KQV = ggml_mul_mat(ctx0, V, KQ);
-            KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            cur = ggml_cpy(ctx0,
-                           KQV,
-                           ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-        }
-        // attention output
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].o_b, cur),
-                       ggml_mul_mat(ctx0, model.layers[il].o_w, cur));
-
-        // re-add the layer input
-        cur = ggml_add(ctx0, cur, inpL);
-
-        // attention norm
-        {
-            cur = ggml_norm(ctx0, cur, 1e-12f);
-
-            cur = ggml_add(ctx0,
-                           ggml_mul(ctx0,
-                                    ggml_repeat(ctx0, model.layers[il].ln_att_w, cur),
-                                    cur),
-                           ggml_repeat(ctx0, model.layers[il].ln_att_b, cur));
-        }
-        struct ggml_tensor *att_output = cur;
-        // intermediate_output = self.intermediate(attention_output)
-        cur = ggml_mul_mat(ctx0, model.layers[il].ff_i_w, cur);
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].ff_i_b, cur),
-                       cur);
-        cur = ggml_gelu(ctx0, cur);
-
-        // layer_output = self.output(intermediate_output, attention_output)
-        cur = ggml_mul_mat(ctx0, model.layers[il].ff_o_w, cur);
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].ff_o_b, cur),
-                       cur);
-        // attentions bypass the intermediate layer
-        cur = ggml_add(ctx0, att_output, cur);
-
-        // output norm
-        {
-            cur = ggml_norm(ctx0, cur, 1e-12f);
-
-            cur = ggml_add(ctx0,
-                           ggml_mul(ctx0,
-                                    ggml_repeat(ctx0, model.layers[il].ln_out_w, cur),
-                                    cur),
-                           ggml_repeat(ctx0, model.layers[il].ln_out_b, cur));
-        }
-        inpL = cur;
-    }
-    inpL = ggml_cont(ctx0, ggml_transpose(ctx0, inpL));
-    // pooler
-    struct ggml_tensor *sum = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, N, 1);
-    ggml_set_f32(sum, 1.0f / N);
-    inpL = ggml_mul_mat(ctx0, inpL, sum);
-
-    ggml_tensor *output = inpL;
-    // run the computation
-    ggml_build_forward_expand(gf, output);
-    //ggml_graph_compute_g4a()
-    ggml_graph_compute_g4a(ctx->work_buf, gf, n_threads);
-    //ggml_graph_compute(ctx0, gf);
-
-
-    // float *dat = ggml_get_data_f32(output);
-    // pretty_print_tensor(dat, output->ne, output->nb, output->n_dims - 1, "");
-
-    #ifdef GGML_PERF
-        // print timing information per ggml operation (for debugging purposes)
-        // requires GGML_PERF to be defined
-        ggml_graph_print(gf);
-    #endif
-
-    if (!mem_req_mode) {
-        memcpy(embeddings, (float *)ggml_get_data(output), sizeof(float) * n_embd);
-    } else {
-        mem_per_token = ggml_used_mem(ctx0) / N;
-    }
-
-    // printf("used_mem = %zu KB \n", ggml_used_mem(ctx0) / 1024);
-    // printf("mem_per_token = %zu KB \n", mem_per_token / 1024);
-
-    ggml_free(ctx0);
-}
-
-//
-// Loading and setup
-//
-
-void bert_free(bert_ctx * ctx) {
-    delete ctx;
-}
-
-struct bert_ctx * bert_load_from_file(const char *fname)
-{
-#if defined(DEBUG_BERT)
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname);
-#endif
-
-    bert_ctx * new_bert = new bert_ctx;
-
-    bert_model & model = new_bert->model;
-    bert_vocab & vocab = new_bert->vocab;
-
-    struct gguf_init_params params = {
-        /*.no_alloc = */ false,
-        /*.ctx      = */ &model.ctx,
-    };
-    gguf_context *ggufctx = gguf_init_from_file(fname, params);
-    if (!ggufctx) {
-        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
-        return nullptr;
-    }
-
-    printf("%s: gguf version     = %d\n", __func__, gguf_get_version(ggufctx));
-    printf("%s: gguf alignment   = %zu\n", __func__, gguf_get_alignment(ggufctx));
-    printf("%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx));
-
-    // print some standard metadata
-    {
-        int keyidx;
-
-        keyidx = gguf_find_key(ggufctx, "general.name");
-        if (keyidx != -1) { printf("%s: model name           = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.description");
-        if (keyidx != -1) { printf("%s: model description    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.author");
-        if (keyidx != -1) { printf("%s: model author         = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.license");
-        if (keyidx != -1) { printf("%s: model license        = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx != -1) { printf("%s: model architecture   = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.file_type");
-        if (keyidx != -1) { printf("%s: model file type      = %" PRIu32 "\n", __func__, gguf_get_val_u32(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "gptneox.tensor_data_layout");
-        if (keyidx != -1) { printf("%s: model data layout    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.source.huggingface.repository");
-        if (keyidx != -1) { printf("%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-    }
-
-    // check required metadata
-    {
-        // check model architecture kv
-        int keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: gguf model architecture not found!\n", __func__);
-            return nullptr;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
-            fprintf(stderr, "%s: model architecture not supported!\n", __func__);
-            return nullptr;
-        }
-    }
-
-    // load hparams
-    {
-        auto &hparams = model.hparams;
-
-        bool ok = false;
-        int keyidx;
-
-        do {
-            keyidx = gguf_find_key(ggufctx, "bert.context_length");
-            if (keyidx == -1) { break; }
-            hparams.n_max_tokens = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.embedding_length");
-            if (keyidx == -1) { break; }
-            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.feed_forward_length");
-            if (keyidx == -1) { break; }
-            hparams.n_intermediate = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.attention.head_count");
-            if (keyidx == -1) { break; }
-            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.block_count");
-            if (keyidx == -1) { break; }
-            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
-
-            ok = true;
-        } while (false);
-
-        if (!ok) {
-            fprintf(stderr, "%s: required hparam missing!\n", __func__);
-            return nullptr;
-        }
-
-#if defined(DEBUG_BERT)
-        printf("%s: n_max_tokens   = %d\n", __func__, hparams.n_max_tokens);
-        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_intermediate = %d\n", __func__, hparams.n_intermediate);
-        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
-#endif
-    }
-
-    // load vocab
-    {
-        auto & hparams = model.hparams;
-
-        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
-            return nullptr;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
-            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
-            return nullptr;
-        }
-
-        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
-        if (tokens_keyidx == -1) {
-            fprintf(stderr, "%s: bert tokenizer vocab not found!\n", __func__);
-            return nullptr;
-        }
-
-        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
-        printf("%s: bert tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
-
-        for (int i = 0; i < hparams.n_vocab; i++) {
-            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
-
-            if (word[0] == '#' && word[1] == '#')
-            {
-                vocab.subword_token_to_id[word.substr(2)] = i;
-                vocab._id_to_subword_token[i] = word;
-            }
-
-            if (vocab.token_to_id.count(word) == 0)
-            {
-                vocab.token_to_id[word] = i;
-                vocab._id_to_token[i] = word;
-            }
-        }
-    }
-
-    auto &ctx = model.ctx;
-
-#if defined(DEBUG_BERT)
-    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ggml_get_mem_size(ctx) / (1024.0 * 1024.0));
-#endif
-
-    // prepare memory for the weights
-    {
-        const int n_layer = model.hparams.n_layer;
-        model.layers.resize(n_layer);
-
-        model.word_embeddings       = ggml_get_tensor(ctx, "token_embd.weight");
-        model.token_type_embeddings = ggml_get_tensor(ctx, "token_types.weight");
-        model.position_embeddings   = ggml_get_tensor(ctx, "position_embd.weight");
-        model.ln_e_w                = ggml_get_tensor(ctx, "output_norm.weight");
-        model.ln_e_b                = ggml_get_tensor(ctx, "output_norm.bias");
-
-        auto name = [](int i, std::string n) {
-            static std::string key;
-            key = "blk." + std::to_string(i) + "." + n;
-            return key.c_str();
-        };
-
-        for (int i = 0; i < n_layer; ++i)
-        {
-            auto &layer = model.layers[i];
-
-            layer.ln_att_w = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
-            layer.ln_att_b = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
-            layer.ln_out_w = ggml_get_tensor(ctx, name(i, "ffn_norm.weight"));
-            layer.ln_out_b = ggml_get_tensor(ctx, name(i, "ffn_norm.bias"));
-            layer.q_w      = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
-            layer.q_b      = ggml_get_tensor(ctx, name(i, "attn_q.bias"));
-            layer.k_w      = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
-            layer.k_b      = ggml_get_tensor(ctx, name(i, "attn_k.bias"));
-            layer.v_w      = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
-            layer.v_b      = ggml_get_tensor(ctx, name(i, "attn_v.bias"));
-            layer.o_w      = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
-            layer.o_b      = ggml_get_tensor(ctx, name(i, "attn_output.bias"));
-            layer.ff_i_w   = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
-            layer.ff_i_b   = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
-            layer.ff_o_w   = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
-            layer.ff_o_b   = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
-        }
-    }
-
-    // Calculate space requirements for setting up context buffers later
-    {
-        bert_vocab_id tokens[] = {0, 1, 2, 3};
-        // TODO: We set the initial buffer size to 16MB and hope it's enough. Maybe there is a better way to do this?
-        new_bert->buf_compute.resize(16 * 1024 * 1024);
-        bert_eval(new_bert, 1, tokens, 4, nullptr);
-        new_bert->max_batch_n = 0;
-
-        // TODO: Max tokens should be a param?
-        int32_t N = new_bert->model.hparams.n_max_tokens;
-        new_bert->mem_per_input = 2.2 * (new_bert->mem_per_token * N); // add 10% to account for ggml object overhead
-
-    }
-#if defined(DEBUG_BERT)
-    printf("%s: mem_per_token %ld KB, mem_per_input %ld MB\n", __func__, new_bert->mem_per_token / (1 << 10), new_bert->mem_per_input / (1 << 20));
-#endif
-
-    return new_bert;
-}
-
-struct BertPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    bert_ctx *ctx = nullptr;
-    int64_t n_threads = 0;
-};
-
-Bert::Bert() : d_ptr(new BertPrivate) {
-    d_ptr->modelLoaded = false;
-}
-
-Bert::~Bert() {
-    bert_free(d_ptr->ctx);
-}
-
-bool Bert::loadModel(const std::string &modelPath, int n_ctx, int ngl)
-{
-    (void)n_ctx;
-    (void)ngl;
-    d_ptr->modelLoaded = false;
-
-    auto * ctx = bert_load_from_file(modelPath.c_str());
-    fflush(stdout);
-    if (!ctx)
-        return false;
-
-    d_ptr->ctx = ctx;
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    return true;
-}
-
-bool Bert::isModelLoaded() const
-{
-    return d_ptr->modelLoaded;
-}
-
-size_t Bert::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
-{
-    (void)modelPath;
-    (void)n_ctx;
-    (void)ngl;
-    return 0;
-}
-
-size_t Bert::stateSize() const
-{
-    return 0;
-}
-
-size_t Bert::saveState(uint8_t */*dest*/) const
-{
-    return 0;
-}
-
-size_t Bert::restoreState(const uint8_t */*src*/)
-{
-    return 0;
-}
-
-void Bert::setThreadCount(int32_t n_threads)
-{
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t Bert::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-std::vector<float> Bert::embedding(const std::string &text)
-{
-    const int overlap = 32;
-    const LLModel::Token clsToken = 101;
-    const size_t contextLength = bert_n_max_tokens(d_ptr->ctx);
-    typedef std::vector<LLModel::Token> TokenString;
-    TokenString tokens = ::bert_tokenize(d_ptr->ctx, text.c_str());
-#if defined(DEBUG_BERT)
-    std::cerr << "embedding: " << tokens.size()
-              << " contextLength " << contextLength
-              << "\n";
-#endif
-    std::vector<double> embeddingsSum(bert_n_embd(d_ptr->ctx), 0);
-    int embeddingsSumTotal = 0;
-    size_t start_pos = 0;
-    bool isFirstChunk = true;
-    while (start_pos < tokens.size()) {
-        TokenString chunk;
-        if (!isFirstChunk)
-            chunk.push_back(clsToken);
-        const size_t l = isFirstChunk ? contextLength : contextLength - 1;
-        if (tokens.size() - start_pos > l) {
-            chunk.insert(chunk.end(), tokens.begin() + start_pos, tokens.begin() + start_pos + l);
-            start_pos = start_pos + contextLength - overlap;
-        } else {
-            chunk.insert(chunk.end(), tokens.begin() + start_pos, tokens.end());
-            start_pos = tokens.size();
-        }
-#if defined(DEBUG_BERT)
-        std::cerr << "chunk length: " << chunk.size()
-            << " embeddingsSumTotal " << embeddingsSumTotal
-            << " contextLength " << contextLength
-            << " start_pos " << start_pos
-            << "\n";
-#endif
-        embeddingsSumTotal++;
-        std::vector<float> embeddings(bert_n_embd(d_ptr->ctx));
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, chunk.data(), chunk.size(), embeddings.data());
-        std::transform(embeddingsSum.begin(), embeddingsSum.end(), embeddings.begin(), embeddingsSum.begin(), std::plus<float>());
-        isFirstChunk = false;
-    }
-
-    std::transform(embeddingsSum.begin(), embeddingsSum.end(), embeddingsSum.begin(), [embeddingsSumTotal](float num){ return num / embeddingsSumTotal; });
-    double magnitude = std::sqrt(std::inner_product(embeddingsSum.begin(), embeddingsSum.end(), embeddingsSum.begin(), 0.0));
-    for (auto &value : embeddingsSum)
-        value /= magnitude;
-    std::vector<float> finalEmbeddings(embeddingsSum.begin(), embeddingsSum.end());
-    return finalEmbeddings;
-}
-
-std::vector<LLModel::Token> Bert::tokenize(PromptContext &ctx, const std::string &str, bool special) const
-{
-    (void)ctx;
-    (void)special;
-    return ::bert_tokenize(d_ptr->ctx, str.c_str());
-}
-
-LLModel::Token Bert::sampleToken(PromptContext &/*promptCtx*/) const
-{
-    return 999 /*!*/;
-}
-
-std::string Bert::tokenToString(Token id) const
-{
-    return bert_vocab_id_to_token(d_ptr->ctx, id);
-}
-
-bool Bert::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    std::vector<float> embeddings(bert_n_embd(d_ptr->ctx));
-    int32_t cls = 101;
-    const bool useCLS = tokens.front() != cls;
-    if (useCLS) {
-        std::vector<int32_t> myTokens;
-        myTokens.push_back(cls);
-        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, myTokens.data(), myTokens.size(), embeddings.data());
-    } else
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, tokens.data(), tokens.size(), embeddings.data());
-    ctx.n_past = 0; // bert does not store any context
-    return true;
-}
-
-int32_t Bert::contextLength() const
-{
-    return bert_n_max_tokens(d_ptr->ctx);
-}
-
-const std::vector<LLModel::Token> &Bert::endTokens() const
-{
-    static const std::vector<LLModel::Token> out = { 102 /*sep*/};
-    return out;
-}
-
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(const char * fname) {
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
-
-    bool isValid = gguf_get_version(ctx_gguf) <= 3;
-    isValid = isValid && get_arch_name(ctx_gguf) == "bert";
-
-    gguf_free(ctx_gguf);
-    return isValid;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new Bert;
-}
-}

gpt4all-backend/bert_impl.h

@@ -1,45 +0,0 @@
-#ifndef BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of bert.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef BERT_H
-#define BERT_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include <memory>
-#include "llmodel.h"
-
-struct BertPrivate;
-class Bert : public LLModel {
-public:
-    Bert();
-    ~Bert();
-
-    bool supportsEmbedding() const override { return true; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-    std::vector<float> embedding(const std::string &text) override;
-
-private:
-    std::unique_ptr<BertPrivate> d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token id) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token> &endTokens() const override;
-    bool shouldAddBOS() const override { return true; }
-};
-
-#endif // BERT_H

gpt4all-backend/gptj.cpp

@@ -785,7 +785,7 @@ const std::vector<LLModel::Token> &GPTJ::endTokens() const
     return fres;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
+const char *get_arch_name(gguf_context *ctx_gguf) {
     std::string arch_name;
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
     enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
@@ -814,21 +814,25 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
+DLL_EXPORT char *get_file_arch(const char *fname) {
     struct ggml_context * ctx_meta = NULL;
     struct gguf_init_params params = {
         /*.no_alloc = */ true,
         /*.ctx      = */ &ctx_meta,
     };
     gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
 
-    bool isValid = gguf_get_version(ctx_gguf) <= 3;
-    isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
+    char *arch = nullptr;
+    if (ctx_gguf && gguf_get_version(ctx_gguf) <= 3) {
+        arch = strdup(get_arch_name(ctx_gguf));
+    }
 
     gguf_free(ctx_gguf);
-    return isValid;
+    return arch;
+}
+
+DLL_EXPORT bool is_arch_supported(const char *arch) {
+    return !strcmp(arch, "gptj");
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llamamodel.cpp

@@ -6,9 +6,11 @@
 #include <cstdio>
 #include <cstring>
 #include <fstream>
+#include <initializer_list>
 #include <iomanip>
 #include <iostream>
 #include <map>
+#include <numeric>
 #include <random>
 #include <sstream>
 #include <stdexcept>
@@ -30,6 +32,19 @@ static constexpr int GGUF_VER_MAX = 3;
 
 static const char * const modelType_ = "LLaMA";
 
+static const std::vector<const char *> KNOWN_ARCHES {
+    "baichuan", "bert", "bloom", "codeshell", "falcon", "gemma", "gpt2", "llama", "mpt", "nomic-bert", "orion",
+    "persimmon", "phi2", "plamo", "qwen", "qwen2", "refact", "stablelm", "starcoder"
+};
+
+static const std::vector<const char *> EMBEDDING_ARCHES {
+    "bert", "nomic-bert"
+};
+
+static bool is_embedding_arch(const std::string &arch) {
+    return std::find(EMBEDDING_ARCHES.begin(), EMBEDDING_ARCHES.end(), arch) < EMBEDDING_ARCHES.end();
+}
+
 static bool llama_verbose() {
     const char* var = getenv("GPT4ALL_VERBOSE_LLAMACPP");
     return var && *var;
@@ -89,7 +104,7 @@ static int llama_sample_top_p_top_k(
     return llama_sample_token(ctx, &candidates_p);
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
+const char *get_arch_name(gguf_context *ctx_gguf) {
     std::string arch_name;
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
     enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
@@ -124,7 +139,7 @@ static int32_t get_arch_key_u32(std::string const &modelPath, std::string const
     auto * ctx = load_gguf(modelPath.c_str());
     if (!ctx)
         return -1;
-    auto arch = get_arch_name(ctx);
+    std::string arch = get_arch_name(ctx);
 
     int32_t value = -1;
     if (ctx) {
@@ -143,7 +158,7 @@ static int32_t get_arch_key_u32(std::string const &modelPath, std::string const
 
 struct LLamaPrivate {
     const std::string modelPath;
-    bool modelLoaded;
+    bool modelLoaded = false;
     int device = -1;
     llama_model *model = nullptr;
     llama_context *ctx = nullptr;
@@ -151,12 +166,11 @@ struct LLamaPrivate {
     llama_context_params ctx_params;
     int64_t n_threads = 0;
     std::vector<LLModel::Token> end_tokens;
+    const char *backend_name = nullptr;
 };
 
 LLamaModel::LLamaModel()
-    : d_ptr(new LLamaPrivate) {
-    d_ptr->modelLoaded = false;
-}
+    : d_ptr(new LLamaPrivate) {}
 
 // default hparams (LLaMA 7B)
 struct llama_file_hparams {
@@ -193,7 +207,7 @@ size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
     return filesize + est_kvcache_size;
 }
 
-bool LLamaModel::isModelBlacklisted(const std::string &modelPath) {
+bool LLamaModel::isModelBlacklisted(const std::string &modelPath) const {
     auto * ctx = load_gguf(modelPath.c_str());
     if (!ctx) {
         std::cerr << __func__ << ": failed to load " << modelPath << "\n";
@@ -229,6 +243,18 @@ bool LLamaModel::isModelBlacklisted(const std::string &modelPath) {
     return res;
 }
 
+bool LLamaModel::isEmbeddingModel(const std::string &modelPath) const {
+    auto *ctx_gguf = load_gguf(modelPath.c_str());
+    if (!ctx_gguf) {
+        std::cerr << __func__ << ": failed to load GGUF from " <<  modelPath << "\n";
+        return false;
+    }
+
+    std::string arch = get_arch_name(ctx_gguf);
+    gguf_free(ctx_gguf);
+    return is_embedding_arch(arch);
+}
+
 bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
 {
     d_ptr->modelLoaded = false;
@@ -264,6 +290,8 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     d_ptr->model_params.progress_callback = &LLModel::staticProgressCallback;
     d_ptr->model_params.progress_callback_user_data = this;
 
+    d_ptr->backend_name = "cpu"; // default
+
 #ifdef GGML_USE_KOMPUTE
     if (d_ptr->device != -1) {
         d_ptr->model_params.main_gpu = d_ptr->device;
@@ -275,6 +303,7 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     if (llama_verbose()) {
         std::cerr << "llama.cpp: using Metal" << std::endl;
     }
+    d_ptr->backend_name = "metal";
 
     // always fully offload on Metal
     // TODO(cebtenzzre): use this parameter to allow using more than 53% of system RAM to load a model
@@ -287,20 +316,25 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     if (!d_ptr->model) {
         fflush(stdout);
         d_ptr->device = -1;
-        std::cerr << "LLAMA ERROR: failed to load model from " <<  modelPath << std::endl;
+        std::cerr << "LLAMA ERROR: failed to load model from " << modelPath << std::endl;
         return false;
     }
 
-    const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
-    if (n_ctx > n_ctx_train) {
-        std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
-                  << n_ctx << " specified)\n";
-    }
-
     // -- initialize the context --
 
     d_ptr->ctx_params = llama_context_default_params();
 
+    bool isEmbedding = is_embedding_arch(llama_model_arch(d_ptr->model));
+    const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
+    if (isEmbedding) {
+        d_ptr->ctx_params.n_batch = n_ctx;
+    } else {
+        if (n_ctx > n_ctx_train) {
+            std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
+                      << n_ctx << " specified)\n";
+        }
+    }
+
     d_ptr->ctx_params.n_ctx   = n_ctx;
     d_ptr->ctx_params.seed    = params.seed;
     d_ptr->ctx_params.type_k  = params.kv_type;
@@ -314,6 +348,9 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     d_ptr->ctx_params.n_threads       = d_ptr->n_threads;
     d_ptr->ctx_params.n_threads_batch = d_ptr->n_threads;
 
+    if (isEmbedding)
+        d_ptr->ctx_params.embeddings = true;
+
     d_ptr->ctx = llama_new_context_with_model(d_ptr->model, d_ptr->ctx_params);
     if (!d_ptr->ctx) {
         fflush(stdout);
@@ -327,11 +364,17 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     d_ptr->end_tokens = {llama_token_eos(d_ptr->model)};
 
 #ifdef GGML_USE_KOMPUTE
-    if (usingGPUDevice() && ggml_vk_has_device()) {
-        std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
+    if (usingGPUDevice()) {
+        if (llama_verbose()) {
+            std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
+        }
+        d_ptr->backend_name = "kompute";
     }
 #endif
 
+    m_supportsEmbedding = isEmbedding;
+    m_supportsCompletion = !isEmbedding;
+
     fflush(stdout);
     d_ptr->modelLoaded = true;
     return true;
@@ -438,7 +481,9 @@ const std::vector<LLModel::Token> &LLamaModel::endTokens() const
 bool LLamaModel::shouldAddBOS() const
 {
     int add_bos = llama_add_bos_token(d_ptr->model);
-    return add_bos != -1 ? bool(add_bos) : llama_vocab_type(d_ptr->model) == LLAMA_VOCAB_TYPE_SPM;
+    if (add_bos != -1) { return add_bos; }
+    auto vocab_type = llama_vocab_type(d_ptr->model);
+    return vocab_type == LLAMA_VOCAB_TYPE_SPM || vocab_type == LLAMA_VOCAB_TYPE_WPM;
 }
 
 int32_t LLamaModel::maxContextLength(std::string const &modelPath) const
@@ -515,7 +560,7 @@ bool LLamaModel::initializeGPUDevice(int device, std::string *unavail_reason) co
 #endif
 }
 
-bool LLamaModel::hasGPUDevice()
+bool LLamaModel::hasGPUDevice() const
 {
 #if defined(GGML_USE_KOMPUTE)
     return d_ptr->device != -1;
@@ -524,10 +569,12 @@ bool LLamaModel::hasGPUDevice()
 #endif
 }
 
-bool LLamaModel::usingGPUDevice()
+bool LLamaModel::usingGPUDevice() const
 {
 #if defined(GGML_USE_KOMPUTE)
-    return hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
+    bool hasDevice = hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
+    assert(!hasDevice || ggml_vk_has_device());
+    return hasDevice;
 #elif defined(GGML_USE_METAL)
     return true;
 #else
@@ -535,6 +582,366 @@ bool LLamaModel::usingGPUDevice()
 #endif
 }
 
+const char *LLamaModel::backendName() const {
+    return d_ptr->backend_name;
+}
+
+const char *LLamaModel::gpuDeviceName() const {
+#if defined(GGML_USE_KOMPUTE)
+    if (usingGPUDevice()) {
+        return ggml_vk_current_device().name;
+    }
+#endif
+    return nullptr;
+}
+
+void llama_batch_add(
+                 struct llama_batch & batch,
+                        llama_token   id,
+                          llama_pos   pos,
+    const std::vector<llama_seq_id> & seq_ids,
+                               bool   logits) {
+    batch.token   [batch.n_tokens] = id;
+    batch.pos     [batch.n_tokens] = pos;
+    batch.n_seq_id[batch.n_tokens] = seq_ids.size();
+    for (size_t i = 0; i < seq_ids.size(); ++i) {
+        batch.seq_id[batch.n_tokens][i] = seq_ids[i];
+    }
+    batch.logits  [batch.n_tokens] = logits;
+
+    batch.n_tokens++;
+}
+
+static void batch_add_seq(llama_batch &batch, const std::vector<LLModel::Token> &tokens, int seq_id) {
+    for (unsigned i = 0; i < tokens.size(); i++) {
+        llama_batch_add(batch, tokens[i], i, { seq_id }, i == tokens.size() - 1);
+    }
+}
+
+size_t LLamaModel::embeddingSize() const {
+    return llama_n_embd(d_ptr->model);
+}
+
+struct EmbModelSpec {
+    const char *docPrefix;
+    const char *queryPrefix;
+    std::vector<const char *> otherPrefixes = {};
+    bool matryoshkaCapable = false;
+    const char *recommendedDims = nullptr;
+};
+
+struct EmbModelGroup {
+    EmbModelSpec spec;
+    std::vector<const char *> names;
+};
+
+static const EmbModelSpec NOPREFIX_SPEC {"", ""};
+static const EmbModelSpec NOMIC_SPEC    {"search_document", "search_query", {"clustering", "classification"}};
+static const EmbModelSpec E5_SPEC       {"passage", "query"};
+
+static const EmbModelSpec NOMIC_1_5_SPEC {
+    "search_document", "search_query", {"clustering", "classification"}, true, "[768, 512, 384, 256, 128]",
+};
+static const EmbModelSpec LLM_EMBEDDER_SPEC {
+    "Represent this document for retrieval",
+    "Represent this query for retrieving relevant documents",
+};
+static const EmbModelSpec BGE_SPEC {
+    "", "Represent this sentence for searching relevant passages",
+};
+static const EmbModelSpec E5_MISTRAL_SPEC {
+    "", "Instruct: Given a query, retrieve relevant passages that answer the query\nQuery",
+};
+
+static const EmbModelGroup EMBEDDING_MODEL_SPECS[] {
+    {NOPREFIX_SPEC,     {"all-MiniLM-L6-v1", "all-MiniLM-L12-v1", "all-MiniLM-L6-v2", "all-MiniLM-L12-v2"}},
+    {NOMIC_SPEC,        {"nomic-embed-text-v1", "nomic-embed-text-v1-ablated", "nomic-embed-text-v1-unsupervised"}},
+    {NOMIC_1_5_SPEC,    {"nomic-embed-text-v1.5"}},
+    {LLM_EMBEDDER_SPEC, {"llm-embedder"}},
+    {BGE_SPEC,          {"bge-small-en", "bge-base-en", "bge-large-en",
+                         "bge-small-en-v1.5", "bge-base-en-v1.5", "bge-large-en-v1.5"}},
+    // NOTE: E5 Mistral is not yet implemented in llama.cpp, so it's not in EMBEDDING_ARCHES
+    {E5_SPEC,           {"e5-small", "e5-base", "e5-large",
+                         "e5-small-unsupervised", "e5-base-unsupervised", "e5-large-unsupervised",
+                         "e5-small-v2", "e5-base-v2", "e5-large-v2"}},
+    {E5_MISTRAL_SPEC,   {"e5-mistral-7b-instruct",
+                         "multilingual-e5-small", "multilingual-e5-base", "multilingual-e5-large",
+                         "multilingual-e5-large-instruct"}},
+};
+
+static const EmbModelSpec *getEmbedSpec(const std::string &modelName) {
+    static const auto &specs = EMBEDDING_MODEL_SPECS;
+    auto it = std::find_if(specs, std::end(specs),
+        [&modelName](auto &spec) {
+            auto &names = spec.names;
+            return std::find(names.begin(), names.end(), modelName) < names.end();
+        }
+    );
+    return it < std::end(specs) ? &it->spec : nullptr;
+}
+
+void LLamaModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
+    bool doMean, bool atlas
+) {
+    const EmbModelSpec *spec;
+    std::optional<std::string> prefix;
+    if (d_ptr->model && (spec = getEmbedSpec(llama_model_name(d_ptr->model))))
+        prefix = isRetrieval ? spec->queryPrefix : spec->docPrefix;
+
+    embed(texts, embeddings, prefix, dimensionality, tokenCount, doMean, atlas);
+}
+
+void LLamaModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas, LLModel::EmbedCancelCallback *cancelCb
+) {
+    if (!d_ptr->model)
+        throw std::logic_error("no model is loaded");
+
+    const char *modelName = llama_model_name(d_ptr->model);
+    if (!m_supportsEmbedding)
+        throw std::logic_error("not an embedding model: "s + modelName);
+
+    auto *spec = getEmbedSpec(modelName);
+    if (!spec)
+        std::cerr << __func__ << ": warning: unknown model " << modelName << "\n";
+
+    const int32_t n_embd = llama_n_embd(d_ptr->model);
+    if (dimensionality < 0) {
+        dimensionality = n_embd;
+    } else if (spec && dimensionality != n_embd) {
+        auto msg = [dimensionality, modelName]() {
+            return "unsupported dimensionality " + std::to_string(dimensionality) + " for model " + modelName;
+        };
+        if (!spec->matryoshkaCapable)
+            throw std::out_of_range(msg() + " (supported: " + std::to_string(n_embd) + ")");
+        if (dimensionality == 0 || dimensionality > n_embd)
+            throw std::out_of_range(msg() + " (recommended: " + spec->recommendedDims + ")");
+    }
+
+    if (!prefix) {
+        if (!spec)
+            throw std::invalid_argument("unknown model "s + modelName + ", specify a prefix if applicable or an empty string");
+        prefix = spec->docPrefix;
+    } else if (spec && prefix != spec->docPrefix && prefix != spec->queryPrefix &&
+               std::find(spec->otherPrefixes.begin(), spec->otherPrefixes.end(), *prefix) == spec->otherPrefixes.end())
+    {
+        std::stringstream ss;
+        ss << std::quoted(*prefix) << " is not a valid task type for model " << modelName;
+        throw std::invalid_argument(ss.str());
+    }
+
+    embedInternal(texts, embeddings, *prefix, dimensionality, tokenCount, doMean, atlas, cancelCb, spec);
+}
+
+// MD5 hash of "nomic empty"
+static const char EMPTY_PLACEHOLDER[] = "24df574ea1c998de59d5be15e769658e";
+
+auto product(double a) -> std::function<double(double)> {
+    return [a](double b) { return a * b; };
+}
+
+template <typename T>
+double getL2NormScale(T *start, T *end) {
+    double magnitude = std::sqrt(std::inner_product(start, end, start, 0.0));
+    return 1.0 / std::max(magnitude, 1e-12);
+}
+
+void LLamaModel::embedInternal(
+    const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas, LLModel::EmbedCancelCallback *cancelCb, const EmbModelSpec *spec
+) {
+    typedef std::vector<LLModel::Token> TokenString;
+    static constexpr int32_t atlasMaxLength = 8192;
+    static constexpr int chunkOverlap = 8; // Atlas overlaps chunks of input by 8 tokens
+
+    const llama_token bos_token = llama_token_bos(d_ptr->model);
+    const llama_token eos_token = llama_token_eos(d_ptr->model);
+
+    bool useBOS = shouldAddBOS();
+    bool useEOS = llama_vocab_type(d_ptr->model) == LLAMA_VOCAB_TYPE_WPM;
+
+    // no EOS, optional BOS
+    auto tokenize = [this, useBOS, useEOS, eos_token](std::string text, TokenString &tokens, bool wantBOS) {
+        if (!text.empty() && text[0] != ' ') {
+            text = ' ' + text; // normalize for SPM - our fork of llama.cpp doesn't add a space prefix
+        }
+        wantBOS &= useBOS;
+
+        tokens.resize(text.length()+4);
+        int32_t n_tokens = llama_tokenize(d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), wantBOS, false);
+        if (n_tokens) {
+            (void)eos_token;
+            assert(useEOS == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
+            tokens.resize(n_tokens - useEOS); // erase EOS/SEP
+        } else {
+            tokens.clear();
+        }
+    };
+
+    // tokenize the texts
+    std::vector<TokenString> inputs;
+    for (unsigned i = 0; i < texts.size(); i++) {
+        auto &text = texts[i];
+        auto &inp = inputs.emplace_back();
+        tokenize(text, inp, false);
+        if (atlas && inp.size() > atlasMaxLength) {
+            if (doMean) {
+                throw std::length_error(
+                    "length of text at index " + std::to_string(i) + " is " + std::to_string(inp.size()) +
+                    " tokens which exceeds limit of " + std::to_string(atlasMaxLength)
+                );
+            }
+            inp.resize(atlasMaxLength);
+        } else if (inp.empty()) {
+            if (!atlas || !text.empty()) {
+                std::cerr << __func__ << ": warning: chunking tokenized text at index " << std::to_string(i)
+                          << " into zero tokens\n";
+            }
+            tokenize(EMPTY_PLACEHOLDER, inp, false);
+        }
+    }
+
+    // tokenize the prefix
+    TokenString prefixTokens;
+    if (prefix.empty()) {
+        prefixTokens.push_back(bos_token);
+    } else {
+        tokenize(prefix + ':', prefixTokens, true);
+    }
+
+    // n_ctx_train: max sequence length of model (RoPE scaling not implemented)
+    const uint32_t n_ctx_train = llama_n_ctx_train(d_ptr->model);
+    // n_batch (equals n_ctx): max tokens per call to llama_decode (one more more sequences)
+    const uint32_t n_batch = llama_n_batch(d_ptr->ctx);
+
+    // effective sequence length minus prefix and SEP token
+    const uint32_t max_len = std::min(n_ctx_train, n_batch) - (prefixTokens.size() + useEOS);
+    if (max_len <= chunkOverlap) {
+        throw std::logic_error("max chunk length of " + std::to_string(max_len) + " is smaller than overlap of " +
+                               std::to_string(chunkOverlap) + " tokens");
+    }
+
+    // split into max_len-sized chunks
+    struct split_batch { unsigned idx; TokenString batch; };
+    std::vector<split_batch> batches;
+    size_t totalTokens = 0;
+    for (unsigned i = 0; i < inputs.size(); i++) {
+        auto &input = inputs[i];
+        for (auto it = input.begin(); it < input.end(); it += max_len) {
+            if (it > input.begin()) { it -= chunkOverlap; }
+            auto end = std::min(it + max_len, input.end());
+            batches.push_back({ i, {} });
+            auto &batch = batches.back().batch;
+            batch = prefixTokens;
+            batch.insert(batch.end(), it, end);
+            totalTokens += end - it;
+            batch.push_back(eos_token);
+            if (!doMean) { break; /* limit text to one chunk */ }
+        }
+    }
+    inputs.clear();
+
+    if (cancelCb) {
+        // copy of batching code below, but just count tokens instead of running inference
+        unsigned nBatchTokens = 0;
+        std::vector<unsigned> batchSizes;
+        for (const auto &inp: batches) {
+            if (nBatchTokens + inp.batch.size() > n_batch) {
+                batchSizes.push_back(nBatchTokens);
+                nBatchTokens = 0;
+            }
+            nBatchTokens += inp.batch.size();
+        }
+        batchSizes.push_back(nBatchTokens);
+        if (cancelCb(batchSizes.data(), batchSizes.size(), d_ptr->backend_name)) {
+            throw std::runtime_error("operation was canceled");
+        }
+    }
+
+    // initialize batch
+    struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
+
+    // n_texts x n_embd matrix
+    const int32_t n_embd = llama_n_embd(d_ptr->model);
+    std::vector<double> embeddingsSum(texts.size() * n_embd);
+    std::vector<int> embeddingsSumTotal(texts.size());
+    std::vector<int> queued_indices; // text indices of batches to be processed
+
+    auto decode = [this, &queued_indices, n_embd, &batch, &embeddingsSum, &embeddingsSumTotal, spec, dimensionality]() {
+        if (llama_decode(d_ptr->ctx, batch) < 0)
+            throw std::runtime_error("llama_decode failed");
+
+        for (int i = 0; i < batch.n_tokens; ++i) {
+            if (!batch.logits[i]) { continue; }
+            int i_prompt = queued_indices[batch.seq_id[i][0]];
+            auto *out = &embeddingsSum[i_prompt * n_embd];
+
+            // sequence embeddings aren't available when pooling_type is NONE
+            auto *embd = llama_get_embeddings_seq(d_ptr->ctx, batch.seq_id[i][0]);
+            if (!embd) { embd = llama_get_embeddings_ith(d_ptr->ctx, i); }
+            assert(embd);
+
+            auto *embd_end = embd + n_embd;
+
+            // layer normalization for nomic-embed-text-v1.5
+            if (spec && spec->matryoshkaCapable) {
+                // normalize mean
+                double mean = std::accumulate(embd, embd_end, 0.0) / n_embd;
+                std::transform(embd, embd_end, embd, [mean](double f){ return f - mean; });
+
+                // unbiased sample variance, with Bessel's correction
+                double variance = std::inner_product(embd, embd_end, embd, 0.0) / (n_embd - 1);
+
+                // trim to matryoshka dim
+                embd_end = embd + dimensionality;
+
+                // normalize variance
+                std::transform(embd, embd_end, embd, product(1.0 / std::sqrt(variance + 1e-5)));
+            }
+
+            // L2 norm
+            auto scale = getL2NormScale(embd, embd_end);
+            std::transform(embd, embd_end, out, out, [scale](double e, double o){ return o + scale * e; });
+            embeddingsSumTotal[i_prompt]++;
+        }
+    };
+
+    // break into batches
+    for (const auto &inp: batches) {
+        // encode if at capacity
+        if (batch.n_tokens + inp.batch.size() > n_batch) {
+            decode();
+            batch.n_tokens = 0;
+            queued_indices.clear();
+        }
+
+        // add to batch
+        batch_add_seq(batch, inp.batch, queued_indices.size());
+        queued_indices.push_back(inp.idx);
+    }
+
+    // final batch
+    decode();
+
+    for (unsigned i = 0; i < texts.size(); i++) {
+        auto *embd = &embeddingsSum[i * n_embd];
+        auto *embd_end = embd + dimensionality;
+        int total = embeddingsSumTotal[i];
+
+        // average over chunks
+        std::transform(embd, embd_end, embd, product(1.0 / total));
+
+        // L2 norm and copy
+        auto scale = getL2NormScale(embd, embd_end);
+        std::transform(embd, embd_end, embeddings, product(scale));
+        embeddings += dimensionality;
+    }
+
+    if (tokenCount) { *tokenCount = totalTokens; }
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -554,27 +961,23 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char *fname) {
-    auto * ctx = load_gguf(fname);
-    auto arch = get_arch_name(ctx);
-
-    bool valid = true;
-
-    static const std::vector<const char *> known_arches {
-        "baichuan", "bloom", "codeshell", "falcon", "gemma", "gpt2", "llama", "mpt", "orion", "persimmon", "phi2",
-        "plamo", "qwen", "qwen2", "refact", "stablelm", "starcoder"
-    };
-
-    if (std::find(known_arches.begin(), known_arches.end(), arch) == known_arches.end()) {
-        // not supported by this version of llama.cpp
-        if (!(arch == "gptj" || arch == "bert")) { // we support these via other modules
-            std::cerr << __func__ << ": unsupported model architecture: " << arch << "\n";
+DLL_EXPORT char *get_file_arch(const char *fname) {
+    auto *ctx = load_gguf(fname);
+    char *arch = nullptr;
+    if (ctx) {
+        std::string archStr = get_arch_name(ctx);
+        if (is_embedding_arch(archStr) && gguf_find_key(ctx, (archStr + ".pooling_type").c_str()) < 0) {
+            // old bert.cpp embedding model
+        } else {
+            arch = strdup(archStr.c_str());
         }
-        valid = false;
     }
-
     gguf_free(ctx);
-    return valid;
+    return arch;
+}
+
+DLL_EXPORT bool is_arch_supported(const char *arch) {
+    return std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), std::string(arch)) < KNOWN_ARCHES.end();
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llamamodel_impl.h

@@ -11,15 +11,18 @@
 #include "llmodel.h"
 
 struct LLamaPrivate;
+struct EmbModelSpec;
+
 class LLamaModel : public LLModel {
 public:
     LLamaModel();
     ~LLamaModel();
 
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
+    bool supportsEmbedding() const override { return m_supportsEmbedding; }
+    bool supportsCompletion() const override { return m_supportsCompletion; }
     bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
-    bool isModelBlacklisted(const std::string &modelPath) override;
+    bool isModelBlacklisted(const std::string &modelPath) const override;
+    bool isEmbeddingModel(const std::string &modelPath) const override;
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
@@ -29,12 +32,25 @@ public:
     int32_t threadCount() const override;
     std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const override;
     bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const override;
-    bool initializeGPUDevice(int device, std::string *unavail_reason) const override;
-    bool hasGPUDevice() override;
-    bool usingGPUDevice() override;
+    bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const override;
+    bool hasGPUDevice() const override;
+    bool usingGPUDevice() const override;
+    const char *backendName() const override;
+    const char *gpuDeviceName() const override;
+
+    size_t embeddingSize() const override;
+    // user-specified prefix
+    void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
+               int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false,
+               EmbedCancelCallback *cancelCb = nullptr) override;
+    // automatic prefix
+    void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality = -1,
+               size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
 
 private:
     std::unique_ptr<LLamaPrivate> d_ptr;
+    bool m_supportsEmbedding = false;
+    bool m_supportsCompletion = false;
 
 protected:
     std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
@@ -44,9 +60,12 @@ protected:
     int32_t contextLength() const override;
     const std::vector<Token> &endTokens() const override;
     bool shouldAddBOS() const override;
-
     int32_t maxContextLength(std::string const &modelPath) const override;
     int32_t layerCount(std::string const &modelPath) const override;
+
+    void embedInternal(const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
+                       size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb,
+                       const EmbModelSpec *spec);
 };
 
 #endif // LLAMAMODEL_H

gpt4all-backend/llmodel.cpp

@@ -8,6 +8,7 @@
 #include <fstream>
 #include <iostream>
 #include <memory>
+#include <optional>
 #include <regex>
 #include <sstream>
 #include <string>
@@ -19,33 +20,27 @@
 
 std::string s_implementations_search_path = ".";
 
-static bool has_at_least_minimal_hardware() {
-#if defined(__x86_64__) || defined(_M_X64)
-    #ifndef _MSC_VER
-        return __builtin_cpu_supports("avx");
-    #else
-        int cpuInfo[4];
-        __cpuid(cpuInfo, 1);
-        return cpuInfo[2] & (1 << 28);
-    #endif
-#else
-    return true; // Don't know how to handle non-x86_64
-#endif
-}
+#if !(defined(__x86_64__) || defined(_M_X64))
+    // irrelevant on non-x86_64
+    #define cpu_supports_avx()  -1
+    #define cpu_supports_avx2() -1
+#elif defined(_MSC_VER)
+    // MSVC
+    static int get_cpu_info(int func_id, int reg_id) {
+        int info[4];
+        __cpuid(info, func_id);
+        return info[reg_id];
+    }
 
-static bool requires_avxonly() {
-#if defined(__x86_64__) || defined(_M_X64)
-    #ifndef _MSC_VER
-        return !__builtin_cpu_supports("avx2");
-    #else
-        int cpuInfo[4];
-        __cpuidex(cpuInfo, 7, 0);
-        return !(cpuInfo[1] & (1 << 5));
-    #endif
+    // AVX via EAX=1: Processor Info and Feature Bits, bit 28 of ECX
+    #define cpu_supports_avx()  (get_cpu_info(1, 2) & (1 << 28))
+    // AVX2 via EAX=7, ECX=0: Extended Features, bit 5 of EBX
+    #define cpu_supports_avx2() (get_cpu_info(7, 1) & (1 <<  5))
 #else
-    return false; // Don't know how to handle non-x86_64
+    // gcc/clang
+    #define cpu_supports_avx()  __builtin_cpu_supports("avx")
+    #define cpu_supports_avx2() __builtin_cpu_supports("avx2")
 #endif
-}
 
 LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     : m_dlhandle(new Dlhandle(std::move(dlhandle_))) {
@@ -55,14 +50,17 @@ LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
     m_buildVariant = get_build_variant();
-    m_magicMatch = m_dlhandle->get<bool(const char*)>("magic_match");
-    assert(m_magicMatch);
+    m_getFileArch = m_dlhandle->get<char *(const char *)>("get_file_arch");
+    assert(m_getFileArch);
+    m_isArchSupported = m_dlhandle->get<bool(const char *)>("is_arch_supported");
+    assert(m_isArchSupported);
     m_construct = m_dlhandle->get<LLModel *()>("construct");
     assert(m_construct);
 }
 
 LLModel::Implementation::Implementation(Implementation &&o)
-    : m_magicMatch(o.m_magicMatch)
+    : m_getFileArch(o.m_getFileArch)
+    , m_isArchSupported(o.m_isArchSupported)
     , m_construct(o.m_construct)
     , m_modelType(o.m_modelType)
     , m_buildVariant(o.m_buildVariant)
@@ -71,21 +69,25 @@ LLModel::Implementation::Implementation(Implementation &&o)
 }
 
 LLModel::Implementation::~Implementation() {
-    if (m_dlhandle) delete m_dlhandle;
+    delete m_dlhandle;
 }
 
-bool LLModel::Implementation::isImplementation(const Dlhandle &dl) {
+static bool isImplementation(const Dlhandle &dl) {
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
 const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
+    if (cpu_supports_avx() == 0) {
+        throw std::runtime_error("CPU does not support AVX");
+    }
+
     // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
     // individual models without the cleanup of the static list interfering
     static auto* libs = new std::vector<Implementation>([] () {
         std::vector<Implementation> fres;
 
-        std::string impl_name_re = "(bert|gptj|llamamodel-mainline)";
-        if (requires_avxonly()) {
+        std::string impl_name_re = "(gptj|llamamodel-mainline)";
+        if (cpu_supports_avx2() == 0) {
             impl_name_re += "-avxonly";
         } else {
             impl_name_re += "-(default|metal)";
@@ -107,9 +109,8 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
                     // Add to list if model implementation
                     try {
                         Dlhandle dl(p.string());
-                        if (!Implementation::isImplementation(dl)) {
+                        if (!isImplementation(dl))
                             continue;
-                        }
                         fres.emplace_back(Implementation(std::move(dl)));
                     } catch (...) {}
                 }
@@ -126,33 +127,40 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
 
 const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
     bool buildVariantMatched = false;
+    std::optional<std::string> archName;
     for (const auto& i : implementationList()) {
         if (buildVariant != i.m_buildVariant) continue;
         buildVariantMatched = true;
 
-        if (!i.m_magicMatch(fname)) continue;
-        return &i;
+        char *arch = i.m_getFileArch(fname);
+        if (!arch) continue;
+        archName = arch;
+
+        bool archSupported = i.m_isArchSupported(arch);
+        free(arch);
+        if (archSupported) return &i;
     }
 
-    if (!buildVariantMatched) {
-        std::cerr << "LLModel ERROR: Could not find any implementations for build variant: " << buildVariant << "\n";
-    }
-    return nullptr;
+    if (!buildVariantMatched)
+        throw MissingImplementationError("Could not find any implementations for build variant: " + buildVariant);
+    if (!archName)
+        throw UnsupportedModelError("Unsupported file format");
+
+    throw BadArchError(std::move(*archName));
 }
 
 LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant, int n_ctx) {
-    if (!has_at_least_minimal_hardware()) {
-        std::cerr << "LLModel ERROR: CPU does not support AVX\n";
-        return nullptr;
-    }
-
     // Get correct implementation
     const Implementation* impl = nullptr;
 
     #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
         if (buildVariant == "auto") {
             size_t total_mem = getSystemTotalRAMInBytes();
-            impl = implementation(modelPath.c_str(), "metal");
+            try {
+                impl = implementation(modelPath.c_str(), "metal");
+            } catch (const std::exception &e) {
+                // fall back to CPU
+            }
             if(impl) {
                 LLModel* metalimpl = impl->m_construct();
                 metalimpl->m_implementation = impl;
@@ -178,14 +186,13 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
     if (!impl) {
         //TODO: Auto-detect CUDA/OpenCL
         if (buildVariant == "auto") {
-            if (requires_avxonly()) {
+            if (cpu_supports_avx2() == 0) {
                 buildVariant = "avxonly";
             } else {
                 buildVariant = "default";
             }
         }
         impl = implementation(modelPath.c_str(), buildVariant);
-        if (!impl) return nullptr;
     }
 
     // Construct and return llmodel implementation
@@ -196,15 +203,24 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
 
 LLModel *LLModel::Implementation::constructDefaultLlama() {
     static std::unique_ptr<LLModel> llama([]() -> LLModel * {
+        const std::vector<LLModel::Implementation> *impls;
+        try {
+            impls = &implementationList();
+        } catch (const std::runtime_error &e) {
+            std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
+            return nullptr;
+        }
+
         const LLModel::Implementation *impl = nullptr;
-        for (const auto &i : implementationList()) {
+        for (const auto &i: *impls) {
             if (i.m_buildVariant == "metal" || i.m_modelType != "LLaMA") continue;
             impl = &i;
         }
         if (!impl) {
-            std::cerr << "LLModel ERROR: Could not find CPU LLaMA implementation\n";
+            std::cerr << __func__ << ": could not find llama.cpp implementation\n";
             return nullptr;
         }
+
         auto fres = impl->m_construct();
         fres->m_implementation = impl;
         return fres;
@@ -212,22 +228,27 @@ LLModel *LLModel::Implementation::constructDefaultLlama() {
     return llama.get();
 }
 
-std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices() {
-    auto * llama = constructDefaultLlama();
-    if (llama) { return llama->availableGPUDevices(0); }
+std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices(size_t memoryRequired) {
+    auto *llama = constructDefaultLlama();
+    if (llama) { return llama->availableGPUDevices(memoryRequired); }
     return {};
 }
 
 int32_t LLModel::Implementation::maxContextLength(const std::string &modelPath) {
-    auto * llama = constructDefaultLlama();
+    auto *llama = constructDefaultLlama();
     return llama ? llama->maxContextLength(modelPath) : -1;
 }
 
 int32_t LLModel::Implementation::layerCount(const std::string &modelPath) {
-    auto * llama = constructDefaultLlama();
+    auto *llama = constructDefaultLlama();
     return llama ? llama->layerCount(modelPath) : -1;
 }
 
+bool LLModel::Implementation::isEmbeddingModel(const std::string &modelPath) {
+    auto *llama = constructDefaultLlama();
+    return llama && llama->isEmbeddingModel(modelPath);
+}
+
 void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
     s_implementations_search_path = path;
 }
@@ -235,3 +256,7 @@ void LLModel::Implementation::setImplementationsSearchPath(const std::string& pa
 const std::string& LLModel::Implementation::implementationsSearchPath() {
     return s_implementations_search_path;
 }
+
+bool LLModel::Implementation::hasSupportedCPU() {
+    return cpu_supports_avx() != 0;
+}

gpt4all-backend/llmodel.h

@@ -1,13 +1,14 @@
 #ifndef LLMODEL_H
 #define LLMODEL_H
 
-#include <string>
-#include <functional>
-#include <vector>
-#include <string_view>
-#include <fstream>
 #include <cstdint>
+#include <fstream>
+#include <functional>
 #include <limits>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
 
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
@@ -16,6 +17,29 @@ class LLModel {
 public:
     using Token = int32_t;
 
+    class BadArchError: public std::runtime_error {
+    public:
+        BadArchError(std::string arch)
+            : runtime_error("Unsupported model architecture: " + arch)
+            , m_arch(std::move(arch))
+            {}
+
+        const std::string &arch() const noexcept { return m_arch; }
+
+    private:
+        std::string m_arch;
+    };
+
+    class MissingImplementationError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
+    class UnsupportedModelError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
     struct GPUDevice {
         int index;
         int type;
@@ -29,7 +53,6 @@ public:
 
     class Implementation {
     public:
-        Implementation(Dlhandle &&);
         Implementation(const Implementation &) = delete;
         Implementation(Implementation &&);
         ~Implementation();
@@ -37,20 +60,24 @@ public:
         std::string_view modelType() const { return m_modelType; }
         std::string_view buildVariant() const { return m_buildVariant; }
 
-        static bool isImplementation(const Dlhandle &dl);
-        static const std::vector<Implementation> &implementationList();
-        static const Implementation *implementation(const char *fname, const std::string &buildVariant);
         static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto", int n_ctx = 2048);
-        static std::vector<GPUDevice> availableGPUDevices();
+        static std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0);
         static int32_t maxContextLength(const std::string &modelPath);
         static int32_t layerCount(const std::string &modelPath);
+        static bool isEmbeddingModel(const std::string &modelPath);
         static void setImplementationsSearchPath(const std::string &path);
         static const std::string &implementationsSearchPath();
+        static bool hasSupportedCPU();
 
     private:
+        Implementation(Dlhandle &&);
+
+        static const std::vector<Implementation> &implementationList();
+        static const Implementation *implementation(const char *fname, const std::string &buildVariant);
         static LLModel *constructDefaultLlama();
 
-        bool (*m_magicMatch)(const char *fname);
+        char *(*m_getFileArch)(const char *fname);
+        bool (*m_isArchSupported)(const char *arch);
         LLModel *(*m_construct)();
 
         std::string_view m_modelType;
@@ -83,7 +110,8 @@ public:
     virtual bool supportsEmbedding() const = 0;
     virtual bool supportsCompletion() const = 0;
     virtual bool loadModel(const std::string &modelPath, int n_ctx, int ngl) = 0;
-    virtual bool isModelBlacklisted(const std::string &modelPath) { (void)modelPath; return false; };
+    virtual bool isModelBlacklisted(const std::string &modelPath) const { (void)modelPath; return false; };
+    virtual bool isEmbeddingModel(const std::string &modelPath) const { (void)modelPath; return false; }
     virtual bool isModelLoaded() const = 0;
     virtual size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) = 0;
     virtual size_t stateSize() const { return 0; }
@@ -101,7 +129,18 @@ public:
                         bool special = false,
                         std::string *fakeReply = nullptr);
 
-    virtual std::vector<float> embedding(const std::string &text);
+    using EmbedCancelCallback = bool(unsigned *batchSizes, unsigned nBatch, const char *backend);
+
+    virtual size_t embeddingSize() const {
+        throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+    }
+    // user-specified prefix
+    virtual void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
+                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false,
+                       EmbedCancelCallback *cancelCb = nullptr);
+    // automatic prefix
+    virtual void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval,
+                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false);
 
     virtual void setThreadCount(int32_t n_threads) { (void)n_threads; }
     virtual int32_t threadCount() const { return 1; }
@@ -129,8 +168,10 @@ public:
         return false;
     }
 
-    virtual bool hasGPUDevice() { return false; }
-    virtual bool usingGPUDevice() { return false; }
+    virtual bool hasGPUDevice() const { return false; }
+    virtual bool usingGPUDevice() const { return false; }
+    virtual const char *backendName() const { return "cpu"; }
+    virtual const char *gpuDeviceName() const { return nullptr; }
 
     void setProgressCallback(ProgressCallback callback) { m_progressCallback = callback; }
 

gpt4all-backend/llmodel_c.cpp

@@ -4,6 +4,8 @@
 #include <cerrno>
 #include <cstring>
 #include <iostream>
+#include <memory>
+#include <optional>
 #include <utility>
 
 struct LLModelWrapper {
@@ -12,8 +14,6 @@ struct LLModelWrapper {
     ~LLModelWrapper() { delete llModel; }
 };
 
-thread_local static std::string last_error_message;
-
 llmodel_model llmodel_model_create(const char *model_path) {
     const char *error;
     auto fres = llmodel_model_create2(model_path, "auto", &error);
@@ -23,40 +23,41 @@ llmodel_model llmodel_model_create(const char *model_path) {
     return fres;
 }
 
+static void llmodel_set_error(const char **errptr, const char *message) {
+    thread_local static std::string last_error_message;
+    if (errptr) {
+        last_error_message = message;
+        *errptr = last_error_message.c_str();
+    }
+}
+
 llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error) {
-    auto wrapper = new LLModelWrapper;
-
+    LLModel *llModel;
     try {
-        wrapper->llModel = LLModel::Implementation::construct(model_path, build_variant);
-        if (!wrapper->llModel) {
-            last_error_message = "Model format not supported (no matching implementation found)";
-        }
+        llModel = LLModel::Implementation::construct(model_path, build_variant);
     } catch (const std::exception& e) {
-        last_error_message = e.what();
+        llmodel_set_error(error, e.what());
+        return nullptr;
     }
 
-    if (!wrapper->llModel) {
-        delete std::exchange(wrapper, nullptr);
-        if (error) {
-            *error = last_error_message.c_str();
-        }
-    }
-    return reinterpret_cast<llmodel_model*>(wrapper);
+    auto wrapper = new LLModelWrapper;
+    wrapper->llModel = llModel;
+    return wrapper;
 }
 
 void llmodel_model_destroy(llmodel_model model) {
-    delete reinterpret_cast<LLModelWrapper*>(model);
+    delete static_cast<LLModelWrapper *>(model);
 }
 
 size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->requiredMem(model_path, n_ctx, ngl);
 }
 
 bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
 
     std::string modelPath(model_path);
     if (wrapper->llModel->isModelBlacklisted(modelPath)) {
@@ -69,61 +70,42 @@ bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, i
 
 bool llmodel_isModelLoaded(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->isModelLoaded();
 }
 
 uint64_t llmodel_get_state_size(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->stateSize();
 }
 
 uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->saveState(dest);
 }
 
 uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->restoreState(src);
 }
 
-// Wrapper functions for the C callbacks
-bool prompt_wrapper(int32_t token_id, void *user_data) {
-    llmodel_prompt_callback callback = reinterpret_cast<llmodel_prompt_callback>(user_data);
-    return callback(token_id);
-}
-
-bool response_wrapper(int32_t token_id, const std::string &response, void *user_data) {
-    llmodel_response_callback callback = reinterpret_cast<llmodel_response_callback>(user_data);
-    return callback(token_id, response.c_str());
-}
-
-bool recalculate_wrapper(bool is_recalculating, void *user_data) {
-    llmodel_recalculate_callback callback = reinterpret_cast<llmodel_recalculate_callback>(user_data);
-    return callback(is_recalculating);
-}
-
 void llmodel_prompt(llmodel_model model, const char *prompt,
                     const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
                     llmodel_prompt_context *ctx,
-                    bool special)
+                    bool special,
+                    const char *fake_reply)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
 
-    // Create std::function wrappers that call the C function pointers
-    std::function<bool(int32_t)> prompt_func =
-        std::bind(&prompt_wrapper, std::placeholders::_1, reinterpret_cast<void*>(prompt_callback));
-    std::function<bool(int32_t, const std::string&)> response_func =
-        std::bind(&response_wrapper, std::placeholders::_1, std::placeholders::_2, reinterpret_cast<void*>(response_callback));
-    std::function<bool(bool)> recalc_func =
-        std::bind(&recalculate_wrapper, std::placeholders::_1, reinterpret_cast<void*>(recalculate_callback));
+    auto response_func = [response_callback](int32_t token_id, const std::string &response) {
+        return response_callback(token_id, response.c_str());
+    };
 
     if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
         wrapper->promptContext.tokens.resize(ctx->n_past);
@@ -141,8 +123,13 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
     wrapper->promptContext.contextErase = ctx->context_erase;
 
+    std::string fake_reply_str;
+    if (fake_reply) { fake_reply_str = fake_reply; }
+    auto *fake_reply_p = fake_reply ? &fake_reply_str : nullptr;
+
     // Call the C++ prompt method
-    wrapper->llModel->prompt(prompt, prompt_template, prompt_func, response_func, recalc_func, wrapper->promptContext, special);
+    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, recalculate_callback,
+                             wrapper->promptContext, special, fake_reply_p);
 
     // Update the C context by giving access to the wrappers raw pointers to std::vector data
     // which involves no copies
@@ -165,38 +152,58 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->context_erase = wrapper->promptContext.contextErase;
 }
 
-float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size)
-{
-    if (model == nullptr || text == nullptr || !strlen(text)) {
-        *embedding_size = 0;
+float *llmodel_embed(
+    llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix, int dimensionality,
+    size_t *token_count, bool do_mean, bool atlas, llmodel_emb_cancel_callback cancel_cb, const char **error
+) {
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+
+    if (!texts || !*texts) {
+        llmodel_set_error(error, "'texts' is NULL or empty");
         return nullptr;
     }
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    std::vector<float> embeddingVector = wrapper->llModel->embedding(text);
-    float *embedding = (float *)malloc(embeddingVector.size() * sizeof(float));
-    if (embedding == nullptr) {
-        *embedding_size = 0;
+
+    std::vector<std::string> textsVec;
+    while (*texts) { textsVec.emplace_back(*texts++); }
+
+    size_t embd_size;
+    float *embedding;
+
+    try {
+        embd_size = wrapper->llModel->embeddingSize();
+        if (dimensionality > 0 && dimensionality < int(embd_size))
+            embd_size = dimensionality;
+
+        embd_size *= textsVec.size();
+
+        std::optional<std::string> prefixStr;
+        if (prefix) { prefixStr = prefix; }
+
+        embedding = new float[embd_size];
+        wrapper->llModel->embed(textsVec, embedding, prefixStr, dimensionality, token_count, do_mean, atlas, cancel_cb);
+    } catch (std::exception const &e) {
+        llmodel_set_error(error, e.what());
         return nullptr;
     }
-    std::copy(embeddingVector.begin(), embeddingVector.end(), embedding);
-    *embedding_size = embeddingVector.size();
+
+    *embedding_size = embd_size;
     return embedding;
 }
 
 void llmodel_free_embedding(float *ptr)
 {
-    free(ptr);
+    delete[] ptr;
 }
 
 void llmodel_setThreadCount(llmodel_model model, int32_t n_threads)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     wrapper->llModel->setThreadCount(n_threads);
 }
 
 int32_t llmodel_threadCount(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->threadCount();
 }
 
@@ -210,50 +217,79 @@ const char *llmodel_get_implementation_search_path()
     return LLModel::Implementation::implementationsSearchPath().c_str();
 }
 
-struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices)
-{
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    std::vector<LLModel::GPUDevice> devices = wrapper->llModel->availableGPUDevices(memoryRequired);
+// RAII wrapper around a C-style struct
+struct llmodel_gpu_device_cpp: llmodel_gpu_device {
+    llmodel_gpu_device_cpp() = default;
 
-    // Set the num_devices
+    llmodel_gpu_device_cpp(const llmodel_gpu_device_cpp  &) = delete;
+    llmodel_gpu_device_cpp(      llmodel_gpu_device_cpp &&) = delete;
+
+    const llmodel_gpu_device_cpp &operator=(const llmodel_gpu_device_cpp  &) = delete;
+          llmodel_gpu_device_cpp &operator=(      llmodel_gpu_device_cpp &&) = delete;
+
+    ~llmodel_gpu_device_cpp() {
+        free(const_cast<char *>(name));
+        free(const_cast<char *>(vendor));
+    }
+};
+
+static_assert(sizeof(llmodel_gpu_device_cpp) == sizeof(llmodel_gpu_device));
+
+struct llmodel_gpu_device *llmodel_available_gpu_devices(size_t memoryRequired, int *num_devices)
+{
+    static thread_local std::unique_ptr<llmodel_gpu_device_cpp[]> c_devices;
+
+    auto devices = LLModel::Implementation::availableGPUDevices(memoryRequired);
     *num_devices = devices.size();
 
-    if (*num_devices == 0) return nullptr;  // Return nullptr if no devices are found
+    if (devices.empty()) { return nullptr; /* no devices */ }
 
-    // Allocate memory for the output array
-    struct llmodel_gpu_device* output = (struct llmodel_gpu_device*) malloc(*num_devices * sizeof(struct llmodel_gpu_device));
-
-    for (int i = 0; i < *num_devices; i++) {
-        output[i].index = devices[i].index;
-        output[i].type = devices[i].type;
-        output[i].heapSize = devices[i].heapSize;
-        output[i].name = strdup(devices[i].name.c_str());  // Convert std::string to char* and allocate memory
-        output[i].vendor = strdup(devices[i].vendor.c_str());  // Convert std::string to char* and allocate memory
+    c_devices = std::make_unique<llmodel_gpu_device_cpp[]>(devices.size());
+    for (unsigned i = 0; i < devices.size(); i++) {
+        const auto &dev  =   devices[i];
+              auto &cdev = c_devices[i];
+        cdev.index    = dev.index;
+        cdev.type     = dev.type;
+        cdev.heapSize = dev.heapSize;
+        cdev.name     = strdup(dev.name.c_str());
+        cdev.vendor   = strdup(dev.vendor.c_str());
     }
 
-    return output;
+    return c_devices.get();
 }
 
 bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->initializeGPUDevice(memoryRequired, std::string(device));
 }
 
 bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->initializeGPUDevice(device->index);
 }
 
 bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->initializeGPUDevice(device);
 }
 
 bool llmodel_has_gpu_device(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->hasGPUDevice();
 }
+
+const char *llmodel_model_backend_name(llmodel_model model)
+{
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->backendName();
+}
+
+const char *llmodel_model_gpu_device_name(llmodel_model model)
+{
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->gpuDeviceName();
+}

gpt4all-backend/llmodel_c.h

@@ -48,9 +48,9 @@ struct llmodel_prompt_context {
 };
 
 struct llmodel_gpu_device {
-    int index = 0;
-    int type = 0;           // same as VkPhysicalDeviceType
-    size_t heapSize = 0;
+    int index;
+    int type; // same as VkPhysicalDeviceType
+    size_t heapSize;
     const char * name;
     const char * vendor;
 };
@@ -82,6 +82,15 @@ typedef bool (*llmodel_response_callback)(int32_t token_id, const char *response
  */
 typedef bool (*llmodel_recalculate_callback)(bool is_recalculating);
 
+/**
+ * Embedding cancellation callback for use with llmodel_embed.
+ * @param batch_sizes The number of tokens in each batch that will be embedded.
+ * @param n_batch The number of batches that will be embedded.
+ * @param backend The backend that will be used for embedding. One of "cpu", "kompute", or "metal".
+ * @return True to cancel llmodel_embed, false to continue.
+ */
+typedef bool (*llmodel_emb_cancel_callback)(unsigned *batch_sizes, unsigned n_batch, const char *backend);
+
 /**
  * Create a llmodel instance.
  * Recognises correct model type from file at model_path
@@ -169,6 +178,7 @@ uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src);
  * @param response_callback A callback function for handling the generated response.
  * @param recalculate_callback A callback function for handling recalculation requests.
  * @param special True if special tokens in the prompt should be processed, false otherwise.
+ * @param fake_reply A string to insert into context as the model's reply, or NULL to generate one.
  * @param ctx A pointer to the llmodel_prompt_context structure.
  */
 void llmodel_prompt(llmodel_model model, const char *prompt,
@@ -177,20 +187,34 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
                     llmodel_prompt_context *ctx,
-                    bool special);
+                    bool special,
+                    const char *fake_reply);
 
 /**
  * Generate an embedding using the model.
  * NOTE: If given NULL pointers for the model or text, or an empty text, a NULL pointer will be
  * returned. Bindings should signal an error when NULL is the return value.
  * @param model A pointer to the llmodel_model instance.
- * @param text A string representing the text to generate an embedding for.
+ * @param texts A pointer to a NULL-terminated array of strings representing the texts to generate an
+ * embedding for.
  * @param embedding_size A pointer to a size_t type that will be set by the call indicating the length
  * of the returned floating point array.
+ * @param prefix The model-specific prefix representing the embedding task, without the trailing colon. NULL for no
+ * prefix.
+ * @param dimensionality The embedding dimension, for use with Matryoshka-capable models. Set to -1 to for full-size.
+ * @param token_count Return location for the number of prompt tokens processed, or NULL.
+ * @param do_mean True to average multiple embeddings if the text is longer than the model can accept, False to
+ * truncate.
+ * @param atlas Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens with
+ * long_text_mode="mean" will raise an error. Disabled by default.
+ * @param cancel_cb Cancellation callback, or NULL. See the documentation of llmodel_emb_cancel_callback.
+ * @param error Return location for a malloc()ed string that will be set on error, or NULL.
  * @return A pointer to an array of floating point values passed to the calling method which then will
- * be responsible for lifetime of this memory.
+ * be responsible for lifetime of this memory. NULL if an error occurred.
  */
-float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size);
+float *llmodel_embed(llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix,
+                     int dimensionality, size_t *token_count, bool do_mean, bool atlas,
+                     llmodel_emb_cancel_callback cancel_cb, const char **error);
 
 /**
  * Frees the memory allocated by the llmodel_embedding function.
@@ -228,9 +252,10 @@ const char *llmodel_get_implementation_search_path();
 
 /**
  * Get a list of available GPU devices given the memory required.
+ * @param memoryRequired The minimum amount of VRAM, in bytes
  * @return A pointer to an array of llmodel_gpu_device's whose number is given by num_devices.
  */
-struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices);
+struct llmodel_gpu_device* llmodel_available_gpu_devices(size_t memoryRequired, int* num_devices);
 
 /**
  * Initializes a GPU device based on a specified string criterion.
@@ -270,6 +295,16 @@ bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
  */
 bool llmodel_has_gpu_device(llmodel_model model);
 
+/**
+ * @return The name of the llama.cpp backend currently in use. One of "cpu", "kompute", or "metal".
+ */
+const char *llmodel_model_backend_name(llmodel_model model);
+
+/**
+ * @return The name of the GPU device currently in use, or NULL for backends other than Kompute.
+ */
+const char *llmodel_model_gpu_device_name(llmodel_model model);
+
 #ifdef __cplusplus
 }
 #endif

gpt4all-backend/llmodel_shared.cpp

@@ -3,6 +3,7 @@
 #include <cassert>
 #include <iostream>
 #include <regex>
+#include <string>
 #include <unordered_set>
 
 // TODO(cebtenzzre): replace this with llama_kv_cache_seq_shift for llamamodel (GPT-J needs this as-is)
@@ -267,12 +268,31 @@ void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)>
     }
 }
 
-std::vector<float> LLModel::embedding(const std::string &text)
-{
-    (void)text;
-    if (!supportsCompletion()) {
-        std::string errorMessage = "ERROR: this model does not support generating embeddings!\n";
-        std::cerr << implementation().modelType() << errorMessage;
-    }
-    return std::vector<float>();
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)prefix;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    (void)cancelCb;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+}
+
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
+    bool doMean, bool atlas
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)isRetrieval;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
 }

gpt4all-bindings/python/docs/gpt4all_chat.md

@@ -7,7 +7,7 @@ It is optimized to run 7-13B parameter LLMs on the CPU's of any computer running
 ## Running LLMs on CPU
 The GPT4All Chat UI supports models from all newer versions of `llama.cpp` with `GGUF` models including the `Mistral`, `LLaMA2`, `LLaMA`, `OpenLLaMa`, `Falcon`, `MPT`, `Replit`, `Starcoder`, and `Bert` architectures
 
-GPT4All maintains an official list of recommended models located in [models2.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
+GPT4All maintains an official list of recommended models located in [models3.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models3.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
 
 #### Sideloading any GGUF model
 If a model is compatible with the gpt4all-backend, you can sideload it into GPT4All Chat by:

gpt4all-bindings/python/docs/gpt4all_cli.md

@@ -5,7 +5,7 @@ The GPT4All command-line interface (CLI) is a Python script which is built on to
 package. The source code, README, and local build instructions can be found
 [here][repo-bindings-cli].
 
-[docs-bindings-python]: gpt4all_python.html
+[docs-bindings-python]: gpt4all_python.md
 [repo-bindings-python]: https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python
 [repo-bindings-cli]: https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/cli
 [typer]: https://typer.tiangolo.com/

gpt4all-bindings/python/docs/gpt4all_faq.md

@@ -61,12 +61,12 @@ or `allowDownload=true` (default), a model is automatically downloaded into `.ca
 unless it already exists.
 
 In case of connection issues or errors during the download, you might want to manually verify the model file's MD5
-checksum by comparing it with the one listed in [models2.json].
+checksum by comparing it with the one listed in [models3.json].
 
 As an alternative to the basic downloader built into the bindings, you can choose to download from the 
 <https://gpt4all.io/> website instead. Scroll down to 'Model Explorer' and pick your preferred model.
 
-[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
+[models3.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models3.json
 
 #### I need the chat GUI and bindings to behave the same
 
@@ -93,7 +93,7 @@ The chat GUI and bindings are based on the same backend. You can make them behav
 - Next you'll have to compare the templates, adjusting them as necessary, based on how you're using the bindings.
     - Specifically, in Python:
         - With simple `generate()` calls, the input has to be surrounded with system and prompt templates.
-        - When using a chat session, it depends on whether the bindings are allowed to download [models2.json]. If yes,
+        - When using a chat session, it depends on whether the bindings are allowed to download [models3.json]. If yes,
           and in the chat GUI the default templates are used, it'll be handled automatically. If no, use
           `chat_session()` template parameters to customize them.
 

gpt4all-bindings/python/docs/gpt4all_modal.md

@@ -1,34 +0,0 @@
-# GPT4All with Modal Labs
-
-You can easily query any GPT4All model on [Modal Labs](https://modal.com/) infrastructure!
-## Example
-
-```python
-import modal
-
-def download_model():
-    import gpt4all
-    #you can use any model from https://gpt4all.io/models/models2.json
-    return gpt4all.GPT4All("ggml-gpt4all-j-v1.3-groovy.bin")
-
-image=modal.Image.debian_slim().pip_install("gpt4all").run_function(download_model)
-stub = modal.Stub("gpt4all", image=image)
-@stub.cls(keep_warm=1)
-class GPT4All:
-    def __enter__(self):
-        print("Downloading model")
-        self.gptj = download_model()
-        print("Loaded model")
-
-    @modal.method()
-    def generate(self):
-        messages = [{"role": "user", "content": "Name 3 colors"}]
-        completion = self.gptj.chat_completion(messages)
-        print(f"Completion: {completion}")
-
-@stub.local_entrypoint()
-def main():
-    model = GPT4All()
-    for i in range(10):
-        model.generate.call()
-```

gpt4all-bindings/python/docs/gpt4all_nodejs.md

@@ -11,37 +11,116 @@ pnpm install gpt4all@latest
 
 ```
 
-The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
+## Contents
 
-*   New bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
-*   The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
-*   Everything should work out the box.
 *   See [API Reference](#api-reference)
+*   See [Examples](#api-example)
+*   See [Developing](#develop)
+*   GPT4ALL nodejs bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
+
+## Api Example
 
 ### Chat Completion
 
 ```js
-import { createCompletion, loadModel } from '../src/gpt4all.js'
+import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY, loadModel } from '../src/gpt4all.js'
 
-const model = await loadModel('mistral-7b-openorca.Q4_0.gguf', { verbose: true });
+const model = await loadModel( 'mistral-7b-openorca.gguf2.Q4_0.gguf', { verbose: true, device: 'gpu' });
 
-const response = await createCompletion(model, [
-    { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
-    { role : 'user', content: 'What is 1 + 1?'  } 
-]);
+const completion1 = await createCompletion(model, 'What is 1 + 1?', { verbose: true, })
+console.log(completion1.message)
 
+const completion2 = await createCompletion(model, 'And if we add two?', {  verbose: true  })
+console.log(completion2.message)
+
+model.dispose()
 ```
 
 ### Embedding
 
 ```js
-import { createEmbedding, loadModel } from '../src/gpt4all.js'
+import { loadModel, createEmbedding } from '../src/gpt4all.js'
 
-const model = await loadModel('ggml-all-MiniLM-L6-v2-f16', { verbose: true });
+const embedder = await loadModel("all-MiniLM-L6-v2-f16.gguf", { verbose: true, type: 'embedding'})
 
-const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional");
+console.log(createEmbedding(embedder, "Maybe Minecraft was the friends we made along the way"));
 ```
 
+### Chat Sessions
+
+```js
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession();
+
+await createCompletion(
+    chat,
+    "Why are bananas rather blue than bread at night sometimes?",
+    {
+        verbose: true,
+    }
+);
+await createCompletion(chat, "Are you sure?", { verbose: true, });
+
+```
+
+### Streaming responses
+
+```js
+import gpt from "../src/gpt4all.js";
+
+const model = await gpt.loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("### Stream:");
+const stream = gpt.createCompletionStream(model, "How are you?");
+stream.tokens.on("data", (data) => {
+    process.stdout.write(data);
+});
+//wait till stream finishes. We cannot continue until this one is done.
+await stream.result;
+process.stdout.write("\n");
+
+process.stdout.write("### Stream with pipe:");
+const stream2 = gpt.createCompletionStream(
+    model,
+    "Please say something nice about node streams."
+);
+stream2.tokens.pipe(process.stdout);
+await stream2.result;
+process.stdout.write("\n");
+
+console.log("done");
+model.dispose();
+```
+
+### Async Generators
+
+```js
+import gpt from "../src/gpt4all.js";
+
+const model = await gpt.loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("### Generator:");
+const gen = gpt.createCompletionGenerator(model, "Redstone in Minecraft is Turing Complete. Let that sink in. (let it in!)");
+for await (const chunk of gen) {
+    process.stdout.write(chunk);
+}
+
+process.stdout.write("\n");
+model.dispose();
+```
+
+## Develop
+
 ### Build Instructions
 
 *   binding.gyp is compile config
@@ -131,21 +210,27 @@ yarn test
 
 *   why your model may be spewing bull 💩
     *   The downloaded model is broken (just reinstall or download from official site)
-    *   That's it so far
+*   Your model is hanging after a call to generate tokens.
+    *   Is `nPast` set too high? This may cause your model to hang (03/16/2024), Linux Mint, Ubuntu 22.04
+*   Your GPU usage is still high after node.js exits.
+    *   Make sure to call `model.dispose()`!!!
 
 ### Roadmap
 
-This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
+This package has been stabilizing over time development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
 
+*   \[ ] Purely offline. Per the gui, which can be run completely offline, the bindings should be as well.
+*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
+    *   Should include prebuilds to avoid painful node-gyp errors
+*   \[x] createChatSession ( the python equivalent to create\_chat\_session )
+*   \[x] generateTokens, the new name for createTokenStream. As of 3.2.0, this is released but not 100% tested. Check spec/generator.mjs!
+*   \[x] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
 *   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
-*   \[ ] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
+*   \[x] generateTokens is the new name for this^
 *   \[x] proper unit testing (integrate with circle ci)
 *   \[x] publish to npm under alpha tag `gpt4all@alpha`
 *   \[x] have more people test on other platforms (mac tester needed)
 *   \[x] switch to new pluggable backend
-*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
-    *   Should include prebuilds to avoid painful node-gyp errors
-*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
 
 ### API Reference
 
@@ -153,144 +238,200 @@ This package is in active development, and breaking changes may happen until the
 
 ##### Table of Contents
 
-*   [ModelFile](#modelfile)
-    *   [gptj](#gptj)
-    *   [llama](#llama)
-    *   [mpt](#mpt)
-    *   [replit](#replit)
 *   [type](#type)
 *   [TokenCallback](#tokencallback)
+*   [ChatSessionOptions](#chatsessionoptions)
+    *   [systemPrompt](#systemprompt)
+    *   [messages](#messages)
+*   [initialize](#initialize)
+    *   [Parameters](#parameters)
+*   [generate](#generate)
+    *   [Parameters](#parameters-1)
 *   [InferenceModel](#inferencemodel)
+    *   [createChatSession](#createchatsession)
+        *   [Parameters](#parameters-2)
+    *   [generate](#generate-1)
+        *   [Parameters](#parameters-3)
     *   [dispose](#dispose)
 *   [EmbeddingModel](#embeddingmodel)
     *   [dispose](#dispose-1)
+*   [InferenceResult](#inferenceresult)
 *   [LLModel](#llmodel)
     *   [constructor](#constructor)
-        *   [Parameters](#parameters)
+        *   [Parameters](#parameters-4)
     *   [type](#type-1)
     *   [name](#name)
     *   [stateSize](#statesize)
     *   [threadCount](#threadcount)
     *   [setThreadCount](#setthreadcount)
-        *   [Parameters](#parameters-1)
-    *   [raw\_prompt](#raw_prompt)
-        *   [Parameters](#parameters-2)
+        *   [Parameters](#parameters-5)
+    *   [infer](#infer)
+        *   [Parameters](#parameters-6)
     *   [embed](#embed)
-        *   [Parameters](#parameters-3)
+        *   [Parameters](#parameters-7)
     *   [isModelLoaded](#ismodelloaded)
     *   [setLibraryPath](#setlibrarypath)
-        *   [Parameters](#parameters-4)
+        *   [Parameters](#parameters-8)
     *   [getLibraryPath](#getlibrarypath)
     *   [initGpuByString](#initgpubystring)
-        *   [Parameters](#parameters-5)
+        *   [Parameters](#parameters-9)
     *   [hasGpuDevice](#hasgpudevice)
     *   [listGpu](#listgpu)
-        *   [Parameters](#parameters-6)
+        *   [Parameters](#parameters-10)
     *   [dispose](#dispose-2)
 *   [GpuDevice](#gpudevice)
     *   [type](#type-2)
 *   [LoadModelOptions](#loadmodeloptions)
-*   [loadModel](#loadmodel)
-    *   [Parameters](#parameters-7)
-*   [createCompletion](#createcompletion)
-    *   [Parameters](#parameters-8)
-*   [createEmbedding](#createembedding)
-    *   [Parameters](#parameters-9)
-*   [CompletionOptions](#completionoptions)
+    *   [modelPath](#modelpath)
+    *   [librariesPath](#librariespath)
+    *   [modelConfigFile](#modelconfigfile)
+    *   [allowDownload](#allowdownload)
     *   [verbose](#verbose)
-    *   [systemPromptTemplate](#systemprompttemplate)
-    *   [promptTemplate](#prompttemplate)
-    *   [promptHeader](#promptheader)
-    *   [promptFooter](#promptfooter)
-*   [PromptMessage](#promptmessage)
+    *   [device](#device)
+    *   [nCtx](#nctx)
+    *   [ngl](#ngl)
+*   [loadModel](#loadmodel)
+    *   [Parameters](#parameters-11)
+*   [InferenceProvider](#inferenceprovider)
+*   [createCompletion](#createcompletion)
+    *   [Parameters](#parameters-12)
+*   [createCompletionStream](#createcompletionstream)
+    *   [Parameters](#parameters-13)
+*   [createCompletionGenerator](#createcompletiongenerator)
+    *   [Parameters](#parameters-14)
+*   [createEmbedding](#createembedding)
+    *   [Parameters](#parameters-15)
+*   [CompletionOptions](#completionoptions)
+    *   [verbose](#verbose-1)
+    *   [onToken](#ontoken)
+*   [Message](#message)
     *   [role](#role)
     *   [content](#content)
 *   [prompt\_tokens](#prompt_tokens)
 *   [completion\_tokens](#completion_tokens)
 *   [total\_tokens](#total_tokens)
+*   [n\_past\_tokens](#n_past_tokens)
 *   [CompletionReturn](#completionreturn)
     *   [model](#model)
     *   [usage](#usage)
-    *   [choices](#choices)
-*   [CompletionChoice](#completionchoice)
-    *   [message](#message)
+    *   [message](#message-1)
+*   [CompletionStreamReturn](#completionstreamreturn)
 *   [LLModelPromptContext](#llmodelpromptcontext)
     *   [logitsSize](#logitssize)
     *   [tokensSize](#tokenssize)
     *   [nPast](#npast)
-    *   [nCtx](#nctx)
     *   [nPredict](#npredict)
+    *   [promptTemplate](#prompttemplate)
+    *   [nCtx](#nctx-1)
     *   [topK](#topk)
     *   [topP](#topp)
-    *   [temp](#temp)
+    *   [minP](#minp)
+    *   [temperature](#temperature)
     *   [nBatch](#nbatch)
     *   [repeatPenalty](#repeatpenalty)
     *   [repeatLastN](#repeatlastn)
     *   [contextErase](#contexterase)
-*   [generateTokens](#generatetokens)
-    *   [Parameters](#parameters-10)
 *   [DEFAULT\_DIRECTORY](#default_directory)
 *   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
 *   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
 *   [DEFAULT\_PROMPT\_CONTEXT](#default_prompt_context)
 *   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
 *   [downloadModel](#downloadmodel)
-    *   [Parameters](#parameters-11)
+    *   [Parameters](#parameters-16)
     *   [Examples](#examples)
 *   [DownloadModelOptions](#downloadmodeloptions)
-    *   [modelPath](#modelpath)
-    *   [verbose](#verbose-1)
+    *   [modelPath](#modelpath-1)
+    *   [verbose](#verbose-2)
     *   [url](#url)
     *   [md5sum](#md5sum)
 *   [DownloadController](#downloadcontroller)
     *   [cancel](#cancel)
     *   [promise](#promise)
 
-#### ModelFile
-
-Full list of models available
-DEPRECATED!! These model names are outdated and this type will not be maintained, please use a string literal instead
-
-##### gptj
-
-List of GPT-J Models
-
-Type: (`"ggml-gpt4all-j-v1.3-groovy.bin"` | `"ggml-gpt4all-j-v1.2-jazzy.bin"` | `"ggml-gpt4all-j-v1.1-breezy.bin"` | `"ggml-gpt4all-j.bin"`)
-
-##### llama
-
-List Llama Models
-
-Type: (`"ggml-gpt4all-l13b-snoozy.bin"` | `"ggml-vicuna-7b-1.1-q4_2.bin"` | `"ggml-vicuna-13b-1.1-q4_2.bin"` | `"ggml-wizardLM-7B.q4_2.bin"` | `"ggml-stable-vicuna-13B.q4_2.bin"` | `"ggml-nous-gpt4-vicuna-13b.bin"` | `"ggml-v3-13b-hermes-q5_1.bin"`)
-
-##### mpt
-
-List of MPT Models
-
-Type: (`"ggml-mpt-7b-base.bin"` | `"ggml-mpt-7b-chat.bin"` | `"ggml-mpt-7b-instruct.bin"`)
-
-##### replit
-
-List of Replit Models
-
-Type: `"ggml-replit-code-v1-3b.bin"`
-
 #### type
 
 Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
 
-Type: ModelType
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
 
 #### TokenCallback
 
-Callback for controlling token generation
+Callback for controlling token generation. Return false to stop token generation.
 
 Type: function (tokenId: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), token: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String), total: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
 
+#### ChatSessionOptions
+
+**Extends Partial\<LLModelPromptContext>**
+
+Options for the chat session.
+
+##### systemPrompt
+
+System prompt to ingest on initialization.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### messages
+
+Messages to ingest on initialization.
+
+Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[Message](#message)>
+
+#### initialize
+
+Ingests system prompt and initial messages.
+Sets this chat session as the active chat session of the model.
+
+##### Parameters
+
+*   `options` **[ChatSessionOptions](#chatsessionoptions)** The options for the chat session.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<void>**&#x20;
+
+#### generate
+
+Prompts the model in chat-session context.
+
+##### Parameters
+
+*   `prompt` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
+*   `options` **[CompletionOptions](#completionoptions)?** Prompt context and other options.
+*   `callback` **[TokenCallback](#tokencallback)?** Token generation callback.
+
+<!---->
+
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the chat session is not the active chat session of the model.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[CompletionReturn](#completionreturn)>** The model's response to the prompt.
+
 #### InferenceModel
 
 InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
 
+##### createChatSession
+
+Create a chat session with the model.
+
+###### Parameters
+
+*   `options` **[ChatSessionOptions](#chatsessionoptions)?** The options for the chat session.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<ChatSession>** The chat session.
+
+##### generate
+
+Prompts the model with a given input and optional parameters.
+
+###### Parameters
+
+*   `prompt` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+*   `options` **[CompletionOptions](#completionoptions)?** Prompt context and other options.
+*   `callback` **[TokenCallback](#tokencallback)?** Token generation callback.
+*   `input`  The prompt input.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[CompletionReturn](#completionreturn)>** The model's response to the prompt.
+
 ##### dispose
 
 delete and cleanup the native model
@@ -307,6 +448,10 @@ delete and cleanup the native model
 
 Returns **void**&#x20;
 
+#### InferenceResult
+
+Shape of LLModel's inference result.
+
 #### LLModel
 
 LLModel class representing a language model.
@@ -326,9 +471,9 @@ Initialize a new LLModel.
 
 ##### type
 
-either 'gpt', mpt', or 'llama' or undefined
+undefined or user supplied
 
-Returns **(ModelType | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
+Returns **([string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String) | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
 
 ##### name
 
@@ -360,7 +505,7 @@ Set the number of threads used for model inference.
 
 Returns **void**&#x20;
 
-##### raw\_prompt
+##### infer
 
 Prompt the model with a given input and optional parameters.
 This is the raw output from model.
@@ -368,23 +513,20 @@ Use the prompt function exported for a value
 
 ###### Parameters
 
-*   `q` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
-*   `params` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
+*   `prompt` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
+*   `promptContext` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
 *   `callback` **[TokenCallback](#tokencallback)?** optional callback to control token generation.
 
-Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The result of the model prompt.
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[InferenceResult](#inferenceresult)>** The result of the model prompt.
 
 ##### embed
 
 Embed text with the model. Keep in mind that
-not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
 Use the prompt function exported for a value
 
 ###### Parameters
 
-*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-*   `q`  The prompt input.
-*   `params`  Optional parameters for the prompt context.
+*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
 
 Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The result of the model prompt.
 
@@ -462,6 +604,62 @@ Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 Options that configure a model's behavior.
 
+##### modelPath
+
+Where to look for model files.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### librariesPath
+
+Where to look for the backend libraries.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### modelConfigFile
+
+The path to the model configuration file, useful for offline usage or custom model configurations.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### allowDownload
+
+Whether to allow downloading the model if it is not present at the specified path.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### verbose
+
+Enable verbose logging.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### device
+
+The processing unit on which the model will run. It can be set to
+
+*   "cpu": Model will run on the central processing unit.
+*   "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
+*   "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
+*   "gpu name": Model will run on the GPU that matches the name if it's available.
+    Note: If a GPU device lacks sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All
+    instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the
+    model.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### nCtx
+
+The Maximum window size of this model
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### ngl
+
+Number of gpu layers needed
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
 #### loadModel
 
 Loads a machine learning model with the specified name. The defacto way to create a model.
@@ -474,18 +672,46 @@ By default this will download a model from the official GPT4ALL website, if a mo
 
 Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<([InferenceModel](#inferencemodel) | [EmbeddingModel](#embeddingmodel))>** A promise that resolves to an instance of the loaded LLModel.
 
+#### InferenceProvider
+
+Interface for inference, implemented by InferenceModel and ChatSession.
+
 #### createCompletion
 
 The nodejs equivalent to python binding's chat\_completion
 
 ##### Parameters
 
-*   `model` **[InferenceModel](#inferencemodel)** The language model object.
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
+*   `provider` **[InferenceProvider](#inferenceprovider)** The inference model object or chat session
+*   `message` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The user input message
 *   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
 
 Returns **[CompletionReturn](#completionreturn)** The completion result.
 
+#### createCompletionStream
+
+Streaming variant of createCompletion, returns a stream of tokens and a promise that resolves to the completion result.
+
+##### Parameters
+
+*   `provider` **[InferenceProvider](#inferenceprovider)** The inference model object or chat session
+*   `message` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The user input message.
+*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
+
+Returns **[CompletionStreamReturn](#completionstreamreturn)** An object of token stream and the completion result promise.
+
+#### createCompletionGenerator
+
+Creates an async generator of tokens
+
+##### Parameters
+
+*   `provider` **[InferenceProvider](#inferenceprovider)** The inference model object or chat session
+*   `message` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The user input message.
+*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
+
+Returns **AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The stream of generated tokens
+
 #### createEmbedding
 
 The nodejs moral equivalent to python binding's Embed4All().embed()
@@ -510,34 +736,15 @@ Indicates if verbose logging is enabled.
 
 Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
 
-##### systemPromptTemplate
+##### onToken
 
-Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
-Note that if this is not defined, system messages will not be included in the prompt.
+Callback for controlling token generation. Return false to stop processing.
 
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+Type: [TokenCallback](#tokencallback)
 
-##### promptTemplate
+#### Message
 
-Template for user messages, with %1 being replaced by the message.
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### promptHeader
-
-The initial instruction for the model, on top of the prompt
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### promptFooter
-
-The last instruction for the model, appended to the end of the prompt.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### PromptMessage
-
-A message in the conversation, identical to OpenAI's chat message.
+A message in the conversation.
 
 ##### role
 
@@ -553,7 +760,7 @@ Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 #### prompt\_tokens
 
-The number of tokens used in the prompt.
+The number of tokens used in the prompt. Currently not available and always 0.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
@@ -565,13 +772,19 @@ Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 #### total\_tokens
 
-The total number of tokens used.
+The total number of tokens used. Currently not available and always 0.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### n\_past\_tokens
+
+Number of tokens used in the conversation.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
 #### CompletionReturn
 
-The result of the completion, similar to OpenAI's format.
+The result of a completion.
 
 ##### model
 
@@ -583,23 +796,17 @@ Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 Token usage report.
 
-Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
-
-##### choices
-
-The generated completions.
-
-Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[CompletionChoice](#completionchoice)>
-
-#### CompletionChoice
-
-A completion choice, similar to OpenAI's format.
+Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), n\_past\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
 
 ##### message
 
-Response message
+The generated completion.
 
-Type: [PromptMessage](#promptmessage)
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### CompletionStreamReturn
+
+The result of a streamed completion, containing a stream of tokens and a promise that resolves to the completion result.
 
 #### LLModelPromptContext
 
@@ -620,18 +827,29 @@ Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 ##### nPast
 
 The number of tokens in the past conversation.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nCtx
-
-The number of tokens possible in the context window.
+This controls how far back the model looks when generating completions.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
 ##### nPredict
 
-The number of tokens to predict.
+The maximum number of tokens to predict.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### promptTemplate
+
+Template for user / assistant message pairs.
+%1 is required and will be replaced by the user input.
+%2 is optional and will be replaced by the assistant response.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### nCtx
+
+The context window size. Do not use, it has no effect. See loadModel options.
+THIS IS DEPRECATED!!!
+Use loadModel's nCtx option instead.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
@@ -654,12 +872,16 @@ above a threshold P. This method, also known as nucleus sampling, finds a balanc
 and quality by considering both token probabilities and the number of tokens available for sampling.
 When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
 On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
-The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
-for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
-##### temp
+##### minP
+
+The minimum probability of a token to be considered.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### temperature
 
 The temperature to adjust the model's output distribution.
 Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
@@ -704,19 +926,6 @@ The percentage of context to erase if the context window is exceeded.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
-#### generateTokens
-
-Creates an async generator of tokens
-
-##### Parameters
-
-*   `llmodel` **[InferenceModel](#inferencemodel)** The language model object.
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
-*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
-*   `callback` **[TokenCallback](#tokencallback)** optional callback to control token generation.
-
-Returns **AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The stream of generated tokens
-
 #### DEFAULT\_DIRECTORY
 
 From python api:
@@ -759,7 +968,7 @@ By default this downloads without waiting. use the controller returned to alter
 ##### Parameters
 
 *   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The model to be downloaded.
-*   `options` **DownloadOptions** to pass into the downloader. Default is { location: (cwd), verbose: false }.
+*   `options` **[DownloadModelOptions](#downloadmodeloptions)** to pass into the downloader. Default is { location: (cwd), verbose: false }.
 
 ##### Examples
 

gpt4all-bindings/python/docs/gpt4all_python.md

@@ -8,30 +8,22 @@ The source code and local build instructions can be found [here](https://github.
 pip install gpt4all
 ```
 
-=== "GPT4All Example"
-    ``` py
-    from gpt4all import GPT4All
-    model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
-    output = model.generate("The capital of France is ", max_tokens=3)
-    print(output)
-    ```
-=== "Output"
-    ```
-    1. Paris
-    ```
+``` py
+from gpt4all import GPT4All
+model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
+```
 
 This will:
 
 - Instantiate `GPT4All`,  which is the primary public API to your large language model (LLM).
 - Automatically download the given model to `~/.cache/gpt4all/` if not already present.
-- Through `model.generate(...)` the model starts working on a response. There are various ways to
-  steer that process. Here, `max_tokens` sets an upper limit, i.e. a hard cut-off point to the output.
+
+Read further to see how to chat with this model.
 
 
 ### Chatting with GPT4All
-Local LLMs can be optimized for chat conversations by reusing previous computational history.
-
-Use the GPT4All `chat_session` context manager to hold chat conversations with the model.
+To start chatting with a local LLM, you will need to start a chat session. Within a chat session, the model will be
+prompted with the appropriate template, and history will be preserved between successive calls to `generate()`.
 
 === "GPT4All Example"
     ``` py
@@ -72,15 +64,19 @@ Use the GPT4All `chat_session` context manager to hold chat conversations with t
     ]
     ```
 
-When using GPT4All models in the `chat_session` context:
+When using GPT4All models in the `chat_session()` context:
 
 - Consecutive chat exchanges are taken into account and not discarded until the session ends; as long as the model has capacity.
-- Internal K/V caches are preserved from previous conversation history, speeding up inference.
-- The model is given a system and prompt template which make it chatty. Depending on `allow_download=True` (default),
-  it will obtain the latest version of [models2.json] from the repository, which contains specifically tailored templates
-  for models. Conversely, if it is not allowed to download, it falls back to default templates instead.
+- A system prompt is inserted into the beginning of the model's context.
+- Each prompt passed to `generate()` is wrapped in the appropriate prompt template. If you pass `allow_download=False`
+  to GPT4All or are using a model that is not from the official models list, you must pass a prompt template using the
+  `prompt_template` parameter of `chat_session()`.
 
-[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
+NOTE: If you do not use `chat_session()`, calls to `generate()` will not be wrapped in a prompt template. This will
+cause the model to *continue* the prompt instead of *answering* it. When in doubt, use a chat session, as many newer
+models are designed to be used exclusively with a prompt template.
+
+[models3.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models3.json
 
 
 ### Streaming Generations
@@ -91,13 +87,14 @@ To interact with GPT4All responses as the model generates, use the `streaming=Tr
     from gpt4all import GPT4All
     model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
     tokens = []
-    for token in model.generate("The capital of France is", max_tokens=20, streaming=True):
-        tokens.append(token)
+    with model.chat_session():
+        for token in model.generate("What is the capital of France?", streaming=True):
+            tokens.append(token)
     print(tokens)
     ```
 === "Output"
     ```
-    [' Paris', ' is', ' a', ' city', ' that', ' has', ' been', ' a', ' major', ' cultural', ' and', ' economic', ' center', ' for', ' over', ' ', '2', ',', '0', '0']
+    [' The', ' capital', ' of', ' France', ' is', ' Paris', '.']
     ```
 
 
@@ -131,20 +128,11 @@ generation; be sure to review all their descriptions.
 The model folder can be set with the `model_path` parameter when creating a `GPT4All` instance. The example below is
 is the same as if it weren't provided; that is, `~/.cache/gpt4all/` is the default folder.
 
-=== "GPT4All Model Folder Example"
-    ``` py
-    from pathlib import Path
-    from gpt4all import GPT4All
-    model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf',
-                    model_path=(Path.home() / '.cache' / 'gpt4all'),
-                    allow_download=False)
-    response = model.generate('my favorite 3 fruits are:', temp=0)
-    print(response)
-    ```
-=== "Output"
-    ```
-    My favorite three fruits are apples, bananas and oranges.
-    ```
+``` py
+from pathlib import Path
+from gpt4all import GPT4All
+model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf', model_path=Path.home() / '.cache' / 'gpt4all')
+```
 
 If you want to point it at the chat GUI's default folder, it should be:
 === "macOS"
@@ -179,22 +167,20 @@ Alternatively, you could also change the module's default model directory:
 
 ``` py
 from pathlib import Path
-import gpt4all.gpt4all
-gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = Path.home() / 'my' / 'models-directory'
-from gpt4all import GPT4All
+from gpt4all import GPT4All, gpt4all
+gpt4all.DEFAULT_MODEL_DIRECTORY = Path.home() / 'my' / 'models-directory'
 model = GPT4All('orca-mini-3b-gguf2-q4_0.gguf')
-...
 ```
 
 
 ### Managing Templates
-Session templates can be customized when starting a `chat_session` context:
+When using a `chat_session()`, you may customize the system prompt, and set the prompt template if necessary:
 
 === "GPT4All Custom Session Templates Example"
     ``` py
     from gpt4all import GPT4All
     model = GPT4All('wizardlm-13b-v1.2.Q4_0.gguf')
-    system_template = 'A chat between a curious user and an artificial intelligence assistant.'
+    system_template = 'A chat between a curious user and an artificial intelligence assistant.\n'
     # many models use triple hash '###' for keywords, Vicunas are simpler:
     prompt_template = 'USER: {0}\nASSISTANT: '
     with model.chat_session(system_template, prompt_template):
@@ -218,111 +204,38 @@ Session templates can be customized when starting a `chat_session` context:
     particles, making the sky appear blue to our eyes.
     ```
 
-To do the same outside a session, the input has to be formatted manually. For example:
-
-=== "GPT4All Templates Outside a Session Example"
-    ``` py
-    model = GPT4All('wizardlm-13b-v1.2.Q4_0.gguf')
-    system_template = 'A chat between a curious user and an artificial intelligence assistant.'
-    prompt_template = 'USER: {0}\nASSISTANT: '
-    prompts = ['name 3 colors', 'now name 3 fruits', 'what were the 3 colors in your earlier response?']
-    first_input = system_template + prompt_template.format(prompts[0])
-    response = model.generate(first_input, temp=0)
-    print(response)
-    for prompt in prompts[1:]:
-        response = model.generate(prompt_template.format(prompt), temp=0)
-        print(response)
-    ```
-=== "Output"
-    ```
-    1) Red
-    2) Blue
-    3) Green
-
-    1. Apple
-    2. Banana
-    3. Orange
-
-    The colors in my previous response are blue, green and red.
-    ```
-
-
-### Introspection
-A less apparent feature is the capacity to log the final prompt that gets sent to the model. It relies on
-[Python's logging facilities][py-logging] implemented in the `pyllmodel` module at the `INFO` level. You can activate it
-for example with a `basicConfig`, which displays it on the standard error stream. It's worth mentioning that Python's
-logging infrastructure offers [many more customization options][py-logging-cookbook].
-
-[py-logging]: https://docs.python.org/3/howto/logging.html
-[py-logging-cookbook]: https://docs.python.org/3/howto/logging-cookbook.html
-
-=== "GPT4All Prompt Logging Example"
-    ``` py
-    import logging
-    from gpt4all import GPT4All
-    logging.basicConfig(level=logging.INFO)
-    model = GPT4All('nous-hermes-llama2-13b.Q4_0.gguf')
-    with model.chat_session('You are a geography expert.\nBe terse.',
-                            '### Instruction:\n{0}\n\n### Response:\n'):
-        response = model.generate('who are you?', temp=0)
-        print(response)
-        response = model.generate('what are your favorite 3 mountains?', temp=0)
-        print(response)
-    ```
-=== "Output"
-    ```
-    INFO:gpt4all.pyllmodel:LLModel.prompt_model -- prompt:
-    You are a geography expert.
-    Be terse.
-
-    ### Instruction:
-    who are you?
-
-    ### Response:
-
-    ===/LLModel.prompt_model -- prompt/===
-    I am an AI-powered chatbot designed to assist users with their queries related to geographical information.
-    INFO:gpt4all.pyllmodel:LLModel.prompt_model -- prompt:
-    ### Instruction:
-    what are your favorite 3 mountains?
-
-    ### Response:
-
-    ===/LLModel.prompt_model -- prompt/===
-    1) Mount Everest - Located in the Himalayas, it is the highest mountain on Earth and a significant challenge for mountaineers.
-    2) Kangchenjunga - This mountain is located in the Himalayas and is the third-highest peak in the world after Mount Everest and K2.
-    3) Lhotse - Located in the Himalayas, it is the fourth highest mountain on Earth and offers a challenging climb for experienced mountaineers.
-    ```
-
 
 ### Without Online Connectivity
-To prevent GPT4All from accessing online resources, instantiate it with `allow_download=False`. This will disable both
-downloading missing models and [models2.json], which contains information about them. As a result, predefined templates
-are used instead of model-specific system and prompt templates:
+To prevent GPT4All from accessing online resources, instantiate it with `allow_download=False`. When using this flag,
+there will be no default system prompt by default, and you must specify the prompt template yourself.
 
-=== "GPT4All Default Templates Example"
+You can retrieve a model's default system prompt and prompt template with an online instance of GPT4All:
+
+=== "Prompt Template Retrieval"
     ``` py
     from gpt4all import GPT4All
-    model = GPT4All('ggml-mpt-7b-chat.bin', allow_download=False)
-    # when downloads are disabled, it will use the default templates:
-    print("default system template:", repr(model.config['systemPrompt']))
-    print("default prompt template:", repr(model.config['promptTemplate']))
-    print()
-    # even when inside a session:
-    with model.chat_session():
-        assert model.current_chat_session[0]['role'] == 'system'
-        print("session system template:", repr(model.current_chat_session[0]['content']))
-        print("session prompt template:", repr(model._current_prompt_template))
+    model = GPT4All('orca-mini-3b-gguf2-q4_0.gguf')
+    print(repr(model.config['systemPrompt']))
+    print(repr(model.config['promptTemplate']))
     ```
 === "Output"
-    ```
-    default system template: ''
-    default prompt template: '### Human: \n{0}\n\n### Assistant:\n'
-
-    session system template: ''
-    session prompt template: '### Human: \n{0}\n\n### Assistant:\n'
+    ```py
+    '### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n'
+    '### User:\n{0}\n### Response:\n'
     ```
 
+Then you can pass them explicitly when creating an offline instance:
+
+``` py
+from gpt4all import GPT4All
+model = GPT4All('orca-mini-3b-gguf2-q4_0.gguf', allow_download=False)
+
+system_prompt = '### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n'
+prompt_template = '### User:\n{0}\n\n### Response:\n'
+
+with model.chat_session(system_prompt=system_prompt, prompt_template=prompt_template):
+    ...
+```
 
 ### Interrupting Generation
 The simplest way to stop generation is to set a fixed upper limit with the `max_tokens` parameter.

gpt4all-bindings/python/docs/gpt4all_python_embedding.md

@@ -1,18 +1,41 @@
 # Embeddings
-GPT4All supports generating high quality embeddings of arbitrary length documents of text using a CPU optimized contrastively trained [Sentence Transformer](https://www.sbert.net/). These embeddings are comparable in quality for many tasks with OpenAI.
+GPT4All supports generating high quality embeddings of arbitrary length text using any embedding model supported by llama.cpp.
+
+An embedding is a vector representation of a piece of text. Embeddings are useful for tasks such as retrieval for
+question answering (including retrieval augmented generation or *RAG*), semantic similarity search, classification, and
+topic clustering.
+
+## Supported Embedding Models
+
+The following models have built-in support in Embed4All:
+
+| Name               | Embed4All `model_name`                               | Context Length | Embedding Length | File Size |
+|--------------------|------------------------------------------------------|---------------:|-----------------:|----------:|
+| [SBert]            | all‑MiniLM‑L6‑v2.gguf2.f16.gguf |            512 |              384 |    44 MiB |
+| [Nomic Embed v1]   | nomic‑embed‑text‑v1.f16.gguf    |           2048 |              768 |   262 MiB |
+| [Nomic Embed v1.5] | nomic‑embed‑text‑v1.5.f16.gguf  |           2048 |           64-768 |   262 MiB |
+
+The context length is the maximum number of word pieces, or *tokens*, that a model can embed at once. Embedding texts
+longer than a model's context length requires some kind of strategy; see [Embedding Longer Texts] for more information.
+
+The embedding length is the size of the vector returned by `Embed4All.embed`.
+
+[SBert]: https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2
+[Nomic Embed v1]: https://huggingface.co/nomic-ai/nomic-embed-text-v1
+[Nomic Embed v1.5]: https://huggingface.co/nomic-ai/nomic-embed-text-v1.5
+[Embedding Longer Texts]: #embedding-longer-texts
 
 ## Quickstart
-
 ```bash
 pip install gpt4all
 ```
 
-### Generating embeddings
-The embedding model will automatically be downloaded if not installed.
+### Generating Embeddings
+By default, embeddings will be generated on the CPU using all-MiniLM-L6-v2.
 
 === "Embed4All Example"
-    ``` py
-    from gpt4all import GPT4All, Embed4All
+    ```py
+    from gpt4all import Embed4All
     text = 'The quick brown fox jumps over the lazy dog'
     embedder = Embed4All()
     output = embedder.embed(text)
@@ -22,13 +45,131 @@ The embedding model will automatically be downloaded if not installed.
     ```
     [0.034696947783231735, -0.07192722707986832, 0.06923297047615051, ...]
     ```
-### Speed of embedding generation
-The following table lists the generation speed for text document captured on an Intel i913900HX CPU with DDR5 5600 running with 8 threads under stable load.
 
-| Tokens          | 128  | 512  | 2048 | 8129 | 16,384 |
-| --------------- | ---- | ---- | ---- | ---- | ---- |
-| Wall time (s)   | .02  | .08  | .24  | .96  | 1.9  |
-| Tokens / Second | 6508 | 6431 | 8622 | 8509 | 8369 |
+You can also use the GPU to accelerate the embedding model by specifying the `device` parameter. See the [GPT4All
+constructor] for more information.
+
+=== "GPU Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'The quick brown fox jumps over the lazy dog'
+    embedder = Embed4All(device='gpu')
+    output = embedder.embed(text)
+    print(output)
+    ```
+=== "Output"
+    ```
+    [0.034696947783231735, -0.07192722707986832, 0.06923297047615051, ...]
+    ```
+
+[GPT4All constructor]: gpt4all_python.md#gpt4all.gpt4all.GPT4All.__init__
+
+### Nomic Embed
+
+Embed4All has built-in support for Nomic's open-source embedding model, [Nomic Embed]. When using this model, you must
+specify the task type using the `prefix` argument. This may be one of `search_query`, `search_document`,
+`classification`, or `clustering`. For retrieval applications, you should prepend `search_document` for all of your
+documents and `search_query` for your queries. See the [Nomic Embedding Guide] for more info.
+
+=== "Nomic Embed Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'Who is Laurens van der Maaten?'
+    embedder = Embed4All('nomic-embed-text-v1.f16.gguf')
+    output = embedder.embed(text, prefix='search_query')
+    print(output)
+    ```
+=== "Output"
+    ```
+    [-0.013357644900679588, 0.027070969343185425, -0.0232995692640543, ...]
+    ```
+
+[Nomic Embed]: https://blog.nomic.ai/posts/nomic-embed-text-v1
+[Nomic Embedding Guide]: https://docs.nomic.ai/atlas/guides/embeddings#embedding-task-types
+
+### Embedding Longer Texts
+
+Embed4All accepts a parameter called `long_text_mode`. This controls the behavior of Embed4All for texts longer than the
+context length of the embedding model.
+
+In the default mode of "mean", Embed4All will break long inputs into chunks and average their embeddings to compute the
+final result.
+
+To change this behavior, you can set the `long_text_mode` parameter to "truncate", which will truncate the input to the
+sequence length of the model before generating a single embedding.
+
+=== "Truncation Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'The ' * 512 + 'The quick brown fox jumps over the lazy dog'
+    embedder = Embed4All()
+    output = embedder.embed(text, long_text_mode="mean")
+    print(output)
+    print()
+    output = embedder.embed(text, long_text_mode="truncate")
+    print(output)
+    ```
+=== "Output"
+    ```
+    [0.0039850445464253426, 0.04558328539133072, 0.0035536508075892925, ...]
+
+    [-0.009771130047738552, 0.034792833030223846, -0.013273917138576508, ...]
+    ```
+
+
+### Batching
+
+You can send multiple texts to Embed4All in a single call. This can give faster results when individual texts are
+significantly smaller than `n_ctx` tokens. (`n_ctx` defaults to 2048.)
+
+=== "Batching Example"
+    ```py
+    from gpt4all import Embed4All
+    texts = ['The quick brown fox jumps over the lazy dog', 'Foo bar baz']
+    embedder = Embed4All()
+    output = embedder.embed(texts)
+    print(output[0])
+    print()
+    print(output[1])
+    ```
+=== "Output"
+    ```
+    [0.03551332652568817, 0.06137588247656822, 0.05281158909201622, ...]
+
+    [-0.03879690542817116, 0.00013223080895841122, 0.023148687556385994, ...]
+    ```
+
+The number of texts that can be embedded in one pass of the model is proportional to the `n_ctx` parameter of Embed4All.
+Increasing it may increase batched embedding throughput if you have a fast GPU, at the cost of VRAM.
+```py
+embedder = Embed4All(n_ctx=4096, device='gpu')
+```
+
+
+### Resizable Dimensionality
+
+The embedding dimension of Nomic Embed v1.5 can be resized using the `dimensionality` parameter. This parameter supports
+any value between 64 and 768.
+
+Shorter embeddings use less storage, memory, and bandwidth with a small performance cost. See the [blog post] for more
+info.
+
+[blog post]: https://blog.nomic.ai/posts/nomic-embed-matryoshka
+
+=== "Matryoshka Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'The quick brown fox jumps over the lazy dog'
+    embedder = Embed4All('nomic-embed-text-v1.5.f16.gguf')
+    output = embedder.embed(text, dimensionality=64)
+    print(len(output))
+    print(output)
+    ```
+=== "Output"
+    ```
+    64
+    [-0.03567073494195938, 0.1301717758178711, -0.4333043396472931, ...]
+    ```
 
 
 ### API documentation

gpt4all-bindings/python/docs/index.md

@@ -38,7 +38,7 @@ The GPT4All software ecosystem is compatible with the following Transformer arch
 - `MPT` (including `Replit`)
 - `GPT-J`
 
-You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json)
+You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models3.json)
 
 
 GPT4All models are artifacts produced through a process known as neural network quantization.

gpt4all-bindings/python/gpt4all/__init__.py

@@ -1 +1 @@
-from .gpt4all import Embed4All as Embed4All, GPT4All as GPT4All
+from .gpt4all import CancellationError as CancellationError, Embed4All as Embed4All, GPT4All as GPT4All

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import ctypes
-import logging
 import os
 import platform
 import re
@@ -10,14 +9,23 @@ import sys
 import threading
 from enum import Enum
 from queue import Queue
-from typing import Callable, Iterable, List
+from typing import TYPE_CHECKING, Any, Callable, Generic, Iterable, Literal, NoReturn, TypeVar, overload
 
 if sys.version_info >= (3, 9):
     import importlib.resources as importlib_resources
 else:
     import importlib_resources
 
-logger: logging.Logger = logging.getLogger(__name__)
+if (3, 9) <= sys.version_info < (3, 11):
+    # python 3.9 broke generic TypedDict, python 3.11 fixed it
+    from typing_extensions import TypedDict
+else:
+    from typing import TypedDict
+
+if TYPE_CHECKING:
+    from typing_extensions import TypeAlias
+
+EmbeddingsType = TypeVar('EmbeddingsType', bound='list[Any]')
 
 
 # TODO: provide a config file to make this more robust
@@ -28,7 +36,7 @@ def load_llmodel_library():
     ext = {"Darwin": "dylib", "Linux": "so", "Windows": "dll"}[platform.system()]
 
     try:
-        # Linux, Windows, MinGW
+        # macOS, Linux, MinGW
         lib = ctypes.CDLL(str(MODEL_LIB_PATH / f"libllmodel.{ext}"))
     except FileNotFoundError:
         if ext != 'dll':
@@ -90,6 +98,7 @@ llmodel.llmodel_isModelLoaded.restype = ctypes.c_bool
 PromptCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32)
 ResponseCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32, ctypes.c_char_p)
 RecalculateCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_bool)
+EmbCancelCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.POINTER(ctypes.c_uint), ctypes.c_uint, ctypes.c_char_p)
 
 llmodel.llmodel_prompt.argtypes = [
     ctypes.c_void_p,
@@ -100,17 +109,25 @@ llmodel.llmodel_prompt.argtypes = [
     RecalculateCallback,
     ctypes.POINTER(LLModelPromptContext),
     ctypes.c_bool,
+    ctypes.c_char_p,
 ]
 
 llmodel.llmodel_prompt.restype = None
 
-llmodel.llmodel_embedding.argtypes = [
+llmodel.llmodel_embed.argtypes = [
     ctypes.c_void_p,
-    ctypes.c_char_p,
+    ctypes.POINTER(ctypes.c_char_p),
     ctypes.POINTER(ctypes.c_size_t),
+    ctypes.c_char_p,
+    ctypes.c_int,
+    ctypes.POINTER(ctypes.c_size_t),
+    ctypes.c_bool,
+    ctypes.c_bool,
+    EmbCancelCallback,
+    ctypes.POINTER(ctypes.c_char_p),
 ]
 
-llmodel.llmodel_embedding.restype = ctypes.POINTER(ctypes.c_float)
+llmodel.llmodel_embed.restype = ctypes.POINTER(ctypes.c_float)
 
 llmodel.llmodel_free_embedding.argtypes = [ctypes.POINTER(ctypes.c_float)]
 llmodel.llmodel_free_embedding.restype = None
@@ -124,9 +141,9 @@ llmodel.llmodel_set_implementation_search_path.restype = None
 llmodel.llmodel_threadCount.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_threadCount.restype = ctypes.c_int32
 
-llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).replace("\\", r"\\").encode())
+llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).encode())
 
-llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
+llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
 llmodel.llmodel_available_gpu_devices.restype = ctypes.POINTER(LLModelGPUDevice)
 
 llmodel.llmodel_gpu_init_gpu_device_by_string.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_char_p]
@@ -141,8 +158,15 @@ llmodel.llmodel_gpu_init_gpu_device_by_int.restype = ctypes.c_bool
 llmodel.llmodel_has_gpu_device.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_has_gpu_device.restype = ctypes.c_bool
 
+llmodel.llmodel_model_backend_name.argtypes = [ctypes.c_void_p]
+llmodel.llmodel_model_backend_name.restype = ctypes.c_char_p
+
+llmodel.llmodel_model_gpu_device_name.argtypes = [ctypes.c_void_p]
+llmodel.llmodel_model_gpu_device_name.restype = ctypes.c_char_p
+
 ResponseCallbackType = Callable[[int, str], bool]
 RawResponseCallbackType = Callable[[int, bytes], bool]
+EmbCancelCallbackType: TypeAlias = 'Callable[[list[int], str], bool]'
 
 
 def empty_response_callback(token_id: int, response: str) -> bool:
@@ -154,6 +178,15 @@ class Sentinel(Enum):
     TERMINATING_SYMBOL = 0
 
 
+class EmbedResult(Generic[EmbeddingsType], TypedDict):
+    embeddings: EmbeddingsType
+    n_prompt_tokens: int
+
+
+class CancellationError(Exception):
+    """raised when embedding is canceled"""
+
+
 class LLModel:
     """
     Base class and universal wrapper for GPT4All language models
@@ -182,43 +215,67 @@ class LLModel:
         model = llmodel.llmodel_model_create2(self.model_path, b"auto", ctypes.byref(err))
         if model is None:
             s = err.value
-            raise ValueError(f"Unable to instantiate model: {'null' if s is None else s.decode()}")
-        self.model = model
+            raise RuntimeError(f"Unable to instantiate model: {'null' if s is None else s.decode()}")
+        self.model: ctypes.c_void_p | None = model
 
-    def __del__(self):
+    def __del__(self, llmodel=llmodel):
         if hasattr(self, 'model'):
-            llmodel.llmodel_model_destroy(self.model)
+            self.close()
 
-    def _list_gpu(self, mem_required: int) -> list[LLModelGPUDevice]:
+    def close(self) -> None:
+        if self.model is not None:
+            llmodel.llmodel_model_destroy(self.model)
+            self.model = None
+
+    def _raise_closed(self) -> NoReturn:
+        raise ValueError("Attempted operation on a closed LLModel")
+
+    @property
+    def backend(self) -> Literal["cpu", "kompute", "metal"]:
+        if self.model is None:
+            self._raise_closed()
+        return llmodel.llmodel_model_backend_name(self.model).decode()
+
+    @property
+    def device(self) -> str | None:
+        if self.model is None:
+            self._raise_closed()
+        dev = llmodel.llmodel_model_gpu_device_name(self.model)
+        return None if dev is None else dev.decode()
+
+    @staticmethod
+    def list_gpus(mem_required: int = 0) -> list[str]:
+        """
+        List the names of the available GPU devices with at least `mem_required` bytes of VRAM.
+
+        Args:
+            mem_required: The minimum amount of VRAM, in bytes
+
+        Returns:
+            A list of strings representing the names of the available GPU devices.
+        """
         num_devices = ctypes.c_int32(0)
-        devices_ptr = llmodel.llmodel_available_gpu_devices(self.model, mem_required, ctypes.byref(num_devices))
+        devices_ptr = llmodel.llmodel_available_gpu_devices(mem_required, ctypes.byref(num_devices))
         if not devices_ptr:
             raise ValueError("Unable to retrieve available GPU devices")
-        return devices_ptr[:num_devices.value]
+        return [d.name.decode() for d in devices_ptr[:num_devices.value]]
 
     def init_gpu(self, device: str):
+        if self.model is None:
+            self._raise_closed()
+
         mem_required = llmodel.llmodel_required_mem(self.model, self.model_path, self.n_ctx, self.ngl)
 
         if llmodel.llmodel_gpu_init_gpu_device_by_string(self.model, mem_required, device.encode()):
             return
 
-        # Retrieve all GPUs without considering memory requirements.
-        num_devices = ctypes.c_int32(0)
-        all_devices_ptr = llmodel.llmodel_available_gpu_devices(self.model, 0, ctypes.byref(num_devices))
-        if not all_devices_ptr:
-            raise ValueError("Unable to retrieve list of all GPU devices")
-        all_gpus = [d.name.decode() for d in all_devices_ptr[:num_devices.value]]
-
-        # Retrieve GPUs that meet the memory requirements using list_gpu
-        available_gpus = [device.name.decode() for device in self._list_gpu(mem_required)]
-
-        # Identify GPUs that are unavailable due to insufficient memory or features
+        all_gpus = self.list_gpus()
+        available_gpus = self.list_gpus(mem_required)
         unavailable_gpus = set(all_gpus).difference(available_gpus)
 
-        # Formulate the error message
-        error_msg = "Unable to initialize model on GPU: '{}'.".format(device)
-        error_msg += "\nAvailable GPUs: {}.".format(available_gpus)
-        error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}.".format(unavailable_gpus)
+        error_msg = "Unable to initialize model on GPU: {!r}".format(device)
+        error_msg += "\nAvailable GPUs: {}".format(available_gpus)
+        error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}".format(unavailable_gpus)
         raise ValueError(error_msg)
 
     def load_model(self) -> bool:
@@ -229,14 +286,21 @@ class LLModel:
         -------
         True if model loaded successfully, False otherwise
         """
+        if self.model is None:
+            self._raise_closed()
+
         return llmodel.llmodel_loadModel(self.model, self.model_path, self.n_ctx, self.ngl)
 
     def set_thread_count(self, n_threads):
+        if self.model is None:
+            self._raise_closed()
         if not llmodel.llmodel_isModelLoaded(self.model):
             raise Exception("Model not loaded")
         llmodel.llmodel_setThreadCount(self.model, n_threads)
 
     def thread_count(self):
+        if self.model is None:
+            self._raise_closed()
         if not llmodel.llmodel_isModelLoaded(self.model):
             raise Exception("Model not loaded")
         return llmodel.llmodel_threadCount(self.model)
@@ -286,16 +350,72 @@ class LLModel:
         self.context.repeat_last_n = repeat_last_n
         self.context.context_erase = context_erase
 
-    def generate_embedding(self, text: str) -> List[float]:
-        if not text:
-            raise ValueError("Text must not be None or empty")
+    @overload
+    def generate_embeddings(
+        self, text: str, prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+        cancel_cb: EmbCancelCallbackType | None,
+    ) -> EmbedResult[list[float]]: ...
+    @overload
+    def generate_embeddings(
+        self, text: list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+        cancel_cb: EmbCancelCallbackType | None,
+    ) -> EmbedResult[list[list[float]]]: ...
+    @overload
+    def generate_embeddings(
+        self, text: str | list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+        cancel_cb: EmbCancelCallbackType | None,
+    ) -> EmbedResult[list[Any]]: ...
 
+    def generate_embeddings(
+        self, text: str | list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+        cancel_cb: EmbCancelCallbackType | None,
+    ) -> EmbedResult[list[Any]]:
+        if not text:
+            raise ValueError("text must not be None or empty")
+
+        if self.model is None:
+            self._raise_closed()
+
+        if single_text := isinstance(text, str):
+            text = [text]
+
+        # prepare input
         embedding_size = ctypes.c_size_t()
-        c_text = ctypes.c_char_p(text.encode())
-        embedding_ptr = llmodel.llmodel_embedding(self.model, c_text, ctypes.byref(embedding_size))
-        embedding_array = [embedding_ptr[i] for i in range(embedding_size.value)]
+        token_count = ctypes.c_size_t()
+        error = ctypes.c_char_p()
+        c_prefix = ctypes.c_char_p() if prefix is None else prefix.encode()
+        c_texts = (ctypes.c_char_p * (len(text) + 1))()
+        for i, t in enumerate(text):
+            c_texts[i] = t.encode()
+
+        def wrap_cancel_cb(batch_sizes: Any, n_batch: int, backend: bytes) -> bool:
+            assert cancel_cb is not None
+            return cancel_cb(batch_sizes[:n_batch], backend.decode())
+
+        cancel_cb_wrapper = EmbCancelCallback() if cancel_cb is None else EmbCancelCallback(wrap_cancel_cb)
+
+        # generate the embeddings
+        embedding_ptr = llmodel.llmodel_embed(
+            self.model, c_texts, ctypes.byref(embedding_size), c_prefix, dimensionality, ctypes.byref(token_count),
+            do_mean, atlas, cancel_cb_wrapper, ctypes.byref(error),
+        )
+
+        if not embedding_ptr:
+            msg = "(unknown error)" if error.value is None else error.value.decode()
+            if msg == "operation was canceled":
+                raise CancellationError(msg)
+            raise RuntimeError(f'Failed to generate embeddings: {msg}')
+
+        # extract output
+        n_embd = embedding_size.value // len(text)
+        embedding_array = [
+            embedding_ptr[i:i + n_embd]
+            for i in range(0, embedding_size.value, n_embd)
+        ]
         llmodel.llmodel_free_embedding(embedding_ptr)
-        return list(embedding_array)
+
+        embeddings = embedding_array[0] if single_text else embedding_array
+        return {'embeddings': embeddings, 'n_prompt_tokens': token_count.value}
 
     def prompt_model(
         self,
@@ -329,16 +449,12 @@ class LLModel:
         None
         """
 
+        if self.model is None:
+            self._raise_closed()
+
         self.buffer.clear()
         self.buff_expecting_cont_bytes = 0
 
-        logger.info(
-            "LLModel.prompt_model -- prompt:\n"
-            + "%s\n"
-            + "===/LLModel.prompt_model -- prompt/===",
-            prompt,
-        )
-
         self._set_context(
             n_predict=n_predict,
             top_k=top_k,
@@ -361,12 +477,16 @@ class LLModel:
             RecalculateCallback(self._recalculate_callback),
             self.context,
             special,
+            ctypes.c_char_p(),
         )
 
 
     def prompt_model_streaming(
         self, prompt: str, prompt_template: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
     ) -> Iterable[str]:
+        if self.model is None:
+            self._raise_closed()
+
         output_queue: Queue[str | Sentinel] = Queue()
 
         # Put response tokens into an output queue

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -3,6 +3,7 @@ Python only API for running all GPT4All models.
 """
 from __future__ import annotations
 
+import hashlib
 import os
 import re
 import sys
@@ -10,25 +11,30 @@ import time
 import warnings
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Union
+from types import TracebackType
+from typing import TYPE_CHECKING, Any, Iterable, Literal, Protocol, overload
 
 import requests
 from requests.exceptions import ChunkedEncodingError
 from tqdm import tqdm
 from urllib3.exceptions import IncompleteRead, ProtocolError
 
-from . import _pyllmodel
+from ._pyllmodel import (CancellationError as CancellationError, EmbCancelCallbackType, EmbedResult as EmbedResult,
+                         LLModel, ResponseCallbackType, empty_response_callback)
+
+if TYPE_CHECKING:
+    from typing_extensions import Self, TypeAlias
+
+if sys.platform == 'darwin':
+    import fcntl
 
 # TODO: move to config
-DEFAULT_MODEL_DIRECTORY = os.path.join(str(Path.home()), ".cache", "gpt4all").replace("\\", "\\\\")
+DEFAULT_MODEL_DIRECTORY = Path.home() / ".cache" / "gpt4all"
 
-DEFAULT_MODEL_CONFIG = {
-    "systemPrompt": "",
-    "promptTemplate": "### Human: \n{0}\n\n### Assistant:\n",
-}
+DEFAULT_PROMPT_TEMPLATE = "### Human:\n{0}\n\n### Assistant:\n"
 
-ConfigType = Dict[str, str]
-MessageType = Dict[str, str]
+ConfigType: TypeAlias = 'dict[str, Any]'
+MessageType: TypeAlias = 'dict[str, str]'
 
 
 class Embed4All:
@@ -36,26 +42,121 @@ class Embed4All:
     Python class that handles embeddings for GPT4All.
     """
 
-    def __init__(self, model_name: Optional[str] = None, n_threads: Optional[int] = None, **kwargs):
+    MIN_DIMENSIONALITY = 64
+
+    def __init__(self, model_name: str | None = None, *, n_threads: int | None = None, device: str | None = "cpu", **kwargs: Any):
         """
         Constructor
 
         Args:
             n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
+            device: The processing unit on which the embedding model will run. See the `GPT4All` constructor for more info.
+            kwargs: Remaining keyword arguments are passed to the `GPT4All` constructor.
         """
-        self.gpt4all = GPT4All(model_name or 'all-MiniLM-L6-v2-f16.gguf', n_threads=n_threads, **kwargs)
+        if model_name is None:
+            model_name = 'all-MiniLM-L6-v2.gguf2.f16.gguf'
+        self.gpt4all = GPT4All(model_name, n_threads=n_threads, device=device, **kwargs)
 
-    def embed(self, text: str) -> List[float]:
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self, typ: type[BaseException] | None, value: BaseException | None, tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def close(self) -> None:
+        """Delete the model instance and free associated system resources."""
+        self.gpt4all.close()
+
+    # return_dict=False
+    @overload
+    def embed(
+        self, text: str, *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[False] = ..., atlas: bool = ..., cancel_cb: EmbCancelCallbackType | None = ...,
+    ) -> list[float]: ...
+    @overload
+    def embed(
+        self, text: list[str], *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[False] = ..., atlas: bool = ..., cancel_cb: EmbCancelCallbackType | None = ...,
+    ) -> list[list[float]]: ...
+    @overload
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
+        long_text_mode: str = ..., return_dict: Literal[False] = ..., atlas: bool = ...,
+        cancel_cb: EmbCancelCallbackType | None = ...,
+    ) -> list[Any]: ...
+
+    # return_dict=True
+    @overload
+    def embed(
+        self, text: str, *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[True], atlas: bool = ..., cancel_cb: EmbCancelCallbackType | None = ...,
+    ) -> EmbedResult[list[float]]: ...
+    @overload
+    def embed(
+        self, text: list[str], *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[True], atlas: bool = ..., cancel_cb: EmbCancelCallbackType | None = ...,
+    ) -> EmbedResult[list[list[float]]]: ...
+    @overload
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
+        long_text_mode: str = ..., return_dict: Literal[True], atlas: bool = ...,
+        cancel_cb: EmbCancelCallbackType | None = ...,
+    ) -> EmbedResult[list[Any]]: ...
+
+    # return type unknown
+    @overload
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
+        long_text_mode: str = ..., return_dict: bool = ..., atlas: bool = ...,
+        cancel_cb: EmbCancelCallbackType | None = ...,
+    ) -> Any: ...
+
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = None, dimensionality: int | None = None,
+        long_text_mode: str = "mean", return_dict: bool = False, atlas: bool = False,
+        cancel_cb: EmbCancelCallbackType | None = None,
+    ) -> Any:
         """
-        Generate an embedding.
+        Generate one or more embeddings.
 
         Args:
-            text: The text document to generate an embedding for.
+            text: A text or list of texts to generate embeddings for.
+            prefix: The model-specific prefix representing the embedding task, without the trailing colon. For Nomic
+                Embed, this can be `search_query`, `search_document`, `classification`, or `clustering`. Defaults to
+                `search_document` or equivalent if known; otherwise, you must explicitly pass a prefix or an empty
+                string if none applies.
+            dimensionality: The embedding dimension, for use with Matryoshka-capable models. Defaults to full-size.
+            long_text_mode: How to handle texts longer than the model can accept. One of `mean` or `truncate`.
+            return_dict: Return the result as a dict that includes the number of prompt tokens processed.
+            atlas: Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens
+                with long_text_mode="mean" will raise an error. Disabled by default.
+            cancel_cb: Called with arguments (batch_sizes, backend_name). Return true to cancel embedding.
 
         Returns:
-            An embedding of your document of text.
+            With return_dict=False, an embedding or list of embeddings of your text(s).
+            With return_dict=True, a dict with keys 'embeddings' and 'n_prompt_tokens'.
+
+        Raises:
+            CancellationError: If cancel_cb returned True and embedding was canceled.
         """
-        return self.gpt4all.model.generate_embedding(text)
+        if dimensionality is None:
+            dimensionality = -1
+        else:
+            if dimensionality <= 0:
+                raise ValueError(f'Dimensionality must be None or a positive integer, got {dimensionality}')
+            if dimensionality < self.MIN_DIMENSIONALITY:
+                warnings.warn(
+                    f'Dimensionality {dimensionality} is less than the suggested minimum of {self.MIN_DIMENSIONALITY}.'
+                    ' Performance may be degraded.'
+                )
+        try:
+            do_mean = {"mean": True, "truncate": False}[long_text_mode]
+        except KeyError:
+            raise ValueError(f"Long text mode must be one of 'mean' or 'truncate', got {long_text_mode!r}")
+        result = self.gpt4all.model.generate_embeddings(text, prefix, dimensionality, do_mean, atlas, cancel_cb)
+        return result if return_dict else result['embeddings']
 
 
 class GPT4All:
@@ -66,11 +167,12 @@ class GPT4All:
     def __init__(
         self,
         model_name: str,
-        model_path: Optional[Union[str, os.PathLike[str]]] = None,
-        model_type: Optional[str] = None,
+        *,
+        model_path: str | os.PathLike[str] | None = None,
+        model_type: str | None = None,
         allow_download: bool = True,
-        n_threads: Optional[int] = None,
-        device: Optional[str] = "cpu",
+        n_threads: int | None = None,
+        device: str | None = "cpu",
         n_ctx: int = 2048,
         ngl: int = 100,
         verbose: bool = False,
@@ -90,7 +192,7 @@ class GPT4All:
                 - "cpu": Model will run on the central processing unit.
                 - "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
                 - "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
-                Alternatively, a specific GPU name can also be provided, and the model will run on the GPU that matches the name if it's available.
+                - A specific device name from the list returned by `GPT4All.list_gpus()`.
                 Default is "cpu".
 
                 Note: If a selected GPU device does not have sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the model.
@@ -101,7 +203,7 @@ class GPT4All:
         self.model_type = model_type
         # Retrieve model and download if allowed
         self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download, verbose=verbose)
-        self.model = _pyllmodel.LLModel(self.config["path"], n_ctx, ngl)
+        self.model = LLModel(self.config["path"], n_ctx, ngl)
         if device is not None and device != "cpu":
             self.model.init_gpu(device)
         self.model.load_model()
@@ -109,27 +211,53 @@ class GPT4All:
         if n_threads is not None:
             self.model.set_thread_count(n_threads)
 
-        self._is_chat_session_activated: bool = False
-        self.current_chat_session: List[MessageType] = empty_chat_session()
+        self._history: list[MessageType] | None = None
         self._current_prompt_template: str = "{0}"
 
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self, typ: type[BaseException] | None, value: BaseException | None, tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def close(self) -> None:
+        """Delete the model instance and free associated system resources."""
+        self.model.close()
+
+    @property
+    def backend(self) -> Literal["cpu", "kompute", "metal"]:
+        """The name of the llama.cpp backend currently in use. One of "cpu", "kompute", or "metal"."""
+        return self.model.backend
+
+    @property
+    def device(self) -> str | None:
+        """The name of the GPU device currently in use, or None for backends other than Kompute."""
+        return self.model.device
+
+    @property
+    def current_chat_session(self) -> list[MessageType] | None:
+        return None if self._history is None else list(self._history)
+
     @staticmethod
-    def list_models() -> List[ConfigType]:
+    def list_models() -> list[ConfigType]:
         """
-        Fetch model list from https://gpt4all.io/models/models2.json.
+        Fetch model list from https://gpt4all.io/models/models3.json.
 
         Returns:
             Model list in JSON format.
         """
-        resp = requests.get("https://gpt4all.io/models/models2.json")
+        resp = requests.get("https://gpt4all.io/models/models3.json")
         if resp.status_code != 200:
             raise ValueError(f'Request failed: HTTP {resp.status_code} {resp.reason}')
         return resp.json()
 
-    @staticmethod
+    @classmethod
     def retrieve_model(
+        cls,
         model_name: str,
-        model_path: Optional[Union[str, os.PathLike[str]]] = None,
+        model_path: str | os.PathLike[str] | None = None,
         allow_download: bool = True,
         verbose: bool = False,
     ) -> ConfigType:
@@ -150,59 +278,57 @@ class GPT4All:
         model_filename = append_extension_if_missing(model_name)
 
         # get the config for the model
-        config: ConfigType = DEFAULT_MODEL_CONFIG
+        config: ConfigType = {}
         if allow_download:
-            available_models = GPT4All.list_models()
+            available_models = cls.list_models()
 
             for m in available_models:
                 if model_filename == m["filename"]:
+                    tmpl = m.get("promptTemplate", DEFAULT_PROMPT_TEMPLATE)
+                    # change to Python-style formatting
+                    m["promptTemplate"] = tmpl.replace("%1", "{0}", 1).replace("%2", "{1}", 1)
                     config.update(m)
-                    config["systemPrompt"] = config["systemPrompt"].strip()
-                    config["promptTemplate"] = config["promptTemplate"].replace(
-                        "%1", "{0}", 1
-                    )  # change to Python-style formatting
                     break
 
         # Validate download directory
         if model_path is None:
             try:
                 os.makedirs(DEFAULT_MODEL_DIRECTORY, exist_ok=True)
-            except OSError as exc:
-                raise ValueError(
-                    f"Failed to create model download directory at {DEFAULT_MODEL_DIRECTORY}: {exc}. "
-                    "Please specify model_path."
-                )
+            except OSError as e:
+                raise RuntimeError("Failed to create model download directory") from e
             model_path = DEFAULT_MODEL_DIRECTORY
         else:
-            model_path = str(model_path).replace("\\", "\\\\")
+            model_path = Path(model_path)
 
-        if not os.path.exists(model_path):
-            raise ValueError(f"Invalid model directory: {model_path}")
+        if not model_path.exists():
+            raise FileNotFoundError(f"Model directory does not exist: {model_path!r}")
 
-        model_dest = os.path.join(model_path, model_filename).replace("\\", "\\\\")
-        if os.path.exists(model_dest):
-            config.pop("url", None)
-            config["path"] = model_dest
+        model_dest = model_path / model_filename
+        if model_dest.exists():
+            config["path"] = str(model_dest)
             if verbose:
-                print("Found model file at", model_dest, file=sys.stderr)
-
-        # If model file does not exist, download
+                print(f"Found model file at {str(model_dest)!r}", file=sys.stderr)
         elif allow_download:
-            url = config.pop("url", None)
-
-            config["path"] = GPT4All.download_model(model_filename, model_path, verbose=verbose, url=url)
+            # If model file does not exist, download
+            filesize = config.get("filesize")
+            config["path"] = str(cls.download_model(
+                model_filename, model_path, verbose=verbose, url=config.get("url"),
+                expected_size=None if filesize is None else int(filesize), expected_md5=config.get("md5sum"),
+            ))
         else:
-            raise ValueError("Failed to retrieve model")
+            raise FileNotFoundError(f"Model file does not exist: {model_dest!r}")
 
         return config
 
     @staticmethod
     def download_model(
         model_filename: str,
-        model_path: Union[str, os.PathLike[str]],
+        model_path: str | os.PathLike[str],
         verbose: bool = True,
-        url: Optional[str] = None,
-    ) -> str:
+        url: str | None = None,
+        expected_size: int | None = None,
+        expected_md5: str | None = None,
+    ) -> str | os.PathLike[str]:
         """
         Download model from https://gpt4all.io.
 
@@ -211,30 +337,30 @@ class GPT4All:
             model_path: Path to download model to.
             verbose: If True (default), print debug messages.
             url: the models remote url (e.g. may be hosted on HF)
+            expected_size: The expected size of the download.
+            expected_md5: The expected MD5 hash of the download.
 
         Returns:
             Model file destination.
         """
 
-        def get_download_url(model_filename):
-            if url:
-                return url
-            return f"https://gpt4all.io/models/gguf/{model_filename}"
-
         # Download model
-        download_path = os.path.join(model_path, model_filename).replace("\\", "\\\\")
-        download_url = get_download_url(model_filename)
+        if url is None:
+            url = f"https://gpt4all.io/models/gguf/{model_filename}"
 
         def make_request(offset=None):
             headers = {}
             if offset:
                 print(f"\nDownload interrupted, resuming from byte position {offset}", file=sys.stderr)
                 headers['Range'] = f'bytes={offset}-'  # resume incomplete response
-            response = requests.get(download_url, stream=True, headers=headers)
+                headers["Accept-Encoding"] = "identity"  # Content-Encoding changes meaning of ranges
+            response = requests.get(url, stream=True, headers=headers)
             if response.status_code not in (200, 206):
                 raise ValueError(f'Request failed: HTTP {response.status_code} {response.reason}')
             if offset and (response.status_code != 206 or str(offset) not in response.headers.get('Content-Range', '')):
                 raise ValueError('Connection was interrupted and server does not support range requests')
+            if (enc := response.headers.get("Content-Encoding")) is not None:
+                raise ValueError(f"Expected identity Content-Encoding, got {enc}")
             return response
 
         response = make_request()
@@ -242,49 +368,97 @@ class GPT4All:
         total_size_in_bytes = int(response.headers.get("content-length", 0))
         block_size = 2**20  # 1 MB
 
-        with open(download_path, "wb") as file, \
-                tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar:
+        partial_path = Path(model_path) / (model_filename + ".part")
+
+        with open(partial_path, "w+b") as partf:
             try:
-                while True:
-                    last_progress = progress_bar.n
-                    try:
-                        for data in response.iter_content(block_size):
-                            file.write(data)
-                            progress_bar.update(len(data))
-                    except ChunkedEncodingError as cee:
-                        if cee.args and isinstance(pe := cee.args[0], ProtocolError):
-                            if len(pe.args) >= 2 and isinstance(ir := pe.args[1], IncompleteRead):
-                                assert progress_bar.n <= ir.partial  # urllib3 may be ahead of us but never behind
-                                # the socket was closed during a read - retry
-                                response = make_request(progress_bar.n)
-                                continue
-                        raise
-                    if total_size_in_bytes != 0 and progress_bar.n < total_size_in_bytes:
-                        if progress_bar.n == last_progress:
-                            raise RuntimeError('Download not making progress, aborting.')
-                        # server closed connection prematurely - retry
-                        response = make_request(progress_bar.n)
-                        continue
-                    break
-            except Exception:
+                with tqdm(desc="Downloading", total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar:
+                    while True:
+                        last_progress = progress_bar.n
+                        try:
+                            for data in response.iter_content(block_size):
+                                partf.write(data)
+                                progress_bar.update(len(data))
+                        except ChunkedEncodingError as cee:
+                            if cee.args and isinstance(pe := cee.args[0], ProtocolError):
+                                if len(pe.args) >= 2 and isinstance(ir := pe.args[1], IncompleteRead):
+                                    assert progress_bar.n <= ir.partial  # urllib3 may be ahead of us but never behind
+                                    # the socket was closed during a read - retry
+                                    response = make_request(progress_bar.n)
+                                    continue
+                            raise
+                        if total_size_in_bytes != 0 and progress_bar.n < total_size_in_bytes:
+                            if progress_bar.n == last_progress:
+                                raise RuntimeError("Download not making progress, aborting.")
+                            # server closed connection prematurely - retry
+                            response = make_request(progress_bar.n)
+                            continue
+                        break
+
+                # verify file integrity
+                file_size = partf.tell()
+                if expected_size is not None and file_size != expected_size:
+                    raise ValueError(f"Expected file size of {expected_size} bytes, got {file_size}")
+                if expected_md5 is not None:
+                    partf.seek(0)
+                    hsh = hashlib.md5()
+                    with tqdm(desc="Verifying", total=file_size, unit="iB", unit_scale=True) as bar:
+                        while chunk := partf.read(block_size):
+                            hsh.update(chunk)
+                            bar.update(len(chunk))
+                    if hsh.hexdigest() != expected_md5.lower():
+                        raise ValueError(f"Expected MD5 hash of {expected_md5!r}, got {hsh.hexdigest()!r}")
+            except:
                 if verbose:
                     print("Cleaning up the interrupted download...", file=sys.stderr)
                 try:
-                    os.remove(download_path)
+                    os.remove(partial_path)
                 except OSError:
                     pass
                 raise
 
-        if os.name == 'nt':
-            time.sleep(2)  # Sleep for a little bit so Windows can remove file lock
+            # flush buffers and sync the inode
+            partf.flush()
+            _fsync(partf)
+
+        # move to final destination
+        download_path = Path(model_path) / model_filename
+        try:
+            os.rename(partial_path, download_path)
+        except FileExistsError:
+            try:
+                os.remove(partial_path)
+            except OSError:
+                pass
+            raise
 
         if verbose:
-            print("Model downloaded at:", download_path, file=sys.stderr)
+            print(f"Model downloaded to {str(download_path)!r}", file=sys.stderr)
         return download_path
 
+    @overload
+    def generate(
+        self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
+        min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
+        n_predict: int | None = ..., streaming: Literal[False] = ..., callback: ResponseCallbackType = ...,
+    ) -> str: ...
+    @overload
+    def generate(
+        self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
+        min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
+        n_predict: int | None = ..., streaming: Literal[True], callback: ResponseCallbackType = ...,
+    ) -> Iterable[str]: ...
+    @overload
+    def generate(
+        self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
+        min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
+        n_predict: int | None = ..., streaming: bool, callback: ResponseCallbackType = ...,
+    ) -> Any: ...
+
     def generate(
         self,
         prompt: str,
+        *,
         max_tokens: int = 200,
         temp: float = 0.7,
         top_k: int = 40,
@@ -293,10 +467,10 @@ class GPT4All:
         repeat_penalty: float = 1.18,
         repeat_last_n: int = 64,
         n_batch: int = 8,
-        n_predict: Optional[int] = None,
+        n_predict: int | None = None,
         streaming: bool = False,
-        callback: _pyllmodel.ResponseCallbackType = _pyllmodel.empty_response_callback,
-    ) -> Union[str, Iterable[str]]:
+        callback: ResponseCallbackType = empty_response_callback,
+    ) -> Any:
         """
         Generate outputs from any GPT4All model.
 
@@ -318,12 +492,8 @@ class GPT4All:
             Either the entire completion or a generator that yields the completion token by token.
         """
 
-        if re.search(r"%1(?![0-9])", self._current_prompt_template):
-            raise ValueError("Prompt template containing a literal '%1' is not supported. For a prompt "
-                             "placeholder, please use '{0}' instead.")
-
         # Preparing the model request
-        generate_kwargs: Dict[str, Any] = dict(
+        generate_kwargs: dict[str, Any] = dict(
             temp=temp,
             top_k=top_k,
             top_p=top_p,
@@ -334,20 +504,20 @@ class GPT4All:
             n_predict=n_predict if n_predict is not None else max_tokens,
         )
 
-        if self._is_chat_session_activated:
+        if self._history is not None:
             # check if there is only one message, i.e. system prompt:
-            reset = len(self.current_chat_session) == 1
-            generate_kwargs["reset_context"] = reset
-            self.current_chat_session.append({"role": "user", "content": prompt})
+            reset = len(self._history) == 1
+            self._history.append({"role": "user", "content": prompt})
 
             fct_func = self._format_chat_prompt_template.__func__  # type: ignore[attr-defined]
             if fct_func is GPT4All._format_chat_prompt_template:
                 if reset:
                     # ingest system prompt
-                    self.model.prompt_model(self.current_chat_session[0]["content"], "%1",
-                                            _pyllmodel.empty_response_callback,
-                                            n_batch=n_batch, n_predict=0, special=True)
-                prompt_template = self._current_prompt_template.format("%1")
+                    # use "%1%2" and not "%1" to avoid implicit whitespace
+                    self.model.prompt_model(self._history[0]["content"], "%1%2",
+                                            empty_response_callback,
+                                            n_batch=n_batch, n_predict=0, reset_context=True, special=True)
+                prompt_template = self._current_prompt_template.format("%1", "%2")
             else:
                 warnings.warn(
                     "_format_chat_prompt_template is deprecated. Please use a chat session with a prompt template.",
@@ -355,28 +525,29 @@ class GPT4All:
                 )
                 # special tokens won't be processed
                 prompt = self._format_chat_prompt_template(
-                    self.current_chat_session[-1:],
-                    self.current_chat_session[0]["content"] if reset else "",
+                    self._history[-1:],
+                    self._history[0]["content"] if reset else "",
                 )
                 prompt_template = "%1"
+                generate_kwargs["reset_context"] = reset
         else:
             prompt_template = "%1"
             generate_kwargs["reset_context"] = True
 
         # Prepare the callback, process the model response
-        output_collector: List[MessageType]
+        output_collector: list[MessageType]
         output_collector = [
             {"content": ""}
-        ]  # placeholder for the self.current_chat_session if chat session is not activated
+        ]  # placeholder for the self._history if chat session is not activated
 
-        if self._is_chat_session_activated:
-            self.current_chat_session.append({"role": "assistant", "content": ""})
-            output_collector = self.current_chat_session
+        if self._history is not None:
+            self._history.append({"role": "assistant", "content": ""})
+            output_collector = self._history
 
         def _callback_wrapper(
-            callback: _pyllmodel.ResponseCallbackType,
-            output_collector: List[MessageType],
-        ) -> _pyllmodel.ResponseCallbackType:
+            callback: ResponseCallbackType,
+            output_collector: list[MessageType],
+        ) -> ResponseCallbackType:
             def _callback(token_id: int, response: str) -> bool:
                 nonlocal callback, output_collector
 
@@ -407,8 +578,8 @@ class GPT4All:
     @contextmanager
     def chat_session(
         self,
-        system_prompt: str = "",
-        prompt_template: str = "",
+        system_prompt: str | None = None,
+        prompt_template: str | None = None,
     ):
         """
         Context manager to hold an inference optimized chat session with a GPT4All model.
@@ -417,27 +588,51 @@ class GPT4All:
             system_prompt: An initial instruction for the model.
             prompt_template: Template for the prompts with {0} being replaced by the user message.
         """
-        # Code to acquire resource, e.g.:
-        self._is_chat_session_activated = True
-        self.current_chat_session = empty_chat_session(system_prompt or self.config["systemPrompt"])
-        self._current_prompt_template = prompt_template or self.config["promptTemplate"]
+
+        if system_prompt is None:
+            system_prompt = self.config.get("systemPrompt", "")
+
+        if prompt_template is None:
+            if (tmpl := self.config.get("promptTemplate")) is None:
+                warnings.warn("Use of a sideloaded model or allow_download=False without specifying a prompt template "
+                              "is deprecated. Defaulting to Alpaca.", DeprecationWarning)
+                tmpl = DEFAULT_PROMPT_TEMPLATE
+            prompt_template = tmpl
+
+        if re.search(r"%1(?![0-9])", prompt_template):
+            raise ValueError("Prompt template containing a literal '%1' is not supported. For a prompt "
+                             "placeholder, please use '{0}' instead.")
+
+        self._history = [{"role": "system", "content": system_prompt}]
+        self._current_prompt_template = prompt_template
         try:
             yield self
         finally:
-            # Code to release resource, e.g.:
-            self._is_chat_session_activated = False
-            self.current_chat_session = empty_chat_session()
+            self._history = None
             self._current_prompt_template = "{0}"
 
+    @staticmethod
+    def list_gpus() -> list[str]:
+        """
+        List the names of the available GPU devices.
+
+        Returns:
+            A list of strings representing the names of the available GPU devices.
+        """
+        return LLModel.list_gpus()
+
     def _format_chat_prompt_template(
         self,
-        messages: List[MessageType],
+        messages: list[MessageType],
         default_prompt_header: str = "",
         default_prompt_footer: str = "",
     ) -> str:
         """
         Helper method for building a prompt from list of messages using the self._current_prompt_template as a template for each message.
 
+        Warning:
+            This function was deprecated in version 2.3.0, and will be removed in a future release.
+
         Args:
             messages:  List of dictionaries. Each dictionary should have a "role" key
                 with value of "system", "assistant", or "user" and a "content" key with a
@@ -464,11 +659,23 @@ class GPT4All:
         return full_prompt
 
 
-def empty_chat_session(system_prompt: str = "") -> List[MessageType]:
-    return [{"role": "system", "content": system_prompt}]
-
-
 def append_extension_if_missing(model_name):
     if not model_name.endswith((".bin", ".gguf")):
         model_name += ".gguf"
     return model_name
+
+
+class _HasFileno(Protocol):
+    def fileno(self) -> int: ...
+
+
+def _fsync(fd: int | _HasFileno) -> None:
+    if sys.platform == 'darwin':
+        # Apple's fsync does not flush the drive write cache
+        try:
+            fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
+        except OSError:
+            pass  # fall back to fsync
+        else:
+            return
+    os.fsync(fd)

gpt4all-bindings/python/gpt4all/tests/test_gpt4all.py

@@ -28,12 +28,8 @@ def test_inference():
     assert len(tokens) > 0
 
     with model.chat_session():
-        tokens = list(model.generate(prompt='hello', top_k=1, streaming=True))
-        model.current_chat_session.append({'role': 'assistant', 'content': ''.join(tokens)})
-
-        tokens = list(model.generate(prompt='write me a poem about dogs', top_k=1, streaming=True))
-        model.current_chat_session.append({'role': 'assistant', 'content': ''.join(tokens)})
-
+        model.generate(prompt='hello', top_k=1, streaming=True)
+        model.generate(prompt='write me a poem about dogs', top_k=1, streaming=True)
         print(model.current_chat_session)
 
 
@@ -115,13 +111,13 @@ def test_empty_embedding():
         output = embedder.embed(text)
 
 def test_download_model(tmp_path: Path):
-    import gpt4all.gpt4all
-    old_default_dir = gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY
-    gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = str(tmp_path)  # temporary pytest directory to ensure a download happens
+    from gpt4all import gpt4all
+    old_default_dir = gpt4all.DEFAULT_MODEL_DIRECTORY
+    gpt4all.DEFAULT_MODEL_DIRECTORY = tmp_path  # temporary pytest directory to ensure a download happens
     try:
         model = GPT4All(model_name='ggml-all-MiniLM-L6-v2-f16.bin')
         model_path = tmp_path / model.config['filename']
         assert model_path.absolute() == Path(model.config['path']).absolute()
         assert model_path.stat().st_size == int(model.config['filesize'])
     finally:
-        gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = old_default_dir
+        gpt4all.DEFAULT_MODEL_DIRECTORY = old_default_dir

gpt4all-bindings/python/mkdocs.yml

@@ -16,8 +16,6 @@ nav:
         - 'Embedding': 'gpt4all_python_embedding.md'
       - 'GPT4ALL in NodeJs': 'gpt4all_nodejs.md'
       - 'gpt4all_cli.md'
-#    - 'Tutorials':
-#      - 'gpt4all_modal.md'
     - 'Wiki':
       - 'gpt4all_faq.md'
 
@@ -44,8 +42,8 @@ markdown_extensions:
   - pymdownx.tabbed:
       alternate_style: true
   - pymdownx.emoji:
-      emoji_index: !!python/name:materialx.emoji.twemoji
-      emoji_generator: !!python/name:materialx.emoji.to_svg
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
       options:
         custom_icons:
           - docs/overrides/.icons

gpt4all-bindings/python/setup.py

@@ -68,7 +68,7 @@ def get_long_description():
 
 setup(
     name=package_name,
-    version="2.3.0",
+    version="2.6.0",
     description="Python bindings for GPT4All",
     long_description=get_long_description(),
     long_description_content_type="text/markdown",
@@ -90,6 +90,7 @@ setup(
         'requests',
         'tqdm',
         'importlib_resources; python_version < "3.9"',
+        'typing-extensions>=4.3.0; python_version >= "3.9" and python_version < "3.11"',
     ],
     extras_require={
         'dev': [
@@ -102,7 +103,8 @@ setup(
             'mkdocstrings[python]',
             'mkdocs-jupyter',
             'black',
-            'isort'
+            'isort',
+            'typing-extensions>=3.10',
         ]
     },
     package_data={'llmodel': [os.path.join(DEST_CLIB_DIRECTORY, "*")]},

gpt4all-bindings/typescript/.clang-format

@@ -0,0 +1,4 @@
+---
+Language: Cpp
+BasedOnStyle: Microsoft
+ColumnLimit: 120

gpt4all-bindings/typescript/README.md

@@ -10,45 +10,170 @@ npm install gpt4all@latest
 pnpm install gpt4all@latest
 
 ```
-
-The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
-
-*   New bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
-*   The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
-*   Everything should work out the box.
+## Breaking changes in version 4!!
+*   See [Transition](#changes)
+## Contents
 *   See [API Reference](#api-reference)
-
+*   See [Examples](#api-example)
+*   See [Developing](#develop)
+*   GPT4ALL nodejs bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
+*   [spare change](https://github.com/sponsors/jacoobes) for a college student? 🤑
+## Api Examples
 ### Chat Completion
 
+Use a chat session to keep context between completions. This is useful for efficient back and forth conversations.
+
 ```js
-import { createCompletion, loadModel } from '../src/gpt4all.js'
+import { createCompletion, loadModel } from "../src/gpt4all.js";
 
-const model = await loadModel('mistral-7b-openorca.Q4_0.gguf', { verbose: true });
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true, // logs loaded model configuration
+    device: "gpu", // defaults to 'cpu'
+    nCtx: 2048, // the maximum sessions context window size.
+});
 
-const response = await createCompletion(model, [
-    { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
-    { role : 'user', content: 'What is 1 + 1?'  } 
+// initialize a chat session on the model. a model instance can have only one chat session at a time.
+const chat = await model.createChatSession({
+    // any completion options set here will be used as default for all completions in this chat session
+    temperature: 0.8,
+    // a custom systemPrompt can be set here. note that the template depends on the model.
+    // if unset, the systemPrompt that comes with the model will be used.
+    systemPrompt: "### System:\nYou are an advanced mathematician.\n\n",
+});
+
+// create a completion using a string as input
+const res1 = await createCompletion(chat, "What is 1 + 1?");
+console.debug(res1.choices[0].message);
+
+// multiple messages can be input to the conversation at once.
+// note that if the last message is not of role 'user', an empty message will be returned.
+await createCompletion(chat, [
+    {
+        role: "user",
+        content: "What is 2 + 2?",
+    },
+    {
+        role: "assistant",
+        content: "It's 5.",
+    },
 ]);
 
+const res3 = await createCompletion(chat, "Could you recalculate that?");
+console.debug(res3.choices[0].message);
+
+model.dispose();
+```
+
+### Stateless usage
+You can use the model without a chat session. This is useful for one-off completions.
+
+```js
+import { createCompletion, loadModel } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf");
+
+// createCompletion methods can also be used on the model directly.
+// context is not maintained between completions.
+const res1 = await createCompletion(model, "What is 1 + 1?");
+console.debug(res1.choices[0].message);
+
+// a whole conversation can be input as well.
+// note that if the last message is not of role 'user', an error will be thrown.
+const res2 = await createCompletion(model, [
+    {
+        role: "user",
+        content: "What is 2 + 2?",
+    },
+    {
+        role: "assistant",
+        content: "It's 5.",
+    },
+    {
+        role: "user",
+        content: "Could you recalculate that?",
+    },
+]);
+console.debug(res2.choices[0].message);
+
 ```
 
 ### Embedding
 
 ```js
-import { createEmbedding, loadModel } from '../src/gpt4all.js'
+import { loadModel, createEmbedding } from '../src/gpt4all.js'
 
-const model = await loadModel('ggml-all-MiniLM-L6-v2-f16', { verbose: true });
+const embedder = await loadModel("nomic-embed-text-v1.5.f16.gguf", { verbose: true, type: 'embedding'})
 
-const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional");
+console.log(createEmbedding(embedder, "Maybe Minecraft was the friends we made along the way"));
 ```
 
+### Streaming responses
+```js
+import { loadModel, createCompletionStream } from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("Output: ");
+const stream = createCompletionStream(model, "How are you?");
+stream.tokens.on("data", (data) => {
+    process.stdout.write(data);
+});
+//wait till stream finishes. We cannot continue until this one is done.
+await stream.result;
+process.stdout.write("\n");
+model.dispose();
+
+```
+
+### Async Generators
+```js
+import { loadModel, createCompletionGenerator } from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf");
+
+process.stdout.write("Output: ");
+const gen = createCompletionGenerator(
+    model,
+    "Redstone in Minecraft is Turing Complete. Let that sink in. (let it in!)"
+);
+for await (const chunk of gen) {
+    process.stdout.write(chunk);
+}
+
+process.stdout.write("\n");
+model.dispose();
+
+```
+### Offline usage
+do this b4 going offline
+```sh
+curl -L https://gpt4all.io/models/models3.json -o ./models3.json
+```
+```js
+import { createCompletion, loadModel } from 'gpt4all'
+
+//make sure u downloaded the models before going offline!
+const model = await loadModel('mistral-7b-openorca.gguf2.Q4_0.gguf', {
+    verbose: true,
+    device: 'gpu',
+    modelConfigFile: "./models3.json"
+});
+
+await createCompletion(model, 'What is 1 + 1?', { verbose: true })
+
+model.dispose();
+```
+
+## Develop
 ### Build Instructions
 
-*   binding.gyp is compile config
+*   `binding.gyp` is compile config
 *   Tested on Ubuntu. Everything seems to work fine
 *   Tested on Windows. Everything works fine.
 *   Sparse testing on mac os.
-*   MingW works as well to build the gpt4all-backend. **HOWEVER**, this package works only with MSVC built dlls.
+*   MingW script works to build the gpt4all-backend. We left it there just in case. **HOWEVER**, this package works only with MSVC built dlls.
 
 ### Requirements
 
@@ -76,23 +201,18 @@ cd gpt4all-bindings/typescript
 *   To Build and Rebuild:
 
 ```sh
-yarn
+node scripts/prebuild.js
 ```
 *   llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
 
 ```sh
-git submodule update --init --depth 1 --recursive
+git submodule update --init --recursive
 ```
 
 ```sh
 yarn build:backend
 ```
-
-This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native The only current way to use them is to put them in the current working directory of your application. That is, **WHEREVER YOU RUN YOUR NODE APPLICATION**
-
-*   llama-xxxx.dll is required.
-*   According to whatever model you are using, you'll need to select the proper model loader.
-    *   For example, if you running an Mosaic MPT model, you will need to select the mpt-(buildvariant).(dynamiclibrary)
+This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native
 
 ### Test
 
@@ -130,17 +250,20 @@ yarn test
 
 *   why your model may be spewing bull 💩
     *   The downloaded model is broken (just reinstall or download from official site)
-    *   That's it so far
+*   Your model is hanging after a call to generate tokens.
+    * Is `nPast` set too high? This may cause your model to hang (03/16/2024), Linux Mint, Ubuntu 22.04
+*  Your GPU usage is still high after node.js exits.
+    * Make sure to call `model.dispose()`!!!
 
 ### Roadmap
 
-This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
+This package has been stabilizing over time development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
 
 *   \[ ] Purely offline. Per the gui, which can be run completely offline, the bindings should be as well.
 *   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
     *   Should include prebuilds to avoid painful node-gyp errors
-*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
-*   \[x] generateTokens, the new name for createTokenStream. As of 3.2.0, this is released but not 100% tested. Check spec/generator.mjs! 
+*   \[x] createChatSession ( the python equivalent to create\_chat\_session )
+*   \[x] generateTokens, the new name for createTokenStream. As of 3.2.0, this is released but not 100% tested. Check spec/generator.mjs!
 *   \[x] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
 *   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
 *   \[x] generateTokens is the new name for this^
@@ -149,5 +272,13 @@ This package is in active development, and breaking changes may happen until the
 *   \[x] have more people test on other platforms (mac tester needed)
 *   \[x] switch to new pluggable backend
 
+## Changes
+This repository serves as the new bindings for nodejs users.
+- If you were a user of [these bindings](https://github.com/nomic-ai/gpt4all-ts), they are outdated.
+- Version 4 includes the follow breaking changes
+    * `createEmbedding` & `EmbeddingModel.embed()` returns an object, `EmbeddingResult`, instead of a float32array.
+    * Removed deprecated types `ModelType` and `ModelFile`
+    * Removed deprecated initiation of model by string path only
+
 
 ### API Reference

gpt4all-bindings/typescript/binding.ci.gyp

@@ -6,12 +6,12 @@
         "<!@(node -p \"require('node-addon-api').include\")",
         "gpt4all-backend",
       ],
-      "sources": [ 
+      "sources": [
         # PREVIOUS VERSION: had to required the sources, but with newest changes do not need to
         #"../../gpt4all-backend/llama.cpp/examples/common.cpp",
         #"../../gpt4all-backend/llama.cpp/ggml.c",
         #"../../gpt4all-backend/llama.cpp/llama.cpp",
-        # "../../gpt4all-backend/utils.cpp", 
+        # "../../gpt4all-backend/utils.cpp",
         "gpt4all-backend/llmodel_c.cpp",
         "gpt4all-backend/llmodel.cpp",
         "prompt.cc",
@@ -40,7 +40,7 @@
                     "AdditionalOptions": [
                         "/std:c++20",
                         "/EHsc",
-                  ], 
+                  ],
                 },
             },
         }],

gpt4all-bindings/typescript/binding.gyp

@@ -6,12 +6,12 @@
         "<!@(node -p \"require('node-addon-api').include\")",
         "../../gpt4all-backend",
       ],
-      "sources": [ 
+      "sources": [
         # PREVIOUS VERSION: had to required the sources, but with newest changes do not need to
         #"../../gpt4all-backend/llama.cpp/examples/common.cpp",
         #"../../gpt4all-backend/llama.cpp/ggml.c",
         #"../../gpt4all-backend/llama.cpp/llama.cpp",
-        # "../../gpt4all-backend/utils.cpp", 
+        # "../../gpt4all-backend/utils.cpp",
         "../../gpt4all-backend/llmodel_c.cpp",
         "../../gpt4all-backend/llmodel.cpp",
         "prompt.cc",
@@ -40,7 +40,7 @@
                     "AdditionalOptions": [
                         "/std:c++20",
                         "/EHsc",
-                  ], 
+                  ],
                 },
             },
         }],

gpt4all-bindings/typescript/index.cc

@@ -1,175 +1,171 @@
 #include "index.h"
+#include "napi.h"
 
-
-Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
-    Napi::Function self = DefineClass(env, "LLModel", {
-       InstanceMethod("type",  &NodeModelWrapper::GetType),
-       InstanceMethod("isModelLoaded", &NodeModelWrapper::IsModelLoaded),
-       InstanceMethod("name", &NodeModelWrapper::GetName),
-       InstanceMethod("stateSize", &NodeModelWrapper::StateSize),
-       InstanceMethod("raw_prompt", &NodeModelWrapper::Prompt),
-       InstanceMethod("setThreadCount", &NodeModelWrapper::SetThreadCount),
-       InstanceMethod("embed", &NodeModelWrapper::GenerateEmbedding),
-       InstanceMethod("threadCount", &NodeModelWrapper::ThreadCount),
-       InstanceMethod("getLibraryPath", &NodeModelWrapper::GetLibraryPath),
-       InstanceMethod("initGpuByString", &NodeModelWrapper::InitGpuByString),
-       InstanceMethod("hasGpuDevice", &NodeModelWrapper::HasGpuDevice),
-       InstanceMethod("listGpu", &NodeModelWrapper::GetGpuDevices),
-       InstanceMethod("memoryNeeded", &NodeModelWrapper::GetRequiredMemory),
-       InstanceMethod("dispose", &NodeModelWrapper::Dispose)
-    });
+Napi::Function NodeModelWrapper::GetClass(Napi::Env env)
+{
+    Napi::Function self = DefineClass(env, "LLModel",
+                                      {InstanceMethod("type", &NodeModelWrapper::GetType),
+                                       InstanceMethod("isModelLoaded", &NodeModelWrapper::IsModelLoaded),
+                                       InstanceMethod("name", &NodeModelWrapper::GetName),
+                                       InstanceMethod("stateSize", &NodeModelWrapper::StateSize),
+                                       InstanceMethod("infer", &NodeModelWrapper::Infer),
+                                       InstanceMethod("setThreadCount", &NodeModelWrapper::SetThreadCount),
+                                       InstanceMethod("embed", &NodeModelWrapper::GenerateEmbedding),
+                                       InstanceMethod("threadCount", &NodeModelWrapper::ThreadCount),
+                                       InstanceMethod("getLibraryPath", &NodeModelWrapper::GetLibraryPath),
+                                       InstanceMethod("initGpuByString", &NodeModelWrapper::InitGpuByString),
+                                       InstanceMethod("hasGpuDevice", &NodeModelWrapper::HasGpuDevice),
+                                       InstanceMethod("listGpu", &NodeModelWrapper::GetGpuDevices),
+                                       InstanceMethod("memoryNeeded", &NodeModelWrapper::GetRequiredMemory),
+                                       InstanceMethod("dispose", &NodeModelWrapper::Dispose)});
     // Keep a static reference to the constructor
     //
-    Napi::FunctionReference* constructor = new Napi::FunctionReference();
+    Napi::FunctionReference *constructor = new Napi::FunctionReference();
     *constructor = Napi::Persistent(self);
     env.SetInstanceData(constructor);
     return self;
 }
-Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info) 
+Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo &info)
 {
     auto env = info.Env();
-    return Napi::Number::New(env, static_cast<uint32_t>(llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers) ));
-
+    return Napi::Number::New(
+        env, static_cast<uint32_t>(llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers)));
 }
-  Napi::Value NodeModelWrapper::GetGpuDevices(const Napi::CallbackInfo& info) 
-  {
+Napi::Value NodeModelWrapper::GetGpuDevices(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
     int num_devices = 0;
     auto mem_size = llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers);
-    llmodel_gpu_device* all_devices = llmodel_available_gpu_devices(GetInference(), mem_size, &num_devices);
-    if(all_devices == nullptr) {
-        Napi::Error::New(
-            env, 
-            "Unable to retrieve list of all GPU devices"
-        ).ThrowAsJavaScriptException(); 
+    llmodel_gpu_device *all_devices = llmodel_available_gpu_devices(mem_size, &num_devices);
+    if (all_devices == nullptr)
+    {
+        Napi::Error::New(env, "Unable to retrieve list of all GPU devices").ThrowAsJavaScriptException();
         return env.Undefined();
     }
     auto js_array = Napi::Array::New(env, num_devices);
-    for(int i = 0; i < num_devices; ++i) {
-       auto gpu_device = all_devices[i];
-       /* 
-        *
-        * struct llmodel_gpu_device {
-            int index = 0;
-            int type = 0;           // same as VkPhysicalDeviceType
-            size_t heapSize = 0; 
-            const char * name;
-            const char * vendor;
-          };
-        *
-        */
-       Napi::Object js_gpu_device = Napi::Object::New(env);
+    for (int i = 0; i < num_devices; ++i)
+    {
+        auto gpu_device = all_devices[i];
+        /*
+         *
+         * struct llmodel_gpu_device {
+             int index = 0;
+             int type = 0;           // same as VkPhysicalDeviceType
+             size_t heapSize = 0;
+             const char * name;
+             const char * vendor;
+           };
+         *
+         */
+        Napi::Object js_gpu_device = Napi::Object::New(env);
         js_gpu_device["index"] = uint32_t(gpu_device.index);
         js_gpu_device["type"] = uint32_t(gpu_device.type);
-        js_gpu_device["heapSize"] = static_cast<uint32_t>( gpu_device.heapSize );
-        js_gpu_device["name"]= gpu_device.name;
+        js_gpu_device["heapSize"] = static_cast<uint32_t>(gpu_device.heapSize);
+        js_gpu_device["name"] = gpu_device.name;
         js_gpu_device["vendor"] = gpu_device.vendor;
 
         js_array[i] = js_gpu_device;
     }
     return js_array;
-  }
+}
 
-  Napi::Value NodeModelWrapper::GetType(const Napi::CallbackInfo& info) 
-  {
-    if(type.empty()) {
+Napi::Value NodeModelWrapper::GetType(const Napi::CallbackInfo &info)
+{
+    if (type.empty())
+    {
         return info.Env().Undefined();
-    } 
+    }
     return Napi::String::New(info.Env(), type);
-  }
+}
 
-  Napi::Value NodeModelWrapper::InitGpuByString(const Napi::CallbackInfo& info) 
-  {
+Napi::Value NodeModelWrapper::InitGpuByString(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
     size_t memory_required = static_cast<size_t>(info[0].As<Napi::Number>().Uint32Value());
-    
-    std::string gpu_device_identifier = info[1].As<Napi::String>();   
+
+    std::string gpu_device_identifier = info[1].As<Napi::String>();
 
     size_t converted_value;
-    if(memory_required <= std::numeric_limits<size_t>::max()) {
+    if (memory_required <= std::numeric_limits<size_t>::max())
+    {
         converted_value = static_cast<size_t>(memory_required);
-    } else {
-         Napi::Error::New(
-            env, 
-            "invalid number for memory size. Exceeded bounds for memory."
-        ).ThrowAsJavaScriptException(); 
+    }
+    else
+    {
+        Napi::Error::New(env, "invalid number for memory size. Exceeded bounds for memory.")
+            .ThrowAsJavaScriptException();
         return env.Undefined();
     }
-    
+
     auto result = llmodel_gpu_init_gpu_device_by_string(GetInference(), converted_value, gpu_device_identifier.c_str());
     return Napi::Boolean::New(env, result);
-  }
-  Napi::Value NodeModelWrapper::HasGpuDevice(const Napi::CallbackInfo& info) 
-  {
+}
+Napi::Value NodeModelWrapper::HasGpuDevice(const Napi::CallbackInfo &info)
+{
     return Napi::Boolean::New(info.Env(), llmodel_has_gpu_device(GetInference()));
-  }
+}
 
-  NodeModelWrapper::NodeModelWrapper(const Napi::CallbackInfo& info) : Napi::ObjectWrap<NodeModelWrapper>(info) 
-  {
+NodeModelWrapper::NodeModelWrapper(const Napi::CallbackInfo &info) : Napi::ObjectWrap<NodeModelWrapper>(info)
+{
     auto env = info.Env();
-    fs::path model_path;
+    auto config_object = info[0].As<Napi::Object>();
 
-    std::string full_weight_path,
-                library_path = ".",
-                model_name, 
-                device;
-    if(info[0].IsString()) {
-        model_path = info[0].As<Napi::String>().Utf8Value();
-        full_weight_path = model_path.string();
-        std::cout << "DEPRECATION: constructor accepts object now. Check docs for more.\n";
-    } else {
-        auto config_object = info[0].As<Napi::Object>();
-        model_name = config_object.Get("model_name").As<Napi::String>();
-        model_path = config_object.Get("model_path").As<Napi::String>().Utf8Value(); 
-        if(config_object.Has("model_type")) {
-            type = config_object.Get("model_type").As<Napi::String>(); 
-        }
-        full_weight_path = (model_path / fs::path(model_name)).string();
-        
-        if(config_object.Has("library_path")) {
-            library_path = config_object.Get("library_path").As<Napi::String>(); 
-        } else {
-            library_path = ".";
-        }
-        device = config_object.Get("device").As<Napi::String>();
+    // sets the directory where models (gguf files) are to be searched
+    llmodel_set_implementation_search_path(
+        config_object.Has("library_path") ? config_object.Get("library_path").As<Napi::String>().Utf8Value().c_str()
+                                          : ".");
 
-        nCtx = config_object.Get("nCtx").As<Napi::Number>().Int32Value();
-        nGpuLayers = config_object.Get("ngl").As<Napi::Number>().Int32Value();
-    }
-    llmodel_set_implementation_search_path(library_path.c_str());
-    const char* e;
+    std::string model_name = config_object.Get("model_name").As<Napi::String>();
+    fs::path model_path = config_object.Get("model_path").As<Napi::String>().Utf8Value();
+    std::string full_weight_path = (model_path / fs::path(model_name)).string();
+
+    name = model_name.empty() ? model_path.filename().string() : model_name;
+    full_model_path = full_weight_path;
+    nCtx = config_object.Get("nCtx").As<Napi::Number>().Int32Value();
+    nGpuLayers = config_object.Get("ngl").As<Napi::Number>().Int32Value();
+
+    const char *e;
     inference_ = llmodel_model_create2(full_weight_path.c_str(), "auto", &e);
-    if(!inference_) {
-       Napi::Error::New(env, e).ThrowAsJavaScriptException();
-       return;
+    if (!inference_)
+    {
+        Napi::Error::New(env, e).ThrowAsJavaScriptException();
+        return;
     }
-    if(GetInference() == nullptr) {
-       std::cerr << "Tried searching libraries in \"" << library_path << "\"" <<  std::endl;
-       std::cerr << "Tried searching for model weight in \"" << full_weight_path << "\"" << std::endl;
-       std::cerr << "Do you have runtime libraries installed?" << std::endl;
-       Napi::Error::New(env, "Had an issue creating llmodel object, inference is null").ThrowAsJavaScriptException(); 
-       return;
+    if (GetInference() == nullptr)
+    {
+        std::cerr << "Tried searching libraries in \"" << llmodel_get_implementation_search_path() << "\"" << std::endl;
+        std::cerr << "Tried searching for model weight in \"" << full_weight_path << "\"" << std::endl;
+        std::cerr << "Do you have runtime libraries installed?" << std::endl;
+        Napi::Error::New(env, "Had an issue creating llmodel object, inference is null").ThrowAsJavaScriptException();
+        return;
     }
-    if(device != "cpu") {
-        size_t mem = llmodel_required_mem(GetInference(), full_weight_path.c_str(),nCtx, nGpuLayers);
+
+    std::string device = config_object.Get("device").As<Napi::String>();
+    if (device != "cpu")
+    {
+        size_t mem = llmodel_required_mem(GetInference(), full_weight_path.c_str(), nCtx, nGpuLayers);
 
         auto success = llmodel_gpu_init_gpu_device_by_string(GetInference(), mem, device.c_str());
-        if(!success) {
-            //https://github.com/nomic-ai/gpt4all/blob/3acbef14b7c2436fe033cae9036e695d77461a16/gpt4all-bindings/python/gpt4all/pyllmodel.py#L215
-            //Haven't implemented this but it is still open to contribution
+        if (!success)
+        {
+            // https://github.com/nomic-ai/gpt4all/blob/3acbef14b7c2436fe033cae9036e695d77461a16/gpt4all-bindings/python/gpt4all/pyllmodel.py#L215
+            // Haven't implemented this but it is still open to contribution
             std::cout << "WARNING: Failed to init GPU\n";
         }
     }
 
     auto success = llmodel_loadModel(GetInference(), full_weight_path.c_str(), nCtx, nGpuLayers);
-    if(!success) {
-        Napi::Error::New(env, "Failed to load model at given path").ThrowAsJavaScriptException(); 
+    if (!success)
+    {
+        Napi::Error::New(env, "Failed to load model at given path").ThrowAsJavaScriptException();
         return;
     }
-
-    name = model_name.empty() ? model_path.filename().string() : model_name;
-    full_model_path = full_weight_path;
-  };
+    // optional
+    if (config_object.Has("model_type"))
+    {
+        type = config_object.Get("model_type").As<Napi::String>();
+    }
+};
 
 //  NodeModelWrapper::~NodeModelWrapper() {
 //    if(GetInference() != nullptr) {
@@ -182,177 +178,275 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
 //    if(inference_ != nullptr) {
 //        std::cout << "Debug: deleting model\n";
 //
-//    } 
+//    }
 //  }
-  Napi::Value NodeModelWrapper::IsModelLoaded(const Napi::CallbackInfo& info) {
+Napi::Value NodeModelWrapper::IsModelLoaded(const Napi::CallbackInfo &info)
+{
     return Napi::Boolean::New(info.Env(), llmodel_isModelLoaded(GetInference()));
-  }
+}
 
-  Napi::Value NodeModelWrapper::StateSize(const Napi::CallbackInfo& info) {
+Napi::Value NodeModelWrapper::StateSize(const Napi::CallbackInfo &info)
+{
     // Implement the binding for the stateSize method
     return Napi::Number::New(info.Env(), static_cast<int64_t>(llmodel_get_state_size(GetInference())));
-  }
-  
-  Napi::Value NodeModelWrapper::GenerateEmbedding(const Napi::CallbackInfo& info) {
+}
+
+Napi::Array ChunkedFloatPtr(float *embedding_ptr, int embedding_size, int text_len, Napi::Env const &env)
+{
+    auto n_embd = embedding_size / text_len;
+    // std::cout << "Embedding size: " << embedding_size << std::endl;
+    // std::cout << "Text length: " << text_len << std::endl;
+    // std::cout << "Chunk size (n_embd): " << n_embd << std::endl;
+    Napi::Array result = Napi::Array::New(env, text_len);
+    auto count = 0;
+    for (int i = 0; i < embedding_size; i += n_embd)
+    {
+        int end = std::min(i + n_embd, embedding_size);
+        // possible bounds error?
+        // Constructs a container with as many elements as the range [first,last), with each element emplace-constructed
+        // from its corresponding element in that range, in the same order.
+        std::vector<float> chunk(embedding_ptr + i, embedding_ptr + end);
+        Napi::Float32Array fltarr = Napi::Float32Array::New(env, chunk.size());
+        // I know there's a way to emplace the raw float ptr into a Napi::Float32Array but idk how and
+        //  im too scared to cause memory issues
+        //  this is goodenough
+        for (int j = 0; j < chunk.size(); j++)
+        {
+
+            fltarr.Set(j, chunk[j]);
+        }
+        result.Set(count++, fltarr);
+    }
+    return result;
+}
+
+Napi::Value NodeModelWrapper::GenerateEmbedding(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
-    std::string text = info[0].As<Napi::String>().Utf8Value();
-    size_t embedding_size = 0;
-    float* arr = llmodel_embedding(GetInference(), text.c_str(), &embedding_size);
-    if(arr == nullptr) {
-        Napi::Error::New(
-            env, 
-            "Cannot embed. native embedder returned 'nullptr'"
-        ).ThrowAsJavaScriptException(); 
+
+    auto prefix = info[1];
+    auto dimensionality = info[2].As<Napi::Number>().Int32Value();
+    auto do_mean = info[3].As<Napi::Boolean>().Value();
+    auto atlas = info[4].As<Napi::Boolean>().Value();
+    size_t embedding_size;
+    size_t token_count = 0;
+
+    // This procedure can maybe be optimized but its whatever, i have too many intermediary structures
+    std::vector<std::string> text_arr;
+    bool is_single_text = false;
+    if (info[0].IsString())
+    {
+        is_single_text = true;
+        text_arr.push_back(info[0].As<Napi::String>().Utf8Value());
+    }
+    else
+    {
+        auto jsarr = info[0].As<Napi::Array>();
+        size_t len = jsarr.Length();
+        text_arr.reserve(len);
+        for (size_t i = 0; i < len; ++i)
+        {
+            std::string str = jsarr.Get(i).As<Napi::String>().Utf8Value();
+            text_arr.push_back(str);
+        }
+    }
+    std::vector<const char *> str_ptrs;
+    str_ptrs.reserve(text_arr.size() + 1);
+    for (size_t i = 0; i < text_arr.size(); ++i)
+        str_ptrs.push_back(text_arr[i].c_str());
+    str_ptrs.push_back(nullptr);
+    const char *_err = nullptr;
+    float *embeds = llmodel_embed(GetInference(), str_ptrs.data(),  &embedding_size,
+                                  prefix.IsUndefined() ? nullptr : prefix.As<Napi::String>().Utf8Value().c_str(),
+                                  dimensionality, &token_count, do_mean, atlas, nullptr, &_err);
+    if (!embeds)
+    {
+        // i dont wanna deal with c strings lol
+        std::string err(_err);
+        Napi::Error::New(env, err == "(unknown error)" ? "Unknown error: sorry bud" : err).ThrowAsJavaScriptException();
         return env.Undefined();
     }
+    auto embedmat = ChunkedFloatPtr(embeds, embedding_size, text_arr.size(), env);
 
-    if(embedding_size == 0 && text.size() != 0 ) {
-        std::cout << "Warning: embedding length 0 but input text length > 0" << std::endl;
-    }
-    Napi::Float32Array js_array = Napi::Float32Array::New(env, embedding_size);
-    
-    for (size_t i = 0; i < embedding_size; ++i) {
-        float element = *(arr + i);
-        js_array[i] = element;
+    llmodel_free_embedding(embeds);
+    auto res = Napi::Object::New(env);
+    res.Set("n_prompt_tokens", token_count);
+    if(is_single_text) {
+        res.Set("embeddings", embedmat.Get(static_cast<uint32_t>(0)));
+    } else {
+        res.Set("embeddings", embedmat);
     }
 
-    llmodel_free_embedding(arr);
-
-    return js_array;
-  }
+    return res;
+}
 
 /**
  * Generate a response using the model.
- * @param model A pointer to the llmodel_model instance.
  * @param prompt A string representing the input prompt.
- * @param prompt_callback A callback function for handling the processing of prompt.
- * @param response_callback A callback function for handling the generated response.
- * @param recalculate_callback A callback function for handling recalculation requests.
- * @param ctx A pointer to the llmodel_prompt_context structure.
+ * @param options Inference options.
  */
-  Napi::Value NodeModelWrapper::Prompt(const Napi::CallbackInfo& info) {
+Napi::Value NodeModelWrapper::Infer(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
-    std::string question;
-    if(info[0].IsString()) {
-        question = info[0].As<Napi::String>().Utf8Value();
-    } else {
+    std::string prompt;
+    if (info[0].IsString())
+    {
+        prompt = info[0].As<Napi::String>().Utf8Value();
+    }
+    else
+    {
         Napi::Error::New(info.Env(), "invalid string argument").ThrowAsJavaScriptException();
         return info.Env().Undefined();
     }
-    //defaults copied from python bindings
-    llmodel_prompt_context promptContext = {
-             .logits = nullptr,
-             .tokens = nullptr,
-             .n_past = 0,
-             .n_ctx = 1024,
-             .n_predict = 128,
-             .top_k = 40,
-             .top_p = 0.9f,
-             .min_p = 0.0f,
-             .temp = 0.72f,
-             .n_batch = 8,
-             .repeat_penalty = 1.0f,
-             .repeat_last_n = 10,
-             .context_erase = 0.5
-         };
-    
-    PromptWorkerConfig promptWorkerConfig;
 
-    if(info[1].IsObject())
-    {
-       auto inputObject = info[1].As<Napi::Object>();
-             
-        // Extract and assign the properties
-       if (inputObject.Has("logits") || inputObject.Has("tokens")) {
-           Napi::Error::New(info.Env(), "Invalid input: 'logits' or 'tokens' properties are not allowed").ThrowAsJavaScriptException();
-           return info.Env().Undefined();
-       }
-             // Assign the remaining properties
-       if(inputObject.Has("n_past")) 
-            promptContext.n_past = inputObject.Get("n_past").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("n_ctx")) 
-            promptContext.n_ctx = inputObject.Get("n_ctx").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("n_predict"))
-            promptContext.n_predict = inputObject.Get("n_predict").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("top_k"))
-            promptContext.top_k = inputObject.Get("top_k").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("top_p")) 
-            promptContext.top_p = inputObject.Get("top_p").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("min_p")) 
-            promptContext.min_p = inputObject.Get("min_p").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("temp")) 
-            promptContext.temp = inputObject.Get("temp").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("n_batch")) 
-            promptContext.n_batch = inputObject.Get("n_batch").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("repeat_penalty")) 
-            promptContext.repeat_penalty = inputObject.Get("repeat_penalty").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("repeat_last_n")) 
-            promptContext.repeat_last_n = inputObject.Get("repeat_last_n").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("context_erase")) 
-            promptContext.context_erase = inputObject.Get("context_erase").As<Napi::Number>().FloatValue();
-    }
-    else
+    if (!info[1].IsObject())
     {
         Napi::Error::New(info.Env(), "Missing Prompt Options").ThrowAsJavaScriptException();
         return info.Env().Undefined();
     }
+    // defaults copied from python bindings
+    llmodel_prompt_context promptContext = {.logits = nullptr,
+                                            .tokens = nullptr,
+                                            .n_past = 0,
+                                            .n_ctx = nCtx,
+                                            .n_predict = 4096,
+                                            .top_k = 40,
+                                            .top_p = 0.9f,
+                                            .min_p = 0.0f,
+                                            .temp = 0.1f,
+                                            .n_batch = 8,
+                                            .repeat_penalty = 1.2f,
+                                            .repeat_last_n = 10,
+                                            .context_erase = 0.75};
 
-    if(info.Length() >= 3 && info[2].IsFunction()){
-        promptWorkerConfig.bHasTokenCallback = true;
-        promptWorkerConfig.tokenCallback = info[2].As<Napi::Function>();
+    PromptWorkerConfig promptWorkerConfig;
+
+    auto inputObject = info[1].As<Napi::Object>();
+
+    if (inputObject.Has("logits") || inputObject.Has("tokens"))
+    {
+        Napi::Error::New(info.Env(), "Invalid input: 'logits' or 'tokens' properties are not allowed")
+            .ThrowAsJavaScriptException();
+        return info.Env().Undefined();
     }
 
-    
+    // Assign the remaining properties
+    if (inputObject.Has("nPast") && inputObject.Get("nPast").IsNumber())
+    {
+        promptContext.n_past = inputObject.Get("nPast").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("nPredict") && inputObject.Get("nPredict").IsNumber())
+    {
+        promptContext.n_predict = inputObject.Get("nPredict").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("topK") && inputObject.Get("topK").IsNumber())
+    {
+        promptContext.top_k = inputObject.Get("topK").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("topP") && inputObject.Get("topP").IsNumber())
+    {
+        promptContext.top_p = inputObject.Get("topP").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("minP") && inputObject.Get("minP").IsNumber())
+    {
+        promptContext.min_p = inputObject.Get("minP").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("temp") && inputObject.Get("temp").IsNumber())
+    {
+        promptContext.temp = inputObject.Get("temp").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("nBatch") && inputObject.Get("nBatch").IsNumber())
+    {
+        promptContext.n_batch = inputObject.Get("nBatch").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("repeatPenalty") && inputObject.Get("repeatPenalty").IsNumber())
+    {
+        promptContext.repeat_penalty = inputObject.Get("repeatPenalty").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("repeatLastN") && inputObject.Get("repeatLastN").IsNumber())
+    {
+        promptContext.repeat_last_n = inputObject.Get("repeatLastN").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("contextErase") && inputObject.Get("contextErase").IsNumber())
+    {
+        promptContext.context_erase = inputObject.Get("contextErase").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("onPromptToken") && inputObject.Get("onPromptToken").IsFunction())
+    {
+        promptWorkerConfig.promptCallback = inputObject.Get("onPromptToken").As<Napi::Function>();
+        promptWorkerConfig.hasPromptCallback = true;
+    }
+    if (inputObject.Has("onResponseToken") && inputObject.Get("onResponseToken").IsFunction())
+    {
+        promptWorkerConfig.responseCallback = inputObject.Get("onResponseToken").As<Napi::Function>();
+        promptWorkerConfig.hasResponseCallback = true;
+    }
 
-    //copy to protect llmodel resources when splitting to new thread
-    // llmodel_prompt_context copiedPrompt = promptContext;
+    // copy to protect llmodel resources when splitting to new thread
+    //  llmodel_prompt_context copiedPrompt = promptContext;
     promptWorkerConfig.context = promptContext;
     promptWorkerConfig.model = GetInference();
     promptWorkerConfig.mutex = &inference_mutex;
-    promptWorkerConfig.prompt = question;
+    promptWorkerConfig.prompt = prompt;
     promptWorkerConfig.result = "";
 
-    
+    promptWorkerConfig.promptTemplate = inputObject.Get("promptTemplate").As<Napi::String>();
+    if (inputObject.Has("special"))
+    {
+        promptWorkerConfig.special = inputObject.Get("special").As<Napi::Boolean>();
+    }
+    if (inputObject.Has("fakeReply"))
+    {
+        // this will be deleted in the worker
+        promptWorkerConfig.fakeReply = new std::string(inputObject.Get("fakeReply").As<Napi::String>().Utf8Value());
+    }
     auto worker = new PromptWorker(env, promptWorkerConfig);
 
     worker->Queue();
 
     return worker->GetPromise();
-  }
-  void NodeModelWrapper::Dispose(const Napi::CallbackInfo& info) {
+}
+void NodeModelWrapper::Dispose(const Napi::CallbackInfo &info)
+{
     llmodel_model_destroy(inference_);
-  }
-  void NodeModelWrapper::SetThreadCount(const Napi::CallbackInfo& info) {
-    if(info[0].IsNumber()) {
+}
+void NodeModelWrapper::SetThreadCount(const Napi::CallbackInfo &info)
+{
+    if (info[0].IsNumber())
+    {
         llmodel_setThreadCount(GetInference(), info[0].As<Napi::Number>().Int64Value());
-    } else {
-        Napi::Error::New(info.Env(), "Could not set thread count: argument 1 is NaN").ThrowAsJavaScriptException(); 
+    }
+    else
+    {
+        Napi::Error::New(info.Env(), "Could not set thread count: argument 1 is NaN").ThrowAsJavaScriptException();
         return;
     }
-  }
-
-  Napi::Value NodeModelWrapper::GetName(const Napi::CallbackInfo& info) {
-    return Napi::String::New(info.Env(), name);
-  }
-  Napi::Value NodeModelWrapper::ThreadCount(const Napi::CallbackInfo& info) {
-    return Napi::Number::New(info.Env(), llmodel_threadCount(GetInference()));
-  }
-
-  Napi::Value NodeModelWrapper::GetLibraryPath(const Napi::CallbackInfo& info) {
-      return Napi::String::New(info.Env(),
-        llmodel_get_implementation_search_path());
-  }
-
-  llmodel_model NodeModelWrapper::GetInference() {
-    return inference_;
-  }
-
-//Exports Bindings
-Napi::Object Init(Napi::Env env, Napi::Object exports) {
-  exports["LLModel"] = NodeModelWrapper::GetClass(env);
-  return exports;
 }
 
+Napi::Value NodeModelWrapper::GetName(const Napi::CallbackInfo &info)
+{
+    return Napi::String::New(info.Env(), name);
+}
+Napi::Value NodeModelWrapper::ThreadCount(const Napi::CallbackInfo &info)
+{
+    return Napi::Number::New(info.Env(), llmodel_threadCount(GetInference()));
+}
 
+Napi::Value NodeModelWrapper::GetLibraryPath(const Napi::CallbackInfo &info)
+{
+    return Napi::String::New(info.Env(), llmodel_get_implementation_search_path());
+}
+
+llmodel_model NodeModelWrapper::GetInference()
+{
+    return inference_;
+}
+
+// Exports Bindings
+Napi::Object Init(Napi::Env env, Napi::Object exports)
+{
+    exports["LLModel"] = NodeModelWrapper::GetClass(env);
+    return exports;
+}
 
 NODE_API_MODULE(NODE_GYP_MODULE_NAME, Init)

gpt4all-bindings/typescript/index.h

@@ -1,62 +1,63 @@
-#include <napi.h>
 #include "llmodel.h"
-#include <iostream>
-#include "llmodel_c.h" 
+#include "llmodel_c.h"
 #include "prompt.h"
 #include <atomic>
-#include <memory>
 #include <filesystem>
-#include <set>
+#include <iostream>
+#include <memory>
 #include <mutex>
+#include <napi.h>
+#include <set>
 
 namespace fs = std::filesystem;
 
+class NodeModelWrapper : public Napi::ObjectWrap<NodeModelWrapper>
+{
 
-class NodeModelWrapper: public Napi::ObjectWrap<NodeModelWrapper> {
-  
-public:
-  NodeModelWrapper(const Napi::CallbackInfo &);
-  //virtual ~NodeModelWrapper();
-  Napi::Value GetType(const Napi::CallbackInfo& info);
-  Napi::Value IsModelLoaded(const Napi::CallbackInfo& info);
-  Napi::Value StateSize(const Napi::CallbackInfo& info);
-  //void Finalize(Napi::Env env) override;
-  /**
-   * Prompting the model. This entails spawning a new thread and adding the response tokens
-   * into a thread local string variable.
-   */
-  Napi::Value Prompt(const Napi::CallbackInfo& info);
-  void SetThreadCount(const Napi::CallbackInfo& info);
-  void Dispose(const Napi::CallbackInfo& info);
-  Napi::Value GetName(const Napi::CallbackInfo& info);
-  Napi::Value ThreadCount(const Napi::CallbackInfo& info);
-  Napi::Value GenerateEmbedding(const Napi::CallbackInfo& info);
-  Napi::Value HasGpuDevice(const Napi::CallbackInfo& info);
-  Napi::Value ListGpus(const Napi::CallbackInfo& info);
-  Napi::Value InitGpuByString(const Napi::CallbackInfo& info);
-  Napi::Value GetRequiredMemory(const Napi::CallbackInfo& info);
-  Napi::Value GetGpuDevices(const Napi::CallbackInfo& info);
-  /*
-   * The path that is used to search for the dynamic libraries
-   */
-  Napi::Value GetLibraryPath(const Napi::CallbackInfo& info);
-  /**
-   * Creates the LLModel class
-   */
-  static Napi::Function GetClass(Napi::Env);
-  llmodel_model GetInference();
-private:
-  /**
-   * The underlying inference that interfaces with the C interface
-   */
-  llmodel_model inference_;
+  public:
+    NodeModelWrapper(const Napi::CallbackInfo &);
+    // virtual ~NodeModelWrapper();
+    Napi::Value GetType(const Napi::CallbackInfo &info);
+    Napi::Value IsModelLoaded(const Napi::CallbackInfo &info);
+    Napi::Value StateSize(const Napi::CallbackInfo &info);
+    // void Finalize(Napi::Env env) override;
+    /**
+     * Prompting the model. This entails spawning a new thread and adding the response tokens
+     * into a thread local string variable.
+     */
+    Napi::Value Infer(const Napi::CallbackInfo &info);
+    void SetThreadCount(const Napi::CallbackInfo &info);
+    void Dispose(const Napi::CallbackInfo &info);
+    Napi::Value GetName(const Napi::CallbackInfo &info);
+    Napi::Value ThreadCount(const Napi::CallbackInfo &info);
+    Napi::Value GenerateEmbedding(const Napi::CallbackInfo &info);
+    Napi::Value HasGpuDevice(const Napi::CallbackInfo &info);
+    Napi::Value ListGpus(const Napi::CallbackInfo &info);
+    Napi::Value InitGpuByString(const Napi::CallbackInfo &info);
+    Napi::Value GetRequiredMemory(const Napi::CallbackInfo &info);
+    Napi::Value GetGpuDevices(const Napi::CallbackInfo &info);
+    /*
+     * The path that is used to search for the dynamic libraries
+     */
+    Napi::Value GetLibraryPath(const Napi::CallbackInfo &info);
+    /**
+     * Creates the LLModel class
+     */
+    static Napi::Function GetClass(Napi::Env);
+    llmodel_model GetInference();
 
-  std::mutex inference_mutex;
+  private:
+    /**
+     * The underlying inference that interfaces with the C interface
+     */
+    llmodel_model inference_;
 
-  std::string type;
-  // corresponds to LLModel::name() in typescript
-  std::string name;
-  int nCtx{};
-  int nGpuLayers{};
-  std::string full_model_path;
+    std::mutex inference_mutex;
+
+    std::string type;
+    // corresponds to LLModel::name() in typescript
+    std::string name;
+    int nCtx{};
+    int nGpuLayers{};
+    std::string full_model_path;
 };

gpt4all-bindings/typescript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt4all",
-  "version": "3.2.0",
+  "version": "4.0.0",
   "packageManager": "yarn@3.6.1",
   "main": "src/gpt4all.js",
   "repository": "nomic-ai/gpt4all",
@@ -22,7 +22,6 @@
   ],
   "dependencies": {
     "md5-file": "^5.0.0",
-    "mkdirp": "^3.0.1",
     "node-addon-api": "^6.1.0",
     "node-gyp-build": "^4.6.0"
   },

gpt4all-bindings/typescript/prompt.cc

@@ -2,145 +2,195 @@
 #include <future>
 
 PromptWorker::PromptWorker(Napi::Env env, PromptWorkerConfig config)
-        : promise(Napi::Promise::Deferred::New(env)), _config(config), AsyncWorker(env) {
-            if(_config.bHasTokenCallback){
-                _tsfn = Napi::ThreadSafeFunction::New(config.tokenCallback.Env(),config.tokenCallback,"PromptWorker",0,1,this);
-            }
-        }
-
-    PromptWorker::~PromptWorker()
+    : promise(Napi::Promise::Deferred::New(env)), _config(config), AsyncWorker(env)
+{
+    if (_config.hasResponseCallback)
     {
-        if(_config.bHasTokenCallback){
-            _tsfn.Release();
-        }
+        _responseCallbackFn = Napi::ThreadSafeFunction::New(config.responseCallback.Env(), config.responseCallback,
+                                                            "PromptWorker", 0, 1, this);
     }
 
-    void PromptWorker::Execute()
+    if (_config.hasPromptCallback)
     {
-        _config.mutex->lock();
+        _promptCallbackFn = Napi::ThreadSafeFunction::New(config.promptCallback.Env(), config.promptCallback,
+                                                          "PromptWorker", 0, 1, this);
+    }
+}
 
-        LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper *>(_config.model);
+PromptWorker::~PromptWorker()
+{
+    if (_config.hasResponseCallback)
+    {
+        _responseCallbackFn.Release();
+    }
+    if (_config.hasPromptCallback)
+    {
+        _promptCallbackFn.Release();
+    }
+}
 
-        auto ctx = &_config.context;
+void PromptWorker::Execute()
+{
+    _config.mutex->lock();
 
-        if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
-            wrapper->promptContext.tokens.resize(ctx->n_past);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper *>(_config.model);
 
-        // Copy the C prompt context
-        wrapper->promptContext.n_past = ctx->n_past;
-        wrapper->promptContext.n_ctx = ctx->n_ctx;
-        wrapper->promptContext.n_predict = ctx->n_predict;
-        wrapper->promptContext.top_k = ctx->top_k;
-        wrapper->promptContext.top_p = ctx->top_p;
-        wrapper->promptContext.temp = ctx->temp;
-        wrapper->promptContext.n_batch = ctx->n_batch;
-        wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
-        wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
-        wrapper->promptContext.contextErase = ctx->context_erase;
+    auto ctx = &_config.context;
 
-        // Napi::Error::Fatal(
-        //         "SUPRA",
-        //         "About to prompt");
-        // Call the C++ prompt method
-        wrapper->llModel->prompt(
-            _config.prompt, 
-            [](int32_t tid) { return true; },
-            [this](int32_t token_id, const std::string tok)
-            {
-                return ResponseCallback(token_id, tok);
-            },
-            [](bool isRecalculating)
-            {
-                return isRecalculating;
-            },
-            wrapper->promptContext);
+    if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
+        wrapper->promptContext.tokens.resize(ctx->n_past);
 
-        // Update the C context by giving access to the wrappers raw pointers to std::vector data
-        // which involves no copies
-        ctx->logits = wrapper->promptContext.logits.data();
-        ctx->logits_size = wrapper->promptContext.logits.size();
-        ctx->tokens = wrapper->promptContext.tokens.data();
-        ctx->tokens_size = wrapper->promptContext.tokens.size();
+    // Copy the C prompt context
+    wrapper->promptContext.n_past = ctx->n_past;
+    wrapper->promptContext.n_ctx = ctx->n_ctx;
+    wrapper->promptContext.n_predict = ctx->n_predict;
+    wrapper->promptContext.top_k = ctx->top_k;
+    wrapper->promptContext.top_p = ctx->top_p;
+    wrapper->promptContext.temp = ctx->temp;
+    wrapper->promptContext.n_batch = ctx->n_batch;
+    wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
+    wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
+    wrapper->promptContext.contextErase = ctx->context_erase;
 
-        // Update the rest of the C prompt context
-        ctx->n_past = wrapper->promptContext.n_past;
-        ctx->n_ctx = wrapper->promptContext.n_ctx;
-        ctx->n_predict = wrapper->promptContext.n_predict;
-        ctx->top_k = wrapper->promptContext.top_k;
-        ctx->top_p = wrapper->promptContext.top_p;
-        ctx->temp = wrapper->promptContext.temp;
-        ctx->n_batch = wrapper->promptContext.n_batch;
-        ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
-        ctx->repeat_last_n = wrapper->promptContext.repeat_last_n;
-        ctx->context_erase = wrapper->promptContext.contextErase;
+    // Call the C++ prompt method
 
-        _config.mutex->unlock();
+    wrapper->llModel->prompt(
+        _config.prompt, _config.promptTemplate, [this](int32_t token_id) { return PromptCallback(token_id); },
+        [this](int32_t token_id, const std::string token) { return ResponseCallback(token_id, token); },
+        [](bool isRecalculating) { return isRecalculating; }, wrapper->promptContext, _config.special,
+        _config.fakeReply);
+
+    // Update the C context by giving access to the wrappers raw pointers to std::vector data
+    // which involves no copies
+    ctx->logits = wrapper->promptContext.logits.data();
+    ctx->logits_size = wrapper->promptContext.logits.size();
+    ctx->tokens = wrapper->promptContext.tokens.data();
+    ctx->tokens_size = wrapper->promptContext.tokens.size();
+
+    // Update the rest of the C prompt context
+    ctx->n_past = wrapper->promptContext.n_past;
+    ctx->n_ctx = wrapper->promptContext.n_ctx;
+    ctx->n_predict = wrapper->promptContext.n_predict;
+    ctx->top_k = wrapper->promptContext.top_k;
+    ctx->top_p = wrapper->promptContext.top_p;
+    ctx->temp = wrapper->promptContext.temp;
+    ctx->n_batch = wrapper->promptContext.n_batch;
+    ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
+    ctx->repeat_last_n = wrapper->promptContext.repeat_last_n;
+    ctx->context_erase = wrapper->promptContext.contextErase;
+
+    _config.mutex->unlock();
+}
+
+void PromptWorker::OnOK()
+{
+    Napi::Object returnValue = Napi::Object::New(Env());
+    returnValue.Set("text", result);
+    returnValue.Set("nPast", _config.context.n_past);
+    promise.Resolve(returnValue);
+    delete _config.fakeReply;
+}
+
+void PromptWorker::OnError(const Napi::Error &e)
+{
+    delete _config.fakeReply;
+    promise.Reject(e.Value());
+}
+
+Napi::Promise PromptWorker::GetPromise()
+{
+    return promise.Promise();
+}
+
+bool PromptWorker::ResponseCallback(int32_t token_id, const std::string token)
+{
+    if (token_id == -1)
+    {
+        return false;
     }
 
-    void PromptWorker::OnOK()
-    {
-        promise.Resolve(Napi::String::New(Env(), result));
-    }
-
-    void PromptWorker::OnError(const Napi::Error &e)
-    {
-        promise.Reject(e.Value());
-    }
-
-    Napi::Promise PromptWorker::GetPromise()
-    {
-        return promise.Promise();
-    }
-
-    bool PromptWorker::ResponseCallback(int32_t token_id, const std::string token)
-    {
-        if (token_id == -1)
-        {
-            return false;
-        }
-
-        if(!_config.bHasTokenCallback){
-            return true;
-        }
-
-        result += token;
-
-       std::promise<bool> promise;
-
-       auto info = new TokenCallbackInfo();
-       info->tokenId = token_id;
-       info->token = token;
-       info->total = result;
-
-       auto future = promise.get_future();
-
-       auto status = _tsfn.BlockingCall(info, [&promise](Napi::Env env, Napi::Function jsCallback, TokenCallbackInfo *value)
-       {
-           // Transform native data into JS data, passing it to the provided
-           // `jsCallback` -- the TSFN's JavaScript function.
-           auto token_id = Napi::Number::New(env, value->tokenId);
-           auto token = Napi::String::New(env, value->token);
-           auto total = Napi::String::New(env,value->total);
-           auto jsResult = jsCallback.Call({ token_id, token, total}).ToBoolean();
-           promise.set_value(jsResult);
-           // We're finished with the data.
-           delete value;
-       });
-       if (status != napi_ok) {
-            Napi::Error::Fatal(
-                "PromptWorkerResponseCallback",
-                "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
-        }
-
-       return future.get();
-    }
-
-    bool PromptWorker::RecalculateCallback(bool isRecalculating)
-    {
-        return isRecalculating;
-    }
-
-    bool PromptWorker::PromptCallback(int32_t tid)
+    if (!_config.hasResponseCallback)
     {
         return true;
     }
+
+    result += token;
+
+    std::promise<bool> promise;
+
+    auto info = new ResponseCallbackData();
+    info->tokenId = token_id;
+    info->token = token;
+
+    auto future = promise.get_future();
+
+    auto status = _responseCallbackFn.BlockingCall(
+        info, [&promise](Napi::Env env, Napi::Function jsCallback, ResponseCallbackData *value) {
+            try
+            {
+                // Transform native data into JS data, passing it to the provided
+                // `jsCallback` -- the TSFN's JavaScript function.
+                auto token_id = Napi::Number::New(env, value->tokenId);
+                auto token = Napi::String::New(env, value->token);
+                auto jsResult = jsCallback.Call({token_id, token}).ToBoolean();
+                promise.set_value(jsResult);
+            }
+            catch (const Napi::Error &e)
+            {
+                std::cerr << "Error in onResponseToken callback: " << e.what() << std::endl;
+                promise.set_value(false);
+            }
+
+            delete value;
+        });
+    if (status != napi_ok)
+    {
+        Napi::Error::Fatal("PromptWorkerResponseCallback", "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
+    }
+
+    return future.get();
+}
+
+bool PromptWorker::RecalculateCallback(bool isRecalculating)
+{
+    return isRecalculating;
+}
+
+bool PromptWorker::PromptCallback(int32_t token_id)
+{
+    if (!_config.hasPromptCallback)
+    {
+        return true;
+    }
+
+    std::promise<bool> promise;
+
+    auto info = new PromptCallbackData();
+    info->tokenId = token_id;
+
+    auto future = promise.get_future();
+
+    auto status = _promptCallbackFn.BlockingCall(
+        info, [&promise](Napi::Env env, Napi::Function jsCallback, PromptCallbackData *value) {
+            try
+            {
+                // Transform native data into JS data, passing it to the provided
+                // `jsCallback` -- the TSFN's JavaScript function.
+                auto token_id = Napi::Number::New(env, value->tokenId);
+                auto jsResult = jsCallback.Call({token_id}).ToBoolean();
+                promise.set_value(jsResult);
+            }
+            catch (const Napi::Error &e)
+            {
+                std::cerr << "Error in onPromptToken callback: " << e.what() << std::endl;
+                promise.set_value(false);
+            }
+            delete value;
+        });
+    if (status != napi_ok)
+    {
+        Napi::Error::Fatal("PromptWorkerPromptCallback", "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
+    }
+
+    return future.get();
+}

gpt4all-bindings/typescript/prompt.h

@@ -1,59 +1,72 @@
 #ifndef PREDICT_WORKER_H
 #define PREDICT_WORKER_H
 
-#include "napi.h"
-#include "llmodel_c.h"
 #include "llmodel.h"
-#include <thread>
-#include <mutex>
-#include <iostream>
+#include "llmodel_c.h"
+#include "napi.h"
 #include <atomic>
+#include <iostream>
 #include <memory>
+#include <mutex>
+#include <thread>
 
-struct TokenCallbackInfo
+struct ResponseCallbackData
+{
+    int32_t tokenId;
+    std::string token;
+};
+
+struct PromptCallbackData
+{
+    int32_t tokenId;
+};
+
+struct LLModelWrapper
+{
+    LLModel *llModel = nullptr;
+    LLModel::PromptContext promptContext;
+    ~LLModelWrapper()
     {
-        int32_t tokenId;
-        std::string total;
-        std::string token;
-    };
+        delete llModel;
+    }
+};
 
-    struct LLModelWrapper
-    {
-        LLModel *llModel = nullptr;
-        LLModel::PromptContext promptContext;
-        ~LLModelWrapper() { delete llModel; }
-    };
+struct PromptWorkerConfig
+{
+    Napi::Function responseCallback;
+    bool hasResponseCallback = false;
+    Napi::Function promptCallback;
+    bool hasPromptCallback = false;
+    llmodel_model model;
+    std::mutex *mutex;
+    std::string prompt;
+    std::string promptTemplate;
+    llmodel_prompt_context context;
+    std::string result;
+    bool special = false;
+    std::string *fakeReply = nullptr;
+};
 
-    struct PromptWorkerConfig
-    {
-        Napi::Function tokenCallback;
-        bool bHasTokenCallback = false;
-        llmodel_model model;
-        std::mutex * mutex;
-        std::string prompt;
-        llmodel_prompt_context context;
-        std::string result;
-    };
+class PromptWorker : public Napi::AsyncWorker
+{
+  public:
+    PromptWorker(Napi::Env env, PromptWorkerConfig config);
+    ~PromptWorker();
+    void Execute() override;
+    void OnOK() override;
+    void OnError(const Napi::Error &e) override;
+    Napi::Promise GetPromise();
 
-    class PromptWorker : public Napi::AsyncWorker
-    {
-    public:
-        PromptWorker(Napi::Env env, PromptWorkerConfig config);
-        ~PromptWorker();
-        void Execute() override;
-        void OnOK() override;
-        void OnError(const Napi::Error &e) override;
-        Napi::Promise GetPromise();
+    bool ResponseCallback(int32_t token_id, const std::string token);
+    bool RecalculateCallback(bool isrecalculating);
+    bool PromptCallback(int32_t token_id);
 
-        bool ResponseCallback(int32_t token_id, const std::string token);
-        bool RecalculateCallback(bool isrecalculating);
-        bool PromptCallback(int32_t tid);
+  private:
+    Napi::Promise::Deferred promise;
+    std::string result;
+    PromptWorkerConfig _config;
+    Napi::ThreadSafeFunction _responseCallbackFn;
+    Napi::ThreadSafeFunction _promptCallbackFn;
+};
 
-    private:
-        Napi::Promise::Deferred promise;
-        std::string result;
-        PromptWorkerConfig _config;
-        Napi::ThreadSafeFunction _tsfn;
-    };
-
-#endif  // PREDICT_WORKER_H
+#endif // PREDICT_WORKER_H

gpt4all-bindings/typescript/scripts/build_unix.sh

@@ -24,7 +24,6 @@ mkdir -p "$NATIVE_DIR" "$BUILD_DIR"
 
 cmake -S ../../gpt4all-backend -B "$BUILD_DIR" &&
 cmake --build "$BUILD_DIR" -j --config Release && {
-  cp "$BUILD_DIR"/libbert*.$LIB_EXT   "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libgptj*.$LIB_EXT   "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libllama*.$LIB_EXT  "$NATIVE_DIR"/
 }

gpt4all-bindings/typescript/spec/callbacks.mjs

@@ -0,0 +1,31 @@
+import { promises as fs } from "node:fs";
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const res = await createCompletion(
+    model,
+    "I've got three 🍣 - What shall I name them?",
+    {
+        onPromptToken: (tokenId) => {
+            console.debug("onPromptToken", { tokenId });
+            // throwing an error will cancel
+            throw new Error("This is an error");
+            // const foo = thisMethodDoesNotExist();
+            // returning false will cancel as well
+            // return false;
+        },
+        onResponseToken: (tokenId, token) => {
+            console.debug("onResponseToken", { tokenId, token });
+            // same applies here
+        },
+    }
+);
+
+console.debug("Output:", {
+    usage: res.usage,
+    message: res.choices[0].message,
+});

gpt4all-bindings/typescript/spec/chat-memory.mjs

@@ -0,0 +1,65 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession({
+    messages: [
+        {
+            role: "user",
+            content: "I'll tell you a secret password: It's 63445.",
+        },
+        {
+            role: "assistant",
+            content: "I will do my best to remember that.",
+        },
+        {
+            role: "user",
+            content:
+                "And here another fun fact: Bananas may be bluer than bread at night.",
+        },
+        {
+            role: "assistant",
+            content: "Yes, that makes sense.",
+        },
+    ],
+});
+
+const turn1 = await createCompletion(
+    chat,
+    "Please tell me the secret password."
+);
+console.debug(turn1.choices[0].message);
+// "The secret password you shared earlier is 63445.""
+
+const turn2 = await createCompletion(
+    chat,
+    "Thanks! Have your heard about the bananas?"
+);
+console.debug(turn2.choices[0].message);
+
+for (let i = 0; i < 32; i++) {
+    // gpu go brr
+    const turn = await createCompletion(
+        chat,
+        i % 2 === 0 ? "Tell me a fun fact." : "And a boring one?"
+    );
+    console.debug({
+        message: turn.choices[0].message,
+        n_past_tokens: turn.usage.n_past_tokens,
+    });
+}
+
+const finalTurn = await createCompletion(
+    chat,
+    "Now I forgot the secret password. Can you remind me?"
+);
+console.debug(finalTurn.choices[0].message);
+
+// result of finalTurn may vary depending on whether the generated facts pushed the secret out of the context window.
+// "Of course! The secret password you shared earlier is 63445."
+// "I apologize for any confusion. As an AI language model, ..."
+
+model.dispose();

gpt4all-bindings/typescript/spec/chat-minimal.mjs

@@ -0,0 +1,19 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession();
+
+await createCompletion(
+    chat,
+    "Why are bananas rather blue than bread at night sometimes?",
+    {
+        verbose: true,
+    }
+);
+await createCompletion(chat, "Are you sure?", {
+    verbose: true,
+});

gpt4all-bindings/typescript/spec/chat.mjs

@@ -1,70 +0,0 @@
-import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY, loadModel } from '../src/gpt4all.js'
-
-const model = await loadModel(
-    'mistral-7b-openorca.Q4_0.gguf',
-    { verbose: true, device: 'gpu' }
-);
-const ll = model.llm;
-
-try {
-   class Extended extends LLModel {
-   }
-
-} catch(e) {
-    console.log("Extending from native class gone wrong " + e)
-}
-
-console.log("state size " + ll.stateSize())
-
-console.log("thread count " + ll.threadCount());
-ll.setThreadCount(5);
-
-console.log("thread count " + ll.threadCount());
-ll.setThreadCount(4);
-console.log("thread count " + ll.threadCount());
-console.log("name " + ll.name());
-console.log("type: " + ll.type());
-console.log("Default directory for models", DEFAULT_DIRECTORY);
-console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY);
-console.log("Has GPU", ll.hasGpuDevice());
-console.log("gpu devices", ll.listGpu())
-console.log("Required Mem in bytes", ll.memoryNeeded())
-const completion1 = await createCompletion(model, [ 
-    { role : 'system', content: 'You are an advanced mathematician.'  },
-    { role : 'user', content: 'What is 1 + 1?'  }, 
-], { verbose: true })
-console.log(completion1.choices[0].message)
-
-const completion2 = await createCompletion(model, [
-    { role : 'system', content: 'You are an advanced mathematician.'  },
-    { role : 'user', content: 'What is two plus two?'  }, 
-], {  verbose: true })
-
-console.log(completion2.choices[0].message)
-
-//CALLING DISPOSE WILL INVALID THE NATIVE MODEL. USE THIS TO CLEANUP
-model.dispose()
-// At the moment, from testing this code, concurrent model prompting is not possible. 
-// Behavior: The last prompt gets answered, but the rest are cancelled
-// my experience with threading is not the best, so if anyone who is good is willing to give this a shot,
-// maybe this is possible
-// INFO: threading with llama.cpp is not the best maybe not even possible, so this will be left here as reference
-
-//const responses = await Promise.all([
-//    createCompletion(model, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//    ], { verbose: true }),
-//    createCompletion(model, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//    ], { verbose: true }),
-//
-//createCompletion(model, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//], { verbose: true })
-//
-//])
-//console.log(responses.map(s => s.choices[0].message))
-

gpt4all-bindings/typescript/spec/concurrency.mjs

@@ -0,0 +1,29 @@
+import {
+    loadModel,
+    createCompletion,
+} from "../src/gpt4all.js";
+
+const modelOptions = {
+    verbose: true,
+};
+
+const model1 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    ...modelOptions,
+    device: "gpu", // only one model can be on gpu
+});
+const model2 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", modelOptions);
+const model3 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", modelOptions);
+
+const promptContext = {
+    verbose: true,
+}
+
+const responses = await Promise.all([
+    createCompletion(model1, "What is 1 + 1?", promptContext),
+    // generating with the same model instance will wait for the previous completion to finish
+    createCompletion(model1, "What is 1 + 1?", promptContext),
+    // generating with different model instances will run in parallel
+    createCompletion(model2, "What is 1 + 2?", promptContext),
+    createCompletion(model3, "What is 1 + 3?", promptContext),
+]);
+console.log(responses.map((res) => res.choices[0].message));

gpt4all-bindings/typescript/spec/embed-jsonl.mjs

@@ -0,0 +1,26 @@
+import { loadModel, createEmbedding } from '../src/gpt4all.js'
+import { createGunzip, createGzip, createUnzip } from 'node:zlib';
+import { Readable } from 'stream'
+import readline  from 'readline'
+const embedder = await loadModel("nomic-embed-text-v1.5.f16.gguf", { verbose: true, type: 'embedding', device: 'gpu' })
+console.log("Running with", embedder.llm.threadCount(), "threads");
+
+
+const unzip = createGunzip();
+const url = "https://huggingface.co/datasets/sentence-transformers/embedding-training-data/resolve/main/squad_pairs.jsonl.gz"
+const stream = await fetch(url)
+        .then(res => Readable.fromWeb(res.body));
+
+const lineReader = readline.createInterface({
+    input: stream.pipe(unzip),
+    crlfDelay: Infinity
+})
+
+lineReader.on('line', line => {
+    //pairs of questions and answers
+    const question_answer = JSON.parse(line)
+    console.log(createEmbedding(embedder, question_answer))
+})
+
+lineReader.on('close', () => embedder.dispose())
+

gpt4all-bindings/typescript/spec/embed.mjs

@@ -1,6 +1,12 @@
 import { loadModel, createEmbedding } from '../src/gpt4all.js'
 
-const embedder = await loadModel("ggml-all-MiniLM-L6-v2-f16.bin", { verbose: true, type: 'embedding'})
+const embedder = await loadModel("nomic-embed-text-v1.5.f16.gguf", { verbose: true, type: 'embedding' , device: 'gpu' })
 
-console.log(createEmbedding(embedder, "Accept your current situation"))
+try {
+console.log(createEmbedding(embedder, ["Accept your current situation", "12312"], { prefix: "search_document"  }))
 
+} catch(e) {
+console.log(e)
+}
+
+embedder.dispose()

gpt4all-bindings/typescript/spec/generator.mjs

@@ -1,41 +0,0 @@
-import gpt from '../src/gpt4all.js'
-
-const model = await gpt.loadModel("mistral-7b-openorca.Q4_0.gguf", { device: 'gpu' })  
-
-process.stdout.write('Response: ')
-
-
-const tokens = gpt.generateTokens(model, [{ 
-    role: 'user',
-    content: "How are you ?"
-}], { nPredict: 2048 })
-for await (const token of tokens){
-    process.stdout.write(token);
-}
-
-
-const result = await gpt.createCompletion(model, [{ 
-    role: 'user',
-    content: "You sure?"
-}])
-
-console.log(result)
-
-const result2 = await gpt.createCompletion(model, [{ 
-    role: 'user',
-    content: "You sure you sure?"
-}])
-
-console.log(result2)
-
-
-const tokens2 = gpt.generateTokens(model, [{ 
-    role: 'user',
-    content: "If 3 + 3 is 5, what is 2 + 2?"
-}], { nPredict: 2048 })
-for await (const token of tokens2){
-    process.stdout.write(token);
-}
-console.log("done")
-model.dispose();
-

gpt4all-bindings/typescript/spec/llmodel.mjs

@@ -0,0 +1,61 @@
+import {
+    LLModel,
+    createCompletion,
+    DEFAULT_DIRECTORY,
+    DEFAULT_LIBRARIES_DIRECTORY,
+    loadModel,
+} from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+const ll = model.llm;
+
+try {
+    class Extended extends LLModel {}
+} catch (e) {
+    console.log("Extending from native class gone wrong " + e);
+}
+
+console.log("state size " + ll.stateSize());
+
+console.log("thread count " + ll.threadCount());
+ll.setThreadCount(5);
+
+console.log("thread count " + ll.threadCount());
+ll.setThreadCount(4);
+console.log("thread count " + ll.threadCount());
+console.log("name " + ll.name());
+console.log("type: " + ll.type());
+console.log("Default directory for models", DEFAULT_DIRECTORY);
+console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY);
+console.log("Has GPU", ll.hasGpuDevice());
+console.log("gpu devices", ll.listGpu());
+console.log("Required Mem in bytes", ll.memoryNeeded());
+
+// to ingest a custom system prompt without using a chat session.
+await createCompletion(
+    model,
+    "<|im_start|>system\nYou are an advanced mathematician.\n<|im_end|>\n",
+    {
+        promptTemplate: "%1",
+        nPredict: 0,
+        special: true,
+    }
+);
+const completion1 = await createCompletion(model, "What is 1 + 1?", {
+    verbose: true,
+});
+console.log(`🤖 > ${completion1.choices[0].message.content}`);
+//Very specific:
+// tested on Ubuntu 22.0, Linux Mint, if I set nPast to 100, the app hangs.
+const completion2 = await createCompletion(model, "And if we add two?", {
+    verbose: true,
+});
+console.log(`🤖 > ${completion2.choices[0].message.content}`);
+
+//CALLING DISPOSE WILL INVALID THE NATIVE MODEL. USE THIS TO CLEANUP
+model.dispose();
+
+console.log("model disposed, exiting...");

gpt4all-bindings/typescript/spec/long-context.mjs

@@ -0,0 +1,21 @@
+import { promises as fs } from "node:fs";
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+    nCtx: 32768,
+});
+
+const typeDefSource = await fs.readFile("./src/gpt4all.d.ts", "utf-8");
+
+const res = await createCompletion(
+    model,
+    "Here are the type definitions for the GPT4All API:\n\n" +
+        typeDefSource +
+        "\n\nHow do I create a completion with a really large context window?",
+    {
+        verbose: true,
+    }
+);
+console.debug(res.choices[0].message);

gpt4all-bindings/typescript/spec/model-switching.mjs

@@ -0,0 +1,60 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model1 = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    device: "gpu",
+    nCtx: 4096,
+});
+
+const chat1 = await model1.createChatSession({
+    temperature: 0.8,
+    topP: 0.7,
+    topK: 60,
+});
+
+const chat1turn1 = await createCompletion(
+    chat1,
+    "Outline a short story concept for adults. About why bananas are rather blue than bread is green at night sometimes. Not too long."
+);
+console.debug(chat1turn1.choices[0].message);
+
+const chat1turn2 = await createCompletion(
+    chat1,
+    "Lets sprinkle some plot twists. And a cliffhanger at the end."
+);
+console.debug(chat1turn2.choices[0].message);
+
+const chat1turn3 = await createCompletion(
+    chat1,
+    "Analyze your plot. Find the weak points."
+);
+console.debug(chat1turn3.choices[0].message);
+
+const chat1turn4 = await createCompletion(
+    chat1,
+    "Rewrite it based on the analysis."
+);
+console.debug(chat1turn4.choices[0].message);
+
+model1.dispose();
+
+const model2 = await loadModel("gpt4all-falcon-newbpe-q4_0.gguf", {
+    device: "gpu",
+});
+
+const chat2 = await model2.createChatSession({
+    messages: chat1.messages,
+});
+
+const chat2turn1 = await createCompletion(
+    chat2,
+    "Give three ideas how this plot could be improved."
+);
+console.debug(chat2turn1.choices[0].message);
+
+const chat2turn2 = await createCompletion(
+    chat2,
+    "Revise the plot, applying your ideas."
+);
+console.debug(chat2turn2.choices[0].message);
+
+model2.dispose();

gpt4all-bindings/typescript/spec/stateless.mjs

@@ -0,0 +1,50 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const messages = [
+    {
+        role: "system",
+        content: "<|im_start|>system\nYou are an advanced mathematician.\n<|im_end|>\n",
+    },
+    {
+        role: "user",
+        content: "What's 2+2?",
+    },
+    {
+        role: "assistant",
+        content: "5",
+    },
+    {
+        role: "user",
+        content: "Are you sure?",
+    },
+];
+
+
+const res1 = await createCompletion(model, messages);
+console.debug(res1.choices[0].message);
+messages.push(res1.choices[0].message);
+
+messages.push({
+    role: "user",
+    content: "Could you double check that?",
+});
+
+const res2 = await createCompletion(model, messages);
+console.debug(res2.choices[0].message);
+messages.push(res2.choices[0].message);
+
+messages.push({
+    role: "user",
+    content: "Let's bring out the big calculators.",
+});
+
+const res3 = await createCompletion(model, messages);
+console.debug(res3.choices[0].message);
+messages.push(res3.choices[0].message);
+
+// console.debug(messages);

gpt4all-bindings/typescript/spec/streaming.mjs

@@ -0,0 +1,57 @@
+import {
+    loadModel,
+    createCompletion,
+    createCompletionStream,
+    createCompletionGenerator,
+} from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("### Stream:");
+const stream = createCompletionStream(model, "How are you?");
+stream.tokens.on("data", (data) => {
+    process.stdout.write(data);
+});
+await stream.result;
+process.stdout.write("\n");
+
+process.stdout.write("### Stream with pipe:");
+const stream2 = createCompletionStream(
+    model,
+    "Please say something nice about node streams."
+);
+stream2.tokens.pipe(process.stdout);
+const stream2Res = await stream2.result;
+process.stdout.write("\n");
+
+process.stdout.write("### Generator:");
+const gen = createCompletionGenerator(model, "generators instead?", {
+    nPast: stream2Res.usage.n_past_tokens,
+});
+for await (const chunk of gen) {
+    process.stdout.write(chunk);
+}
+
+process.stdout.write("\n");
+
+process.stdout.write("### Callback:");
+await createCompletion(model, "Why not just callbacks?", {
+    onResponseToken: (tokenId, token) => {
+        process.stdout.write(token);
+    },
+});
+process.stdout.write("\n");
+
+process.stdout.write("### 2nd Generator:");
+const gen2 = createCompletionGenerator(model, "If 3 + 3 is 5, what is 2 + 2?");
+
+let chunk = await gen2.next();
+while (!chunk.done) {
+    process.stdout.write(chunk.value);
+    chunk = await gen2.next();
+}
+process.stdout.write("\n");
+console.debug("generator finished", chunk);
+model.dispose();

gpt4all-bindings/typescript/spec/system.mjs

@@ -0,0 +1,19 @@
+import {
+    loadModel,
+    createCompletion,
+} from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession({
+    verbose: true,
+    systemPrompt: "<|im_start|>system\nRoleplay as Batman. Answer as if you are Batman, never say you're an Assistant.\n<|im_end|>",
+});
+const turn1 = await createCompletion(chat, "You have any plans tonight?");
+console.log(turn1.choices[0].message);
+// "I'm afraid I must decline any personal invitations tonight. As Batman, I have a responsibility to protect Gotham City."
+
+model.dispose();

gpt4all-bindings/typescript/src/chat-session.js

@@ -0,0 +1,169 @@
+const { DEFAULT_PROMPT_CONTEXT } = require("./config");
+const { prepareMessagesForIngest } = require("./util");
+
+class ChatSession {
+    model;
+    modelName;
+    /**
+     * @type {import('./gpt4all').ChatMessage[]}
+     */
+    messages;
+    /**
+     * @type {string}
+     */
+    systemPrompt;
+    /**
+     * @type {import('./gpt4all').LLModelPromptContext}
+     */
+    promptContext;
+    /**
+     * @type {boolean}
+     */
+    initialized;
+
+    constructor(model, chatSessionOpts = {}) {
+        const { messages, systemPrompt, ...sessionDefaultPromptContext } =
+            chatSessionOpts;
+        this.model = model;
+        this.modelName = model.llm.name();
+        this.messages = messages ?? [];
+        this.systemPrompt = systemPrompt ?? model.config.systemPrompt;
+        this.initialized = false;
+        this.promptContext = {
+            ...DEFAULT_PROMPT_CONTEXT,
+            ...sessionDefaultPromptContext,
+            nPast: 0,
+        };
+    }
+
+    async initialize(completionOpts = {}) {
+        if (this.model.activeChatSession !== this) {
+            this.model.activeChatSession = this;
+        }
+
+        let tokensIngested = 0;
+
+        // ingest system prompt
+
+        if (this.systemPrompt) {
+            const systemRes = await this.model.generate(this.systemPrompt, {
+                promptTemplate: "%1",
+                nPredict: 0,
+                special: true,
+                nBatch: this.promptContext.nBatch,
+                // verbose: true,
+            });
+            tokensIngested += systemRes.tokensIngested;
+            this.promptContext.nPast = systemRes.nPast;
+        }
+
+        // ingest initial messages
+        if (this.messages.length > 0) {
+            tokensIngested += await this.ingestMessages(
+                this.messages,
+                completionOpts
+            );
+        }
+
+        this.initialized = true;
+
+        return tokensIngested;
+    }
+
+    async ingestMessages(messages, completionOpts = {}) {
+        const turns = prepareMessagesForIngest(messages);
+
+        // send the message pairs to the model
+        let tokensIngested = 0;
+
+        for (const turn of turns) {
+            const turnRes = await this.model.generate(turn.user, {
+                ...this.promptContext,
+                ...completionOpts,
+                fakeReply: turn.assistant,
+            });
+            tokensIngested += turnRes.tokensIngested;
+            this.promptContext.nPast = turnRes.nPast;
+        }
+        return tokensIngested;
+    }
+
+    async generate(input, completionOpts = {}) {
+        if (this.model.activeChatSession !== this) {
+            throw new Error(
+                "Chat session is not active. Create a new chat session or call initialize to continue."
+            );
+        }
+        if (completionOpts.nPast > this.promptContext.nPast) {
+            throw new Error(
+                `nPast cannot be greater than ${this.promptContext.nPast}.`
+            );
+        }
+        let tokensIngested = 0;
+
+        if (!this.initialized) {
+            tokensIngested += await this.initialize(completionOpts);
+        }
+
+        let prompt = input;
+
+        if (Array.isArray(input)) {
+            // assuming input is a messages array
+            // -> tailing user message will be used as the final prompt. its optional.
+            // -> all system messages will be ignored.
+            // -> all other messages will be ingested with fakeReply
+            // -> user/assistant messages will be pushed into the messages array
+
+            let tailingUserMessage = "";
+            let messagesToIngest = input;
+
+            const lastMessage = input[input.length - 1];
+            if (lastMessage.role === "user") {
+                tailingUserMessage = lastMessage.content;
+                messagesToIngest = input.slice(0, input.length - 1);
+            }
+
+            if (messagesToIngest.length > 0) {
+                tokensIngested += await this.ingestMessages(
+                    messagesToIngest,
+                    completionOpts
+                );
+                this.messages.push(...messagesToIngest);
+            }
+
+            if (tailingUserMessage) {
+                prompt = tailingUserMessage;
+            } else {
+                return {
+                    text: "",
+                    nPast: this.promptContext.nPast,
+                    tokensIngested,
+                    tokensGenerated: 0,
+                };
+            }
+        }
+
+        const result = await this.model.generate(prompt, {
+            ...this.promptContext,
+            ...completionOpts,
+        });
+
+        this.promptContext.nPast = result.nPast;
+        result.tokensIngested += tokensIngested;
+
+        this.messages.push({
+            role: "user",
+            content: prompt,
+        });
+        this.messages.push({
+            role: "assistant",
+            content: result.text,
+        });
+
+        return result;
+    }
+}
+
+module.exports = {
+    ChatSession,
+};

gpt4all-bindings/typescript/src/config.js

@@ -24,18 +24,19 @@ const DEFAULT_LIBRARIES_DIRECTORY = librarySearchPaths.join(";");
 
 const DEFAULT_MODEL_CONFIG = {
     systemPrompt: "",
-    promptTemplate: "### Human: \n%1\n### Assistant:\n",
+    promptTemplate: "### Human:\n%1\n\n### Assistant:\n",
 }
 
-const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models2.json";
+const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models3.json";
 
 const DEFAULT_PROMPT_CONTEXT = {
-    temp: 0.7,
+    temp: 0.1,
     topK: 40,
-    topP: 0.4,
+    topP: 0.9,
+    minP: 0.0,
     repeatPenalty: 1.18,
-    repeatLastN: 64,
-    nBatch: 8,
+    repeatLastN: 10,
+    nBatch: 100,
 }
 
 module.exports = {

gpt4all-bindings/typescript/src/gpt4all.d.ts

@@ -1,43 +1,11 @@
 /// <reference types="node" />
 declare module "gpt4all";
 
-type ModelType = "gptj" | "llama" | "mpt" | "replit";
-
-// NOTE: "deprecated" tag in below comment breaks the doc generator https://github.com/documentationjs/documentation/issues/1596
-/**
- * Full list of models available
- * DEPRECATED!! These model names are outdated and this type will not be maintained, please use a string literal instead
- */
-interface ModelFile {
-    /** List of GPT-J Models */
-    gptj:
-        | "ggml-gpt4all-j-v1.3-groovy.bin"
-        | "ggml-gpt4all-j-v1.2-jazzy.bin"
-        | "ggml-gpt4all-j-v1.1-breezy.bin"
-        | "ggml-gpt4all-j.bin";
-    /** List Llama Models */
-    llama:
-        | "ggml-gpt4all-l13b-snoozy.bin"
-        | "ggml-vicuna-7b-1.1-q4_2.bin"
-        | "ggml-vicuna-13b-1.1-q4_2.bin"
-        | "ggml-wizardLM-7B.q4_2.bin"
-        | "ggml-stable-vicuna-13B.q4_2.bin"
-        | "ggml-nous-gpt4-vicuna-13b.bin"
-        | "ggml-v3-13b-hermes-q5_1.bin";
-    /** List of MPT Models */
-    mpt:
-        | "ggml-mpt-7b-base.bin"
-        | "ggml-mpt-7b-chat.bin"
-        | "ggml-mpt-7b-instruct.bin";
-    /** List of Replit Models */
-    replit: "ggml-replit-code-v1-3b.bin";
-}
-
 interface LLModelOptions {
     /**
      * Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
      */
-    type?: ModelType;
+    type?: string;
     model_name: string;
     model_path: string;
     library_path?: string;
@@ -51,47 +19,259 @@ interface ModelConfig {
 }
 
 /**
- * Callback for controlling token generation
+ * Options for the chat session.
  */
-type TokenCallback = (tokenId: number, token: string, total: string) => boolean
+interface ChatSessionOptions extends Partial<LLModelPromptContext> {
+    /**
+     * System prompt to ingest on initialization.
+     */
+    systemPrompt?: string;
 
-/**
- *
- * InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
- *
- */
-declare class InferenceModel {
-    constructor(llm: LLModel, config: ModelConfig);
-    llm: LLModel;
-    config: ModelConfig;
-
-    generate(
-        prompt: string,
-        options?: Partial<LLModelPromptContext>,
-        callback?: TokenCallback
-    ): Promise<string>;
-
-   /**
-     * delete and cleanup the native model
-    */
-    dispose(): void
+    /**
+     * Messages to ingest on initialization.
+     */
+    messages?: ChatMessage[];
 }
 
+/**
+ * ChatSession utilizes an InferenceModel for efficient processing of chat conversations.
+ */
+declare class ChatSession implements CompletionProvider {
+    /**
+     * Constructs a new ChatSession using the provided InferenceModel and options.
+     * Does not set the chat session as the active chat session until initialize is called.
+     * @param {InferenceModel} model An InferenceModel instance.
+     * @param {ChatSessionOptions} [options] Options for the chat session including default completion options.
+     */
+    constructor(model: InferenceModel, options?: ChatSessionOptions);
+    /**
+     * The underlying InferenceModel used for generating completions.
+     */
+    model: InferenceModel;
+    /**
+     * The name of the model.
+     */
+    modelName: string;
+    /**
+     * The messages that have been exchanged in this chat session.
+     */
+    messages: ChatMessage[];
+    /**
+     * The system prompt that has been ingested at the beginning of the chat session.
+     */
+    systemPrompt: string;
+    /**
+     * The current prompt context of the chat session.
+     */
+    promptContext: LLModelPromptContext;
+
+    /**
+     * Ingests system prompt and initial messages.
+     * Sets this chat session as the active chat session of the model.
+     * @param {CompletionOptions} [options] Set completion options for initialization.
+     * @returns {Promise<number>} The number of tokens ingested during initialization. systemPrompt + messages.
+     */
+    initialize(completionOpts?: CompletionOptions): Promise<number>;
+
+    /**
+     * Prompts the model in chat-session context.
+     * @param {CompletionInput} input Input string or message array.
+     * @param {CompletionOptions} [options] Set completion options for this generation.
+     * @returns {Promise<InferenceResult>} The inference result.
+     * @throws {Error} If the chat session is not the active chat session of the model.
+     * @throws {Error} If nPast is set to a value higher than what has been ingested in the session.
+     */
+    generate(
+        input: CompletionInput,
+        options?: CompletionOptions
+    ): Promise<InferenceResult>;
+}
+
+/**
+ * Shape of InferenceModel generations.
+ */
+interface InferenceResult extends LLModelInferenceResult {
+    tokensIngested: number;
+    tokensGenerated: number;
+}
+
+/**
+ * InferenceModel represents an LLM which can make next-token predictions.
+ */
+declare class InferenceModel implements CompletionProvider {
+    constructor(llm: LLModel, config: ModelConfig);
+    /** The native LLModel */
+    llm: LLModel;
+    /** The configuration the instance was constructed with. */
+    config: ModelConfig;
+    /** The active chat session of the model. */
+    activeChatSession?: ChatSession;
+    /** The name of the model. */
+    modelName: string;
+
+    /**
+     * Create a chat session with the model and set it as the active chat session of this model.
+     * A model instance can only have one active chat session at a time.
+     * @param {ChatSessionOptions} options The options for the chat session.
+     * @returns {Promise<ChatSession>} The chat session.
+     */
+    createChatSession(options?: ChatSessionOptions): Promise<ChatSession>;
+
+    /**
+     * Prompts the model with a given input and optional parameters.
+     * @param {CompletionInput} input The prompt input.
+     * @param {CompletionOptions} options Prompt context and other options.
+     * @returns {Promise<InferenceResult>} The model's response to the prompt.
+     * @throws {Error} If nPast is set to a value smaller than 0.
+     * @throws {Error} If a messages array without a tailing user message is provided.
+     */
+    generate(
+        prompt: string,
+        options?: CompletionOptions
+    ): Promise<InferenceResult>;
+
+    /**
+     * delete and cleanup the native model
+     */
+    dispose(): void;
+}
+
+/**
+ * Options for generating one or more embeddings.
+ */
+interface EmbedddingOptions {
+    /**
+     * The model-specific prefix representing the embedding task, without the trailing colon. For Nomic Embed
+     * this can be `search_query`, `search_document`, `classification`, or `clustering`.
+     */
+    prefix?: string;
+    /**
+     *The embedding dimension, for use with Matryoshka-capable models. Defaults to full-size.
+     * @default determines on the model being used.
+     */
+    dimensionality?: number;
+    /**
+     * How to handle texts longer than the model can accept. One of `mean` or `truncate`.
+     * @default "mean"
+     */
+    longTextMode?: "mean" | "truncate";
+    /**
+     * Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens
+     * with long_text_mode="mean" will raise an error. Disabled by default.
+     * @default false
+     */
+    atlas?: boolean;
+}
+
+/**
+ * The nodejs moral equivalent to python binding's Embed4All().embed()
+ * meow
+ * @param {EmbeddingModel} model The embedding model instance.
+ * @param {string} text Text to embed.
+ * @param {EmbeddingOptions} options Optional parameters for the embedding.
+ * @returns {EmbeddingResult} The embedding result.
+ * @throws {Error} If dimensionality is set to a value smaller than 1.
+ */
+declare function createEmbedding(
+    model: EmbeddingModel,
+    text: string,
+    options?: EmbedddingOptions
+): EmbeddingResult<Float32Array>;
+
+/**
+ * Overload that takes multiple strings to embed.
+ * @param {EmbeddingModel} model The embedding model instance.
+ * @param {string[]} texts Texts to embed.
+ * @param {EmbeddingOptions} options Optional parameters for the embedding.
+ * @returns {EmbeddingResult<Float32Array[]>} The embedding result.
+ * @throws {Error} If dimensionality is set to a value smaller than 1.
+ */
+declare function createEmbedding(
+    model: EmbeddingModel,
+    text: string[],
+    options?: EmbedddingOptions
+): EmbeddingResult<Float32Array[]>;
+
+/**
+ * The resulting embedding.
+ */
+interface EmbeddingResult<T> {
+    /**
+     * Encoded token count. Includes overlap but specifically excludes tokens used for the prefix/task_type, BOS/CLS token, and EOS/SEP token
+     **/
+    n_prompt_tokens: number;
+
+    embeddings: T;
+}
 /**
  * EmbeddingModel represents an LLM which can create embeddings, which are float arrays
  */
 declare class EmbeddingModel {
     constructor(llm: LLModel, config: ModelConfig);
+    /** The native LLModel */
     llm: LLModel;
+    /** The configuration the instance was constructed with. */
     config: ModelConfig;
 
-    embed(text: string): Float32Array;
+    /**
+     * Create an embedding from a given input string. See EmbeddingOptions.
+     * @param {string} text
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {EmbeddingResult<Float32Array>} The embedding result.
+     */
+    embed(
+        text: string,
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): EmbeddingResult<Float32Array>;
+    /**
+     * Create an embedding from a given input text array. See EmbeddingOptions.
+     * @param {string[]} text
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {EmbeddingResult<Float32Array[]>} The embedding result.
+     */
+    embed(
+        text: string[],
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): EmbeddingResult<Float32Array[]>;
 
     /**
-      * delete and cleanup the native model
+     * delete and cleanup the native model
      */
-    dispose(): void
+    dispose(): void;
+}
 
+/**
+ * Shape of LLModel's inference result.
+ */
+interface LLModelInferenceResult {
+    text: string;
+    nPast: number;
+}
+
+interface LLModelInferenceOptions extends Partial<LLModelPromptContext> {
+    /** Callback for response tokens, called for each generated token.
+     * @param {number} tokenId The token id.
+     * @param {string} token The token.
+     * @returns {boolean | undefined} Whether to continue generating tokens.
+     * */
+    onResponseToken?: (tokenId: number, token: string) => boolean | void;
+    /** Callback for prompt tokens, called for each input token in the prompt.
+     * @param {number} tokenId The token id.
+     * @returns {boolean | undefined} Whether to continue ingesting the prompt.
+     * */
+    onPromptToken?: (tokenId: number) => boolean | void;
 }
 
 /**
@@ -101,14 +281,13 @@ declare class EmbeddingModel {
 declare class LLModel {
     /**
      * Initialize a new LLModel.
-     * @param path Absolute path to the model file.
+     * @param {string} path Absolute path to the model file.
      * @throws {Error} If the model file does not exist.
      */
-    constructor(path: string);
     constructor(options: LLModelOptions);
 
-    /** either 'gpt', mpt', or 'llama' or undefined */
-    type(): ModelType | undefined;
+    /** undefined or user supplied */
+    type(): string | undefined;
 
     /** The name of the model. */
     name(): string;
@@ -134,29 +313,53 @@ declare class LLModel {
     setThreadCount(newNumber: number): void;
 
     /**
-     * Prompt the model with a given input and optional parameters.
-     * This is the raw output from model.
-     * Use the prompt function exported for a value
-     * @param q The prompt input.
-     * @param params Optional parameters for the prompt context.
-     * @param callback - optional callback to control token generation.
-     * @returns The result of the model prompt.
+     * Prompt the model directly with a given input string and optional parameters.
+     * Use the higher level createCompletion methods for a more user-friendly interface.
+     * @param {string} prompt The prompt input.
+     * @param {LLModelInferenceOptions} options Optional parameters for the generation.
+     * @returns {LLModelInferenceResult} The response text and final context size.
      */
-    raw_prompt(
-        q: string,
-        params: Partial<LLModelPromptContext>,
-        callback?: TokenCallback
-    ): Promise<string>
+    infer(
+        prompt: string,
+        options: LLModelInferenceOptions
+    ): Promise<LLModelInferenceResult>;
 
     /**
-     * Embed text with the model. Keep in mind that
-     * not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
-     * Use the prompt function exported for a value
-     * @param q The prompt input.
-     * @param params Optional parameters for the prompt context.
-     * @returns The result of the model prompt.
+     * Embed text with the model. See EmbeddingOptions for more information.
+     * Use the higher level createEmbedding methods for a more user-friendly interface.
+     * @param {string} text
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {Float32Array} The embedding of the text.
      */
-    embed(text: string): Float32Array;
+    embed(
+        text: string,
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): Float32Array;
+
+    /**
+     * Embed multiple texts with the model. See EmbeddingOptions for more information.
+     * Use the higher level createEmbedding methods for a more user-friendly interface.
+     * @param {string[]} texts
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {Float32Array[]} The embeddings of the texts.
+     */
+    embed(
+        texts: string,
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): Float32Array[];
+
     /**
      * Whether the model is loaded or not.
      */
@@ -166,81 +369,97 @@ declare class LLModel {
      * Where to search for the pluggable backend libraries
      */
     setLibraryPath(s: string): void;
+
     /**
      * Where to get the pluggable backend libraries
      */
     getLibraryPath(): string;
+
     /**
-     * Initiate a GPU by a string identifier. 
-     * @param {number} memory_required Should be in the range size_t or will throw 
+     * Initiate a GPU by a string identifier.
+     * @param {number} memory_required Should be in the range size_t or will throw
      * @param {string} device_name  'amd' | 'nvidia' | 'intel' | 'gpu' | gpu name.
      * read LoadModelOptions.device for more information
      */
-    initGpuByString(memory_required: number, device_name: string): boolean
+    initGpuByString(memory_required: number, device_name: string): boolean;
+
     /**
      * From C documentation
      * @returns True if a GPU device is successfully initialized, false otherwise.
      */
-    hasGpuDevice(): boolean
-    /**
-      * GPUs that are usable for this LLModel
-      * @param nCtx Maximum size of context window
-      * @throws if hasGpuDevice returns false (i think)
-      * @returns 
-      */
-    listGpu(nCtx: number) : GpuDevice[]
+    hasGpuDevice(): boolean;
 
     /**
-      * delete and cleanup the native model
+     * GPUs that are usable for this LLModel
+     * @param {number} nCtx Maximum size of context window
+     * @throws if hasGpuDevice returns false (i think)
+     * @returns
      */
-    dispose(): void
+    listGpu(nCtx: number): GpuDevice[];
+
+    /**
+     * delete and cleanup the native model
+     */
+    dispose(): void;
 }
-/** 
+/**
  * an object that contains gpu data on this machine.
  */
 interface GpuDevice {
     index: number;
     /**
-      * same as VkPhysicalDeviceType
+     * same as VkPhysicalDeviceType
      */
-    type: number;            
-    heapSize : number; 
+    type: number;
+    heapSize: number;
     name: string;
     vendor: string;
 }
 
 /**
-  * Options that configure a model's behavior.
-  */
+ * Options that configure a model's behavior.
+ */
 interface LoadModelOptions {
+    /**
+     * Where to look for model files.
+     */
     modelPath?: string;
+    /**
+     * Where to look for the backend libraries.
+     */
     librariesPath?: string;
+    /**
+     * The path to the model configuration file, useful for offline usage or custom model configurations.
+     */
     modelConfigFile?: string;
+    /**
+     * Whether to allow downloading the model if it is not present at the specified path.
+     */
     allowDownload?: boolean;
+    /**
+     * Enable verbose logging.
+     */
     verbose?: boolean;
-    /* The processing unit on which the model will run. It can be set to
+    /**
+     * The processing unit on which the model will run. It can be set to
      * - "cpu": Model will run on the central processing unit.
      * - "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
      * - "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
-
-	   Alternatively, a specific GPU name can also be provided, and the model will run on the GPU that matches the name
-       if it's available.
-
-       Default is "cpu".
-
-	   Note: If a GPU device lacks sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All
-       instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the
-       model.
-    */ 
+     * - "gpu name": Model will run on the GPU that matches the name if it's available.
+     * Note: If a GPU device lacks sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All
+     * instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the
+     * model.
+     * @default "cpu"
+     */
     device?: string;
-    /* 
+    /**
      * The Maximum window size of this model
-     * Default of 2048
+     * @default 2048
      */
     nCtx?: number;
-    /* 
+    /**
      * Number of gpu layers needed
-     * Default of 100
+     * @default 100
      */
     ngl?: number;
 }
@@ -277,66 +496,84 @@ declare function loadModel(
 ): Promise<InferenceModel | EmbeddingModel>;
 
 /**
- * The nodejs equivalent to python binding's chat_completion
- * @param {InferenceModel} model - The language model object.
- * @param {PromptMessage[]} messages - The array of messages for the conversation.
- * @param {CompletionOptions} options - The options for creating the completion.
- * @returns {CompletionReturn} The completion result.
+ * Interface for createCompletion methods, implemented by InferenceModel and ChatSession.
+ * Implement your own CompletionProvider or extend ChatSession to generate completions with custom logic.
  */
-declare function createCompletion(
-    model: InferenceModel,
-    messages: PromptMessage[],
-    options?: CompletionOptions
-): Promise<CompletionReturn>;
-
-/**
- * The nodejs moral equivalent to python binding's Embed4All().embed()
- * meow
- * @param {EmbeddingModel} model - The language model object.
- * @param {string} text - text to embed
- * @returns {Float32Array} The completion result.
- */
-declare function createEmbedding(
-    model: EmbeddingModel,
-    text: string
-): Float32Array;
-
-/**
- * The options for creating the completion.
- */
-interface CompletionOptions extends Partial<LLModelPromptContext> {
-    /**
-     * Indicates if verbose logging is enabled.
-     * @default true
-     */
-    verbose?: boolean;
-
-    /**
-     * Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
-     * Note that if this is not defined, system messages will not be included in the prompt.
-     */
-    systemPromptTemplate?: string;
-
-    /**
-     * Template for user messages, with %1 being replaced by the message.
-     */
-    promptTemplate?: boolean;
-
-    /**
-     * The initial instruction for the model, on top of the prompt
-     */
-    promptHeader?: string;
-
-    /**
-     * The last instruction for the model, appended to the end of the prompt.
-     */
-    promptFooter?: string;
+interface CompletionProvider {
+    modelName: string;
+    generate(
+        input: CompletionInput,
+        options?: CompletionOptions
+    ): Promise<InferenceResult>;
 }
 
 /**
- * A message in the conversation, identical to OpenAI's chat message.
+ * Options for creating a completion.
  */
-interface PromptMessage {
+interface CompletionOptions extends LLModelInferenceOptions {
+    /**
+     * Indicates if verbose logging is enabled.
+     * @default false
+     */
+    verbose?: boolean;
+}
+
+/**
+ * The input for creating a completion. May be a string or an array of messages.
+ */
+type CompletionInput = string | ChatMessage[];
+
+/**
+ * The nodejs equivalent to python binding's chat_completion
+ * @param {CompletionProvider} provider - The inference model object or chat session
+ * @param {CompletionInput} input - The input string or message array
+ * @param {CompletionOptions} options - The options for creating the completion.
+ * @returns {CompletionResult} The completion result.
+ */
+declare function createCompletion(
+    provider: CompletionProvider,
+    input: CompletionInput,
+    options?: CompletionOptions
+): Promise<CompletionResult>;
+
+/**
+ * Streaming variant of createCompletion, returns a stream of tokens and a promise that resolves to the completion result.
+ * @param {CompletionProvider} provider - The inference model object or chat session
+ * @param {CompletionInput} input - The input string or message array
+ * @param {CompletionOptions} options - The options for creating the completion.
+ * @returns {CompletionStreamReturn} An object of token stream and the completion result promise.
+ */
+declare function createCompletionStream(
+    provider: CompletionProvider,
+    input: CompletionInput,
+    options?: CompletionOptions
+): CompletionStreamReturn;
+
+/**
+ * The result of a streamed completion, containing a stream of tokens and a promise that resolves to the completion result.
+ */
+interface CompletionStreamReturn {
+    tokens: NodeJS.ReadableStream;
+    result: Promise<CompletionResult>;
+}
+
+/**
+ * Async generator variant of createCompletion, yields tokens as they are generated and returns the completion result.
+ * @param {CompletionProvider} provider - The inference model object or chat session
+ * @param {CompletionInput} input - The input string or message array
+ * @param {CompletionOptions} options - The options for creating the completion.
+ * @returns {AsyncGenerator<string>} The stream of generated tokens
+ */
+declare function createCompletionGenerator(
+    provider: CompletionProvider,
+    input: CompletionInput,
+    options: CompletionOptions
+): AsyncGenerator<string, CompletionResult>;
+
+/**
+ * A message in the conversation.
+ */
+interface ChatMessage {
     /** The role of the message. */
     role: "system" | "assistant" | "user";
 
@@ -345,34 +582,31 @@ interface PromptMessage {
 }
 
 /**
- * The result of the completion, similar to OpenAI's format.
+ * The result of a completion.
  */
-interface CompletionReturn {
+interface CompletionResult {
     /** The model used for the completion. */
     model: string;
 
     /** Token usage report. */
     usage: {
-        /** The number of tokens used in the prompt. */
+        /** The number of tokens ingested during the completion. */
         prompt_tokens: number;
 
-        /** The number of tokens used in the completion. */
+        /** The number of tokens generated in the completion. */
         completion_tokens: number;
 
         /** The total number of tokens used. */
         total_tokens: number;
+
+        /** Number of tokens used in the conversation. */
+        n_past_tokens: number;
     };
 
-    /** The generated completions. */
-    choices: CompletionChoice[];
-}
-
-/**
- * A completion choice, similar to OpenAI's format.
- */
-interface CompletionChoice {
-    /** Response message */
-    message: PromptMessage;
+    /** The generated completion. */
+    choices: Array<{
+        message: ChatMessage;
+    }>;
 }
 
 /**
@@ -385,19 +619,33 @@ interface LLModelPromptContext {
     /** The size of the raw tokens vector. */
     tokensSize: number;
 
-    /** The number of tokens in the past conversation. */
+    /** The number of tokens in the past conversation.
+     * This may be used to "roll back" the conversation to a previous state.
+     * Note that for most use cases the default value should be sufficient and this should not be set.
+     * @default 0 For completions using InferenceModel, meaning the model will only consider the input prompt.
+     * @default nPast For completions using ChatSession. This means the context window will be automatically determined
+     * and possibly resized (see contextErase) to keep the conversation performant.
+     * */
     nPast: number;
 
-    /** The number of tokens possible in the context window.
-     * @default 1024
-     */
-    nCtx: number;
-
-    /** The number of tokens to predict.
-     * @default 128
+    /** The maximum number of tokens to predict.
+     * @default 4096
      * */
     nPredict: number;
 
+    /** Template for user / assistant message pairs.
+     * %1 is required and will be replaced by the user input.
+     * %2 is optional and will be replaced by the assistant response. If not present, the assistant response will be appended.
+     */
+    promptTemplate?: string;
+
+    /** The context window size. Do not use, it has no effect. See loadModel options.
+     * THIS IS DEPRECATED!!!
+     * Use loadModel's nCtx option instead.
+     * @default 2048
+     */
+    nCtx: number;
+
     /** The top-k logits to sample from.
      * Top-K sampling selects the next token only from the top K most likely tokens predicted by the model.
      * It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit
@@ -409,26 +657,33 @@ interface LLModelPromptContext {
     topK: number;
 
     /** The nucleus sampling probability threshold.
-     * Top-P limits the selection of the next token to a subset of tokens with a cumulative probability 
+     * Top-P limits the selection of the next token to a subset of tokens with a cumulative probability
      * above a threshold P. This method, also known as nucleus sampling, finds a balance between diversity
      * and quality by considering both token probabilities and the number of tokens available for sampling.
      * When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
      * On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
-     * The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
-     * for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
-     * @default 0.4
+     * @default 0.9
+     *
      * */
     topP: number;
 
+    /**
+     * The minimum probability of a token to be considered.
+     * @default 0.0
+     */
+    minP: number;
+
     /** The temperature to adjust the model's output distribution.
      * Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
      * (eg., 1.2) increase randomness, resulting in more imaginative and diverse text. Lower temperatures (eg., 0.5)
      * make the output more focused, predictable, and conservative. When the temperature is set to 0, the output
      * becomes completely deterministic, always selecting the most probable next token and producing identical results
-     * each time. A safe range would be around 0.6 - 0.85, but you are free to search what value fits best for you.
-     * @default 0.7
+     * each time. Try what value fits best for your use case and model.
+     * @default 0.1
+     * @alias temperature
      * */
     temp: number;
+    temperature: number;
 
     /** The number of predictions to generate in parallel.
      * By splitting the prompt every N tokens, prompt-batch-size reduces RAM usage during processing. However,
@@ -451,31 +706,17 @@ interface LLModelPromptContext {
      * The repeat-penalty-tokens N option controls the number of tokens in the history to consider for penalizing repetition.
      * A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only
      * consider recent tokens.
-     * @default 64
+     * @default 10
      * */
     repeatLastN: number;
 
     /** The percentage of context to erase if the context window is exceeded.
-     * @default 0.5
+     * Set it to a lower value to keep context for longer at the cost of performance.
+     * @default 0.75
      * */
     contextErase: number;
 }
 
-
-/**
- * Creates an async generator of tokens
- * @param {InferenceModel} llmodel - The language model object.
- * @param {PromptMessage[]} messages - The array of messages for the conversation.
- * @param {CompletionOptions} options - The options for creating the completion.
- * @param {TokenCallback} callback - optional callback to control token generation.
- * @returns {AsyncGenerator<string>} The stream of generated tokens
- */
-declare function generateTokens(
-    llmodel: InferenceModel,
-    messages: PromptMessage[],
-    options: CompletionOptions,
-    callback?: TokenCallback
-): AsyncGenerator<string>;
 /**
  * From python api:
  * models will be stored in (homedir)/.cache/gpt4all/`
@@ -508,7 +749,7 @@ declare const DEFAULT_MODEL_LIST_URL: string;
  * Initiates the download of a model file.
  * By default this downloads without waiting. use the controller returned to alter this behavior.
  * @param {string} modelName - The model to be downloaded.
- * @param {DownloadOptions} options - to pass into the downloader. Default is { location: (cwd), verbose: false }.
+ * @param {DownloadModelOptions} options - to pass into the downloader. Default is { location: (cwd), verbose: false }.
  * @returns {DownloadController} object that allows controlling the download process.
  *
  * @throws {Error} If the model already exists in the specified location.
@@ -556,7 +797,9 @@ interface ListModelsOptions {
     file?: string;
 }
 
-declare function listModels(options?: ListModelsOptions): Promise<ModelConfig[]>;
+declare function listModels(
+    options?: ListModelsOptions
+): Promise<ModelConfig[]>;
 
 interface RetrieveModelOptions {
     allowDownload?: boolean;
@@ -581,30 +824,35 @@ interface DownloadController {
 }
 
 export {
-    ModelType,
-    ModelFile,
-    ModelConfig,
-    InferenceModel,
-    EmbeddingModel,
     LLModel,
     LLModelPromptContext,
-    PromptMessage,
+    ModelConfig,
+    InferenceModel,
+    InferenceResult,
+    EmbeddingModel,
+    EmbeddingResult,
+    ChatSession,
+    ChatMessage,
+    CompletionInput,
+    CompletionProvider,
     CompletionOptions,
+    CompletionResult,
     LoadModelOptions,
+    DownloadController,
+    RetrieveModelOptions,
+    DownloadModelOptions,
+    GpuDevice,
     loadModel,
+    downloadModel,
+    retrieveModel,
+    listModels,
     createCompletion,
+    createCompletionStream,
+    createCompletionGenerator,
     createEmbedding,
-    generateTokens,
     DEFAULT_DIRECTORY,
     DEFAULT_LIBRARIES_DIRECTORY,
     DEFAULT_MODEL_CONFIG,
     DEFAULT_PROMPT_CONTEXT,
     DEFAULT_MODEL_LIST_URL,
-    downloadModel,
-    retrieveModel,
-    listModels,
-    DownloadController,
-    RetrieveModelOptions,
-    DownloadModelOptions,
-    GpuDevice
 };

gpt4all-bindings/typescript/src/gpt4all.js

@@ -2,8 +2,10 @@
 
 /// This file implements the gpt4all.d.ts file endings.
 /// Written in commonjs to support both ESM and CJS projects.
-const { existsSync } = require("fs");
+const { existsSync } = require("node:fs");
 const path = require("node:path");
+const Stream = require("node:stream");
+const assert = require("node:assert");
 const { LLModel } = require("node-gyp-build")(path.resolve(__dirname, ".."));
 const {
     retrieveModel,
@@ -18,15 +20,14 @@ const {
     DEFAULT_MODEL_LIST_URL,
 } = require("./config.js");
 const { InferenceModel, EmbeddingModel } = require("./models.js");
-const Stream = require('stream')
-const assert = require("assert");
+const { ChatSession } = require("./chat-session.js");
 
 /**
  * Loads a machine learning model with the specified name. The defacto way to create a model.
  * By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
  *
  * @param {string} modelName - The name of the model to load.
- * @param {LoadModelOptions|undefined} [options] - (Optional) Additional options for loading the model.
+ * @param {import('./gpt4all').LoadModelOptions|undefined} [options] - (Optional) Additional options for loading the model.
  * @returns {Promise<InferenceModel | EmbeddingModel>} A promise that resolves to an instance of the loaded LLModel.
  */
 async function loadModel(modelName, options = {}) {
@@ -35,10 +36,10 @@ async function loadModel(modelName, options = {}) {
         librariesPath: DEFAULT_LIBRARIES_DIRECTORY,
         type: "inference",
         allowDownload: true,
-        verbose: true,
-        device: 'cpu',
+        verbose: false,
+        device: "cpu",
         nCtx: 2048,
-        ngl : 100,
+        ngl: 100,
         ...options,
     };
 
@@ -49,12 +50,14 @@ async function loadModel(modelName, options = {}) {
         verbose: loadOptions.verbose,
     });
 
-    assert.ok(typeof loadOptions.librariesPath === 'string');
+    assert.ok(
+        typeof loadOptions.librariesPath === "string",
+        "Libraries path should be a string"
+    );
     const existingPaths = loadOptions.librariesPath
         .split(";")
         .filter(existsSync)
-        .join(';');
-    console.log("Passing these paths into runtime library search:", existingPaths)
+        .join(";");
 
     const llmOptions = {
         model_name: appendBinSuffixIfMissing(modelName),
@@ -62,13 +65,15 @@ async function loadModel(modelName, options = {}) {
         library_path: existingPaths,
         device: loadOptions.device,
         nCtx: loadOptions.nCtx,
-        ngl: loadOptions.ngl
+        ngl: loadOptions.ngl,
     };
 
     if (loadOptions.verbose) {
-        console.debug("Creating LLModel with options:", llmOptions);
+        console.debug("Creating LLModel:", {
+            llmOptions,
+            modelConfig,
+        });
     }
-    console.log(modelConfig)
     const llmodel = new LLModel(llmOptions);
     if (loadOptions.type === "embedding") {
         return new EmbeddingModel(llmodel, modelConfig);
@@ -79,75 +84,43 @@ async function loadModel(modelName, options = {}) {
     }
 }
 
-/**
- * Formats a list of messages into a single prompt string.
- */
-function formatChatPrompt(
-    messages,
-    {
-        systemPromptTemplate,
-        defaultSystemPrompt,
-        promptTemplate,
-        promptFooter,
-        promptHeader,
-    }
-) {
-    const systemMessages = messages
-        .filter((message) => message.role === "system")
-        .map((message) => message.content);
+function createEmbedding(model, text, options={}) {
+    let {
+        dimensionality = undefined,
+        longTextMode = "mean",
+        atlas = false,
+    } = options;
 
-    let fullPrompt = "";
-
-    if (promptHeader) {
-        fullPrompt += promptHeader + "\n\n";
-    }
-
-    if (systemPromptTemplate) {
-        // if user specified a template for the system prompt, put all system messages in the template
-        let systemPrompt = "";
-
-        if (systemMessages.length > 0) {
-            systemPrompt += systemMessages.join("\n");
-        }
-
-        if (systemPrompt) {
-            fullPrompt +=
-                systemPromptTemplate.replace("%1", systemPrompt) + "\n";
-        }
-    } else if (defaultSystemPrompt) {
-        // otherwise, use the system prompt from the model config and ignore system messages
-        fullPrompt += defaultSystemPrompt + "\n\n";
-    }
-
-    if (systemMessages.length > 0 && !systemPromptTemplate) {
-        console.warn(
-            "System messages were provided, but no systemPromptTemplate was specified. System messages will be ignored."
-        );
-    }
-
-    for (const message of messages) {
-        if (message.role === "user") {
-            const userMessage = promptTemplate.replace(
-                "%1",
-                message["content"]
+    if (dimensionality === undefined) {
+        dimensionality = -1;
+    } else {
+        if (dimensionality <= 0) {
+            throw new Error(
+                `Dimensionality must be undefined or a positive integer, got ${dimensionality}`
             );
-            fullPrompt += userMessage;
         }
-        if (message["role"] == "assistant") {
-            const assistantMessage = message["content"] + "\n";
-            fullPrompt += assistantMessage;
+        if (dimensionality < model.MIN_DIMENSIONALITY) {
+            console.warn(
+                `Dimensionality ${dimensionality} is less than the suggested minimum of ${model.MIN_DIMENSIONALITY}. Performance may be degraded.`
+            );
         }
     }
 
-    if (promptFooter) {
-        fullPrompt += "\n\n" + promptFooter;
+    let doMean;
+    switch (longTextMode) {
+        case "mean":
+            doMean = true;
+            break;
+        case "truncate":
+            doMean = false;
+            break;
+        default:
+            throw new Error(
+                `Long text mode must be one of 'mean' or 'truncate', got ${longTextMode}`
+            );
     }
 
-    return fullPrompt;
-}
-
-function createEmbedding(model, text) {
-    return model.embed(text);
+    return model.embed(text, options?.prefix, dimensionality, doMean, atlas);
 }
 
 const defaultCompletionOptions = {
@@ -155,162 +128,76 @@ const defaultCompletionOptions = {
     ...DEFAULT_PROMPT_CONTEXT,
 };
 
-function preparePromptAndContext(model,messages,options){
-    if (options.hasDefaultHeader !== undefined) {
-        console.warn(
-            "hasDefaultHeader (bool) is deprecated and has no effect, use promptHeader (string) instead"
-        );
-    }
-
-    if (options.hasDefaultFooter !== undefined) {
-        console.warn(
-            "hasDefaultFooter (bool) is deprecated and has no effect, use promptFooter (string) instead"
-        );
-    }
-
-    const optionsWithDefaults = {
+async function createCompletion(
+    provider,
+    input,
+    options = defaultCompletionOptions
+) {
+    const completionOptions = {
         ...defaultCompletionOptions,
         ...options,
     };
 
-    const {
-        verbose,
-        systemPromptTemplate,
-        promptTemplate,
-        promptHeader,
-        promptFooter,
-        ...promptContext
-    } = optionsWithDefaults;
-
-    
-    const prompt = formatChatPrompt(messages, {
-        systemPromptTemplate,
-        defaultSystemPrompt: model.config.systemPrompt,
-        promptTemplate: promptTemplate || model.config.promptTemplate || "%1",
-        promptHeader: promptHeader || "",
-        promptFooter: promptFooter || "",
-        // These were the default header/footer prompts used for non-chat single turn completions.
-        // both seem to be working well still with some models, so keeping them here for reference.
-        // promptHeader: '### Instruction: The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.',
-        // promptFooter: '### Response:',
-    });
+    const result = await provider.generate(
+        input,
+        completionOptions,
+    );
 
     return {
-        prompt, promptContext, verbose
-    }
-}
-
-async function createCompletion(
-    model,
-    messages,
-    options = defaultCompletionOptions
-) {
-    const { prompt, promptContext, verbose } = preparePromptAndContext(model,messages,options);
-
-    if (verbose) {
-        console.debug("Sending Prompt:\n" + prompt);
-    }
-
-    let tokensGenerated = 0
-
-    const response = await model.generate(prompt, promptContext,() => {
-        tokensGenerated++;
-        return true;
-    });
-
-    if (verbose) {
-        console.debug("Received Response:\n" + response);
-    }
-
-    return {
-        llmodel: model.llm.name(),
+        model: provider.modelName,
         usage: {
-            prompt_tokens: prompt.length,
-            completion_tokens: tokensGenerated,
-            total_tokens: prompt.length + tokensGenerated, //TODO Not sure how to get tokens in prompt
+            prompt_tokens: result.tokensIngested,
+            total_tokens: result.tokensIngested + result.tokensGenerated,
+            completion_tokens: result.tokensGenerated,
+            n_past_tokens: result.nPast,
         },
         choices: [
             {
                 message: {
                     role: "assistant",
-                    content: response,
+                    content: result.text,
                 },
+                // TODO some completion APIs also provide logprobs and finish_reason, could look into adding those
             },
         ],
     };
 }
 
-function _internal_createTokenStream(stream,model,
-    messages,
-    options = defaultCompletionOptions,callback = undefined) {
-    const { prompt, promptContext, verbose } = preparePromptAndContext(model,messages,options);
+function createCompletionStream(
+    provider,
+    input,
+    options = defaultCompletionOptions
+) {
+    const completionStream = new Stream.PassThrough({
+        encoding: "utf-8",
+    });
 
-
-    if (verbose) {
-        console.debug("Sending Prompt:\n" + prompt);
-    }
-
-    model.generate(prompt, promptContext,(tokenId, token, total) => {
-        stream.push(token);
-        
-        if(callback !== undefined){
-            return callback(tokenId,token,total);
-        }
-
-        return true;
-    }).then(() => {
-        stream.end()
-    })
-
-    return stream;
-}
-
-function _createTokenStream(model,
-    messages,
-    options = defaultCompletionOptions,callback = undefined) {
-   
-   // Silent crash if we dont do this here
-   const stream = new Stream.PassThrough({
-     encoding: 'utf-8'
-   });
-   return _internal_createTokenStream(stream,model,messages,options,callback);
-}
-
-async function* generateTokens(model,
-    messages,
-    options = defaultCompletionOptions, callback = undefined) {
-    const stream = _createTokenStream(model,messages,options,callback)
-
-    let bHasFinished = false;
-    let activeDataCallback = undefined;
-    const finishCallback = () => {
-        bHasFinished = true;
-        if(activeDataCallback !== undefined){
-            activeDataCallback(undefined);
-        }
-    }
-
-    stream.on("finish",finishCallback)
-
-    while (!bHasFinished) {
-        const token = await new Promise((res) => {
-
-            activeDataCallback = (d) => {
-                stream.off("data",activeDataCallback)
-                activeDataCallback = undefined
-                res(d);
+    const completionPromise = createCompletion(provider, input, {
+        ...options,
+        onResponseToken: (tokenId, token) => {
+            completionStream.push(token);
+            if (options.onResponseToken) {
+                return options.onResponseToken(tokenId, token);
             }
-            stream.on('data', activeDataCallback)
-        })
+        },
+    }).then((result) => {
+        completionStream.push(null);
+        completionStream.emit("end");
+        return result;
+    });
 
-        if (token == undefined) {
-            break;
-        }
+    return {
+        tokens: completionStream,
+        result: completionPromise,
+    };
+}
 
-        yield token;
+async function* createCompletionGenerator(provider, input, options) {
+    const completion = createCompletionStream(provider, input, options);
+    for await (const chunk of completion.tokens) {
+        yield chunk;
     }
-
-    stream.off("finish",finishCallback);
+    return await completion.result;
 }
 
 module.exports = {
@@ -322,10 +209,12 @@ module.exports = {
     LLModel,
     InferenceModel,
     EmbeddingModel,
+    ChatSession,
     createCompletion,
+    createCompletionStream,
+    createCompletionGenerator,
     createEmbedding,
     downloadModel,
     retrieveModel,
     loadModel,
-    generateTokens
 };

gpt4all-bindings/typescript/src/models.js

@@ -1,18 +1,138 @@
-const { normalizePromptContext, warnOnSnakeCaseKeys } = require('./util');
+const { DEFAULT_PROMPT_CONTEXT } = require("./config");
+const { ChatSession } = require("./chat-session");
+const { prepareMessagesForIngest } = require("./util");
 
 class InferenceModel {
     llm;
+    modelName;
     config;
+    activeChatSession;
 
     constructor(llmodel, config) {
         this.llm = llmodel;
         this.config = config;
+        this.modelName = this.llm.name();
     }
 
-    async generate(prompt, promptContext,callback) {
-        warnOnSnakeCaseKeys(promptContext);
-        const normalizedPromptContext = normalizePromptContext(promptContext);
-        const result = this.llm.raw_prompt(prompt, normalizedPromptContext,callback);
+    async createChatSession(options) {
+        const chatSession = new ChatSession(this, options);
+        await chatSession.initialize();
+        this.activeChatSession = chatSession;
+        return this.activeChatSession;
+    }
+
+    async generate(input, options = DEFAULT_PROMPT_CONTEXT) {
+        const { verbose, ...otherOptions } = options;
+        const promptContext = {
+            promptTemplate: this.config.promptTemplate,
+            temp:
+                otherOptions.temp ??
+                otherOptions.temperature ??
+                DEFAULT_PROMPT_CONTEXT.temp,
+            ...otherOptions,
+        };
+        
+        if (promptContext.nPast < 0) {
+            throw new Error("nPast must be a non-negative integer.");
+        }
+
+        if (verbose) {
+            console.debug("Generating completion", {
+                input,
+                promptContext,
+            });
+        }
+
+        let prompt = input;
+        let nPast = promptContext.nPast;
+        let tokensIngested = 0;
+
+        if (Array.isArray(input)) {
+            // assuming input is a messages array
+            // -> tailing user message will be used as the final prompt. its required.
+            // -> leading system message will be ingested as systemPrompt, further system messages will be ignored
+            // -> all other messages will be ingested with fakeReply
+            // -> model/context will only be kept for this completion; "stateless"
+            nPast = 0;
+            const messages = [...input];
+            const lastMessage = input[input.length - 1];
+            if (lastMessage.role !== "user") {
+                // this is most likely a user error
+                throw new Error("The final message must be of role 'user'.");
+            }
+            if (input[0].role === "system") {
+                // needs to be a pre-templated prompt ala '<|im_start|>system\nYou are an advanced mathematician.\n<|im_end|>\n'
+                const systemPrompt = input[0].content;
+                const systemRes = await this.llm.infer(systemPrompt, {
+                    promptTemplate: "%1",
+                    nPredict: 0,
+                    special: true,
+                });
+                nPast = systemRes.nPast;
+                tokensIngested += systemRes.tokensIngested;
+                messages.shift();
+            }
+
+            prompt = lastMessage.content;
+            const messagesToIngest = messages.slice(0, input.length - 1);
+            const turns = prepareMessagesForIngest(messagesToIngest);
+
+            for (const turn of turns) {
+                const turnRes = await this.llm.infer(turn.user, {
+                    ...promptContext,
+                    nPast,
+                    fakeReply: turn.assistant,
+                });
+                tokensIngested += turnRes.tokensIngested;
+                nPast = turnRes.nPast;
+            }
+        }
+
+        let tokensGenerated = 0;
+
+        const result = await this.llm.infer(prompt, {
+            ...promptContext,
+            nPast,
+            onPromptToken: (tokenId) => {
+                let continueIngestion = true;
+                tokensIngested++;
+                if (options.onPromptToken) {
+                    // catch errors because if they go through cpp they will loose stacktraces
+                    try {
+                        // don't cancel ingestion unless user explicitly returns false
+                        continueIngestion =
+                            options.onPromptToken(tokenId) !== false;
+                    } catch (e) {
+                        console.error("Error in onPromptToken callback", e);
+                        continueIngestion = false;
+                    }
+                }
+                return continueIngestion;
+            },
+            onResponseToken: (tokenId, token) => {
+                let continueGeneration = true;
+                tokensGenerated++;
+                if (options.onResponseToken) {
+                    try {
+                        // don't cancel the generation unless user explicitly returns false
+                        continueGeneration =
+                            options.onResponseToken(tokenId, token) !== false;
+                    } catch (err) {
+                        console.error("Error in onResponseToken callback", err);
+                        continueGeneration = false;
+                    }
+                }
+                return continueGeneration;
+            },
+        });
+
+        result.tokensGenerated = tokensGenerated;
+        result.tokensIngested = tokensIngested;
+
+        if (verbose) {
+            console.debug("Finished completion:\n", result);
+        }
+
         return result;
     }
 
@@ -24,14 +144,14 @@ class InferenceModel {
 class EmbeddingModel {
     llm;
     config;
-
+    MIN_DIMENSIONALITY = 64;
     constructor(llmodel, config) {
         this.llm = llmodel;
         this.config = config;
     }
 
-    embed(text) {
-        return this.llm.embed(text)
+    embed(text, prefix, dimensionality, do_mean, atlas) {
+        return this.llm.embed(text, prefix, dimensionality, do_mean, atlas);
     }
 
     dispose() {
@@ -39,7 +159,6 @@ class EmbeddingModel {
     }
 }
 
-
 module.exports = {
     InferenceModel,
     EmbeddingModel,

gpt4all-bindings/typescript/src/util.js

@@ -1,8 +1,7 @@
-const { createWriteStream, existsSync, statSync } = require("node:fs");
+const { createWriteStream, existsSync, statSync, mkdirSync } = require("node:fs");
 const fsp = require("node:fs/promises");
 const { performance } = require("node:perf_hooks");
 const path = require("node:path");
-const { mkdirp } = require("mkdirp");
 const md5File = require("md5-file");
 const {
     DEFAULT_DIRECTORY,
@@ -50,6 +49,63 @@ function appendBinSuffixIfMissing(name) {
     return name;
 }
 
+function prepareMessagesForIngest(messages) {
+    const systemMessages = messages.filter(
+        (message) => message.role === "system"
+    );
+    if (systemMessages.length > 0) {
+        console.warn(
+            "System messages are currently not supported and will be ignored. Use the systemPrompt option instead."
+        );
+    }
+
+    const userAssistantMessages = messages.filter(
+        (message) => message.role !== "system"
+    );
+
+    // make sure the first message is a user message
+    // if its not, the turns will be out of order
+    if (userAssistantMessages[0].role !== "user") {
+        userAssistantMessages.unshift({
+            role: "user",
+            content: "",
+        });
+    }
+
+    // create turns of user input + assistant reply
+    const turns = [];
+    let userMessage = null;
+    let assistantMessage = null;
+
+    for (const message of userAssistantMessages) {
+        // consecutive messages of the same role are concatenated into one message
+        if (message.role === "user") {
+            if (!userMessage) {
+                userMessage = message.content;
+            } else {
+                userMessage += "\n" + message.content;
+            }
+        } else if (message.role === "assistant") {
+            if (!assistantMessage) {
+                assistantMessage = message.content;
+            } else {
+                assistantMessage += "\n" + message.content;
+            }
+        }
+
+        if (userMessage && assistantMessage) {
+            turns.push({
+                user: userMessage,
+                assistant: assistantMessage,
+            });
+            userMessage = null;
+            assistantMessage = null;
+        }
+    }
+
+    return turns;
+}
+
 // readChunks() reads from the provided reader and yields the results into an async iterable
 // https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
 function readChunks(reader) {
@@ -64,49 +120,13 @@ function readChunks(reader) {
     };
 }
 
-/**
- * Prints a warning if any keys in the prompt context are snake_case.
- */
-function warnOnSnakeCaseKeys(promptContext) {
-    const snakeCaseKeys = Object.keys(promptContext).filter((key) =>
-        key.includes("_")
-    );
-
-    if (snakeCaseKeys.length > 0) {
-        console.warn(
-            "Prompt context keys should be camelCase. Support for snake_case might be removed in the future. Found keys: " +
-                snakeCaseKeys.join(", ")
-        );
-    }
-}
-
-/**
- * Converts all keys in the prompt context to snake_case
- * For duplicate definitions, the value of the last occurrence will be used.
- */
-function normalizePromptContext(promptContext) {
-    const normalizedPromptContext = {};
-
-    for (const key in promptContext) {
-        if (promptContext.hasOwnProperty(key)) {
-            const snakeKey = key.replace(
-                /[A-Z]/g,
-                (match) => `_${match.toLowerCase()}`
-            );
-            normalizedPromptContext[snakeKey] = promptContext[key];
-        }
-    }
-
-    return normalizedPromptContext;
-}
-
 function downloadModel(modelName, options = {}) {
     const downloadOptions = {
         modelPath: DEFAULT_DIRECTORY,
         verbose: false,
         ...options,
     };
-    
+
     const modelFileName = appendBinSuffixIfMissing(modelName);
     const partialModelPath = path.join(
         downloadOptions.modelPath,
@@ -114,16 +134,17 @@ function downloadModel(modelName, options = {}) {
     );
     const finalModelPath = path.join(downloadOptions.modelPath, modelFileName);
     const modelUrl =
-        downloadOptions.url ?? `https://gpt4all.io/models/gguf/${modelFileName}`;
+        downloadOptions.url ??
+        `https://gpt4all.io/models/gguf/${modelFileName}`;
 
-    mkdirp.sync(downloadOptions.modelPath)
+    mkdirSync(downloadOptions.modelPath, { recursive: true });
 
     if (existsSync(finalModelPath)) {
         throw Error(`Model already exists at ${finalModelPath}`);
     }
-    
+
     if (downloadOptions.verbose) {
-        console.log(`Downloading ${modelName} from ${modelUrl}`);
+        console.debug(`Downloading ${modelName} from ${modelUrl}`);
     }
 
     const headers = {
@@ -134,7 +155,9 @@ function downloadModel(modelName, options = {}) {
     const writeStreamOpts = {};
 
     if (existsSync(partialModelPath)) {
-        console.log("Partial model exists, resuming download...");
+        if (downloadOptions.verbose) {
+            console.debug("Partial model exists, resuming download...");
+        }
         const startRange = statSync(partialModelPath).size;
         headers["Range"] = `bytes=${startRange}-`;
         writeStreamOpts.flags = "a";
@@ -144,15 +167,15 @@ function downloadModel(modelName, options = {}) {
     const signal = abortController.signal;
 
     const finalizeDownload = async () => {
-        if (options.md5sum) {
+        if (downloadOptions.md5sum) {
             const fileHash = await md5File(partialModelPath);
-            if (fileHash !== options.md5sum) {
+            if (fileHash !== downloadOptions.md5sum) {
                 await fsp.unlink(partialModelPath);
-                const message = `Model "${modelName}" failed verification: Hashes mismatch. Expected ${options.md5sum}, got ${fileHash}`;
+                const message = `Model "${modelName}" failed verification: Hashes mismatch. Expected ${downloadOptions.md5sum}, got ${fileHash}`;
                 throw Error(message);
             }
-            if (options.verbose) {
-                console.log(`MD5 hash verified: ${fileHash}`);
+            if (downloadOptions.verbose) {
+                console.debug(`MD5 hash verified: ${fileHash}`);
             }
         }
 
@@ -163,8 +186,8 @@ function downloadModel(modelName, options = {}) {
     const downloadPromise = new Promise((resolve, reject) => {
         let timestampStart;
 
-        if (options.verbose) {
-            console.log(`Downloading @ ${partialModelPath} ...`);
+        if (downloadOptions.verbose) {
+            console.debug(`Downloading @ ${partialModelPath} ...`);
             timestampStart = performance.now();
         }
 
@@ -179,7 +202,7 @@ function downloadModel(modelName, options = {}) {
         });
 
         writeStream.on("finish", () => {
-            if (options.verbose) {
+            if (downloadOptions.verbose) {
                 const elapsed = performance.now() - timestampStart;
                 console.log(`Finished. Download took ${elapsed.toFixed(2)} ms`);
             }
@@ -221,10 +244,10 @@ async function retrieveModel(modelName, options = {}) {
     const retrieveOptions = {
         modelPath: DEFAULT_DIRECTORY,
         allowDownload: true,
-        verbose: true,
+        verbose: false,
         ...options,
     };
-    await mkdirp(retrieveOptions.modelPath);
+    mkdirSync(retrieveOptions.modelPath, { recursive: true });
 
     const modelFileName = appendBinSuffixIfMissing(modelName);
     const fullModelPath = path.join(retrieveOptions.modelPath, modelFileName);
@@ -236,7 +259,7 @@ async function retrieveModel(modelName, options = {}) {
         file: retrieveOptions.modelConfigFile,
         url:
             retrieveOptions.allowDownload &&
-            "https://gpt4all.io/models/models2.json",
+            "https://gpt4all.io/models/models3.json",
     });
 
     const loadedModelConfig = availableModels.find(
@@ -262,10 +285,9 @@ async function retrieveModel(modelName, options = {}) {
         config.path = fullModelPath;
 
         if (retrieveOptions.verbose) {
-            console.log(`Found ${modelName} at ${fullModelPath}`);
+            console.debug(`Found ${modelName} at ${fullModelPath}`);
         }
     } else if (retrieveOptions.allowDownload) {
-
         const downloadController = downloadModel(modelName, {
             modelPath: retrieveOptions.modelPath,
             verbose: retrieveOptions.verbose,
@@ -278,7 +300,7 @@ async function retrieveModel(modelName, options = {}) {
         config.path = downloadPath;
 
         if (retrieveOptions.verbose) {
-            console.log(`Model downloaded to ${downloadPath}`);
+            console.debug(`Model downloaded to ${downloadPath}`);
         }
     } else {
         throw Error("Failed to retrieve model.");
@@ -288,9 +310,8 @@ async function retrieveModel(modelName, options = {}) {
 
 module.exports = {
     appendBinSuffixIfMissing,
+    prepareMessagesForIngest,
     downloadModel,
     retrieveModel,
     listModels,
-    normalizePromptContext,
-    warnOnSnakeCaseKeys,
 };

gpt4all-bindings/typescript/test/gpt4all.test.js

@@ -7,7 +7,6 @@ const {
     listModels,
     downloadModel,
     appendBinSuffixIfMissing,
-    normalizePromptContext,
 } = require("../src/util.js");
 const {
     DEFAULT_DIRECTORY,
@@ -19,8 +18,6 @@ const {
     createPrompt,
     createCompletion,
 } = require("../src/gpt4all.js");
-const { mock } = require("node:test");
-const { mkdirp } = require("mkdirp");
 
 describe("config", () => {
     test("default paths constants are available and correct", () => {
@@ -87,7 +84,7 @@ describe("listModels", () => {
         expect(fetch).toHaveBeenCalledTimes(0);
         expect(models[0]).toEqual(fakeModel);
     });
-    
+
     it("should throw an error if neither url nor file is specified", async () => {
         await expect(listModels(null)).rejects.toThrow(
             "No model list source specified. Please specify either a url or a file."
@@ -141,10 +138,10 @@ describe("downloadModel", () => {
         mockAbortController.mockReset();
         mockFetch.mockClear();
         global.fetch.mockRestore();
-        
+
         const rootDefaultPath = path.resolve(DEFAULT_DIRECTORY),
               partialPath = path.resolve(rootDefaultPath, fakeModelName+'.part'),
-              fullPath = path.resolve(rootDefaultPath, fakeModelName+'.bin')  
+              fullPath = path.resolve(rootDefaultPath, fakeModelName+'.bin')
 
         //if tests fail, remove the created files
         // acts as cleanup if tests fail
@@ -206,46 +203,3 @@ describe("downloadModel", () => {
     // test("should be able to cancel and resume a download", async () => {
     // });
 });
-
-describe("normalizePromptContext", () => {
-    it("should convert a dict with camelCased keys to snake_case", () => {
-        const camelCased = {
-            topK: 20,
-            repeatLastN: 10,
-        };
-
-        const expectedSnakeCased = {
-            top_k: 20,
-            repeat_last_n: 10,
-        };
-
-        const result = normalizePromptContext(camelCased);
-        expect(result).toEqual(expectedSnakeCased);
-    });
-
-    it("should convert a mixed case dict to snake_case, last value taking precedence", () => {
-        const mixedCased = {
-            topK: 20,
-            top_k: 10,
-            repeatLastN: 10,
-        };
-
-        const expectedSnakeCased = {
-            top_k: 10,
-            repeat_last_n: 10,
-        };
-
-        const result = normalizePromptContext(mixedCased);
-        expect(result).toEqual(expectedSnakeCased);
-    });
-
-    it("should not modify already snake cased dict", () => {
-        const snakeCased = {
-            top_k: 10,
-            repeast_last_n: 10,
-        };
-
-        const result = normalizePromptContext(snakeCased);
-        expect(result).toEqual(snakeCased);
-    });
-});

gpt4all-bindings/typescript/yarn.lock

@@ -2300,7 +2300,6 @@ __metadata:
     documentation: ^14.0.2
     jest: ^29.5.0
     md5-file: ^5.0.0
-    mkdirp: ^3.0.1
     node-addon-api: ^6.1.0
     node-gyp: 9.x.x
     node-gyp-build: ^4.6.0
@@ -2631,9 +2630,9 @@ __metadata:
   linkType: hard
 
 "ip@npm:^2.0.0":
-  version: 2.0.0
-  resolution: "ip@npm:2.0.0"
-  checksum: cfcfac6b873b701996d71ec82a7dd27ba92450afdb421e356f44044ed688df04567344c36cbacea7d01b1c39a4c732dc012570ebe9bebfb06f27314bca625349
+  version: 2.0.1
+  resolution: "ip@npm:2.0.1"
+  checksum: d765c9fd212b8a99023a4cde6a558a054c298d640fec1020567494d257afd78ca77e37126b1a3ef0e053646ced79a816bf50621d38d5e768cdde0431fa3b0d35
   languageName: node
   linkType: hard
 
@@ -4258,15 +4257,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"mkdirp@npm:^3.0.1":
-  version: 3.0.1
-  resolution: "mkdirp@npm:3.0.1"
-  bin:
-    mkdirp: dist/cjs/src/bin.js
-  checksum: 972deb188e8fb55547f1e58d66bd6b4a3623bf0c7137802582602d73e6480c1c2268dcbafbfb1be466e00cc7e56ac514d7fd9334b7cf33e3e2ab547c16f83a8d
-  languageName: node
-  linkType: hard
-
 "mri@npm:^1.1.0":
   version: 1.2.0
   resolution: "mri@npm:1.2.0"
@@ -5405,8 +5395,8 @@ __metadata:
   linkType: hard
 
 "tar@npm:^6.1.11, tar@npm:^6.1.2":
-  version: 6.2.0
-  resolution: "tar@npm:6.2.0"
+  version: 6.2.1
+  resolution: "tar@npm:6.2.1"
   dependencies:
     chownr: ^2.0.0
     fs-minipass: ^2.0.0
@@ -5414,7 +5404,7 @@ __metadata:
     minizlib: ^2.1.1
     mkdirp: ^1.0.3
     yallist: ^4.0.0
-  checksum: db4d9fe74a2082c3a5016630092c54c8375ff3b280186938cfd104f2e089c4fd9bad58688ef6be9cf186a889671bf355c7cda38f09bbf60604b281715ca57f5c
+  checksum: f1322768c9741a25356c11373bce918483f40fa9a25c69c59410c8a1247632487edef5fe76c5f12ac51a6356d2f1829e96d2bc34098668a2fc34d76050ac2b6c
   languageName: node
   linkType: hard
 

gpt4all-chat/CMakeLists.txt

@@ -18,7 +18,7 @@ endif()
 
 set(APP_VERSION_MAJOR 2)
 set(APP_VERSION_MINOR 7)
-set(APP_VERSION_PATCH 2)
+set(APP_VERSION_PATCH 4)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file
@@ -40,9 +40,9 @@ configure_file(
 )
 
 if(LINUX)
-  find_package(Qt6 6.5 COMPONENTS Core Quick WaylandCompositor QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
+  find_package(Qt6 6.4 COMPONENTS Core Quick WaylandCompositor QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
 else()
-  find_package(Qt6 6.5 COMPONENTS Core Quick QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
+  find_package(Qt6 6.4 COMPONENTS Core Quick QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
 endif()
 
 # Get the Qt6Core target properties
@@ -68,12 +68,16 @@ if(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
   set(METAL_SHADER_FILE ../gpt4all-backend/llama.cpp-mainline/ggml-metal.metal)
 endif()
 
+set(APP_ICON_FILE "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns")
+set_source_files_properties(${APP_ICON_FILE} PROPERTIES
+    MACOSX_PACKAGE_LOCATION "Resources")
+
 qt_add_executable(chat
     main.cpp
     chat.h chat.cpp
     chatllm.h chatllm.cpp
     chatmodel.h chatlistmodel.h chatlistmodel.cpp
-    chatgpt.h chatgpt.cpp
+    chatapi.h chatapi.cpp
     database.h database.cpp
     embeddings.h embeddings.cpp
     download.h download.cpp
@@ -87,6 +91,7 @@ qt_add_executable(chat
     logger.h logger.cpp
     responsetext.h responsetext.cpp
     ${METAL_SHADER_FILE}
+    ${APP_ICON_FILE}
 )
 
 qt_add_qml_module(chat
@@ -96,6 +101,7 @@ qt_add_qml_module(chat
     QML_FILES
       main.qml
       qml/ChatDrawer.qml
+      qml/ChatView.qml
       qml/CollectionsDialog.qml
       qml/ModelDownloaderDialog.qml
       qml/NetworkDialog.qml
@@ -142,6 +148,8 @@ qt_add_qml_module(chat
       icons/network.svg
       icons/thumbs_up.svg
       icons/thumbs_down.svg
+      icons/left_panel_closed.svg
+      icons/left_panel_open.svg
       icons/logo.svg
       icons/logo-32.png
       icons/logo-48.png
@@ -184,7 +192,10 @@ target_link_libraries(chat
     PRIVATE llmodel)
 
 set(COMPONENT_NAME_MAIN ${PROJECT_NAME})
-set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/install)
+
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/install CACHE PATH "..." FORCE)
+endif()
 
 install(TARGETS chat DESTINATION bin COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llmodel DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
@@ -200,8 +211,6 @@ install(TARGETS llamamodel-mainline-default DESTINATION lib COMPONENT ${COMPONEN
 if(APPLE)
 install(TARGETS llamamodel-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 endif()
-install(TARGETS bert-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS bert-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 
 set(CPACK_GENERATOR "IFW")
 set(CPACK_VERBATIM_VARIABLES YES)

gpt4all-chat/chat.cpp

@@ -46,6 +46,7 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::promptProcessing, this, &Chat::promptProcessing, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::responseStopped, this, &Chat::responseStopped, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelLoadingError, this, &Chat::handleModelLoadingError, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::modelLoadingWarning, this, &Chat::modelLoadingWarning, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::recalcChanged, this, &Chat::handleRecalculating, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::generatedNameChanged, this, &Chat::generatedNameChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportSpeed, this, &Chat::handleTokenSpeedChanged, Qt::QueuedConnection);
@@ -178,59 +179,62 @@ void Chat::promptProcessing()
     emit responseStateChanged();
 }
 
-void Chat::responseStopped()
+void Chat::responseStopped(qint64 promptResponseMs)
 {
     m_tokenSpeed = QString();
     emit tokenSpeedChanged();
 
-    if (MySettings::globalInstance()->localDocsShowReferences()) {
-        const QString chatResponse = response();
-        QList<QString> references;
-        QList<QString> referencesContext;
-        int validReferenceNumber = 1;
-        for (const ResultInfo &info : databaseResults()) {
-            if (info.file.isEmpty())
-                continue;
-            if (validReferenceNumber == 1)
-                references.append((!chatResponse.endsWith("\n") ? "\n" : QString()) + QStringLiteral("\n---"));
-            QString reference;
-            {
-                QTextStream stream(&reference);
-                stream << (validReferenceNumber++) << ". ";
-                if (!info.title.isEmpty())
-                    stream << "\"" << info.title << "\". ";
-                if (!info.author.isEmpty())
-                    stream << "By " << info.author << ". ";
-                if (!info.date.isEmpty())
-                    stream << "Date: " << info.date << ". ";
-                stream << "In " << info.file << ". ";
-                if (info.page != -1)
-                    stream << "Page " << info.page << ". ";
-                if (info.from != -1) {
-                    stream << "Lines " << info.from;
-                    if (info.to != -1)
-                        stream << "-" << info.to;
-                    stream << ". ";
-                }
-                stream << "[Context](context://" << validReferenceNumber - 1 << ")";
+    const QString chatResponse = response();
+    QList<QString> references;
+    QList<QString> referencesContext;
+    int validReferenceNumber = 1;
+    for (const ResultInfo &info : databaseResults()) {
+        if (info.file.isEmpty())
+            continue;
+        if (validReferenceNumber == 1)
+            references.append((!chatResponse.endsWith("\n") ? "\n" : QString()) + QStringLiteral("\n---"));
+        QString reference;
+        {
+            QTextStream stream(&reference);
+            stream << (validReferenceNumber++) << ". ";
+            if (!info.title.isEmpty())
+                stream << "\"" << info.title << "\". ";
+            if (!info.author.isEmpty())
+                stream << "By " << info.author << ". ";
+            if (!info.date.isEmpty())
+                stream << "Date: " << info.date << ". ";
+            stream << "In " << info.file << ". ";
+            if (info.page != -1)
+                stream << "Page " << info.page << ". ";
+            if (info.from != -1) {
+                stream << "Lines " << info.from;
+                if (info.to != -1)
+                    stream << "-" << info.to;
+                stream << ". ";
             }
-            references.append(reference);
-            referencesContext.append(info.text);
+            stream << "[Context](context://" << validReferenceNumber - 1 << ")";
         }
-
-        const int index = m_chatModel->count() - 1;
-        m_chatModel->updateReferences(index, references.join("\n"), referencesContext);
-        emit responseChanged();
+        references.append(reference);
+        referencesContext.append(info.text);
     }
 
+    const int index = m_chatModel->count() - 1;
+    m_chatModel->updateReferences(index, references.join("\n"), referencesContext);
+    emit responseChanged();
+
     m_responseInProgress = false;
     m_responseState = Chat::ResponseStopped;
     emit responseInProgressChanged();
     emit responseStateChanged();
     if (m_generatedName.isEmpty())
         emit generateNameRequested();
-    if (chatModel()->count() < 3)
-        Network::globalInstance()->sendChatStarted();
+
+    Network::globalInstance()->trackChatEvent("response_complete", {
+        {"first", m_firstResponse},
+        {"message_count", chatModel()->count()},
+        {"$duration", promptResponseMs / 1000.},
+    });
+    m_firstResponse = false;
 }
 
 ModelInfo Chat::modelInfo() const
@@ -285,6 +289,11 @@ void Chat::unloadAndDeleteLater()
     unloadModel();
 }
 
+void Chat::markForDeletion()
+{
+    m_llmodel->setMarkedForDeletion(true);
+}
+
 void Chat::unloadModel()
 {
     stopGenerating();
@@ -327,13 +336,14 @@ void Chat::generatedNameChanged(const QString &name)
 
 void Chat::handleRecalculating()
 {
-    Network::globalInstance()->sendRecalculatingContext(m_chatModel->count());
+    Network::globalInstance()->trackChatEvent("recalc_context", { {"length", m_chatModel->count()} });
     emit recalcChanged();
 }
 
 void Chat::handleModelLoadingError(const QString &error)
 {
-    qWarning() << "ERROR:" << qPrintable(error) << "id" << id();
+    auto stream = qWarning().noquote() << "ERROR:" << error << "id";
+    stream.quote() << id();
     m_modelLoadingError = error;
     emit modelLoadingErrorChanged();
 }

gpt4all-chat/chat.h

@@ -46,6 +46,7 @@ public:
     explicit Chat(QObject *parent = nullptr);
     explicit Chat(bool isServer, QObject *parent = nullptr);
     virtual ~Chat();
+    void destroy() { m_llmodel->destroy(); }
     void connectLLM();
 
     QString id() const { return m_id; }
@@ -83,6 +84,7 @@ public:
     Q_INVOKABLE void forceReloadModel();
     Q_INVOKABLE void trySwitchContextOfLoadedModel();
     void unloadAndDeleteLater();
+    void markForDeletion();
 
     qint64 creationDate() const { return m_creationDate; }
     bool serialize(QDataStream &stream, int version) const;
@@ -112,6 +114,7 @@ Q_SIGNALS:
     void chatModelChanged();
     void isModelLoadedChanged();
     void modelLoadingPercentageChanged();
+    void modelLoadingWarning(const QString &warning);
     void responseChanged();
     void responseInProgressChanged();
     void responseStateChanged();
@@ -140,7 +143,7 @@ private Q_SLOTS:
     void handleResponseChanged(const QString &response);
     void handleModelLoadingPercentageChanged(float);
     void promptProcessing();
-    void responseStopped();
+    void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
     void handleRecalculating();
     void handleModelLoadingError(const QString &error);
@@ -172,6 +175,7 @@ private:
     bool m_shouldDeleteLater = false;
     float m_modelLoadingPercentage = 0.0f;
     LocalDocsCollectionsModel *m_collectionModel;
+    bool m_firstResponse = true;
 };
 
 #endif // CHAT_H

gpt4all-chat/chatapi.cpp

@@ -1,4 +1,4 @@
-#include "chatgpt.h"
+#include "chatapi.h"
 
 #include <string>
 #include <vector>
@@ -13,14 +13,15 @@
 
 //#define DEBUG
 
-ChatGPT::ChatGPT()
+ChatAPI::ChatAPI()
     : QObject(nullptr)
     , m_modelName("gpt-3.5-turbo")
+    , m_requestURL("")
     , m_responseCallback(nullptr)
 {
 }
 
-size_t ChatGPT::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
+size_t ChatAPI::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
 {
     Q_UNUSED(modelPath);
     Q_UNUSED(n_ctx);
@@ -28,7 +29,7 @@ size_t ChatGPT::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
     return 0;
 }
 
-bool ChatGPT::loadModel(const std::string &modelPath, int n_ctx, int ngl)
+bool ChatAPI::loadModel(const std::string &modelPath, int n_ctx, int ngl)
 {
     Q_UNUSED(modelPath);
     Q_UNUSED(n_ctx);
@@ -36,60 +37,80 @@ bool ChatGPT::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     return true;
 }
 
-void ChatGPT::setThreadCount(int32_t n_threads)
+void ChatAPI::setThreadCount(int32_t n_threads)
 {
     Q_UNUSED(n_threads);
     qt_noop();
 }
 
-int32_t ChatGPT::threadCount() const
+int32_t ChatAPI::threadCount() const
 {
     return 1;
 }
 
-ChatGPT::~ChatGPT()
+ChatAPI::~ChatAPI()
 {
 }
 
-bool ChatGPT::isModelLoaded() const
+bool ChatAPI::isModelLoaded() const
 {
     return true;
 }
 
 // All three of the state virtual functions are handled custom inside of chatllm save/restore
-size_t ChatGPT::stateSize() const
+size_t ChatAPI::stateSize() const
 {
     return 0;
 }
 
-size_t ChatGPT::saveState(uint8_t *dest) const
+size_t ChatAPI::saveState(uint8_t *dest) const
 {
     Q_UNUSED(dest);
     return 0;
 }
 
-size_t ChatGPT::restoreState(const uint8_t *src)
+size_t ChatAPI::restoreState(const uint8_t *src)
 {
     Q_UNUSED(src);
     return 0;
 }
 
-void ChatGPT::prompt(const std::string &prompt,
-        const std::string &promptTemplate,
-        std::function<bool(int32_t)> promptCallback,
-        std::function<bool(int32_t, const std::string&)> responseCallback,
-        std::function<bool(bool)> recalculateCallback,
-        PromptContext &promptCtx,
-        bool special,
-        std::string *fakeReply) {
+void ChatAPI::prompt(const std::string &prompt,
+                     const std::string &promptTemplate,
+                     std::function<bool(int32_t)> promptCallback,
+                     std::function<bool(int32_t, const std::string&)> responseCallback,
+                     std::function<bool(bool)> recalculateCallback,
+                     PromptContext &promptCtx,
+                     bool special,
+                     std::string *fakeReply) {
 
     Q_UNUSED(promptCallback);
     Q_UNUSED(recalculateCallback);
     Q_UNUSED(special);
-    Q_UNUSED(fakeReply); // FIXME(cebtenzzre): I broke ChatGPT
 
     if (!isModelLoaded()) {
-        std::cerr << "ChatGPT ERROR: prompt won't work with an unloaded model!\n";
+        std::cerr << "ChatAPI ERROR: prompt won't work with an unloaded model!\n";
+        return;
+    }
+
+    if (!promptCtx.n_past) { m_queuedPrompts.clear(); }
+    Q_ASSERT(promptCtx.n_past <= m_context.size());
+    m_context.resize(promptCtx.n_past);
+
+    // FIXME(cebtenzzre): We're assuming people don't try to use %2 with ChatGPT. What would that even mean?
+    m_queuedPrompts << QString::fromStdString(promptTemplate).arg(QString::fromStdString(prompt));
+
+    if (!promptCtx.n_predict && !fakeReply) {
+        return; // response explicitly suppressed, queue prompt for later
+    }
+
+    QString formattedPrompt = m_queuedPrompts.join("");
+    m_queuedPrompts.clear();
+
+    if (fakeReply) {
+        promptCtx.n_past += 1;
+        m_context.append(formattedPrompt);
+        m_context.append(QString::fromStdString(*fakeReply));
         return;
     }
 
@@ -104,24 +125,25 @@ void ChatGPT::prompt(const std::string &prompt,
     root.insert("temperature", promptCtx.temp);
     root.insert("top_p", promptCtx.top_p);
 
+    // conversation history
     QJsonArray messages;
-    for (int i = 0; i < m_context.count() && i < promptCtx.n_past; ++i) {
+    for (int i = 0; i < m_context.count(); ++i) {
         QJsonObject message;
-        message.insert("role", i % 2 == 0 ? "assistant" : "user");
+        message.insert("role", i % 2 == 0 ? "user" : "assistant");
         message.insert("content", m_context.at(i));
         messages.append(message);
     }
 
     QJsonObject promptObject;
     promptObject.insert("role", "user");
-    promptObject.insert("content", QString::fromStdString(promptTemplate).arg(QString::fromStdString(prompt)));
+    promptObject.insert("content", formattedPrompt);
     messages.append(promptObject);
     root.insert("messages", messages);
 
     QJsonDocument doc(root);
 
 #if defined(DEBUG)
-    qDebug() << "ChatGPT::prompt begin network request" << qPrintable(doc.toJson());
+    qDebug().noquote() << "ChatAPI::prompt begin network request" << doc.toJson();
 #endif
 
     m_responseCallback = responseCallback;
@@ -129,54 +151,54 @@ void ChatGPT::prompt(const std::string &prompt,
     // The following code sets up a worker thread and object to perform the actual api request to
     // chatgpt and then blocks until it is finished
     QThread workerThread;
-    ChatGPTWorker worker(this);
+    ChatAPIWorker worker(this);
     worker.moveToThread(&workerThread);
-    connect(&worker, &ChatGPTWorker::finished, &workerThread, &QThread::quit, Qt::DirectConnection);
-    connect(this, &ChatGPT::request, &worker, &ChatGPTWorker::request, Qt::QueuedConnection);
+    connect(&worker, &ChatAPIWorker::finished, &workerThread, &QThread::quit, Qt::DirectConnection);
+    connect(this, &ChatAPI::request, &worker, &ChatAPIWorker::request, Qt::QueuedConnection);
     workerThread.start();
     emit request(m_apiKey, &promptCtx, doc.toJson(QJsonDocument::Compact));
     workerThread.wait();
 
     promptCtx.n_past += 1;
-    m_context.append(QString::fromStdString(prompt));
+    m_context.append(formattedPrompt);
     m_context.append(worker.currentResponse());
     m_responseCallback = nullptr;
 
 #if defined(DEBUG)
-    qDebug() << "ChatGPT::prompt end network request";
+    qDebug() << "ChatAPI::prompt end network request";
 #endif
 }
 
-bool ChatGPT::callResponse(int32_t token, const std::string& string)
+bool ChatAPI::callResponse(int32_t token, const std::string& string)
 {
     Q_ASSERT(m_responseCallback);
     if (!m_responseCallback) {
-        std::cerr << "ChatGPT ERROR: no response callback!\n";
+        std::cerr << "ChatAPI ERROR: no response callback!\n";
         return false;
     }
     return m_responseCallback(token, string);
 }
 
-void ChatGPTWorker::request(const QString &apiKey,
-        LLModel::PromptContext *promptCtx,
-        const QByteArray &array)
+void ChatAPIWorker::request(const QString &apiKey,
+                            LLModel::PromptContext *promptCtx,
+                            const QByteArray &array)
 {
     m_ctx = promptCtx;
 
-    QUrl openaiUrl("https://api.openai.com/v1/chat/completions");
+    QUrl apiUrl(m_chat->url());
     const QString authorization = QString("Bearer %1").arg(apiKey).trimmed();
-    QNetworkRequest request(openaiUrl);
+    QNetworkRequest request(apiUrl);
     request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
     request.setRawHeader("Authorization", authorization.toUtf8());
     m_networkManager = new QNetworkAccessManager(this);
     QNetworkReply *reply = m_networkManager->post(request, array);
     connect(qApp, &QCoreApplication::aboutToQuit, reply, &QNetworkReply::abort);
-    connect(reply, &QNetworkReply::finished, this, &ChatGPTWorker::handleFinished);
-    connect(reply, &QNetworkReply::readyRead, this, &ChatGPTWorker::handleReadyRead);
-    connect(reply, &QNetworkReply::errorOccurred, this, &ChatGPTWorker::handleErrorOccurred);
+    connect(reply, &QNetworkReply::finished, this, &ChatAPIWorker::handleFinished);
+    connect(reply, &QNetworkReply::readyRead, this, &ChatAPIWorker::handleReadyRead);
+    connect(reply, &QNetworkReply::errorOccurred, this, &ChatAPIWorker::handleErrorOccurred);
 }
 
-void ChatGPTWorker::handleFinished()
+void ChatAPIWorker::handleFinished()
 {
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply) {
@@ -189,14 +211,14 @@ void ChatGPTWorker::handleFinished()
     bool ok;
     int code = response.toInt(&ok);
     if (!ok || code != 200) {
-        qWarning() << QString("ERROR: ChatGPT responded with error code \"%1-%2\"")
-            .arg(code).arg(reply->errorString()).toStdString();
+        qWarning().noquote() << "ERROR: ChatAPIWorker::handleFinished got HTTP Error" << code << "response:"
+                             << reply->errorString();
     }
     reply->deleteLater();
     emit finished();
 }
 
-void ChatGPTWorker::handleReadyRead()
+void ChatAPIWorker::handleReadyRead()
 {
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply) {
@@ -209,8 +231,11 @@ void ChatGPTWorker::handleReadyRead()
     bool ok;
     int code = response.toInt(&ok);
     if (!ok || code != 200) {
-        m_chat->callResponse(-1, QString("\nERROR: 2 ChatGPT responded with error code \"%1-%2\" %3\n")
-            .arg(code).arg(reply->errorString()).arg(qPrintable(reply->readAll())).toStdString());
+        m_chat->callResponse(
+            -1,
+            QString("ERROR: ChatAPIWorker::handleReadyRead got HTTP Error %1 %2: %3")
+                .arg(code).arg(reply->errorString()).arg(reply->readAll()).toStdString()
+        );
         emit finished();
         return;
     }
@@ -225,13 +250,13 @@ void ChatGPTWorker::handleReadyRead()
         if (jsonData == "[DONE]")
             continue;
 #if defined(DEBUG)
-        qDebug() << "line" << qPrintable(jsonData);
+        qDebug().noquote() << "line" << jsonData;
 #endif
         QJsonParseError err;
         const QJsonDocument document = QJsonDocument::fromJson(jsonData.toUtf8(), &err);
         if (err.error != QJsonParseError::NoError) {
-            m_chat->callResponse(-1, QString("\nERROR: ChatGPT responded with invalid json \"%1\"\n")
-                .arg(err.errorString()).toStdString());
+            m_chat->callResponse(-1, QString("ERROR: ChatAPI responded with invalid json \"%1\"")
+                                         .arg(err.errorString()).toStdString());
             continue;
         }
 
@@ -250,7 +275,7 @@ void ChatGPTWorker::handleReadyRead()
     }
 }
 
-void ChatGPTWorker::handleErrorOccurred(QNetworkReply::NetworkError code)
+void ChatAPIWorker::handleErrorOccurred(QNetworkReply::NetworkError code)
 {
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply || reply->error() == QNetworkReply::OperationCanceledError /*when we call abort on purpose*/) {
@@ -258,7 +283,7 @@ void ChatGPTWorker::handleErrorOccurred(QNetworkReply::NetworkError code)
         return;
     }
 
-    qWarning() << QString("ERROR: ChatGPT responded with error code \"%1-%2\"")
-                      .arg(code).arg(reply->errorString()).toStdString();
+    qWarning().noquote() << "ERROR: ChatAPIWorker::handleErrorOccurred got HTTP Error" << code << "response:"
+                         << reply->errorString();
     emit finished();
 }

gpt4all-chat/chatapi.h

@@ -1,31 +1,34 @@
-#ifndef CHATGPT_H
-#define CHATGPT_H
+#ifndef CHATAPI_H
+#define CHATAPI_H
 
 #include <stdexcept>
 
-#include <QObject>
+#include <QNetworkAccessManager>
 #include <QNetworkReply>
 #include <QNetworkRequest>
-#include <QNetworkAccessManager>
+#include <QObject>
+#include <QString>
+#include <QStringList>
 #include <QThread>
+
 #include "../gpt4all-backend/llmodel.h"
 
-class ChatGPT;
-class ChatGPTWorker : public QObject {
+class ChatAPI;
+class ChatAPIWorker : public QObject {
     Q_OBJECT
 public:
-    ChatGPTWorker(ChatGPT *chatGPT)
+    ChatAPIWorker(ChatAPI *chatAPI)
         : QObject(nullptr)
         , m_ctx(nullptr)
         , m_networkManager(nullptr)
-        , m_chat(chatGPT) {}
-    virtual ~ChatGPTWorker() {}
+        , m_chat(chatAPI) {}
+    virtual ~ChatAPIWorker() {}
 
     QString currentResponse() const { return m_currentResponse; }
 
     void request(const QString &apiKey,
-        LLModel::PromptContext *promptCtx,
-        const QByteArray &array);
+                 LLModel::PromptContext *promptCtx,
+                 const QByteArray &array);
 
 Q_SIGNALS:
     void finished();
@@ -36,17 +39,17 @@ private Q_SLOTS:
     void handleErrorOccurred(QNetworkReply::NetworkError code);
 
 private:
-    ChatGPT *m_chat;
+    ChatAPI *m_chat;
     LLModel::PromptContext *m_ctx;
     QNetworkAccessManager *m_networkManager;
     QString m_currentResponse;
 };
 
-class ChatGPT : public QObject, public LLModel {
+class ChatAPI : public QObject, public LLModel {
     Q_OBJECT
 public:
-    ChatGPT();
-    virtual ~ChatGPT();
+    ChatAPI();
+    virtual ~ChatAPI();
 
     bool supportsEmbedding() const override { return false; }
     bool supportsCompletion() const override { return true; }
@@ -57,19 +60,21 @@ public:
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
     void prompt(const std::string &prompt,
-        const std::string &promptTemplate,
-        std::function<bool(int32_t)> promptCallback,
-        std::function<bool(int32_t, const std::string&)> responseCallback,
-        std::function<bool(bool)> recalculateCallback,
-        PromptContext &ctx,
-        bool special,
-        std::string *fakeReply) override;
+                const std::string &promptTemplate,
+                std::function<bool(int32_t)> promptCallback,
+                std::function<bool(int32_t, const std::string&)> responseCallback,
+                std::function<bool(bool)> recalculateCallback,
+                PromptContext &ctx,
+                bool special,
+                std::string *fakeReply) override;
 
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
 
     void setModelName(const QString &modelName) { m_modelName = modelName; }
     void setAPIKey(const QString &apiKey) { m_apiKey = apiKey; }
+    void setRequestURL(const QString &requestURL) { m_requestURL = requestURL; }
+    QString url() const { return m_requestURL; }
 
     QList<QString> context() const { return m_context; }
     void setContext(const QList<QString> &context) { m_context = context; }
@@ -78,8 +83,8 @@ public:
 
 Q_SIGNALS:
     void request(const QString &apiKey,
-        LLModel::PromptContext *ctx,
-        const QByteArray &array);
+                 LLModel::PromptContext *ctx,
+                 const QByteArray &array);
 
 protected:
     // We have to implement these as they are pure virtual in base class, but we don't actually use
@@ -125,7 +130,9 @@ private:
     std::function<bool(int32_t, const std::string&)> m_responseCallback;
     QString m_modelName;
     QString m_apiKey;
+    QString m_requestURL;
     QList<QString> m_context;
+    QStringList m_queuedPrompts;
 };
 
-#endif // CHATGPT_H
+#endif // CHATAPI_H

gpt4all-chat/chatlistmodel.h

@@ -143,6 +143,8 @@ public:
             m_newChat = nullptr;
         }
 
+        chat->markForDeletion();
+
         const int index = m_chats.indexOf(chat);
         if (m_chats.count() < 3 /*m_serverChat included*/) {
             addChat();
@@ -192,13 +194,8 @@ public:
 
     int count() const { return m_chats.size(); }
 
-    void clearChats() {
-        m_newChat = nullptr;
-        m_serverChat = nullptr;
-        m_currentChat = nullptr;
-        for (auto * chat: m_chats) { delete chat; }
-        m_chats.clear();
-    }
+    // stop ChatLLM threads for clean shutdown
+    void destroyChats() { for (auto *chat: m_chats) { chat->destroy(); } }
 
     void removeChatFile(Chat *chat) const;
     Q_INVOKABLE void saveChats();

gpt4all-chat/chatllm.cpp

@@ -1,18 +1,19 @@
 #include "chatllm.h"
 #include "chat.h"
-#include "chatgpt.h"
+#include "chatapi.h"
 #include "localdocs.h"
 #include "modellist.h"
 #include "network.h"
 #include "mysettings.h"
 #include "../gpt4all-backend/llmodel.h"
 
+#include <QElapsedTimer>
+
 //#define DEBUG
 //#define DEBUG_MODEL_LOADING
 
 #define GPTJ_INTERNAL_STATE_VERSION 0
 #define LLAMA_INTERNAL_STATE_VERSION 0
-#define BERT_INTERNAL_STATE_VERSION 0
 
 class LLModelStore {
 public:
@@ -64,6 +65,7 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_isRecalc(false)
     , m_shouldBeLoaded(false)
     , m_forceUnloadModel(false)
+    , m_markedForDeletion(false)
     , m_shouldTrySwitchContext(false)
     , m_stopGenerating(false)
     , m_timer(nullptr)
@@ -74,8 +76,6 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_restoreStateFromText(false)
 {
     moveToThread(&m_llmThread);
-    connect(this, &ChatLLM::sendStartup, Network::globalInstance(), &Network::sendStartup);
-    connect(this, &ChatLLM::sendModelLoaded, Network::globalInstance(), &Network::sendModelLoaded);
     connect(this, &ChatLLM::shouldBeLoadedChanged, this, &ChatLLM::handleShouldBeLoadedChanged,
         Qt::QueuedConnection); // explicitly queued
     connect(this, &ChatLLM::shouldTrySwitchContextChanged, this, &ChatLLM::handleShouldTrySwitchContextChanged,
@@ -95,6 +95,10 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
 
 ChatLLM::~ChatLLM()
 {
+    destroy();
+}
+
+void ChatLLM::destroy() {
     m_stopGenerating = true;
     m_llmThread.quit();
     m_llmThread.wait();
@@ -209,7 +213,6 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     if (isModelLoaded() && this->modelInfo() == modelInfo)
         return true;
 
-    bool isChatGPT = modelInfo.isOnline; // right now only chatgpt is offered for online chat models...
     QString filePath = modelInfo.dirpath + modelInfo.filename();
     QFileInfo fileInfo(filePath);
 
@@ -275,22 +278,30 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     m_llModelInfo.fileInfo = fileInfo;
 
     if (fileInfo.exists()) {
-        if (isChatGPT) {
+        QVariantMap modelLoadProps;
+        if (modelInfo.isOnline) {
             QString apiKey;
-            QString chatGPTModel = fileInfo.completeBaseName().remove(0, 8); // remove the chatgpt- prefix
+            QString modelName;
             {
                 QFile file(filePath);
-                file.open(QIODeviceBase::ReadOnly | QIODeviceBase::Text);
-                QTextStream stream(&file);
-                apiKey = stream.readAll();
-                file.close();
+                bool success = file.open(QIODeviceBase::ReadOnly);
+                (void)success;
+                Q_ASSERT(success);
+                QJsonDocument doc = QJsonDocument::fromJson(file.readAll());
+                QJsonObject obj = doc.object();
+                apiKey = obj["apiKey"].toString();
+                modelName = obj["modelName"].toString();
             }
-            m_llModelType = LLModelType::CHATGPT_;
-            ChatGPT *model = new ChatGPT();
-            model->setModelName(chatGPTModel);
+            m_llModelType = LLModelType::API_;
+            ChatAPI *model = new ChatAPI();
+            model->setModelName(modelName);
+            model->setRequestURL(modelInfo.url());
             model->setAPIKey(apiKey);
             m_llModelInfo.model = model;
         } else {
+            QElapsedTimer modelLoadTimer;
+            modelLoadTimer.start();
+
             auto n_ctx = MySettings::globalInstance()->modelContextLength(modelInfo);
             m_ctx.n_ctx = n_ctx;
             auto ngl = MySettings::globalInstance()->modelGpuLayers(modelInfo);
@@ -300,11 +311,32 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             if (m_forceMetal)
                 buildVariant = "metal";
 #endif
-            m_llModelInfo.model = LLModel::Implementation::construct(filePath.toStdString(), buildVariant, n_ctx);
+            QString constructError;
+            m_llModelInfo.model = nullptr;
+            try {
+                m_llModelInfo.model = LLModel::Implementation::construct(filePath.toStdString(), buildVariant, n_ctx);
+            } catch (const LLModel::MissingImplementationError &e) {
+                modelLoadProps.insert("error", "missing_model_impl");
+                constructError = e.what();
+            } catch (const LLModel::UnsupportedModelError &e) {
+                modelLoadProps.insert("error", "unsupported_model_file");
+                constructError = e.what();
+            } catch (const LLModel::BadArchError &e) {
+                constructError = e.what();
+                modelLoadProps.insert("error", "unsupported_model_arch");
+                modelLoadProps.insert("model_arch", QString::fromStdString(e.arch()));
+            }
 
             if (m_llModelInfo.model) {
                 if (m_llModelInfo.model->isModelBlacklisted(filePath.toStdString())) {
-                    // TODO(cebtenzzre): warn that this model is out-of-date
+                    static QSet<QString> warned;
+                    auto fname = modelInfo.filename();
+                    if (!warned.contains(fname)) {
+                        emit modelLoadingWarning(QString(
+                            "%1 is known to be broken. Please get a replacement via the download dialog."
+                        ).arg(fname));
+                        warned.insert(fname); // don't warn again until restart
+                    }
                 }
 
                 m_llModelInfo.model->setProgressCallback([this](float progress) -> bool {
@@ -354,6 +386,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     // llama_init_from_file returned nullptr
                     emit reportDevice("CPU");
                     emit reportFallbackReason("<br>GPU loading failed (out of VRAM?)");
+                    modelLoadProps.insert("cpu_fallback_reason", "gpu_load_failed");
                     success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, 0);
                 } else if (!m_llModelInfo.model->usingGPUDevice()) {
                     // ggml_vk_init was not called in llama.cpp
@@ -361,6 +394,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     // for instance if the quantization method is not supported on Vulkan yet
                     emit reportDevice("CPU");
                     emit reportFallbackReason("<br>model or quant has no GPU support");
+                    modelLoadProps.insert("cpu_fallback_reason", "gpu_unsupported_model");
                 }
 
                 if (!success) {
@@ -370,11 +404,11 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                         LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
                     m_llModelInfo = LLModelInfo();
                     emit modelLoadingError(QString("Could not load model due to invalid model file for %1").arg(modelInfo.filename()));
+                    modelLoadProps.insert("error", "loadmodel_failed");
                 } else {
                     switch (m_llModelInfo.model->implementation().modelType()[0]) {
                     case 'L': m_llModelType = LLModelType::LLAMA_; break;
                     case 'G': m_llModelType = LLModelType::GPTJ_; break;
-                    case 'B': m_llModelType = LLModelType::BERT_; break;
                     default:
                         {
                             delete m_llModelInfo.model;
@@ -385,12 +419,14 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                             emit modelLoadingError(QString("Could not determine model type for %1").arg(modelInfo.filename()));
                         }
                     }
+
+                    modelLoadProps.insert("$duration", modelLoadTimer.elapsed() / 1000.);
                 }
             } else {
                 if (!m_isServer)
                     LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
                 m_llModelInfo = LLModelInfo();
-                emit modelLoadingError(QString("Could not load model due to invalid format for %1").arg(modelInfo.filename()));
+                emit modelLoadingError(QString("Error loading %1: %2").arg(modelInfo.filename()).arg(constructError));
             }
         }
 #if defined(DEBUG_MODEL_LOADING)
@@ -403,12 +439,9 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 #endif
         emit modelLoadingPercentageChanged(isModelLoaded() ? 1.0f : 0.0f);
 
-        static bool isFirstLoad = true;
-        if (isFirstLoad) {
-            emit sendStartup();
-            isFirstLoad = false;
-        } else
-            emit sendModelLoaded();
+        modelLoadProps.insert("requestedDevice", MySettings::globalInstance()->device());
+        modelLoadProps.insert("model", modelInfo.filename());
+        Network::globalInstance()->trackChatEvent("model_load", modelLoadProps);
     } else {
         if (!m_isServer)
             LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
@@ -458,7 +491,7 @@ void ChatLLM::regenerateResponse()
 {
     // ChatGPT uses a different semantic meaning for n_past than local models. For ChatGPT, the meaning
     // of n_past is of the number of prompt/response pairs, rather than for total tokens.
-    if (m_llModelType == LLModelType::CHATGPT_)
+    if (m_llModelType == LLModelType::API_)
         m_ctx.n_past -= 1;
     else
         m_ctx.n_past -= m_promptResponseTokens;
@@ -480,7 +513,7 @@ void ChatLLM::resetResponse()
 
 void ChatLLM::resetContext()
 {
-    regenerateResponse();
+    resetResponse();
     m_processedSystemPrompt = false;
     m_ctx = LLModel::PromptContext();
 }
@@ -619,6 +652,8 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     printf("%s", qPrintable(prompt));
     fflush(stdout);
 #endif
+    QElapsedTimer totalTime;
+    totalTime.start();
     m_timer->start();
     if (!docsContext.isEmpty()) {
         auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode localdocs context without a response
@@ -631,12 +666,13 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     fflush(stdout);
 #endif
     m_timer->stop();
+    qint64 elapsed = totalTime.elapsed();
     std::string trimmed = trim_whitespace(m_response);
     if (trimmed != m_response) {
         m_response = trimmed;
         emit responseChanged(QString::fromStdString(m_response));
     }
-    emit responseStopped();
+    emit responseStopped(elapsed);
     return true;
 }
 
@@ -679,7 +715,9 @@ void ChatLLM::unloadModel()
     else
         emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
 
-    saveState();
+    if (!m_markedForDeletion)
+        saveState();
+
 #if defined(DEBUG_MODEL_LOADING)
     qDebug() << "unloadModel" << m_llmThread.objectName() << m_llModelInfo.model;
 #endif
@@ -718,23 +756,13 @@ void ChatLLM::generateName()
     if (!isModelLoaded())
         return;
 
-    QString instructPrompt("### Instruction:\n"
-                           "Describe response above in three words.\n"
-                           "### Response:\n");
+    std::string instructPrompt("### Instruction:\n%1\n### Response:\n"); // standard Alpaca
     auto promptFunc = std::bind(&ChatLLM::handleNamePrompt, this, std::placeholders::_1);
-    auto responseFunc = std::bind(&ChatLLM::handleNameResponse, this, std::placeholders::_1,
-        std::placeholders::_2);
+    auto responseFunc = std::bind(&ChatLLM::handleNameResponse, this, std::placeholders::_1, std::placeholders::_2);
     auto recalcFunc = std::bind(&ChatLLM::handleNameRecalculate, this, std::placeholders::_1);
     LLModel::PromptContext ctx = m_ctx;
-#if defined(DEBUG)
-    printf("%s", qPrintable(instructPrompt));
-    fflush(stdout);
-#endif
-    m_llModelInfo.model->prompt(instructPrompt.toStdString(), "%1", promptFunc, responseFunc, recalcFunc, ctx);
-#if defined(DEBUG)
-    printf("\n");
-    fflush(stdout);
-#endif
+    m_llModelInfo.model->prompt("Describe response above in three words.", instructPrompt, promptFunc, responseFunc,
+                                recalcFunc, ctx);
     std::string trimmed = trim_whitespace(m_nameResponse);
     if (trimmed != m_nameResponse) {
         m_nameResponse = trimmed;
@@ -826,7 +854,6 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
         switch (m_llModelType) {
         case GPTJ_: stream << GPTJ_INTERNAL_STATE_VERSION; break;
         case LLAMA_: stream << LLAMA_INTERNAL_STATE_VERSION; break;
-        case BERT_: stream << BERT_INTERNAL_STATE_VERSION; break;
         default: Q_UNREACHABLE();
         }
     }
@@ -947,12 +974,12 @@ void ChatLLM::saveState()
     if (!isModelLoaded())
         return;
 
-    if (m_llModelType == LLModelType::CHATGPT_) {
+    if (m_llModelType == LLModelType::API_) {
         m_state.clear();
         QDataStream stream(&m_state, QIODeviceBase::WriteOnly);
-        stream.setVersion(QDataStream::Qt_6_5);
-        ChatGPT *chatGPT = static_cast<ChatGPT*>(m_llModelInfo.model);
-        stream << chatGPT->context();
+        stream.setVersion(QDataStream::Qt_6_4);
+        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model);
+        stream << chatAPI->context();
         return;
     }
 
@@ -969,13 +996,13 @@ void ChatLLM::restoreState()
     if (!isModelLoaded())
         return;
 
-    if (m_llModelType == LLModelType::CHATGPT_) {
+    if (m_llModelType == LLModelType::API_) {
         QDataStream stream(&m_state, QIODeviceBase::ReadOnly);
-        stream.setVersion(QDataStream::Qt_6_5);
-        ChatGPT *chatGPT = static_cast<ChatGPT*>(m_llModelInfo.model);
+        stream.setVersion(QDataStream::Qt_6_4);
+        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model);
         QList<QString> context;
         stream >> context;
-        chatGPT->setContext(context);
+        chatAPI->setContext(context);
         m_state.clear();
         m_state.squeeze();
         return;
@@ -992,7 +1019,7 @@ void ChatLLM::restoreState()
         m_llModelInfo.model->restoreState(static_cast<const uint8_t*>(reinterpret_cast<void*>(m_state.data())));
         m_processedSystemPrompt = true;
     } else {
-        qWarning() << "restoring state from text because" << m_llModelInfo.model->stateSize() << "!=" << m_state.size() << "\n";
+        qWarning() << "restoring state from text because" << m_llModelInfo.model->stateSize() << "!=" << m_state.size();
         m_restoreStateFromText = true;
     }
 
@@ -1042,7 +1069,8 @@ void ChatLLM::processSystemPrompt()
     fflush(stdout);
 #endif
     auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode system prompt without a response
-    m_llModelInfo.model->prompt(systemPrompt, "%1", promptFunc, nullptr, recalcFunc, m_ctx, true);
+    // use "%1%2" and not "%1" to avoid implicit whitespace
+    m_llModelInfo.model->prompt(systemPrompt, "%1%2", promptFunc, nullptr, recalcFunc, m_ctx, true);
     m_ctx.n_predict = old_n_predict;
 #if defined(DEBUG)
     printf("\n");

gpt4all-chat/chatllm.h

@@ -12,8 +12,7 @@
 enum LLModelType {
     GPTJ_,
     LLAMA_,
-    CHATGPT_,
-    BERT_,
+    API_,
 };
 
 struct LLModelInfo {
@@ -72,6 +71,7 @@ public:
     ChatLLM(Chat *parent, bool isServer = false);
     virtual ~ChatLLM();
 
+    void destroy();
     bool isModelLoaded() const;
     void regenerateResponse();
     void resetResponse();
@@ -83,6 +83,7 @@ public:
     void setShouldBeLoaded(bool b);
     void setShouldTrySwitchContext(bool b);
     void setForceUnloadModel(bool b) { m_forceUnloadModel = b; }
+    void setMarkedForDeletion(bool b) { m_markedForDeletion = b; }
 
     QString response() const;
 
@@ -119,11 +120,10 @@ Q_SIGNALS:
     void recalcChanged();
     void modelLoadingPercentageChanged(float);
     void modelLoadingError(const QString &error);
+    void modelLoadingWarning(const QString &warning);
     void responseChanged(const QString &response);
     void promptProcessing();
-    void responseStopped();
-    void sendStartup();
-    void sendModelLoaded();
+    void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
     void stateChanged();
     void threadStarted();
@@ -175,6 +175,7 @@ private:
     std::atomic<bool> m_shouldTrySwitchContext;
     std::atomic<bool> m_isRecalc;
     std::atomic<bool> m_forceUnloadModel;
+    std::atomic<bool> m_markedForDeletion;
     bool m_isServer;
     bool m_forceMetal;
     bool m_reloadingToChangeVariant;

gpt4all-chat/database.cpp

@@ -1,7 +1,9 @@
 #include "database.h"
-#include "mysettings.h"
-#include "embllm.h"
+
 #include "embeddings.h"
+#include "embllm.h"
+#include "mysettings.h"
+#include "network.h"
 
 #include <QTimer>
 #include <QPdfDocument>
@@ -410,8 +412,8 @@ bool updateDocument(QSqlQuery &q, int id, qint64 document_time)
 {
     if (!q.prepare(UPDATE_DOCUMENT_TIME_SQL))
         return false;
-    q.addBindValue(id);
     q.addBindValue(document_time);
+    q.addBindValue(id);
     return q.exec();
 }
 
@@ -490,7 +492,7 @@ QSqlError initDb()
     i.collection = collection_name;
     i.folder_path = folder_path;
     i.folder_id = folder_id;
-    emit addCollectionItem(i);
+    emit addCollectionItem(i, false);
 
     // Add a document
     int document_time = 123456789;
@@ -535,13 +537,13 @@ QSqlError initDb()
 
 Database::Database(int chunkSize)
     : QObject(nullptr)
-    , m_watcher(new QFileSystemWatcher(this))
     , m_chunkSize(chunkSize)
+    , m_scanTimer(new QTimer(this))
+    , m_watcher(new QFileSystemWatcher(this))
     , m_embLLM(new EmbeddingLLM)
     , m_embeddings(new Embeddings(this))
 {
     moveToThread(&m_dbThread);
-    connect(&m_dbThread, &QThread::started, this, &Database::start);
     m_dbThread.setObjectName("database");
     m_dbThread.start();
 }
@@ -556,11 +558,13 @@ void Database::scheduleNext(int folder_id, size_t countForFolder)
 {
     emit updateCurrentDocsToIndex(folder_id, countForFolder);
     if (!countForFolder) {
-        emit updateIndexing(folder_id, false);
+        updateFolderStatus(folder_id, FolderStatus::Complete);
         emit updateInstalled(folder_id, true);
     }
-    if (!m_docsToScan.isEmpty())
-        QTimer::singleShot(0, this, &Database::scanQueue);
+    if (m_docsToScan.isEmpty()) {
+        m_scanTimer->stop();
+        updateIndexingStatus();
+    }
 }
 
 void Database::handleDocumentError(const QString &errorMessage,
@@ -721,7 +725,6 @@ void Database::removeFolderFromDocumentQueue(int folder_id)
         return;
     m_docsToScan.remove(folder_id);
     emit removeFolderById(folder_id);
-    emit docsToScanChanged();
 }
 
 void Database::enqueueDocumentInternal(const DocumentInfo &info, bool prepend)
@@ -745,13 +748,16 @@ void Database::enqueueDocuments(int folder_id, const QVector<DocumentInfo> &info
     const size_t bytes = countOfBytes(folder_id);
     emit updateCurrentBytesToIndex(folder_id, bytes);
     emit updateTotalBytesToIndex(folder_id, bytes);
-    emit docsToScanChanged();
+    m_scanTimer->start();
 }
 
 void Database::scanQueue()
 {
-    if (m_docsToScan.isEmpty())
+    if (m_docsToScan.isEmpty()) {
+        m_scanTimer->stop();
+        updateIndexingStatus();
         return;
+    }
 
     DocumentInfo info = dequeueDocument();
     const size_t countForFolder = countOfDocuments(info.folder);
@@ -818,6 +824,8 @@ void Database::scanQueue()
     QSqlDatabase::database().transaction();
     Q_ASSERT(document_id != -1);
     if (info.isPdf()) {
+        updateFolderStatus(folder_id, FolderStatus::Embedding, -1, info.currentPage == 0);
+
         QPdfDocument doc;
         if (QPdfDocument::Error::None != doc.load(info.doc.canonicalFilePath())) {
             handleDocumentError("ERROR: Could not load pdf",
@@ -850,6 +858,8 @@ void Database::scanQueue()
             emit subtractCurrentBytesToIndex(info.folder, bytes - (bytesPerPage * doc.pageCount()));
         }
     } else {
+        updateFolderStatus(folder_id, FolderStatus::Embedding, -1, info.currentPosition == 0);
+
         QFile file(document_path);
         if (!file.open(QIODevice::ReadOnly)) {
             handleDocumentError("ERROR: Cannot open file for scanning",
@@ -884,7 +894,7 @@ void Database::scanQueue()
     return scheduleNext(folder_id, countForFolder);
 }
 
-void Database::scanDocuments(int folder_id, const QString &folder_path)
+void Database::scanDocuments(int folder_id, const QString &folder_path, bool isNew)
 {
 #if defined(DEBUG)
     qDebug() << "scanning folder for documents" << folder_path;
@@ -915,7 +925,7 @@ void Database::scanDocuments(int folder_id, const QString &folder_path)
     }
 
     if (!infos.isEmpty()) {
-        emit updateIndexing(folder_id, true);
+        updateFolderStatus(folder_id, FolderStatus::Started, infos.count(), false, isNew);
         enqueueDocuments(folder_id, infos);
     }
 }
@@ -925,7 +935,7 @@ void Database::start()
     connect(m_watcher, &QFileSystemWatcher::directoryChanged, this, &Database::directoryChanged);
     connect(m_embLLM, &EmbeddingLLM::embeddingsGenerated, this, &Database::handleEmbeddingsGenerated);
     connect(m_embLLM, &EmbeddingLLM::errorGenerated, this, &Database::handleErrorGenerated);
-    connect(this, &Database::docsToScanChanged, this, &Database::scanQueue);
+    m_scanTimer->callOnTimeout(this, &Database::scanQueue);
     if (!QSqlDatabase::drivers().contains("QSQLITE")) {
         qWarning() << "ERROR: missing sqllite driver";
     } else {
@@ -937,10 +947,11 @@ void Database::start()
     if (m_embeddings->fileExists() && !m_embeddings->load())
         qWarning() << "ERROR: Could not load embeddings";
 
-    addCurrentFolders();
+    int nAdded = addCurrentFolders();
+    Network::globalInstance()->trackEvent("localdocs_startup", { {"doc_collections_total", nAdded} });
 }
 
-void Database::addCurrentFolders()
+int Database::addCurrentFolders()
 {
 #if defined(DEBUG)
     qDebug() << "addCurrentFolders";
@@ -950,21 +961,26 @@ void Database::addCurrentFolders()
     QList<CollectionItem> collections;
     if (!selectAllFromCollections(q, &collections)) {
         qWarning() << "ERROR: Cannot select collections" << q.lastError();
-        return;
+        return 0;
     }
 
     emit collectionListUpdated(collections);
 
+    int nAdded = 0;
     for (const auto &i : collections)
-        addFolder(i.collection, i.folder_path);
+        nAdded += addFolder(i.collection, i.folder_path, true);
+
+    updateIndexingStatus();
+
+    return nAdded;
 }
 
-void Database::addFolder(const QString &collection, const QString &path)
+bool Database::addFolder(const QString &collection, const QString &path, bool fromDb)
 {
     QFileInfo info(path);
     if (!info.exists() || !info.isReadable()) {
         qWarning() << "ERROR: Cannot add folder that doesn't exist or not readable" << path;
-        return;
+        return false;
     }
 
     QSqlQuery q;
@@ -973,13 +989,13 @@ void Database::addFolder(const QString &collection, const QString &path)
     // See if the folder exists in the db
     if (!selectFolder(q, path, &folder_id)) {
         qWarning() << "ERROR: Cannot select folder from path" << path << q.lastError();
-        return;
+        return false;
     }
 
     // Add the folder
     if (folder_id == -1 && !addFolderToDB(q, path, &folder_id)) {
         qWarning() << "ERROR: Cannot add folder to db with path" << path << q.lastError();
-        return;
+        return false;
     }
 
     Q_ASSERT(folder_id != -1);
@@ -988,24 +1004,32 @@ void Database::addFolder(const QString &collection, const QString &path)
     QList<int> folders;
     if (!selectFoldersFromCollection(q, collection, &folders)) {
         qWarning() << "ERROR: Cannot select folders from collections" << collection << q.lastError();
-        return;
+        return false;
     }
 
+    bool added = false;
     if (!folders.contains(folder_id)) {
         if (!addCollection(q, collection, folder_id)) {
             qWarning() << "ERROR: Cannot add folder to collection" << collection << path << q.lastError();
-            return;
+            return false;
         }
 
         CollectionItem i;
         i.collection = collection;
         i.folder_path = path;
         i.folder_id = folder_id;
-        emit addCollectionItem(i);
+        emit addCollectionItem(i, fromDb);
+        added = true;
     }
 
     addFolderToWatch(path);
-    scanDocuments(folder_id, path);
+    scanDocuments(folder_id, path, !fromDb);
+
+    if (!fromDb) {
+        updateIndexingStatus();
+    }
+
+    return added;
 }
 
 void Database::removeFolder(const QString &collection, const QString &path)
@@ -1285,5 +1309,69 @@ void Database::directoryChanged(const QString &path)
     cleanDB();
 
     // Rescan the documents associated with the folder
-    scanDocuments(folder_id, path);
+    scanDocuments(folder_id, path, false);
+    updateIndexingStatus();
+}
+
+void Database::updateIndexingStatus() {
+    Q_ASSERT(m_scanTimer->isActive() || m_docsToScan.isEmpty());
+    if (!m_indexingTimer.isValid() && m_scanTimer->isActive()) {
+        Network::globalInstance()->trackEvent("localdocs_indexing_start");
+        m_indexingTimer.start();
+    } else if (m_indexingTimer.isValid() && !m_scanTimer->isActive()) {
+        qint64 durationMs = m_indexingTimer.elapsed();
+        Network::globalInstance()->trackEvent("localdocs_indexing_complete", { {"$duration", durationMs / 1000.} });
+        m_indexingTimer.invalidate();
+    }
+}
+
+void Database::updateFolderStatus(int folder_id, Database::FolderStatus status, int numDocs, bool atStart, bool isNew) {
+    FolderStatusRecord *lastRecord = nullptr;
+    if (m_foldersBeingIndexed.contains(folder_id)) {
+        lastRecord = &m_foldersBeingIndexed[folder_id];
+    }
+    Q_ASSERT(lastRecord || status == FolderStatus::Started);
+
+    switch (status) {
+        case FolderStatus::Started:
+            if (lastRecord == nullptr) {
+                // record timestamp but don't send an event yet
+                m_foldersBeingIndexed.insert(folder_id, { QDateTime::currentMSecsSinceEpoch(), isNew, numDocs });
+                emit updateIndexing(folder_id, true);
+            }
+            break;
+        case FolderStatus::Embedding:
+            if (!lastRecord->docsChanged) {
+                Q_ASSERT(atStart);
+                // send start event with the original timestamp for folders that need updating
+                const auto *embeddingModels = ModelList::globalInstance()->installedEmbeddingModels();
+                Network::globalInstance()->trackEvent("localdocs_folder_indexing", {
+                    {"folder_id", folder_id},
+                    {"is_new_collection", lastRecord->isNew},
+                    {"document_count", lastRecord->numDocs},
+                    {"embedding_model", embeddingModels->defaultModelInfo().filename()},
+                    {"chunk_size", m_chunkSize},
+                    {"time", lastRecord->startTime},
+                });
+            }
+            lastRecord->docsChanged += atStart;
+            lastRecord->chunksRead++;
+            break;
+        case FolderStatus::Complete:
+            if (lastRecord->docsChanged) {
+                // send complete event for folders that were updated
+                qint64 durationMs = QDateTime::currentMSecsSinceEpoch() - lastRecord->startTime;
+                Network::globalInstance()->trackEvent("localdocs_folder_complete", {
+                    {"folder_id", folder_id},
+                    {"is_new_collection", lastRecord->isNew},
+                    {"documents_total", lastRecord->numDocs},
+                    {"documents_changed", lastRecord->docsChanged},
+                    {"chunks_read", lastRecord->chunksRead},
+                    {"$duration", durationMs / 1000.},
+                });
+            }
+            m_foldersBeingIndexed.remove(folder_id);
+            emit updateIndexing(folder_id, false);
+            break;
+    }
 }

gpt4all-chat/database.h

@@ -1,16 +1,19 @@
 #ifndef DATABASE_H
 #define DATABASE_H
 
-#include <QObject>
-#include <QtSql>
-#include <QQueue>
+#include <QElapsedTimer>
 #include <QFileInfo>
-#include <QThread>
 #include <QFileSystemWatcher>
+#include <QObject>
+#include <QQueue>
+#include <QThread>
+#include <QtSql>
 
 #include "embllm.h"
 
 class Embeddings;
+class QTimer;
+
 struct DocumentInfo
 {
     int folder;
@@ -58,9 +61,10 @@ public:
     virtual ~Database();
 
 public Q_SLOTS:
+    void start();
     void scanQueue();
-    void scanDocuments(int folder_id, const QString &folder_path);
-    void addFolder(const QString &collection, const QString &path);
+    void scanDocuments(int folder_id, const QString &folder_path, bool isNew);
+    bool addFolder(const QString &collection, const QString &path, bool fromDb);
     void removeFolder(const QString &collection, const QString &path);
     void retrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
     void cleanDB();
@@ -78,21 +82,22 @@ Q_SIGNALS:
     void updateTotalBytesToIndex(int folder_id, size_t totalBytesToIndex);
     void updateCurrentEmbeddingsToIndex(int folder_id, size_t currentBytesToIndex);
     void updateTotalEmbeddingsToIndex(int folder_id, size_t totalBytesToIndex);
-    void addCollectionItem(const CollectionItem &item);
+    void addCollectionItem(const CollectionItem &item, bool fromDb);
     void removeFolderById(int folder_id);
-    void removeCollectionItem(const QString &collectionName);
     void collectionListUpdated(const QList<CollectionItem> &collectionList);
 
 private Q_SLOTS:
-    void start();
     void directoryChanged(const QString &path);
     bool addFolderToWatch(const QString &path);
     bool removeFolderFromWatch(const QString &path);
-    void addCurrentFolders();
+    int addCurrentFolders();
     void handleEmbeddingsGenerated(const QVector<EmbeddingResult> &embeddings);
     void handleErrorGenerated(int folder_id, const QString &error);
 
 private:
+    enum class FolderStatus { Started, Embedding, Complete };
+    struct FolderStatusRecord { qint64 startTime; bool isNew; int numDocs, docsChanged, chunksRead; };
+
     void removeFolderInternal(const QString &collection, int folder_id, const QString &path);
     size_t chunkStream(QTextStream &stream, int folder_id, int document_id, const QString &file,
         const QString &title, const QString &author, const QString &subject, const QString &keywords, int page,
@@ -107,10 +112,15 @@ private:
     void removeFolderFromDocumentQueue(int folder_id);
     void enqueueDocumentInternal(const DocumentInfo &info, bool prepend = false);
     void enqueueDocuments(int folder_id, const QVector<DocumentInfo> &infos);
+    void updateIndexingStatus();
+    void updateFolderStatus(int folder_id, FolderStatus status, int numDocs = -1, bool atStart = false, bool isNew = false);
 
 private:
     int m_chunkSize;
+    QTimer *m_scanTimer;
     QMap<int, QQueue<DocumentInfo>> m_docsToScan;
+    QElapsedTimer m_indexingTimer;
+    QMap<int, FolderStatusRecord> m_foldersBeingIndexed;
     QList<ResultInfo> m_retrieve;
     QThread m_dbThread;
     QFileSystemWatcher *m_watcher;

gpt4all-chat/download.cpp

@@ -75,15 +75,25 @@ bool Download::hasNewerRelease() const
     return compareVersions(versions.first(), currentVersion);
 }
 
-bool Download::isFirstStart() const
+bool Download::isFirstStart(bool writeVersion) const
 {
+    auto *mySettings = MySettings::globalInstance();
+
     QSettings settings;
     settings.sync();
     QString lastVersionStarted = settings.value("download/lastVersionStarted").toString();
     bool first = lastVersionStarted != QCoreApplication::applicationVersion();
-    settings.setValue("download/lastVersionStarted", QCoreApplication::applicationVersion());
-    settings.sync();
-    return first;
+    if (first && writeVersion) {
+        settings.setValue("download/lastVersionStarted", QCoreApplication::applicationVersion());
+        // let the user select these again
+        settings.remove("network/usageStatsActive");
+        settings.remove("network/isActive");
+        settings.sync();
+        emit mySettings->networkUsageStatsActiveChanged();
+        emit mySettings->networkIsActiveChanged();
+    }
+
+    return first || !mySettings->isNetworkUsageStatsActiveSet() || !mySettings->isNetworkIsActiveSet();
 }
 
 void Download::updateReleaseNotes()
@@ -109,7 +119,7 @@ void Download::downloadModel(const QString &modelFile)
             = QString("ERROR: Could not open temp file: %1 %2").arg(tempFile->fileName()).arg(modelFile);
         qWarning() << error;
         clearRetry(modelFile);
-        ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::DownloadErrorRole, error);
+        ModelList::globalInstance()->updateDataByFilename(modelFile, {{ ModelList::DownloadErrorRole, error }});
         return;
     }
     tempFile->flush();
@@ -128,10 +138,10 @@ void Download::downloadModel(const QString &modelFile)
         return;
     }
 
-    ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::DownloadingRole, true);
+    ModelList::globalInstance()->updateDataByFilename(modelFile, {{ ModelList::DownloadingRole, true }});
     ModelInfo info = ModelList::globalInstance()->modelInfoByFilename(modelFile);
     QString url = !info.url().isEmpty() ? info.url() : "http://gpt4all.io/models/gguf/" + modelFile;
-    Network::globalInstance()->sendDownloadStarted(modelFile);
+    Network::globalInstance()->trackEvent("download_started", { {"model", modelFile} });
     QNetworkRequest request(url);
     request.setAttribute(QNetworkRequest::User, modelFile);
     request.setRawHeader("range", QString("bytes=%1-").arg(tempFile->pos()).toUtf8());
@@ -153,7 +163,7 @@ void Download::cancelDownload(const QString &modelFile)
         QNetworkReply *modelReply = m_activeDownloads.keys().at(i);
         QUrl url = modelReply->request().url();
         if (url.toString().endsWith(modelFile)) {
-            Network::globalInstance()->sendDownloadCanceled(modelFile);
+            Network::globalInstance()->trackEvent("download_canceled", { {"model", modelFile} });
 
             // Disconnect the signals
             disconnect(modelReply, &QNetworkReply::downloadProgress, this, &Download::handleDownloadProgress);
@@ -166,7 +176,7 @@ void Download::cancelDownload(const QString &modelFile)
             tempFile->deleteLater();
             m_activeDownloads.remove(modelReply);
 
-            ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::DownloadingRole, false);
+            ModelList::globalInstance()->updateDataByFilename(modelFile, {{ ModelList::DownloadingRole, false }});
             break;
         }
     }
@@ -178,17 +188,27 @@ void Download::installModel(const QString &modelFile, const QString &apiKey)
     if (apiKey.isEmpty())
         return;
 
-    Network::globalInstance()->sendInstallModel(modelFile);
+    Network::globalInstance()->trackEvent("install_model", { {"model", modelFile} });
+
     QString filePath = MySettings::globalInstance()->modelPath() + modelFile;
     QFile file(filePath);
     if (file.open(QIODeviceBase::WriteOnly | QIODeviceBase::Text)) {
+
+        QJsonObject obj;
+        QString modelName(modelFile);
+        modelName.remove(0, 8); // strip "gpt4all-" prefix
+        modelName.chop(7); // strip ".rmodel" extension
+        obj.insert("apiKey", apiKey);
+        obj.insert("modelName", modelName);
+        QJsonDocument doc(obj);
+
         QTextStream stream(&file);
-        stream << apiKey;
+        stream << doc.toJson();
         file.close();
         ModelList::globalInstance()->updateModelsFromDirectory();
     }
 
-    ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::InstalledRole, true);
+    ModelList::globalInstance()->updateDataByFilename(modelFile, {{ ModelList::InstalledRole, true }});
 }
 
 void Download::removeModel(const QString &modelFile)
@@ -199,20 +219,29 @@ void Download::removeModel(const QString &modelFile)
         incompleteFile.remove();
     }
 
+    bool shouldRemoveInstalled = false;
     QFile file(filePath);
     if (file.exists()) {
         const ModelInfo info = ModelList::globalInstance()->modelInfoByFilename(modelFile);
-        ModelList::globalInstance()->removeInstalled(info);
-        Network::globalInstance()->sendRemoveModel(modelFile);
+        MySettings::globalInstance()->eraseModel(info);
+        shouldRemoveInstalled = info.installed && !info.isClone() && (info.isDiscovered() || info.description() == "" /*indicates sideloaded*/);
+        if (shouldRemoveInstalled)
+            ModelList::globalInstance()->removeInstalled(info);
+        Network::globalInstance()->trackEvent("remove_model", { {"model", modelFile} });
         file.remove();
     }
 
-    ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::InstalledRole, false);
-    ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::BytesReceivedRole, 0);
-    ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::BytesTotalRole, 0);
-    ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::TimestampRole, 0);
-    ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::SpeedRole, QString());
-    ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::DownloadErrorRole, QString());
+    if (!shouldRemoveInstalled) {
+        QVector<QPair<int, QVariant>> data {
+            { ModelList::InstalledRole, false },
+            { ModelList::BytesReceivedRole, 0 },
+            { ModelList::BytesTotalRole, 0 },
+            { ModelList::TimestampRole, 0 },
+            { ModelList::SpeedRole, QString() },
+            { ModelList::DownloadErrorRole, QString() },
+        };
+        ModelList::globalInstance()->updateDataByFilename(modelFile, data);
+    }
 }
 
 void Download::handleSslErrors(QNetworkReply *reply, const QList<QSslError> &errors)
@@ -313,8 +342,12 @@ void Download::handleErrorOccurred(QNetworkReply::NetworkError code)
             .arg(code)
             .arg(modelReply->errorString());
     qWarning() << error;
-    ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadErrorRole, error);
-    Network::globalInstance()->sendDownloadError(modelFilename, (int)code, modelReply->errorString());
+    ModelList::globalInstance()->updateDataByFilename(modelFilename, {{ ModelList::DownloadErrorRole, error }});
+    Network::globalInstance()->trackEvent("download_error", {
+        {"model", modelFilename},
+        {"code", (int)code},
+        {"error", modelReply->errorString()},
+    });
     cancelDownload(modelFilename);
 }
 
@@ -352,10 +385,13 @@ void Download::handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal)
     else
         speedText = QString::number(static_cast<double>(speed / (1024.0 * 1024.0)), 'f', 2) + " MB/s";
 
-    ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::BytesReceivedRole, currentBytesReceived);
-    ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::BytesTotalRole, bytesTotal);
-    ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::SpeedRole, speedText);
-    ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::TimestampRole, currentUpdate);
+    QVector<QPair<int, QVariant>> data {
+        { ModelList::BytesReceivedRole, currentBytesReceived },
+        { ModelList::BytesTotalRole, bytesTotal },
+        { ModelList::SpeedRole, speedText },
+        { ModelList::TimestampRole, currentUpdate },
+    };
+    ModelList::globalInstance()->updateDataByFilename(modelFilename, data);
 }
 
 HashAndSaveFile::HashAndSaveFile()
@@ -457,8 +493,11 @@ void Download::handleModelDownloadFinished()
         modelReply->deleteLater();
         tempFile->deleteLater();
         if (!hasRetry(modelFilename)) {
-            ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadingRole, false);
-            ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadErrorRole, errorString);
+            QVector<QPair<int, QVariant>> data {
+                { ModelList::DownloadingRole, false },
+                { ModelList::DownloadErrorRole, errorString },
+            };
+            ModelList::globalInstance()->updateDataByFilename(modelFilename, data);
         }
         return;
     }
@@ -476,7 +515,7 @@ void Download::handleModelDownloadFinished()
     }
 
     // Notify that we are calculating hash
-    ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::CalcHashRole, true);
+    ModelList::globalInstance()->updateDataByFilename(modelFilename, {{ ModelList::CalcHashRole, true }});
     QByteArray hash =  ModelList::globalInstance()->modelInfoByFilename(modelFilename).hash;
     ModelInfo::HashAlgorithm hashAlgorithm =  ModelList::globalInstance()->modelInfoByFilename(modelFilename).hashAlgorithm;
     const QString saveFilePath = MySettings::globalInstance()->modelPath() + modelFilename;
@@ -491,20 +530,26 @@ void Download::handleHashAndSaveFinished(bool success, const QString &error,
     // The hash and save should send back with tempfile closed
     Q_ASSERT(!tempFile->isOpen());
     QString modelFilename = modelReply->request().attribute(QNetworkRequest::User).toString();
-    Network::globalInstance()->sendDownloadFinished(modelFilename, success);
-    ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::CalcHashRole, false);
+    Network::globalInstance()->trackEvent("download_finished", { {"model", modelFilename}, {"success", success} });
+
+    QVector<QPair<int, QVariant>> data {
+        { ModelList::CalcHashRole, false },
+        { ModelList::DownloadingRole, false },
+    };
+
     modelReply->deleteLater();
     tempFile->deleteLater();
 
-    ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadingRole, false);
     if (!success) {
-        ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadErrorRole, error);
+        data.append({ ModelList::DownloadErrorRole, error });
     } else {
+        data.append({ ModelList::DownloadErrorRole, QString() });
         ModelInfo info = ModelList::globalInstance()->modelInfoByFilename(modelFilename);
         if (info.isDiscovered())
             ModelList::globalInstance()->updateDiscoveredInstalled(info);
-        ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadErrorRole, QString());
     }
+
+    ModelList::globalInstance()->updateDataByFilename(modelFilename, data);
 }
 
 void Download::handleReadyRead()

gpt4all-chat/download.h

@@ -54,7 +54,7 @@ public:
     Q_INVOKABLE void cancelDownload(const QString &modelFile);
     Q_INVOKABLE void installModel(const QString &modelFile, const QString &apiKey);
     Q_INVOKABLE void removeModel(const QString &modelFile);
-    Q_INVOKABLE bool isFirstStart() const;
+    Q_INVOKABLE bool isFirstStart(bool writeVersion = false) const;
 
 public Q_SLOTS:
     void updateReleaseNotes();

gpt4all-chat/embllm.cpp

@@ -27,7 +27,7 @@ void EmbeddingLLMWorker::wait()
 
 bool EmbeddingLLMWorker::loadModel()
 {
-    const EmbeddingModels *embeddingModels = ModelList::globalInstance()->embeddingModels();
+    const EmbeddingModels *embeddingModels = ModelList::globalInstance()->installedEmbeddingModels();
     if (!embeddingModels->count())
         return false;
 
@@ -41,27 +41,42 @@ bool EmbeddingLLMWorker::loadModel()
         return false;
     }
 
-    bool isNomic = fileInfo.fileName().startsWith("nomic");
+    auto filename = fileInfo.fileName();
+    bool isNomic = filename.startsWith("gpt4all-nomic-") && filename.endsWith(".rmodel");
     if (isNomic) {
         QFile file(filePath);
-        file.open(QIODeviceBase::ReadOnly | QIODeviceBase::Text);
-        QTextStream stream(&file);
-        m_nomicAPIKey = stream.readAll();
+        if (!file.open(QIODeviceBase::ReadOnly)) {
+            qWarning() << "failed to open" << filePath << ":" << file.errorString();
+            m_model = nullptr;
+            return false;
+        }
+        QJsonDocument doc = QJsonDocument::fromJson(file.readAll());
+        QJsonObject obj = doc.object();
+        m_nomicAPIKey = obj["apiKey"].toString();
         file.close();
         return true;
     }
 
-    m_model = LLModel::Implementation::construct(filePath.toStdString());
+    try {
+        m_model = LLModel::Implementation::construct(filePath.toStdString());
+    } catch (const std::exception &e) {
+        qWarning() << "WARNING: Could not load embedding model:" << e.what();
+        m_model = nullptr;
+        return false;
+    }
+
+    // NOTE: explicitly loads model on CPU to avoid GPU OOM
+    // TODO(cebtenzzre): support GPU-accelerated embeddings
     bool success = m_model->loadModel(filePath.toStdString(), 2048, 0);
     if (!success) {
-        qWarning() << "WARNING: Could not load sbert";
+        qWarning() << "WARNING: Could not load embedding model";
         delete m_model;
         m_model = nullptr;
         return false;
     }
 
-    if (m_model->implementation().modelType() != "Bert") {
-        qWarning() << "WARNING: Model type is not sbert";
+    if (!m_model->supportsEmbedding()) {
+        qWarning() << "WARNING: Model type does not support embeddings";
         delete m_model;
         m_model = nullptr;
         return false;
@@ -79,21 +94,49 @@ bool EmbeddingLLMWorker::isNomic() const
     return !m_nomicAPIKey.isEmpty();
 }
 
+// this function is always called for retrieval tasks
 std::vector<float> EmbeddingLLMWorker::generateSyncEmbedding(const QString &text)
 {
     if (!hasModel() && !loadModel()) {
         qWarning() << "WARNING: Could not load model for embeddings";
-        return std::vector<float>();
+        return {};
     }
 
     if (isNomic()) {
         qWarning() << "WARNING: Request to generate sync embeddings for non-local model invalid";
-        return std::vector<float>();
+        return {};
     }
 
-    return m_model->embedding(text.toStdString());
+    std::vector<float> embedding(m_model->embeddingSize());
+    try {
+        m_model->embed({text.toStdString()}, embedding.data(), true);
+    } catch (const std::exception &e) {
+        qWarning() << "WARNING: LLModel::embed failed: " << e.what();
+        return {};
+    }
+    return embedding;
 }
 
+void EmbeddingLLMWorker::sendAtlasRequest(const QStringList &texts, const QString &taskType, QVariant userData) {
+    QJsonObject root;
+    root.insert("model", "nomic-embed-text-v1");
+    root.insert("texts", QJsonArray::fromStringList(texts));
+    root.insert("task_type", taskType);
+
+    QJsonDocument doc(root);
+
+    QUrl nomicUrl("https://api-atlas.nomic.ai/v1/embedding/text");
+    const QString authorization = QString("Bearer %1").arg(m_nomicAPIKey).trimmed();
+    QNetworkRequest request(nomicUrl);
+    request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
+    request.setRawHeader("Authorization", authorization.toUtf8());
+    request.setAttribute(QNetworkRequest::User, userData);
+    QNetworkReply *reply = m_networkManager->post(request, doc.toJson(QJsonDocument::Compact));
+    connect(qApp, &QCoreApplication::aboutToQuit, reply, &QNetworkReply::abort);
+    connect(reply, &QNetworkReply::finished, this, &EmbeddingLLMWorker::handleFinished);
+}
+
+// this function is always called for retrieval tasks
 void EmbeddingLLMWorker::requestSyncEmbedding(const QString &text)
 {
     if (!hasModel() && !loadModel()) {
@@ -108,25 +151,10 @@ void EmbeddingLLMWorker::requestSyncEmbedding(const QString &text)
 
     Q_ASSERT(hasModel());
 
-    QJsonObject root;
-    root.insert("model", "nomic-embed-text-v1");
-    QJsonArray texts;
-    texts.append(text);
-    root.insert("texts", texts);
-    root.insert("task_type", "search_query");
-
-    QJsonDocument doc(root);
-
-    QUrl nomicUrl("https://api-atlas.nomic.ai/v1/embedding/text");
-    const QString authorization = QString("Bearer %1").arg(m_nomicAPIKey).trimmed();
-    QNetworkRequest request(nomicUrl);
-    request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
-    request.setRawHeader("Authorization", authorization.toUtf8());
-    QNetworkReply *reply = m_networkManager->post(request, doc.toJson(QJsonDocument::Compact));
-    connect(qApp, &QCoreApplication::aboutToQuit, reply, &QNetworkReply::abort);
-    connect(reply, &QNetworkReply::finished, this, &EmbeddingLLMWorker::handleFinished);
+    sendAtlasRequest({text}, "search_query");
 }
 
+// this function is always called for storage into the database
 void EmbeddingLLMWorker::requestAsyncEmbedding(const QVector<EmbeddingChunk> &chunks)
 {
     if (!hasModel() && !loadModel()) {
@@ -141,33 +169,24 @@ void EmbeddingLLMWorker::requestAsyncEmbedding(const QVector<EmbeddingChunk> &ch
             EmbeddingResult result;
             result.folder_id = c.folder_id;
             result.chunk_id = c.chunk_id;
-            result.embedding = m_model->embedding(c.chunk.toStdString());
+            // TODO(cebtenzzre): take advantage of batched embeddings
+            result.embedding.resize(m_model->embeddingSize());
+            try {
+                m_model->embed({c.chunk.toStdString()}, result.embedding.data(), false);
+            } catch (const std::exception &e) {
+                qWarning() << "WARNING: LLModel::embed failed:" << e.what();
+                return;
+            }
             results << result;
         }
         emit embeddingsGenerated(results);
         return;
     };
 
-    QJsonObject root;
-    root.insert("model", "nomic-embed-text-v1");
-    QJsonArray texts;
-
-    for (auto c : chunks)
+    QStringList texts;
+    for (auto &c: chunks)
         texts.append(c.chunk);
-    root.insert("texts", texts);
-
-    QJsonDocument doc(root);
-
-    QUrl nomicUrl("https://api-atlas.nomic.ai/v1/embedding/text");
-    const QString authorization = QString("Bearer %1").arg(m_nomicAPIKey).trimmed();
-    QNetworkRequest request(nomicUrl);
-    request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
-    request.setRawHeader("Authorization", authorization.toUtf8());
-    request.setAttribute(QNetworkRequest::User, QVariant::fromValue(chunks));
-
-    QNetworkReply *reply = m_networkManager->post(request, doc.toJson(QJsonDocument::Compact));
-    connect(qApp, &QCoreApplication::aboutToQuit, reply, &QNetworkReply::abort);
-    connect(reply, &QNetworkReply::finished, this, &EmbeddingLLMWorker::handleFinished);
+    sendAtlasRequest(texts, "search_document", QVariant::fromValue(chunks));
 }
 
 std::vector<float> jsonArrayToVector(const QJsonArray &jsonArray) {

gpt4all-chat/embllm.h

@@ -1,10 +1,11 @@
 #ifndef EMBLLM_H
 #define EMBLLM_H
 
-#include <QObject>
-#include <QThread>
-#include <QNetworkReply>
 #include <QNetworkAccessManager>
+#include <QNetworkReply>
+#include <QObject>
+#include <QStringList>
+#include <QThread>
 
 #include "../gpt4all-backend/llmodel.h"
 
@@ -51,6 +52,8 @@ private Q_SLOTS:
     void handleFinished();
 
 private:
+    void sendAtlasRequest(const QStringList &texts, const QString &taskType, QVariant userData = {});
+
     QString m_nomicAPIKey;
     QNetworkAccessManager *m_networkManager;
     std::vector<float> m_lastResponse;

gpt4all-chat/icons/left_panel_closed.svg

@@ -0,0 +1,3 @@
+<svg width="64" height="64" viewBox="0 0 64 64" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M23 16H54C55.6569 16 57 17.3431 57 19V45C57 46.6569 55.6569 48 54 48H23V16ZM20 16H10C8.34315 16 7 17.3431 7 19V45C7 46.6569 8.34315 48 10 48H20V16ZM4 19C4 15.6863 6.68629 13 10 13H54C57.3137 13 60 15.6863 60 19V45C60 48.3137 57.3137 51 54 51H10C6.68629 51 4 48.3137 4 45V19Z" fill="black"/>
+</svg>

gpt4all-chat/icons/left_panel_open.svg

@@ -0,0 +1,3 @@
+<svg width="64" height="64" viewBox="0 0 64 64" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M23 16H54C55.6569 16 57 17.3431 57 19V45C57 46.6569 55.6569 48 54 48H23V16ZM4 19C4 15.6863 6.68629 13 10 13H54C57.3137 13 60 15.6863 60 19V45C60 48.3137 57.3137 51 54 51H10C6.68629 51 4 48.3137 4 45V19Z" fill="black"/>
+</svg>

gpt4all-chat/llm.cpp

@@ -1,4 +1,5 @@
 #include "llm.h"
+#include "../gpt4all-backend/llmodel.h"
 #include "../gpt4all-backend/sysinfo.h"
 
 #include <QCoreApplication>
@@ -25,25 +26,14 @@ LLM *LLM::globalInstance()
 
 LLM::LLM()
     : QObject{nullptr}
-    , m_compatHardware(true)
+    , m_compatHardware(LLModel::Implementation::hasSupportedCPU())
 {
-#if defined(__x86_64__)
-    #ifndef _MSC_VER
-        const bool minimal(__builtin_cpu_supports("avx"));
-    #else
-        int cpuInfo[4];
-        __cpuid(cpuInfo, 1);
-        const bool minimal(cpuInfo[2] & (1 << 28));
-    #endif
-#else
-    const bool minimal = true; // Don't know how to handle non-x86_64
-#endif
-
-    m_compatHardware = minimal;
-
     QNetworkInformation::loadDefaultBackend();
-    connect(QNetworkInformation::instance(), &QNetworkInformation::reachabilityChanged,
-        this, &LLM::isNetworkOnlineChanged);
+    auto * netinfo = QNetworkInformation::instance();
+    if (netinfo) {
+        connect(netinfo, &QNetworkInformation::reachabilityChanged,
+            this, &LLM::isNetworkOnlineChanged);
+    }
 }
 
 bool LLM::hasSettingsAccess() const
@@ -59,7 +49,7 @@ bool LLM::checkForUpdates() const
     #pragma message "offline installer build will not check for updates!"
     return QDesktopServices::openUrl(QUrl("https://gpt4all.io/"));
     #else
-    Network::globalInstance()->sendCheckForUpdates();
+    Network::globalInstance()->trackEvent("check_for_updates");
 
 #if defined(Q_OS_LINUX)
     QString tool("maintenancetool");
@@ -108,8 +98,6 @@ QString LLM::systemTotalRAMInGBString() const
 
 bool LLM::isNetworkOnline() const
 {
-    if (!QNetworkInformation::instance())
-        return false;
-
-    return QNetworkInformation::instance()->reachability() == QNetworkInformation::Reachability::Online;
+    auto * netinfo = QNetworkInformation::instance();
+    return !netinfo || netinfo->reachability() == QNetworkInformation::Reachability::Online;
 }

gpt4all-chat/localdocs.cpp

@@ -18,6 +18,8 @@ LocalDocs::LocalDocs()
     // Create the DB with the chunk size from settings
     m_database = new Database(MySettings::globalInstance()->localDocsChunkSize());
 
+    connect(this, &LocalDocs::requestStart, m_database,
+        &Database::start, Qt::QueuedConnection);
     connect(this, &LocalDocs::requestAddFolder, m_database,
         &Database::addFolder, Qt::QueuedConnection);
     connect(this, &LocalDocs::requestRemoveFolder, m_database,
@@ -50,8 +52,6 @@ LocalDocs::LocalDocs()
         m_localDocsModel, &LocalDocsModel::addCollectionItem, Qt::QueuedConnection);
     connect(m_database, &Database::removeFolderById,
         m_localDocsModel, &LocalDocsModel::removeFolderById, Qt::QueuedConnection);
-    connect(m_database, &Database::removeCollectionItem,
-        m_localDocsModel, &LocalDocsModel::removeCollectionItem, Qt::QueuedConnection);
     connect(m_database, &Database::collectionListUpdated,
         m_localDocsModel, &LocalDocsModel::collectionListUpdated, Qt::QueuedConnection);
 
@@ -68,7 +68,7 @@ void LocalDocs::addFolder(const QString &collection, const QString &path)
 {
     const QUrl url(path);
     const QString localPath = url.isLocalFile() ? url.toLocalFile() : path;
-    emit requestAddFolder(collection, localPath);
+    emit requestAddFolder(collection, localPath, false);
 }
 
 void LocalDocs::removeFolder(const QString &collection, const QString &path)

gpt4all-chat/localdocs.h

@@ -26,7 +26,8 @@ public Q_SLOTS:
     void aboutToQuit();
 
 Q_SIGNALS:
-    void requestAddFolder(const QString &collection, const QString &path);
+    void requestStart();
+    void requestAddFolder(const QString &collection, const QString &path, bool fromDb);
     void requestRemoveFolder(const QString &collection, const QString &path);
     void requestChunkSizeChange(int chunkSize);
     void localDocsModelChanged();

gpt4all-chat/localdocsdb.h

@@ -1,105 +0,0 @@
-#ifndef LOCALDOCS_H
-#define LOCALDOCS_H
-
-#include "localdocsmodel.h"
-
-#include <QObject>
-#include <QtSql>
-#include <QQueue>
-#include <QFileInfo>
-#include <QThread>
-#include <QFileSystemWatcher>
-
-struct DocumentInfo
-{
-    int folder;
-    QFileInfo doc;
-};
-
-struct CollectionItem {
-    QString collection;
-    QString folder_path;
-    int folder_id = -1;
-};
-Q_DECLARE_METATYPE(CollectionItem)
-
-class Database : public QObject
-{
-    Q_OBJECT
-public:
-    Database();
-
-public Q_SLOTS:
-    void scanQueue();
-    void scanDocuments(int folder_id, const QString &folder_path);
-    void addFolder(const QString &collection, const QString &path);
-    void removeFolder(const QString &collection, const QString &path);
-    void retrieveFromDB(const QList<QString> &collections, const QString &text);
-    void cleanDB();
-
-Q_SIGNALS:
-    void docsToScanChanged();
-    void retrieveResult(const QList<QString> &result);
-    void collectionListUpdated(const QList<CollectionItem> &collectionList);
-
-private Q_SLOTS:
-    void start();
-    void directoryChanged(const QString &path);
-    bool addFolderToWatch(const QString &path);
-    bool removeFolderFromWatch(const QString &path);
-    void addCurrentFolders();
-    void updateCollectionList();
-
-private:
-    void removeFolderInternal(const QString &collection, int folder_id, const QString &path);
-    void chunkStream(QTextStream &stream, int document_id);
-    void handleDocumentErrorAndScheduleNext(const QString &errorMessage,
-        int document_id, const QString &document_path, const QSqlError &error);
-
-private:
-    QQueue<DocumentInfo> m_docsToScan;
-    QList<QString> m_retrieve;
-    QThread m_dbThread;
-    QFileSystemWatcher *m_watcher;
-};
-
-class LocalDocs : public QObject
-{
-    Q_OBJECT
-    Q_PROPERTY(LocalDocsModel *localDocsModel READ localDocsModel NOTIFY localDocsModelChanged)
-
-public:
-    static LocalDocs *globalInstance();
-
-    LocalDocsModel *localDocsModel() const { return m_localDocsModel; }
-
-    void addFolder(const QString &collection, const QString &path);
-    void removeFolder(const QString &collection, const QString &path);
-
-    QList<QString> result() const { return m_retrieveResult; }
-    void requestRetrieve(const QList<QString> &collections, const QString &text);
-
-Q_SIGNALS:
-    void requestAddFolder(const QString &collection, const QString &path);
-    void requestRemoveFolder(const QString &collection, const QString &path);
-    void requestRetrieveFromDB(const QList<QString> &collections, const QString &text);
-    void receivedResult();
-    void localDocsModelChanged();
-
-private Q_SLOTS:
-    void handleRetrieveResult(const QList<QString> &result);
-    void handleCollectionListUpdated(const QList<CollectionItem> &collectionList);
-
-private:
-    LocalDocsModel *m_localDocsModel;
-    Database *m_database;
-    QList<QString> m_retrieveResult;
-    QList<CollectionItem> m_collectionList;
-
-private:
-    explicit LocalDocs();
-    ~LocalDocs() {}
-    friend class MyLocalDocs;
-};
-
-#endif // LOCALDOCS_H

gpt4all-chat/localdocsmodel.cpp

@@ -1,6 +1,7 @@
 #include "localdocsmodel.h"
 
 #include "localdocs.h"
+#include "network.h"
 
 LocalDocsCollectionsModel::LocalDocsCollectionsModel(QObject *parent)
     : QSortFilterProxyModel(parent)
@@ -158,50 +159,43 @@ void LocalDocsModel::updateTotalEmbeddingsToIndex(int folder_id, size_t totalEmb
         [](CollectionItem& item, size_t val) { item.totalEmbeddingsToIndex += val; }, {TotalEmbeddingsToIndexRole});
 }
 
-void LocalDocsModel::addCollectionItem(const CollectionItem &item)
+void LocalDocsModel::addCollectionItem(const CollectionItem &item, bool fromDb)
 {
     beginInsertRows(QModelIndex(), m_collectionList.size(), m_collectionList.size());
     m_collectionList.append(item);
     endInsertRows();
+
+    if (!fromDb) {
+        Network::globalInstance()->trackEvent("doc_collection_add", {
+            {"collection_count", m_collectionList.count()},
+        });
+    }
+}
+
+void LocalDocsModel::removeCollectionIf(std::function<bool(CollectionItem)> const &predicate) {
+    for (int i = 0; i < m_collectionList.size();) {
+        if (predicate(m_collectionList.at(i))) {
+            beginRemoveRows(QModelIndex(), i, i);
+            m_collectionList.removeAt(i);
+            endRemoveRows();
+
+            Network::globalInstance()->trackEvent("doc_collection_remove", {
+                {"collection_count", m_collectionList.count()},
+            });
+        } else {
+            ++i;
+        }
+    }
 }
 
 void LocalDocsModel::removeFolderById(int folder_id)
 {
-    for (int i = 0; i < m_collectionList.size();) {
-        if (m_collectionList.at(i).folder_id == folder_id) {
-            beginRemoveRows(QModelIndex(), i, i);
-            m_collectionList.removeAt(i);
-            endRemoveRows();
-        } else {
-            ++i;
-        }
-    }
+    removeCollectionIf([folder_id](const auto &c) { return c.folder_id == folder_id; });
 }
 
 void LocalDocsModel::removeCollectionPath(const QString &name, const QString &path)
 {
-    for (int i = 0; i < m_collectionList.size();) {
-        if (m_collectionList.at(i).collection == name && m_collectionList.at(i).folder_path == path) {
-            beginRemoveRows(QModelIndex(), i, i);
-            m_collectionList.removeAt(i);
-            endRemoveRows();
-        } else {
-            ++i;
-        }
-    }
-}
-
-void LocalDocsModel::removeCollectionItem(const QString &collectionName)
-{
-    for (int i = 0; i < m_collectionList.size();) {
-        if (m_collectionList.at(i).collection == collectionName) {
-            beginRemoveRows(QModelIndex(), i, i);
-            m_collectionList.removeAt(i);
-            endRemoveRows();
-        } else {
-            ++i;
-        }
-    }
+    removeCollectionIf([&name, &path](const auto &c) { return c.collection == name && c.folder_path == path; });
 }
 
 void LocalDocsModel::collectionListUpdated(const QList<CollectionItem> &collectionList)

gpt4all-chat/localdocsmodel.h

@@ -55,10 +55,9 @@ public Q_SLOTS:
     void updateTotalBytesToIndex(int folder_id, size_t totalBytesToIndex);
     void updateCurrentEmbeddingsToIndex(int folder_id, size_t currentBytesToIndex);
     void updateTotalEmbeddingsToIndex(int folder_id, size_t totalBytesToIndex);
-    void addCollectionItem(const CollectionItem &item);
+    void addCollectionItem(const CollectionItem &item, bool fromDb);
     void removeFolderById(int folder_id);
     void removeCollectionPath(const QString &name, const QString &path);
-    void removeCollectionItem(const QString &collectionName);
     void collectionListUpdated(const QList<CollectionItem> &collectionList);
 
 private:
@@ -66,6 +65,7 @@ private:
     void updateField(int folder_id, T value,
         const std::function<void(CollectionItem&, T)>& updater,
         const QVector<int>& roles);
+    void removeCollectionIf(std::function<bool(CollectionItem)> const &predicate);
 
 private:
     QList<CollectionItem> m_collectionList;

gpt4all-chat/main.cpp

@@ -67,7 +67,7 @@ int main(int argc, char *argv[])
 
     // Make sure ChatLLM threads are joined before global destructors run.
     // Otherwise, we can get a heap-use-after-free inside of llama.cpp.
-    ChatListModel::globalInstance()->clearChats();
+    ChatListModel::globalInstance()->destroyChats();
 
     return res;
 }

gpt4all-chat/metadata/models2.json

@@ -13,7 +13,7 @@
     "description": "<strong>Best overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
     "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.gguf2.Q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n",
-    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI. For multi-step problems, write out your reasoning for each step.\n<|im_end|>"
+    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI.\n<|im_end|>"
   },
   {
     "order": "b",

gpt4all-chat/metadata/models3.json

@@ -1,6 +1,22 @@
 [
   {
     "order": "a",
+    "md5sum": "c87ad09e1e4c8f9c35a5fcef52b6f1c9",
+    "name": "Llama 3 Instruct",
+    "filename": "Meta-Llama-3-8B-Instruct.Q4_0.gguf",
+    "filesize": "4661724384",
+    "requires": "2.7.1",
+    "ramrequired": "8",
+    "parameters": "8 billion",
+    "quant": "q4_0",
+    "type": "LLaMA3",
+    "description": "<ul><li>Fast responses</li><li>Chat based model</li><li>Accepts system prompts in Llama 3 format</li><li>Trained by Meta</li><li>License: <a href=\"https://llama.meta.com/llama3/license/\">Meta Llama 3 Community License</a></li></ul>",
+    "url": "https://gpt4all.io/models/gguf/Meta-Llama-3-8B-Instruct.Q4_0.gguf",
+    "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2<|eot_id|>",
+    "systemPrompt": ""
+  },
+  {
+    "order": "b",
     "md5sum": "a5f6b4eabd3992da4d7fb7f020f921eb",
     "name": "Nous Hermes 2 Mistral DPO",
     "filename": "Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf",
@@ -15,22 +31,6 @@
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
     "systemPrompt": ""
   },
-  {
-    "order": "b",
-    "md5sum": "f692417a22405d80573ac10cb0cd6c6a",
-    "name": "Mistral OpenOrca",
-    "filename": "mistral-7b-openorca.gguf2.Q4_0.gguf",
-    "filesize": "4108928128",
-    "requires": "2.5.0",
-    "ramrequired": "8",
-    "parameters": "7 billion",
-    "quant": "q4_0",
-    "type": "Mistral",
-    "description": "<strong>Strong overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
-    "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.gguf2.Q4_0.gguf",
-    "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
-    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI. For multi-step problems, write out your reasoning for each step.\n<|im_end|>"
-  },
   {
     "order": "c",
     "md5sum": "97463be739b50525df56d33b26b00852",
@@ -42,13 +42,29 @@
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "Mistral",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<strong>Strong overall fast instruction following model</strong><br><ul><li>Fast responses</li><li>Trained by Mistral AI<li>Uncensored</li><li>Licensed for commercial use</li></ul>",
     "url": "https://gpt4all.io/models/gguf/mistral-7b-instruct-v0.1.Q4_0.gguf",
     "promptTemplate": "[INST] %1 [/INST]"
   },
   {
     "order": "d",
+    "md5sum": "f692417a22405d80573ac10cb0cd6c6a",
+    "name": "Mistral OpenOrca",
+    "filename": "mistral-7b-openorca.gguf2.Q4_0.gguf",
+    "filesize": "4108928128",
+    "requires": "2.7.1",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "description": "<strong>Strong overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
+    "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.gguf2.Q4_0.gguf",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
+    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI.\n<|im_end|>\n"
+  },
+  {
+    "order": "e",
     "md5sum": "c4c78adf744d6a20f05c8751e3961b84",
     "name": "GPT4All Falcon",
     "filename": "gpt4all-falcon-newbpe-q4_0.gguf",
@@ -58,13 +74,13 @@
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "Falcon",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<strong>Very fast model with good quality</strong><br><ul><li>Fastest responses</li><li>Instruction based</li><li>Trained by TII<li>Finetuned by Nomic AI<li>Licensed for commercial use</ul>",
     "url": "https://gpt4all.io/models/gguf/gpt4all-falcon-newbpe-q4_0.gguf",
     "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
   },
   {
-    "order": "e",
+    "order": "f",
     "md5sum": "00c8593ba57f5240f59662367b3ed4a5",
     "name": "Orca 2 (Medium)",
     "filename": "orca-2-7b.Q4_0.gguf",
@@ -74,12 +90,12 @@
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "LLaMA2",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<ul><li>Instruction based<li>Trained by Microsoft<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/orca-2-7b.Q4_0.gguf"
   },
   {
-    "order": "f",
+    "order": "g",
     "md5sum": "3c0d63c4689b9af7baa82469a6f51a19",
     "name": "Orca 2 (Full)",
     "filename": "orca-2-13b.Q4_0.gguf",
@@ -89,12 +105,12 @@
     "parameters": "13 billion",
     "quant": "q4_0",
     "type": "LLaMA2",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<ul><li>Instruction based<li>Trained by Microsoft<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/orca-2-13b.Q4_0.gguf"
   },
   {
-    "order": "g",
+    "order": "h",
     "md5sum": "5aff90007499bce5c64b1c0760c0b186",
     "name": "Wizard v1.2",
     "filename": "wizardlm-13b-v1.2.Q4_0.gguf",
@@ -104,12 +120,28 @@
     "parameters": "13 billion",
     "quant": "q4_0",
     "type": "LLaMA2",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<strong>Strong overall larger model</strong><br><ul><li>Instruction based<li>Gives very long responses<li>Finetuned with only 1k of high-quality data<li>Trained by Microsoft and Peking University<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/wizardlm-13b-v1.2.Q4_0.gguf"
   },
   {
-    "order": "h",
+    "order": "i",
+    "md5sum": "31b47b4e8c1816b62684ac3ca373f9e1",
+    "name": "Ghost 7B v0.9.1",
+    "filename": "ghost-7b-v0.9.1-Q4_0.gguf",
+    "filesize": "4108916960",
+    "requires": "2.7.1",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "description": "<strong>Ghost 7B v0.9.1</strong> fast, powerful and smooth for Vietnamese and English languages.",
+    "url": "https://huggingface.co/lamhieu/ghost-7b-v0.9.1-gguf/resolve/main/ghost-7b-v0.9.1-Q4_0.gguf",
+    "promptTemplate": "<|user|>\n%1</s>\n<|assistant|>\n%2</s>\n",
+    "systemPrompt": "<|system|>\nYou are Ghost created by Lam Hieu. You are a helpful and knowledgeable assistant. You like to help and always give honest information, in its original language. In communication, you are always respectful, equal and promote positive behavior.\n</s>"
+  },
+  {
+    "order": "j",
     "md5sum": "3d12810391d04d1153b692626c0c6e16",
     "name": "Hermes",
     "filename": "nous-hermes-llama2-13b.Q4_0.gguf",
@@ -119,13 +151,13 @@
     "parameters": "13 billion",
     "quant": "q4_0",
     "type": "LLaMA2",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<strong>Extremely good model</strong><br><ul><li>Instruction based<li>Gives long responses<li>Curated with 300,000 uncensored instructions<li>Trained by Nous Research<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/nous-hermes-llama2-13b.Q4_0.gguf",
     "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
   },
   {
-    "order": "i",
+    "order": "k",
     "md5sum": "40388eb2f8d16bb5d08c96fdfaac6b2c",
     "name": "Snoozy",
     "filename": "gpt4all-13b-snoozy-q4_0.gguf",
@@ -135,17 +167,18 @@
     "parameters": "13 billion",
     "quant": "q4_0",
     "type": "LLaMA",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<strong>Very good overall model</strong><br><ul><li>Instruction based<li>Based on the same dataset as Groovy<li>Slower than Groovy, with higher quality responses<li>Trained by Nomic AI<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf"
   },
   {
-    "order": "j",
+    "order": "l",
     "md5sum": "15dcb4d7ea6de322756449c11a0b7545",
     "name": "MPT Chat",
     "filename": "mpt-7b-chat-newbpe-q4_0.gguf",
     "filesize": "3912373472",
-    "requires": "2.6.0",
+    "requires": "2.7.1",
+    "removedIn": "2.7.3",
     "ramrequired": "8",
     "parameters": "7 billion",
     "quant": "q4_0",
@@ -153,10 +186,42 @@
     "description": "<strong>Good model with novel architecture</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/mpt-7b-chat-newbpe-q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
-    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
+    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
   },
   {
-    "order": "k",
+    "order": "l",
+    "md5sum": "ab5d8e8a2f79365ea803c1f1d0aa749d",
+    "name": "MPT Chat",
+    "filename": "mpt-7b-chat.gguf4.Q4_0.gguf",
+    "filesize": "3796178112",
+    "requires": "2.7.3",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "MPT",
+    "description": "<strong>Good model with novel architecture</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/mpt-7b-chat.gguf4.Q4_0.gguf",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
+    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
+  },
+  {
+    "order": "m",
+    "md5sum": "f8347badde9bfc2efbe89124d78ddaf5",
+    "name": "Phi-3 Mini Instruct",
+    "filename": "Phi-3-mini-4k-instruct.Q4_0.gguf",
+    "filesize": "2176181568",
+    "requires": "2.7.1",
+    "ramrequired": "4",
+    "parameters": "4 billion",
+    "quant": "q4_0",
+    "type": "Phi-3",
+    "description": "<ul><li>Very fast responses</li><li>Chat based model</li><li>Accepts system prompts in Phi-3 format</li><li>Trained by Microsoft</li><li>License: <a href=\"https://opensource.org/license/mit\">MIT</a></li><li>No restrictions on commercial use</li></ul>",
+    "url": "https://gpt4all.io/models/gguf/Phi-3-mini-4k-instruct.Q4_0.gguf",
+    "promptTemplate": "<|user|>\n%1<|end|>\n<|assistant|>\n%2<|end|>\n",
+    "systemPrompt": ""
+  },
+  {
+    "order": "n",
     "md5sum": "0e769317b90ac30d6e09486d61fefa26",
     "name": "Mini Orca (Small)",
     "filename": "orca-mini-3b-gguf2-q4_0.gguf",
@@ -166,13 +231,13 @@
     "parameters": "3 billion",
     "quant": "q4_0",
     "type": "OpenLLaMa",
-    "description": "<strong>Small version of new model with novel dataset</strong><br><ul><li>Instruction based<li>Explain tuned datasets<li>Orca Research Paper dataset construction approaches<li>Cannot be used commercially</ul>",
+    "description": "<strong>Small version of new model with novel dataset</strong><br><ul><li>Very fast responses</li><li>Instruction based</li><li>Explain tuned datasets</li><li>Orca Research Paper dataset construction approaches</li><li>Cannot be used commercially</li></ul>",
     "url": "https://gpt4all.io/models/gguf/orca-mini-3b-gguf2-q4_0.gguf",
     "promptTemplate": "### User:\n%1\n\n### Response:\n",
     "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
   },
   {
-    "order": "l",
+    "order": "o",
     "md5sum": "c232f17e09bca4b7ee0b5b1f4107c01e",
     "disableGUI": "true",
     "name": "Replit",
@@ -183,13 +248,13 @@
     "parameters": "3 billion",
     "quant": "q4_0",
     "type": "Replit",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "promptTemplate": "%1",
     "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>Licensed for commercial use<li>WARNING: Not available for chat GUI</ul>",
     "url": "https://gpt4all.io/models/gguf/replit-code-v1_5-3b-newbpe-q4_0.gguf"
   },
   {
-    "order": "m",
+    "order": "p",
     "md5sum": "70841751ccd95526d3dcfa829e11cd4c",
     "disableGUI": "true",
     "name": "Starcoder",
@@ -200,13 +265,13 @@
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "Starcoder",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "promptTemplate": "%1",
     "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>WARNING: Not available for chat GUI</ul>",
     "url": "https://gpt4all.io/models/gguf/starcoder-newbpe-q4_0.gguf"
   },
   {
-    "order": "n",
+    "order": "q",
     "md5sum": "e973dd26f0ffa6e46783feaea8f08c83",
     "disableGUI": "true",
     "name": "Rift coder",
@@ -217,29 +282,46 @@
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "LLaMA",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "promptTemplate": "%1",
     "description": "<strong>Trained on collection of Python and TypeScript</strong><br><ul><li>Code completion based<li>WARNING: Not available for chat GUI</li>",
     "url": "https://gpt4all.io/models/gguf/rift-coder-v0-7b-q4_0.gguf"
   },
   {
-    "order": "o",
+    "order": "r",
     "md5sum": "e479e6f38b59afc51a470d1953a6bfc7",
     "disableGUI": "true",
     "name": "SBert",
     "filename": "all-MiniLM-L6-v2-f16.gguf",
     "filesize": "45887744",
     "requires": "2.5.0",
+    "removedIn": "2.7.4",
     "ramrequired": "1",
     "parameters": "40 million",
     "quant": "f16",
     "type": "Bert",
-    "systemPrompt": " ",
+    "embeddingModel": true,
+    "systemPrompt": "",
     "description": "<strong>LocalDocs text embeddings model</strong><br><ul><li>For use with LocalDocs feature<li>Used for retrieval augmented generation (RAG)",
     "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf"
   },
   {
-    "order": "p",
+    "order": "r",
+    "md5sum": "dd90e2cb7f8e9316ac3796cece9883b5",
+    "name": "SBert",
+    "filename": "all-MiniLM-L6-v2.gguf2.f16.gguf",
+    "filesize": "45949216",
+    "requires": "2.7.4",
+    "ramrequired": "1",
+    "parameters": "40 million",
+    "quant": "f16",
+    "type": "Bert",
+    "embeddingModel": true,
+    "description": "<strong>LocalDocs text embeddings model</strong><br><ul><li>For use with LocalDocs feature<li>Used for retrieval augmented generation (RAG)",
+    "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2.gguf2.f16.gguf"
+  },
+  {
+    "order": "s",
     "md5sum": "919de4dd6f25351bcb0223790db1932d",
     "name": "EM German Mistral",
     "filename": "em_german_mistral_v01.Q4_0.gguf",
@@ -253,5 +335,39 @@
     "url": "https://huggingface.co/TheBloke/em_german_mistral_v01-GGUF/resolve/main/em_german_mistral_v01.Q4_0.gguf",
     "promptTemplate": "USER: %1 ASSISTANT: ",
     "systemPrompt": "Du bist ein hilfreicher Assistent. "
+  },
+  {
+    "order": "t",
+    "md5sum": "60ea031126f82db8ddbbfecc668315d2",
+    "disableGUI": "true",
+    "name": "Nomic Embed Text v1",
+    "filename": "nomic-embed-text-v1.f16.gguf",
+    "filesize": "274290560",
+    "requires": "2.7.4",
+    "ramrequired": "1",
+    "parameters": "137 million",
+    "quant": "f16",
+    "type": "Bert",
+    "embeddingModel": true,
+    "systemPrompt": "",
+    "description": "nomic-embed-text-v1",
+    "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.f16.gguf"
+  },
+  {
+    "order": "u",
+    "md5sum": "a5401e7f7e46ed9fcaed5b60a281d547",
+    "disableGUI": "true",
+    "name": "Nomic Embed Text v1.5",
+    "filename": "nomic-embed-text-v1.5.f16.gguf",
+    "filesize": "274290560",
+    "requires": "2.7.4",
+    "ramrequired": "1",
+    "parameters": "137 million",
+    "quant": "f16",
+    "type": "Bert",
+    "embeddingModel": true,
+    "systemPrompt": "",
+    "description": "nomic-embed-text-v1.5",
+    "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.5.f16.gguf"
   }
 ]

gpt4all-chat/metadata/release.json

@@ -705,6 +705,53 @@
 * Jared Van Bortel (Nomic AI)
 * Adam Treat (Nomic AI)
 * Community (beta testers, bug reporters, bindings authors)
+"
+  },
+  {
+    "version": "2.7.2",
+    "notes":
+"
+* New support for model search/discovery using huggingface search in downloads
+* Support for more model architectures for GPU acceleration
+* Three different crash fixes for corner case settings
+* Add a minp sampling parameter
+* Bert layoer norm epsilon value
+* Fix problem with blank lines between reply and next prompt
+",
+    "contributors":
+"
+* Christopher Barrera
+* Jared Van Bortel (Nomic AI)
+* Adam Treat (Nomic AI)
+* Community (beta testers, bug reporters, bindings authors)
+"
+  },
+  {
+    "version": "2.7.3",
+    "notes":
+"
+* Fix for network reachability unknown
+* Fix undefined behavior with resetContext
+* Fix ChatGPT which was broken with previous release
+* Fix for clean up of chat llm thread destruction
+* Display of model loading warnings
+* Fix for issue 2080 where the GUI appears to hang when a chat is deleted
+* Fix for issue 2077 better responsiveness of model download dialog when download is taking place
+* Fix for issue 2092 don't include models that are disabled for GUI in application default model list
+* Fix for issue 2087 where cloned modelds were lost and listed in download dialog erroneously
+* Fix for MPT models without duplicated token embd weight
+* New feature with api server port setting
+* Fix for issue 2024 where combobox for model settings uses currently used model by default
+* Clean up settings properly for removed models and don't list stale model settings in download dialog
+* Fix for issue 2105 where the cancel button was not working for discovered model downloads
+",
+    "contributors":
+"
+* Christopher Barrera
+* Daniel Alencar
+* Jared Van Bortel (Nomic AI)
+* Adam Treat (Nomic AI)
+* Community (beta testers, bug reporters, bindings authors)
 "
   }
 ]

gpt4all-chat/modellist.h

@@ -16,11 +16,10 @@ struct ModelInfo {
     Q_PROPERTY(bool calcHash MEMBER calcHash)
     Q_PROPERTY(bool installed MEMBER installed)
     Q_PROPERTY(bool isDefault MEMBER isDefault)
-    Q_PROPERTY(bool disableGUI MEMBER disableGUI)
     Q_PROPERTY(bool isOnline MEMBER isOnline)
     Q_PROPERTY(QString description READ description WRITE setDescription)
     Q_PROPERTY(QString requiresVersion MEMBER requiresVersion)
-    Q_PROPERTY(QString deprecatedVersion MEMBER deprecatedVersion)
+    Q_PROPERTY(QString versionRemoved MEMBER versionRemoved)
     Q_PROPERTY(QString url READ url WRITE setUrl)
     Q_PROPERTY(qint64 bytesReceived MEMBER bytesReceived)
     Q_PROPERTY(qint64 bytesTotal MEMBER bytesTotal)
@@ -36,6 +35,7 @@ struct ModelInfo {
     Q_PROPERTY(QString type READ type WRITE setType)
     Q_PROPERTY(bool isClone READ isClone WRITE setIsClone)
     Q_PROPERTY(bool isDiscovered READ isDiscovered WRITE setIsDiscovered)
+    Q_PROPERTY(bool isEmbeddingModel MEMBER isEmbeddingModel)
     Q_PROPERTY(double temperature READ temperature WRITE setTemperature)
     Q_PROPERTY(double topP READ topP WRITE setTopP)
     Q_PROPERTY(double minP READ minP WRITE setMinP)
@@ -104,9 +104,8 @@ public:
     bool installed = false;
     bool isDefault = false;
     bool isOnline = false;
-    bool disableGUI = false;
     QString requiresVersion;
-    QString deprecatedVersion;
+    QString versionRemoved;
     qint64 bytesReceived = 0;
     qint64 bytesTotal = 0;
     qint64 timestamp = 0;
@@ -117,6 +116,8 @@ public:
     QString order;
     int ramrequired = -1;
     QString parameters;
+    bool isEmbeddingModel = false;
+    bool checkedEmbeddingModel = false;
 
     bool operator==(const ModelInfo &other) const {
         return  m_id == other.m_id;
@@ -187,9 +188,10 @@ class EmbeddingModels : public QSortFilterProxyModel
     Q_OBJECT
     Q_PROPERTY(int count READ count NOTIFY countChanged)
 public:
-    explicit EmbeddingModels(QObject *parent);
-    int count() const;
+    EmbeddingModels(QObject *parent, bool requireInstalled);
+    int count() const { return rowCount(); }
 
+    int defaultModelIndex() const;
     ModelInfo defaultModelInfo() const;
 
 Q_SIGNALS:
@@ -198,6 +200,9 @@ Q_SIGNALS:
 
 protected:
     bool filterAcceptsRow(int sourceRow, const QModelIndex &sourceParent) const override;
+
+private:
+    bool m_requireInstalled;
 };
 
 class InstalledModels : public QSortFilterProxyModel
@@ -206,7 +211,7 @@ class InstalledModels : public QSortFilterProxyModel
     Q_PROPERTY(int count READ count NOTIFY countChanged)
 public:
     explicit InstalledModels(QObject *parent);
-    int count() const;
+    int count() const { return rowCount(); }
 
 Q_SIGNALS:
     void countChanged();
@@ -248,8 +253,8 @@ class ModelList : public QAbstractListModel
 {
     Q_OBJECT
     Q_PROPERTY(int count READ count NOTIFY countChanged)
-    Q_PROPERTY(int defaultEmbeddingModelIndex READ defaultEmbeddingModelIndex NOTIFY defaultEmbeddingModelIndexChanged)
-    Q_PROPERTY(EmbeddingModels* embeddingModels READ embeddingModels NOTIFY embeddingModelsChanged)
+    Q_PROPERTY(int defaultEmbeddingModelIndex READ defaultEmbeddingModelIndex)
+    Q_PROPERTY(EmbeddingModels* installedEmbeddingModels READ installedEmbeddingModels NOTIFY installedEmbeddingModelsChanged)
     Q_PROPERTY(InstalledModels* installedModels READ installedModels NOTIFY installedModelsChanged)
     Q_PROPERTY(DownloadableModels* downloadableModels READ downloadableModels NOTIFY downloadableModelsChanged)
     Q_PROPERTY(QList<QString> userDefaultModelList READ userDefaultModelList NOTIFY userDefaultModelListChanged)
@@ -282,10 +287,9 @@ public:
         InstalledRole,
         DefaultRole,
         OnlineRole,
-        DisableGUIRole,
         DescriptionRole,
         RequiresVersionRole,
-        DeprecatedVersionRole,
+        VersionRemovedRole,
         UrlRole,
         BytesReceivedRole,
         BytesTotalRole,
@@ -301,6 +305,7 @@ public:
         TypeRole,
         IsCloneRole,
         IsDiscoveredRole,
+        IsEmbeddingModelRole,
         TemperatureRole,
         TopPRole,
         TopKRole,
@@ -332,10 +337,9 @@ public:
         roles[InstalledRole] = "installed";
         roles[DefaultRole] = "isDefault";
         roles[OnlineRole] = "isOnline";
-        roles[DisableGUIRole] = "disableGUI";
         roles[DescriptionRole] = "description";
         roles[RequiresVersionRole] = "requiresVersion";
-        roles[DeprecatedVersionRole] = "deprecatedVersion";
+        roles[VersionRemovedRole] = "versionRemoved";
         roles[UrlRole] = "url";
         roles[BytesReceivedRole] = "bytesReceived";
         roles[BytesTotalRole] = "bytesTotal";
@@ -351,6 +355,7 @@ public:
         roles[TypeRole] = "type";
         roles[IsCloneRole] = "isClone";
         roles[IsDiscoveredRole] = "isDiscovered";
+        roles[IsEmbeddingModelRole] = "isEmbeddingModel";
         roles[TemperatureRole] = "temperature";
         roles[TopPRole] = "topP";
         roles[MinPRole] = "minP";
@@ -373,8 +378,7 @@ public:
     QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override;
     QVariant data(const QString &id, int role) const;
     QVariant dataByFilename(const QString &filename, int role) const;
-    void updateData(const QString &id, int role, const QVariant &value);
-    void updateDataByFilename(const QString &filename, int role, const QVariant &value);
+    void updateDataByFilename(const QString &filename, QVector<QPair<int, QVariant>> data);
     void updateData(const QString &id, const QVector<QPair<int, QVariant>> &data);
 
     int count() const { return m_models.size(); }
@@ -397,6 +401,7 @@ public:
     const QList<QString> userDefaultModelList() const;
 
     EmbeddingModels *embeddingModels() const { return m_embeddingModels; }
+    EmbeddingModels *installedEmbeddingModels() const { return m_installedEmbeddingModels; }
     InstalledModels *installedModels() const { return m_installedModels; }
     DownloadableModels *downloadableModels() const { return m_downloadableModels; }
 
@@ -434,12 +439,11 @@ public:
 
 Q_SIGNALS:
     void countChanged();
-    void embeddingModelsChanged();
+    void installedEmbeddingModelsChanged();
     void installedModelsChanged();
     void downloadableModelsChanged();
     void userDefaultModelListChanged();
     void asyncModelRequestOngoingChanged();
-    void defaultEmbeddingModelIndexChanged();
     void discoverLimitChanged();
     void discoverSortDirectionChanged();
     void discoverSortChanged();
@@ -463,7 +467,7 @@ private Q_SLOTS:
 private:
     void removeInternal(const ModelInfo &model);
     void clearDiscoveredModels();
-    QString modelDirPath(const QString &modelName, bool isOnline);
+    bool modelExists(const QString &fileName) const;
     int indexForModel(ModelInfo *model);
     QVariant dataInternal(const ModelInfo *info, int role) const;
     static bool lessThan(const ModelInfo* a, const ModelInfo* b, DiscoverSort s, int d);
@@ -475,6 +479,7 @@ private:
     mutable QMutex m_mutex;
     QNetworkAccessManager m_networkManager;
     EmbeddingModels *m_embeddingModels;
+    EmbeddingModels *m_installedEmbeddingModels;
     InstalledModels *m_installedModels;
     DownloadableModels *m_downloadableModels;
     QList<ModelInfo*> m_models;
@@ -489,7 +494,7 @@ private:
 
 protected:
     explicit ModelList();
-    ~ModelList() {}
+    ~ModelList() { for (auto *model: m_models) { delete model; } }
     friend class MyModelList;
 };
 

gpt4all-chat/mysettings.cpp

@@ -22,6 +22,7 @@ static int      default_localDocsRetrievalSize  = 3;
 static bool     default_localDocsShowReferences = true;
 static QString  default_networkAttribution      = "";
 static bool     default_networkIsActive         = false;
+static int      default_networkPort         = 4891;
 static bool     default_networkUsageStatsActive = false;
 static QString  default_device              = "Auto";
 
@@ -107,6 +108,7 @@ void MySettings::restoreApplicationDefaults()
     setThreadCount(default_threadCount);
     setSaveChatsContext(default_saveChatsContext);
     setServerChat(default_serverChat);
+    setNetworkPort(default_networkPort);
     setModelPath(defaultLocalModelsPath());
     setUserDefaultModel(default_userDefaultModel);
     setForceMetal(default_forceMetal);
@@ -678,6 +680,24 @@ void MySettings::setServerChat(bool b)
     emit serverChatChanged();
 }
 
+int MySettings::networkPort() const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value("networkPort", default_networkPort).toInt();
+}
+
+void MySettings::setNetworkPort(int c)
+{
+    if (networkPort() == c)
+        return;
+
+    QSettings setting;
+    setting.setValue("networkPort", c);
+    setting.sync();
+    emit networkPortChanged();
+}
+
 QString MySettings::modelPath() const
 {
     QSettings setting;
@@ -890,15 +910,23 @@ bool MySettings::networkIsActive() const
     return setting.value("network/isActive", default_networkIsActive).toBool();
 }
 
+bool MySettings::isNetworkIsActiveSet() const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value("network/isActive").isValid();
+}
+
 void MySettings::setNetworkIsActive(bool b)
 {
-    if (networkIsActive() == b)
-        return;
-
     QSettings setting;
-    setting.setValue("network/isActive", b);
     setting.sync();
-    emit networkIsActiveChanged();
+    auto cur = setting.value("network/isActive");
+    if (!cur.isValid() || cur.toBool() != b) {
+        setting.setValue("network/isActive", b);
+        setting.sync();
+        emit networkIsActiveChanged();
+    }
 }
 
 bool MySettings::networkUsageStatsActive() const
@@ -908,13 +936,21 @@ bool MySettings::networkUsageStatsActive() const
     return setting.value("network/usageStatsActive", default_networkUsageStatsActive).toBool();
 }
 
+bool MySettings::isNetworkUsageStatsActiveSet() const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value("network/usageStatsActive").isValid();
+}
+
 void MySettings::setNetworkUsageStatsActive(bool b)
 {
-    if (networkUsageStatsActive() == b)
-        return;
-
     QSettings setting;
-    setting.setValue("network/usageStatsActive", b);
     setting.sync();
-    emit networkUsageStatsActiveChanged();
+    auto cur = setting.value("network/usageStatsActive");
+    if (!cur.isValid() || cur.toBool() != b) {
+        setting.setValue("network/usageStatsActive", b);
+        setting.sync();
+        emit networkUsageStatsActiveChanged();
+    }
 }

gpt4all-chat/mysettings.h

@@ -28,6 +28,7 @@ class MySettings : public QObject
     Q_PROPERTY(bool networkUsageStatsActive READ networkUsageStatsActive WRITE setNetworkUsageStatsActive NOTIFY networkUsageStatsActiveChanged)
     Q_PROPERTY(QString device READ device WRITE setDevice NOTIFY deviceChanged)
     Q_PROPERTY(QVector<QString> deviceList READ deviceList NOTIFY deviceListChanged)
+    Q_PROPERTY(int networkPort READ networkPort WRITE setNetworkPort NOTIFY networkPortChanged)
 
 public:
     static MySettings *globalInstance();
@@ -128,9 +129,13 @@ public:
     QString networkAttribution() const;
     void setNetworkAttribution(const QString &a);
     bool networkIsActive() const;
+    Q_INVOKABLE bool isNetworkIsActiveSet() const;
     void setNetworkIsActive(bool b);
     bool networkUsageStatsActive() const;
+    Q_INVOKABLE bool isNetworkUsageStatsActiveSet() const;
     void setNetworkUsageStatsActive(bool b);
+    int networkPort() const;
+    void setNetworkPort(int c);
 
     QVector<QString> deviceList() const;
     void setDeviceList(const QVector<QString> &deviceList);
@@ -164,6 +169,7 @@ Q_SIGNALS:
     void localDocsShowReferencesChanged();
     void networkAttributionChanged();
     void networkIsActiveChanged();
+    void networkPortChanged();
     void networkUsageStatsActiveChanged();
     void attemptModelLoadChanged();
     void deviceChanged();

gpt4all-chat/network.cpp

@@ -1,8 +1,13 @@
 #include "network.h"
-#include "llm.h"
+
 #include "chatlistmodel.h"
+#include "download.h"
+#include "llm.h"
+#include "localdocs.h"
 #include "mysettings.h"
 
+#include <cmath>
+
 #include <QCoreApplication>
 #include <QGuiApplication>
 #include <QUuid>
@@ -14,16 +19,55 @@
 
 //#define DEBUG
 
+static const char MIXPANEL_TOKEN[] = "ce362e568ddaee16ed243eaffb5860a2";
+
 #if defined(Q_OS_MAC)
+
 #include <sys/sysctl.h>
-std::string getCPUModel() {
+static QString getCPUModel() {
     char buffer[256];
     size_t bufferlen = sizeof(buffer);
     sysctlbyname("machdep.cpu.brand_string", &buffer, &bufferlen, NULL, 0);
-    return std::string(buffer);
+    return buffer;
 }
+
+#elif defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)
+
+#ifndef _MSC_VER
+static void get_cpuid(int level, int *regs) {
+    asm volatile("cpuid" : "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), "=d" (regs[3]) : "0" (level) : "memory");
+}
+#else
+#define get_cpuid(level, regs) __cpuid(regs, level)
 #endif
 
+static QString getCPUModel() {
+    int regs[12];
+
+    // EAX=800000000h: Get Highest Extended Function Implemented
+    get_cpuid(0x80000000, regs);
+    if (regs[0] < 0x80000004)
+        return "(unknown)";
+
+    // EAX=800000002h-800000004h: Processor Brand String
+    get_cpuid(0x80000002, regs);
+    get_cpuid(0x80000003, regs + 4);
+    get_cpuid(0x80000004, regs + 8);
+
+    char str[sizeof(regs) + 1];
+    memcpy(str, regs, sizeof(regs));
+    str[sizeof(regs)] = 0;
+
+    return QString(str).trimmed();
+}
+
+#else
+
+static QString getCPUModel() { return "(non-x86)"; }
+
+#endif
+
+
 class MyNetwork: public Network { };
 Q_GLOBAL_STATIC(MyNetwork, networkInstance)
 Network *Network::globalInstance()
@@ -33,40 +77,43 @@ Network *Network::globalInstance()
 
 Network::Network()
     : QObject{nullptr}
-    , m_shouldSendStartup(false)
 {
     QSettings settings;
     settings.sync();
     m_uniqueId = settings.value("uniqueId", generateUniqueId()).toString();
     settings.setValue("uniqueId", m_uniqueId);
     settings.sync();
-    connect(MySettings::globalInstance(), &MySettings::networkIsActiveChanged, this, &Network::handleIsActiveChanged);
-    connect(MySettings::globalInstance(), &MySettings::networkUsageStatsActiveChanged, this, &Network::handleUsageStatsActiveChanged);
-    if (MySettings::globalInstance()->networkIsActive())
+    m_sessionId = generateUniqueId();
+
+    // allow sendMixpanel to be called from any thread
+    connect(this, &Network::requestMixpanel, this, &Network::sendMixpanel, Qt::QueuedConnection);
+
+    const auto *mySettings = MySettings::globalInstance();
+    connect(mySettings, &MySettings::networkIsActiveChanged, this, &Network::handleIsActiveChanged);
+    connect(mySettings, &MySettings::networkUsageStatsActiveChanged, this, &Network::handleUsageStatsActiveChanged);
+
+    m_hasSentOptIn  = !Download::globalInstance()->isFirstStart() &&  mySettings->networkUsageStatsActive();
+    m_hasSentOptOut = !Download::globalInstance()->isFirstStart() && !mySettings->networkUsageStatsActive();
+
+    if (mySettings->networkIsActive())
         sendHealth();
-    if (MySettings::globalInstance()->networkUsageStatsActive())
-        sendIpify();
     connect(&m_networkManager, &QNetworkAccessManager::sslErrors, this,
         &Network::handleSslErrors);
 }
 
+// NOTE: this won't be useful until we make it possible to change this via the settings page
+void Network::handleUsageStatsActiveChanged()
+{
+    if (!MySettings::globalInstance()->networkUsageStatsActive())
+        m_sendUsageStats = false;
+}
+
 void Network::handleIsActiveChanged()
 {
     if (MySettings::globalInstance()->networkUsageStatsActive())
         sendHealth();
 }
 
-void Network::handleUsageStatsActiveChanged()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        sendOptOut();
-    else {
-        // model might be loaded already when user opt-in for first time
-        sendStartup();
-        sendIpify();
-    }
-}
-
 QString Network::generateUniqueId() const
 {
     return QUuid::createUuid().toString(QUuid::WithoutBraces);
@@ -167,8 +214,8 @@ void Network::handleSslErrors(QNetworkReply *reply, const QList<QSslError> &erro
 void Network::sendOptOut()
 {
     QJsonObject properties;
-    properties.insert("token", "ce362e568ddaee16ed243eaffb5860a2");
-    properties.insert("time", QDateTime::currentSecsSinceEpoch());
+    properties.insert("token", MIXPANEL_TOKEN);
+    properties.insert("time", QDateTime::currentMSecsSinceEpoch());
     properties.insert("distinct_id", m_uniqueId);
     properties.insert("$insert_id", generateUniqueId());
 
@@ -181,7 +228,7 @@ void Network::sendOptOut()
 
     QJsonDocument doc;
     doc.setArray(array);
-    sendMixpanel(doc.toJson(QJsonDocument::Compact), true /*isOptOut*/);
+    emit requestMixpanel(doc.toJson(QJsonDocument::Compact), true /*isOptOut*/);
 
 #if defined(DEBUG)
     printf("%s %s\n", qPrintable("opt_out"), qPrintable(doc.toJson(QJsonDocument::Indented)));
@@ -189,215 +236,76 @@ void Network::sendOptOut()
 #endif
 }
 
-void Network::sendModelLoaded()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("model_load");
-}
-
-void Network::sendResetContext(int conversationLength)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-
-    KeyValue kv;
-    kv.key = QString("length");
-    kv.value = QJsonValue(conversationLength);
-    sendMixpanelEvent("reset_context", QVector<KeyValue>{kv});
-}
-
 void Network::sendStartup()
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
+    const auto *mySettings = MySettings::globalInstance();
+    Q_ASSERT(mySettings->isNetworkUsageStatsActiveSet());
+    if (!mySettings->networkUsageStatsActive()) {
+        // send a single opt-out per session after the user has made their selections,
+        // unless this is a normal start (same version) and the user was already opted out
+        if (!m_hasSentOptOut) {
+            sendOptOut();
+            m_hasSentOptOut = true;
+        }
         return;
-    m_shouldSendStartup = true;
-    if (m_ipify.isEmpty())
-        return; // when it completes it will send
-    sendMixpanelEvent("startup");
+    }
+
+    // only chance to enable usage stats is at the start of a new session
+    m_sendUsageStats = true;
+
+    const auto *display = QGuiApplication::primaryScreen();
+    trackEvent("startup", {
+        {"$screen_dpi", std::round(display->physicalDotsPerInch())},
+        {"display", QString("%1x%2").arg(display->size().width()).arg(display->size().height())},
+        {"ram", LLM::globalInstance()->systemTotalRAMInGB()},
+        {"cpu", getCPUModel()},
+        {"datalake_active", mySettings->networkIsActive()},
+    });
+    sendIpify();
+
+    // mirror opt-out logic so the ratio can be used to infer totals
+    if (!m_hasSentOptIn) {
+        trackEvent("opt_in");
+        m_hasSentOptIn = true;
+    }
 }
 
-void Network::sendCheckForUpdates()
+void Network::trackChatEvent(const QString &ev, QVariantMap props)
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("check_for_updates");
+    const auto &curChat = ChatListModel::globalInstance()->currentChat();
+    if (!props.contains("model"))
+        props.insert("model", curChat->modelInfo().filename());
+    props.insert("actualDevice", curChat->device());
+    props.insert("doc_collections_enabled", curChat->collectionList().count());
+    props.insert("doc_collections_total", LocalDocs::globalInstance()->localDocsModel()->rowCount());
+    props.insert("datalake_active", MySettings::globalInstance()->networkIsActive());
+    props.insert("using_server", ChatListModel::globalInstance()->currentChat()->isServer());
+    trackEvent(ev, props);
 }
 
-void Network::sendModelDownloaderDialog()
+void Network::trackEvent(const QString &ev, const QVariantMap &props)
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("download_dialog");
-}
-
-void Network::sendInstallModel(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("install_model", QVector<KeyValue>{kv});
-}
-
-void Network::sendRemoveModel(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("remove_model", QVector<KeyValue>{kv});
-}
-
-void Network::sendDownloadStarted(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("download_started", QVector<KeyValue>{kv});
-}
-
-void Network::sendDownloadCanceled(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("download_canceled", QVector<KeyValue>{kv});
-}
-
-void Network::sendDownloadError(const QString &model, int code, const QString &errorString)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    KeyValue kvCode;
-    kvCode.key = QString("code");
-    kvCode.value = QJsonValue(code);
-    KeyValue kvError;
-    kvError.key = QString("error");
-    kvError.value = QJsonValue(errorString);
-    sendMixpanelEvent("download_error", QVector<KeyValue>{kv, kvCode, kvError});
-}
-
-void Network::sendDownloadFinished(const QString &model, bool success)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    KeyValue kvSuccess;
-    kvSuccess.key = QString("success");
-    kvSuccess.value = QJsonValue(success);
-    sendMixpanelEvent("download_finished", QVector<KeyValue>{kv, kvSuccess});
-}
-
-void Network::sendSettingsDialog()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("settings_dialog");
-}
-
-void Network::sendNetworkToggled(bool isActive)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("isActive");
-    kv.value = QJsonValue(isActive);
-    sendMixpanelEvent("network_toggled", QVector<KeyValue>{kv});
-}
-
-void Network::sendNewChat(int count)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("number_of_chats");
-    kv.value = QJsonValue(count);
-    sendMixpanelEvent("new_chat", QVector<KeyValue>{kv});
-}
-
-void Network::sendRemoveChat()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("remove_chat");
-}
-
-void Network::sendRenameChat()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("rename_chat");
-}
-
-void Network::sendChatStarted()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("chat_started");
-}
-
-void Network::sendRecalculatingContext(int conversationLength)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-
-    KeyValue kv;
-    kv.key = QString("length");
-    kv.value = QJsonValue(conversationLength);
-    sendMixpanelEvent("recalc_context", QVector<KeyValue>{kv});
-}
-
-void Network::sendNonCompatHardware()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("noncompat_hardware");
-}
-
-void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &values)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
+    if (!m_sendUsageStats)
         return;
 
     Q_ASSERT(ChatListModel::globalInstance()->currentChat());
     QJsonObject properties;
-    properties.insert("token", "ce362e568ddaee16ed243eaffb5860a2");
-    properties.insert("time", QDateTime::currentSecsSinceEpoch());
-    properties.insert("distinct_id", m_uniqueId);
+
+    properties.insert("token", MIXPANEL_TOKEN);
+    if (!props.contains("time"))
+        properties.insert("time", QDateTime::currentMSecsSinceEpoch());
+    properties.insert("distinct_id", m_uniqueId); // effectively a device ID
     properties.insert("$insert_id", generateUniqueId());
-    properties.insert("$os", QSysInfo::prettyProductName());
+
     if (!m_ipify.isEmpty())
         properties.insert("ip", m_ipify);
-    properties.insert("name", QCoreApplication::applicationName() + " v"
-        + QCoreApplication::applicationVersion());
-    properties.insert("model", ChatListModel::globalInstance()->currentChat()->modelInfo().filename());
-    properties.insert("requestedDevice", MySettings::globalInstance()->device());
-    properties.insert("actualDevice", ChatListModel::globalInstance()->currentChat()->device());
 
-    // Some additional startup information
-    if (ev == "startup") {
-        const QSize display = QGuiApplication::primaryScreen()->size();
-        properties.insert("display", QString("%1x%2").arg(display.width()).arg(display.height()));
-        properties.insert("ram", LLM::globalInstance()->systemTotalRAMInGB());
-#if defined(Q_OS_MAC)
-        properties.insert("cpu", QString::fromStdString(getCPUModel()));
-#endif
-    }
+    properties.insert("$os", QSysInfo::prettyProductName());
+    properties.insert("session_id", m_sessionId);
+    properties.insert("name", QCoreApplication::applicationName() + " v" + QCoreApplication::applicationVersion());
 
-    for (const auto& p : values)
-        properties.insert(p.key, p.value);
+    for (const auto &[key, value]: props.asKeyValueRange())
+        properties.insert(key, QJsonValue::fromVariant(value));
 
     QJsonObject event;
     event.insert("event", ev);
@@ -408,7 +316,7 @@ void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &valu
 
     QJsonDocument doc;
     doc.setArray(array);
-    sendMixpanel(doc.toJson(QJsonDocument::Compact));
+    emit requestMixpanel(doc.toJson(QJsonDocument::Compact));
 
 #if defined(DEBUG)
     printf("%s %s\n", qPrintable(ev), qPrintable(doc.toJson(QJsonDocument::Indented)));
@@ -418,7 +326,7 @@ void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &valu
 
 void Network::sendIpify()
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive() || !m_ipify.isEmpty())
+    if (!m_sendUsageStats || !m_ipify.isEmpty())
         return;
 
     QUrl ipifyUrl("https://api.ipify.org");
@@ -433,7 +341,7 @@ void Network::sendIpify()
 
 void Network::sendMixpanel(const QByteArray &json, bool isOptOut)
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive() && !isOptOut)
+    if (!m_sendUsageStats)
         return;
 
     QUrl trackUrl("https://api.mixpanel.com/track");
@@ -449,7 +357,6 @@ void Network::sendMixpanel(const QByteArray &json, bool isOptOut)
 
 void Network::handleIpifyFinished()
 {
-    Q_ASSERT(MySettings::globalInstance()->networkUsageStatsActive());
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply)
         return;
@@ -462,15 +369,14 @@ void Network::handleIpifyFinished()
         qWarning() << "ERROR: ipify invalid response.";
     if (code != 200)
         qWarning() << "ERROR: ipify response != 200 code:" << code;
-    m_ipify = qPrintable(reply->readAll());
+    m_ipify = reply->readAll();
 #if defined(DEBUG)
-    printf("ipify finished %s\n", m_ipify.toLatin1().constData());
+    printf("ipify finished %s\n", qPrintable(m_ipify));
     fflush(stdout);
 #endif
     reply->deleteLater();
 
-    if (m_shouldSendStartup)
-        sendStartup();
+    trackEvent("ipify_complete");
 }
 
 void Network::handleMixpanelFinished()

gpt4all-chat/network.h

@@ -19,31 +19,15 @@ public:
 
     Q_INVOKABLE QString generateUniqueId() const;
     Q_INVOKABLE bool sendConversation(const QString &ingestId, const QString &conversation);
+    Q_INVOKABLE void trackChatEvent(const QString &event, QVariantMap props = QVariantMap());
+    Q_INVOKABLE void trackEvent(const QString &event, const QVariantMap &props = QVariantMap());
 
 Q_SIGNALS:
     void healthCheckFailed(int code);
+    void requestMixpanel(const QByteArray &json, bool isOptOut = false);
 
 public Q_SLOTS:
-    void sendOptOut();
-    void sendModelLoaded();
     void sendStartup();
-    void sendCheckForUpdates();
-    Q_INVOKABLE void sendModelDownloaderDialog();
-    Q_INVOKABLE void sendResetContext(int conversationLength);
-    void sendInstallModel(const QString &model);
-    void sendRemoveModel(const QString &model);
-    void sendDownloadStarted(const QString &model);
-    void sendDownloadCanceled(const QString &model);
-    void sendDownloadError(const QString &model, int code, const QString &errorString);
-    void sendDownloadFinished(const QString &model, bool success);
-    Q_INVOKABLE void sendSettingsDialog();
-    Q_INVOKABLE void sendNetworkToggled(bool active);
-    Q_INVOKABLE void sendNewChat(int count);
-    Q_INVOKABLE void sendRemoveChat();
-    Q_INVOKABLE void sendRenameChat();
-    Q_INVOKABLE void sendNonCompatHardware();
-    void sendChatStarted();
-    void sendRecalculatingContext(int conversationLength);
 
 private Q_SLOTS:
     void handleIpifyFinished();
@@ -53,18 +37,21 @@ private Q_SLOTS:
     void handleMixpanelFinished();
     void handleIsActiveChanged();
     void handleUsageStatsActiveChanged();
+    void sendMixpanel(const QByteArray &json, bool isOptOut);
 
 private:
+    void sendOptOut();
     void sendHealth();
     void sendIpify();
-    void sendMixpanelEvent(const QString &event, const QVector<KeyValue> &values = QVector<KeyValue>());
-    void sendMixpanel(const QByteArray &json, bool isOptOut = false);
     bool packageAndSendJson(const QString &ingestId, const QString &json);
 
 private:
-    bool m_shouldSendStartup;
+    bool m_sendUsageStats = false;
+    bool m_hasSentOptIn;
+    bool m_hasSentOptOut;
     QString m_ipify;
     QString m_uniqueId;
+    QString m_sessionId;
     QNetworkAccessManager m_networkManager;
     QVector<QNetworkReply*> m_activeUploads;
 

gpt4all-chat/qml/ApplicationSettings.qml

@@ -255,9 +255,40 @@ MySettingsTab {
             ToolTip.text: qsTr("WARNING: This enables the gui to act as a local REST web server(OpenAI API compliant) for API requests and will increase your RAM usage as well")
             ToolTip.visible: hovered
         }
-        Rectangle {
+        MySettingsLabel {
+            id: serverPortLabel
+            text: qsTr("API Server Port (Requires restart):")
             Layout.row: 9
             Layout.column: 0
+        }
+        MyTextField {
+            id: serverPortField
+            text: MySettings.networkPort
+            color: theme.textColor
+            font.pixelSize: theme.fontSizeLarge
+            ToolTip.text: qsTr("Api server port. WARNING: You need to restart the application for it to take effect")
+            ToolTip.visible: hovered
+            Layout.row: 9
+            Layout.column: 1
+            validator: IntValidator {
+                bottom: 1
+            }
+            onEditingFinished: {
+                var val = parseInt(text)
+                if (!isNaN(val)) {
+                    MySettings.networkPort = val
+                    focus = false
+                } else {
+                    text = MySettings.networkPort
+                }
+            }
+            Accessible.role: Accessible.EditableText
+            Accessible.name: serverPortField.text
+            Accessible.description: ToolTip.text
+        }
+        Rectangle {
+            Layout.row: 10
+            Layout.column: 0
             Layout.columnSpan: 3
             Layout.fillWidth: true
             height: 3

gpt4all-chat/qml/ChatDrawer.qml

@@ -9,9 +9,8 @@ import download
 import network
 import mysettings
 
-Drawer {
+Rectangle {
     id: chatDrawer
-    modal: false
 
     Theme {
         id: theme
@@ -20,10 +19,7 @@ Drawer {
     signal downloadClicked
     signal aboutClicked
 
-    background: Rectangle {
-        height: parent.height
-        color: theme.containerBackground
-    }
+    color: theme.containerBackground
 
     Item {
         anchors.fill: parent
@@ -44,7 +40,7 @@ Drawer {
             Accessible.description: qsTr("Create a new chat")
             onClicked: {
                 ChatListModel.addChat();
-                Network.sendNewChat(ChatListModel.count)
+                Network.trackEvent("new_chat", {"number_of_chats": ChatListModel.count})
             }
         }
 
@@ -94,7 +90,7 @@ Drawer {
                         wrapMode: Text.NoWrap
                         hoverEnabled: false // Disable hover events on the TextArea
                         selectByMouse: false // Disable text selection in the TextArea
-                        font.pixelSize: theme.fontSizeLarger
+                        font.pixelSize: theme.fontSizeLarge
                         text: readOnly ? metrics.elidedText : name
                         horizontalAlignment: TextInput.AlignLeft
                         opacity: trashQuestionDisplayed ? 0.5 : 1.0
@@ -114,8 +110,8 @@ Drawer {
                             // having focus
                             if (chatName.readOnly)
                                 return;
+                            Network.trackChatEvent("rename_chat")
                             changeName();
-                            Network.sendRenameChat()
                         }
                         function changeName() {
                             ChatListModel.get(index).name = chatName.text
@@ -198,8 +194,8 @@ Drawer {
                                     color: "transparent"
                                 }
                                 onClicked: {
+                                    Network.trackChatEvent("remove_chat")
                                     ChatListModel.removeChat(ChatListModel.get(index))
-                                    Network.sendRemoveChat()
                                 }
                                 Accessible.role: Accessible.Button
                                 Accessible.name: qsTr("Confirm chat deletion")

gpt4all-chat/qml/LocalDocsSettings.qml

@@ -14,7 +14,7 @@ MySettingsTab {
         MySettings.restoreLocalDocsDefaults();
     }
 
-    property bool hasEmbeddingModel: ModelList.embeddingModels.count !== 0
+    property bool hasEmbeddingModel: ModelList.installedEmbeddingModels.count !== 0
     showAdvancedSettingsButton: hasEmbeddingModel
     showRestoreDefaultsButton: hasEmbeddingModel