python: add list_gpus to the GPT4All API (#2194)

Other changes:
* fix memory leak in llmodel_available_gpu_devices
* drop model argument from llmodel_available_gpu_devices
* breaking: make GPT4All/Embed4All arguments past model_name keyword-only

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/continue_config.yml

@@ -370,13 +370,13 @@ jobs:
       - run:
           name: Make Documentation
           command: |
-            cd gpt4all-bindings/python/
+            cd gpt4all-bindings/python
             mkdocs build
       - run:
           name: Deploy Documentation
           command: |
-            cd gpt4all-bindings/python/
-            aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat
+            cd gpt4all-bindings/python
+            aws s3 sync --delete site/ s3://docs.gpt4all.io/
       - run:
           name: Invalidate docs.gpt4all.io cloudfront
           command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"

gpt4all-backend/llmodel.cpp

@@ -213,9 +213,9 @@ LLModel *LLModel::Implementation::constructDefaultLlama() {
     return llama.get();
 }
 
-std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices() {
+std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices(size_t memoryRequired) {
     auto *llama = constructDefaultLlama();
-    if (llama) { return llama->availableGPUDevices(0); }
+    if (llama) { return llama->availableGPUDevices(memoryRequired); }
     return {};
 }
 

gpt4all-backend/llmodel.h

@@ -38,7 +38,7 @@ public:
         std::string_view buildVariant() const { return m_buildVariant; }
 
         static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto", int n_ctx = 2048);
-        static std::vector<GPUDevice> availableGPUDevices();
+        static std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0);
         static int32_t maxContextLength(const std::string &modelPath);
         static int32_t layerCount(const std::string &modelPath);
         static bool isEmbeddingModel(const std::string &modelPath);

gpt4all-backend/llmodel_c.cpp

@@ -4,6 +4,7 @@
 #include <cerrno>
 #include <cstring>
 #include <iostream>
+#include <memory>
 #include <optional>
 #include <utility>
 
@@ -221,28 +222,45 @@ const char *llmodel_get_implementation_search_path()
     return LLModel::Implementation::implementationsSearchPath().c_str();
 }
 
-struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices)
-{
-    auto *wrapper = static_cast<LLModelWrapper *>(model);
-    std::vector<LLModel::GPUDevice> devices = wrapper->llModel->availableGPUDevices(memoryRequired);
+// RAII wrapper around a C-style struct
+struct llmodel_gpu_device_cpp: llmodel_gpu_device {
+    llmodel_gpu_device_cpp() = default;
 
-    // Set the num_devices
+    llmodel_gpu_device_cpp(const llmodel_gpu_device_cpp  &) = delete;
+    llmodel_gpu_device_cpp(      llmodel_gpu_device_cpp &&) = delete;
+
+    const llmodel_gpu_device_cpp &operator=(const llmodel_gpu_device_cpp  &) = delete;
+          llmodel_gpu_device_cpp &operator=(      llmodel_gpu_device_cpp &&) = delete;
+
+    ~llmodel_gpu_device_cpp() {
+        free(const_cast<char *>(name));
+        free(const_cast<char *>(vendor));
+    }
+};
+
+static_assert(sizeof(llmodel_gpu_device_cpp) == sizeof(llmodel_gpu_device));
+
+struct llmodel_gpu_device *llmodel_available_gpu_devices(size_t memoryRequired, int *num_devices)
+{
+    static thread_local std::unique_ptr<llmodel_gpu_device_cpp[]> c_devices;
+
+    auto devices = LLModel::Implementation::availableGPUDevices(memoryRequired);
     *num_devices = devices.size();
 
-    if (*num_devices == 0) return nullptr;  // Return nullptr if no devices are found
+    if (devices.empty()) { return nullptr; /* no devices */ }
 
-    // Allocate memory for the output array
-    struct llmodel_gpu_device* output = (struct llmodel_gpu_device*) malloc(*num_devices * sizeof(struct llmodel_gpu_device));
-
-    for (int i = 0; i < *num_devices; i++) {
-        output[i].index = devices[i].index;
-        output[i].type = devices[i].type;
-        output[i].heapSize = devices[i].heapSize;
-        output[i].name = strdup(devices[i].name.c_str());  // Convert std::string to char* and allocate memory
-        output[i].vendor = strdup(devices[i].vendor.c_str());  // Convert std::string to char* and allocate memory
+    c_devices = std::make_unique<llmodel_gpu_device_cpp[]>(devices.size());
+    for (unsigned i = 0; i < devices.size(); i++) {
+        const auto &dev  =   devices[i];
+              auto &cdev = c_devices[i];
+        cdev.index    = dev.index;
+        cdev.type     = dev.type;
+        cdev.heapSize = dev.heapSize;
+        cdev.name     = strdup(dev.name.c_str());
+        cdev.vendor   = strdup(dev.vendor.c_str());
     }
 
-    return output;
+    return c_devices.get();
 }
 
 bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)

gpt4all-backend/llmodel_c.h

@@ -48,9 +48,9 @@ struct llmodel_prompt_context {
 };
 
 struct llmodel_gpu_device {
-    int index = 0;
-    int type = 0;           // same as VkPhysicalDeviceType
-    size_t heapSize = 0;
+    int index;
+    int type; // same as VkPhysicalDeviceType
+    size_t heapSize;
     const char * name;
     const char * vendor;
 };
@@ -241,9 +241,10 @@ const char *llmodel_get_implementation_search_path();
 
 /**
  * Get a list of available GPU devices given the memory required.
+ * @param memoryRequired The minimum amount of VRAM, in bytes
  * @return A pointer to an array of llmodel_gpu_device's whose number is given by num_devices.
  */
-struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices);
+struct llmodel_gpu_device* llmodel_available_gpu_devices(size_t memoryRequired, int* num_devices);
 
 /**
  * Initializes a GPU device based on a specified string criterion.

gpt4all-bindings/python/docs/gpt4all_nodejs.md

@@ -11,37 +11,116 @@ pnpm install gpt4all@latest
 
 ```
 
-The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
+## Contents
 
-*   New bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
-*   The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
-*   Everything should work out the box.
 *   See [API Reference](#api-reference)
+*   See [Examples](#api-example)
+*   See [Developing](#develop)
+*   GPT4ALL nodejs bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
+
+## Api Example
 
 ### Chat Completion
 
 ```js
-import { createCompletion, loadModel } from '../src/gpt4all.js'
+import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY, loadModel } from '../src/gpt4all.js'
 
-const model = await loadModel('mistral-7b-openorca.Q4_0.gguf', { verbose: true });
+const model = await loadModel( 'mistral-7b-openorca.gguf2.Q4_0.gguf', { verbose: true, device: 'gpu' });
 
-const response = await createCompletion(model, [
-    { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
-    { role : 'user', content: 'What is 1 + 1?'  } 
-]);
+const completion1 = await createCompletion(model, 'What is 1 + 1?', { verbose: true, })
+console.log(completion1.message)
 
+const completion2 = await createCompletion(model, 'And if we add two?', {  verbose: true  })
+console.log(completion2.message)
+
+model.dispose()
 ```
 
 ### Embedding
 
 ```js
-import { createEmbedding, loadModel } from '../src/gpt4all.js'
+import { loadModel, createEmbedding } from '../src/gpt4all.js'
 
-const model = await loadModel('ggml-all-MiniLM-L6-v2-f16', { verbose: true });
+const embedder = await loadModel("all-MiniLM-L6-v2-f16.gguf", { verbose: true, type: 'embedding'})
 
-const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional");
+console.log(createEmbedding(embedder, "Maybe Minecraft was the friends we made along the way"));
 ```
 
+### Chat Sessions
+
+```js
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession();
+
+await createCompletion(
+    chat,
+    "Why are bananas rather blue than bread at night sometimes?",
+    {
+        verbose: true,
+    }
+);
+await createCompletion(chat, "Are you sure?", { verbose: true, });
+
+```
+
+### Streaming responses
+
+```js
+import gpt from "../src/gpt4all.js";
+
+const model = await gpt.loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("### Stream:");
+const stream = gpt.createCompletionStream(model, "How are you?");
+stream.tokens.on("data", (data) => {
+    process.stdout.write(data);
+});
+//wait till stream finishes. We cannot continue until this one is done.
+await stream.result;
+process.stdout.write("\n");
+
+process.stdout.write("### Stream with pipe:");
+const stream2 = gpt.createCompletionStream(
+    model,
+    "Please say something nice about node streams."
+);
+stream2.tokens.pipe(process.stdout);
+await stream2.result;
+process.stdout.write("\n");
+
+console.log("done");
+model.dispose();
+```
+
+### Async Generators
+
+```js
+import gpt from "../src/gpt4all.js";
+
+const model = await gpt.loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("### Generator:");
+const gen = gpt.createCompletionGenerator(model, "Redstone in Minecraft is Turing Complete. Let that sink in. (let it in!)");
+for await (const chunk of gen) {
+    process.stdout.write(chunk);
+}
+
+process.stdout.write("\n");
+model.dispose();
+```
+
+## Develop
+
 ### Build Instructions
 
 *   binding.gyp is compile config
@@ -131,21 +210,27 @@ yarn test
 
 *   why your model may be spewing bull 💩
     *   The downloaded model is broken (just reinstall or download from official site)
-    *   That's it so far
+*   Your model is hanging after a call to generate tokens.
+    *   Is `nPast` set too high? This may cause your model to hang (03/16/2024), Linux Mint, Ubuntu 22.04
+*   Your GPU usage is still high after node.js exits.
+    *   Make sure to call `model.dispose()`!!!
 
 ### Roadmap
 
-This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
+This package has been stabilizing over time development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
 
+*   \[ ] Purely offline. Per the gui, which can be run completely offline, the bindings should be as well.
+*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
+    *   Should include prebuilds to avoid painful node-gyp errors
+*   \[x] createChatSession ( the python equivalent to create\_chat\_session )
+*   \[x] generateTokens, the new name for createTokenStream. As of 3.2.0, this is released but not 100% tested. Check spec/generator.mjs!
+*   \[x] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
 *   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
-*   \[ ] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
+*   \[x] generateTokens is the new name for this^
 *   \[x] proper unit testing (integrate with circle ci)
 *   \[x] publish to npm under alpha tag `gpt4all@alpha`
 *   \[x] have more people test on other platforms (mac tester needed)
 *   \[x] switch to new pluggable backend
-*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
-    *   Should include prebuilds to avoid painful node-gyp errors
-*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
 
 ### API Reference
 
@@ -153,144 +238,200 @@ This package is in active development, and breaking changes may happen until the
 
 ##### Table of Contents
 
-*   [ModelFile](#modelfile)
-    *   [gptj](#gptj)
-    *   [llama](#llama)
-    *   [mpt](#mpt)
-    *   [replit](#replit)
 *   [type](#type)
 *   [TokenCallback](#tokencallback)
+*   [ChatSessionOptions](#chatsessionoptions)
+    *   [systemPrompt](#systemprompt)
+    *   [messages](#messages)
+*   [initialize](#initialize)
+    *   [Parameters](#parameters)
+*   [generate](#generate)
+    *   [Parameters](#parameters-1)
 *   [InferenceModel](#inferencemodel)
+    *   [createChatSession](#createchatsession)
+        *   [Parameters](#parameters-2)
+    *   [generate](#generate-1)
+        *   [Parameters](#parameters-3)
     *   [dispose](#dispose)
 *   [EmbeddingModel](#embeddingmodel)
     *   [dispose](#dispose-1)
+*   [InferenceResult](#inferenceresult)
 *   [LLModel](#llmodel)
     *   [constructor](#constructor)
-        *   [Parameters](#parameters)
+        *   [Parameters](#parameters-4)
     *   [type](#type-1)
     *   [name](#name)
     *   [stateSize](#statesize)
     *   [threadCount](#threadcount)
     *   [setThreadCount](#setthreadcount)
-        *   [Parameters](#parameters-1)
-    *   [raw\_prompt](#raw_prompt)
-        *   [Parameters](#parameters-2)
+        *   [Parameters](#parameters-5)
+    *   [infer](#infer)
+        *   [Parameters](#parameters-6)
     *   [embed](#embed)
-        *   [Parameters](#parameters-3)
+        *   [Parameters](#parameters-7)
     *   [isModelLoaded](#ismodelloaded)
     *   [setLibraryPath](#setlibrarypath)
-        *   [Parameters](#parameters-4)
+        *   [Parameters](#parameters-8)
     *   [getLibraryPath](#getlibrarypath)
     *   [initGpuByString](#initgpubystring)
-        *   [Parameters](#parameters-5)
+        *   [Parameters](#parameters-9)
     *   [hasGpuDevice](#hasgpudevice)
     *   [listGpu](#listgpu)
-        *   [Parameters](#parameters-6)
+        *   [Parameters](#parameters-10)
     *   [dispose](#dispose-2)
 *   [GpuDevice](#gpudevice)
     *   [type](#type-2)
 *   [LoadModelOptions](#loadmodeloptions)
-*   [loadModel](#loadmodel)
-    *   [Parameters](#parameters-7)
-*   [createCompletion](#createcompletion)
-    *   [Parameters](#parameters-8)
-*   [createEmbedding](#createembedding)
-    *   [Parameters](#parameters-9)
-*   [CompletionOptions](#completionoptions)
+    *   [modelPath](#modelpath)
+    *   [librariesPath](#librariespath)
+    *   [modelConfigFile](#modelconfigfile)
+    *   [allowDownload](#allowdownload)
     *   [verbose](#verbose)
-    *   [systemPromptTemplate](#systemprompttemplate)
-    *   [promptTemplate](#prompttemplate)
-    *   [promptHeader](#promptheader)
-    *   [promptFooter](#promptfooter)
-*   [PromptMessage](#promptmessage)
+    *   [device](#device)
+    *   [nCtx](#nctx)
+    *   [ngl](#ngl)
+*   [loadModel](#loadmodel)
+    *   [Parameters](#parameters-11)
+*   [InferenceProvider](#inferenceprovider)
+*   [createCompletion](#createcompletion)
+    *   [Parameters](#parameters-12)
+*   [createCompletionStream](#createcompletionstream)
+    *   [Parameters](#parameters-13)
+*   [createCompletionGenerator](#createcompletiongenerator)
+    *   [Parameters](#parameters-14)
+*   [createEmbedding](#createembedding)
+    *   [Parameters](#parameters-15)
+*   [CompletionOptions](#completionoptions)
+    *   [verbose](#verbose-1)
+    *   [onToken](#ontoken)
+*   [Message](#message)
     *   [role](#role)
     *   [content](#content)
 *   [prompt\_tokens](#prompt_tokens)
 *   [completion\_tokens](#completion_tokens)
 *   [total\_tokens](#total_tokens)
+*   [n\_past\_tokens](#n_past_tokens)
 *   [CompletionReturn](#completionreturn)
     *   [model](#model)
     *   [usage](#usage)
-    *   [choices](#choices)
-*   [CompletionChoice](#completionchoice)
-    *   [message](#message)
+    *   [message](#message-1)
+*   [CompletionStreamReturn](#completionstreamreturn)
 *   [LLModelPromptContext](#llmodelpromptcontext)
     *   [logitsSize](#logitssize)
     *   [tokensSize](#tokenssize)
     *   [nPast](#npast)
-    *   [nCtx](#nctx)
     *   [nPredict](#npredict)
+    *   [promptTemplate](#prompttemplate)
+    *   [nCtx](#nctx-1)
     *   [topK](#topk)
     *   [topP](#topp)
-    *   [temp](#temp)
+    *   [minP](#minp)
+    *   [temperature](#temperature)
     *   [nBatch](#nbatch)
     *   [repeatPenalty](#repeatpenalty)
     *   [repeatLastN](#repeatlastn)
     *   [contextErase](#contexterase)
-*   [generateTokens](#generatetokens)
-    *   [Parameters](#parameters-10)
 *   [DEFAULT\_DIRECTORY](#default_directory)
 *   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
 *   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
 *   [DEFAULT\_PROMPT\_CONTEXT](#default_prompt_context)
 *   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
 *   [downloadModel](#downloadmodel)
-    *   [Parameters](#parameters-11)
+    *   [Parameters](#parameters-16)
     *   [Examples](#examples)
 *   [DownloadModelOptions](#downloadmodeloptions)
-    *   [modelPath](#modelpath)
-    *   [verbose](#verbose-1)
+    *   [modelPath](#modelpath-1)
+    *   [verbose](#verbose-2)
     *   [url](#url)
     *   [md5sum](#md5sum)
 *   [DownloadController](#downloadcontroller)
     *   [cancel](#cancel)
     *   [promise](#promise)
 
-#### ModelFile
-
-Full list of models available
-DEPRECATED!! These model names are outdated and this type will not be maintained, please use a string literal instead
-
-##### gptj
-
-List of GPT-J Models
-
-Type: (`"ggml-gpt4all-j-v1.3-groovy.bin"` | `"ggml-gpt4all-j-v1.2-jazzy.bin"` | `"ggml-gpt4all-j-v1.1-breezy.bin"` | `"ggml-gpt4all-j.bin"`)
-
-##### llama
-
-List Llama Models
-
-Type: (`"ggml-gpt4all-l13b-snoozy.bin"` | `"ggml-vicuna-7b-1.1-q4_2.bin"` | `"ggml-vicuna-13b-1.1-q4_2.bin"` | `"ggml-wizardLM-7B.q4_2.bin"` | `"ggml-stable-vicuna-13B.q4_2.bin"` | `"ggml-nous-gpt4-vicuna-13b.bin"` | `"ggml-v3-13b-hermes-q5_1.bin"`)
-
-##### mpt
-
-List of MPT Models
-
-Type: (`"ggml-mpt-7b-base.bin"` | `"ggml-mpt-7b-chat.bin"` | `"ggml-mpt-7b-instruct.bin"`)
-
-##### replit
-
-List of Replit Models
-
-Type: `"ggml-replit-code-v1-3b.bin"`
-
 #### type
 
 Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
 
-Type: ModelType
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
 
 #### TokenCallback
 
-Callback for controlling token generation
+Callback for controlling token generation. Return false to stop token generation.
 
 Type: function (tokenId: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), token: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String), total: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
 
+#### ChatSessionOptions
+
+**Extends Partial\<LLModelPromptContext>**
+
+Options for the chat session.
+
+##### systemPrompt
+
+System prompt to ingest on initialization.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### messages
+
+Messages to ingest on initialization.
+
+Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[Message](#message)>
+
+#### initialize
+
+Ingests system prompt and initial messages.
+Sets this chat session as the active chat session of the model.
+
+##### Parameters
+
+*   `options` **[ChatSessionOptions](#chatsessionoptions)** The options for the chat session.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<void>**&#x20;
+
+#### generate
+
+Prompts the model in chat-session context.
+
+##### Parameters
+
+*   `prompt` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
+*   `options` **[CompletionOptions](#completionoptions)?** Prompt context and other options.
+*   `callback` **[TokenCallback](#tokencallback)?** Token generation callback.
+
+<!---->
+
+*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the chat session is not the active chat session of the model.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[CompletionReturn](#completionreturn)>** The model's response to the prompt.
+
 #### InferenceModel
 
 InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
 
+##### createChatSession
+
+Create a chat session with the model.
+
+###### Parameters
+
+*   `options` **[ChatSessionOptions](#chatsessionoptions)?** The options for the chat session.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<ChatSession>** The chat session.
+
+##### generate
+
+Prompts the model with a given input and optional parameters.
+
+###### Parameters
+
+*   `prompt` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
+*   `options` **[CompletionOptions](#completionoptions)?** Prompt context and other options.
+*   `callback` **[TokenCallback](#tokencallback)?** Token generation callback.
+*   `input`  The prompt input.
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[CompletionReturn](#completionreturn)>** The model's response to the prompt.
+
 ##### dispose
 
 delete and cleanup the native model
@@ -307,6 +448,10 @@ delete and cleanup the native model
 
 Returns **void**&#x20;
 
+#### InferenceResult
+
+Shape of LLModel's inference result.
+
 #### LLModel
 
 LLModel class representing a language model.
@@ -326,9 +471,9 @@ Initialize a new LLModel.
 
 ##### type
 
-either 'gpt', mpt', or 'llama' or undefined
+undefined or user supplied
 
-Returns **(ModelType | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
+Returns **([string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String) | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
 
 ##### name
 
@@ -360,7 +505,7 @@ Set the number of threads used for model inference.
 
 Returns **void**&#x20;
 
-##### raw\_prompt
+##### infer
 
 Prompt the model with a given input and optional parameters.
 This is the raw output from model.
@@ -368,23 +513,20 @@ Use the prompt function exported for a value
 
 ###### Parameters
 
-*   `q` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
-*   `params` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
+*   `prompt` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
+*   `promptContext` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
 *   `callback` **[TokenCallback](#tokencallback)?** optional callback to control token generation.
 
-Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The result of the model prompt.
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[InferenceResult](#inferenceresult)>** The result of the model prompt.
 
 ##### embed
 
 Embed text with the model. Keep in mind that
-not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
 Use the prompt function exported for a value
 
 ###### Parameters
 
-*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-*   `q`  The prompt input.
-*   `params`  Optional parameters for the prompt context.
+*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
 
 Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The result of the model prompt.
 
@@ -462,6 +604,62 @@ Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 Options that configure a model's behavior.
 
+##### modelPath
+
+Where to look for model files.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### librariesPath
+
+Where to look for the backend libraries.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### modelConfigFile
+
+The path to the model configuration file, useful for offline usage or custom model configurations.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### allowDownload
+
+Whether to allow downloading the model if it is not present at the specified path.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### verbose
+
+Enable verbose logging.
+
+Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
+##### device
+
+The processing unit on which the model will run. It can be set to
+
+*   "cpu": Model will run on the central processing unit.
+*   "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
+*   "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
+*   "gpu name": Model will run on the GPU that matches the name if it's available.
+    Note: If a GPU device lacks sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All
+    instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the
+    model.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### nCtx
+
+The Maximum window size of this model
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### ngl
+
+Number of gpu layers needed
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
 #### loadModel
 
 Loads a machine learning model with the specified name. The defacto way to create a model.
@@ -474,18 +672,46 @@ By default this will download a model from the official GPT4ALL website, if a mo
 
 Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<([InferenceModel](#inferencemodel) | [EmbeddingModel](#embeddingmodel))>** A promise that resolves to an instance of the loaded LLModel.
 
+#### InferenceProvider
+
+Interface for inference, implemented by InferenceModel and ChatSession.
+
 #### createCompletion
 
 The nodejs equivalent to python binding's chat\_completion
 
 ##### Parameters
 
-*   `model` **[InferenceModel](#inferencemodel)** The language model object.
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
+*   `provider` **[InferenceProvider](#inferenceprovider)** The inference model object or chat session
+*   `message` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The user input message
 *   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
 
 Returns **[CompletionReturn](#completionreturn)** The completion result.
 
+#### createCompletionStream
+
+Streaming variant of createCompletion, returns a stream of tokens and a promise that resolves to the completion result.
+
+##### Parameters
+
+*   `provider` **[InferenceProvider](#inferenceprovider)** The inference model object or chat session
+*   `message` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The user input message.
+*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
+
+Returns **[CompletionStreamReturn](#completionstreamreturn)** An object of token stream and the completion result promise.
+
+#### createCompletionGenerator
+
+Creates an async generator of tokens
+
+##### Parameters
+
+*   `provider` **[InferenceProvider](#inferenceprovider)** The inference model object or chat session
+*   `message` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The user input message.
+*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
+
+Returns **AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The stream of generated tokens
+
 #### createEmbedding
 
 The nodejs moral equivalent to python binding's Embed4All().embed()
@@ -510,34 +736,15 @@ Indicates if verbose logging is enabled.
 
 Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
 
-##### systemPromptTemplate
+##### onToken
 
-Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
-Note that if this is not defined, system messages will not be included in the prompt.
+Callback for controlling token generation. Return false to stop processing.
 
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+Type: [TokenCallback](#tokencallback)
 
-##### promptTemplate
+#### Message
 
-Template for user messages, with %1 being replaced by the message.
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### promptHeader
-
-The initial instruction for the model, on top of the prompt
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### promptFooter
-
-The last instruction for the model, appended to the end of the prompt.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### PromptMessage
-
-A message in the conversation, identical to OpenAI's chat message.
+A message in the conversation.
 
 ##### role
 
@@ -553,7 +760,7 @@ Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 #### prompt\_tokens
 
-The number of tokens used in the prompt.
+The number of tokens used in the prompt. Currently not available and always 0.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
@@ -565,13 +772,19 @@ Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 #### total\_tokens
 
-The total number of tokens used.
+The total number of tokens used. Currently not available and always 0.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### n\_past\_tokens
+
+Number of tokens used in the conversation.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
 #### CompletionReturn
 
-The result of the completion, similar to OpenAI's format.
+The result of a completion.
 
 ##### model
 
@@ -583,23 +796,17 @@ Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 
 Token usage report.
 
-Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
-
-##### choices
-
-The generated completions.
-
-Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[CompletionChoice](#completionchoice)>
-
-#### CompletionChoice
-
-A completion choice, similar to OpenAI's format.
+Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), n\_past\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
 
 ##### message
 
-Response message
+The generated completion.
 
-Type: [PromptMessage](#promptmessage)
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+#### CompletionStreamReturn
+
+The result of a streamed completion, containing a stream of tokens and a promise that resolves to the completion result.
 
 #### LLModelPromptContext
 
@@ -620,18 +827,29 @@ Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Globa
 ##### nPast
 
 The number of tokens in the past conversation.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nCtx
-
-The number of tokens possible in the context window.
+This controls how far back the model looks when generating completions.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
 ##### nPredict
 
-The number of tokens to predict.
+The maximum number of tokens to predict.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### promptTemplate
+
+Template for user / assistant message pairs.
+%1 is required and will be replaced by the user input.
+%2 is optional and will be replaced by the assistant response.
+
+Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
+
+##### nCtx
+
+The context window size. Do not use, it has no effect. See loadModel options.
+THIS IS DEPRECATED!!!
+Use loadModel's nCtx option instead.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
@@ -654,12 +872,16 @@ above a threshold P. This method, also known as nucleus sampling, finds a balanc
 and quality by considering both token probabilities and the number of tokens available for sampling.
 When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
 On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
-The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
-for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
-##### temp
+##### minP
+
+The minimum probability of a token to be considered.
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+##### temperature
 
 The temperature to adjust the model's output distribution.
 Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
@@ -704,19 +926,6 @@ The percentage of context to erase if the context window is exceeded.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
-#### generateTokens
-
-Creates an async generator of tokens
-
-##### Parameters
-
-*   `llmodel` **[InferenceModel](#inferencemodel)** The language model object.
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
-*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
-*   `callback` **[TokenCallback](#tokencallback)** optional callback to control token generation.
-
-Returns **AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The stream of generated tokens
-
 #### DEFAULT\_DIRECTORY
 
 From python api:
@@ -759,7 +968,7 @@ By default this downloads without waiting. use the controller returned to alter
 ##### Parameters
 
 *   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The model to be downloaded.
-*   `options` **DownloadOptions** to pass into the downloader. Default is { location: (cwd), verbose: false }.
+*   `options` **[DownloadModelOptions](#downloadmodeloptions)** to pass into the downloader. Default is { location: (cwd), verbose: false }.
 
 ##### Examples
 

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -9,7 +9,7 @@ import sys
 import threading
 from enum import Enum
 from queue import Queue
-from typing import Any, Callable, Generic, Iterable, TypeVar, overload
+from typing import Any, Callable, Generic, Iterable, NoReturn, TypeVar, overload
 
 if sys.version_info >= (3, 9):
     import importlib.resources as importlib_resources
@@ -138,7 +138,7 @@ llmodel.llmodel_threadCount.restype = ctypes.c_int32
 
 llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).encode())
 
-llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
+llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
 llmodel.llmodel_available_gpu_devices.restype = ctypes.POINTER(LLModelGPUDevice)
 
 llmodel.llmodel_gpu_init_gpu_device_by_string.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_char_p]
@@ -200,42 +200,53 @@ class LLModel:
         if model is None:
             s = err.value
             raise RuntimeError(f"Unable to instantiate model: {'null' if s is None else s.decode()}")
-        self.model = model
+        self.model: ctypes.c_void_p | None = model
 
     def __del__(self, llmodel=llmodel):
         if hasattr(self, 'model'):
-            llmodel.llmodel_model_destroy(self.model)
+            self.close()
 
-    def _list_gpu(self, mem_required: int) -> list[LLModelGPUDevice]:
+    def close(self) -> None:
+        if self.model is not None:
+            llmodel.llmodel_model_destroy(self.model)
+            self.model = None
+
+    def _raise_closed(self) -> NoReturn:
+        raise ValueError("Attempted operation on a closed LLModel")
+
+    @staticmethod
+    def list_gpus(mem_required: int = 0) -> list[str]:
+        """
+        List the names of the available GPU devices with at least `mem_required` bytes of VRAM.
+
+        Args:
+            mem_required: The minimum amount of VRAM, in bytes
+
+        Returns:
+            A list of strings representing the names of the available GPU devices.
+        """
         num_devices = ctypes.c_int32(0)
-        devices_ptr = llmodel.llmodel_available_gpu_devices(self.model, mem_required, ctypes.byref(num_devices))
+        devices_ptr = llmodel.llmodel_available_gpu_devices(mem_required, ctypes.byref(num_devices))
         if not devices_ptr:
             raise ValueError("Unable to retrieve available GPU devices")
-        return devices_ptr[:num_devices.value]
+        return [d.name.decode() for d in devices_ptr[:num_devices.value]]
 
     def init_gpu(self, device: str):
+        if self.model is None:
+            self._raise_closed()
+
         mem_required = llmodel.llmodel_required_mem(self.model, self.model_path, self.n_ctx, self.ngl)
 
         if llmodel.llmodel_gpu_init_gpu_device_by_string(self.model, mem_required, device.encode()):
             return
 
-        # Retrieve all GPUs without considering memory requirements.
-        num_devices = ctypes.c_int32(0)
-        all_devices_ptr = llmodel.llmodel_available_gpu_devices(self.model, 0, ctypes.byref(num_devices))
-        if not all_devices_ptr:
-            raise ValueError("Unable to retrieve list of all GPU devices")
-        all_gpus = [d.name.decode() for d in all_devices_ptr[:num_devices.value]]
-
-        # Retrieve GPUs that meet the memory requirements using list_gpu
-        available_gpus = [device.name.decode() for device in self._list_gpu(mem_required)]
-
-        # Identify GPUs that are unavailable due to insufficient memory or features
+        all_gpus = self.list_gpus()
+        available_gpus = self.list_gpus(mem_required)
         unavailable_gpus = set(all_gpus).difference(available_gpus)
 
-        # Formulate the error message
-        error_msg = "Unable to initialize model on GPU: '{}'.".format(device)
-        error_msg += "\nAvailable GPUs: {}.".format(available_gpus)
-        error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}.".format(unavailable_gpus)
+        error_msg = "Unable to initialize model on GPU: {!r}".format(device)
+        error_msg += "\nAvailable GPUs: {}".format(available_gpus)
+        error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}".format(unavailable_gpus)
         raise ValueError(error_msg)
 
     def load_model(self) -> bool:
@@ -246,14 +257,21 @@ class LLModel:
         -------
         True if model loaded successfully, False otherwise
         """
+        if self.model is None:
+            self._raise_closed()
+
         return llmodel.llmodel_loadModel(self.model, self.model_path, self.n_ctx, self.ngl)
 
     def set_thread_count(self, n_threads):
+        if self.model is None:
+            self._raise_closed()
         if not llmodel.llmodel_isModelLoaded(self.model):
             raise Exception("Model not loaded")
         llmodel.llmodel_setThreadCount(self.model, n_threads)
 
     def thread_count(self):
+        if self.model is None:
+            self._raise_closed()
         if not llmodel.llmodel_isModelLoaded(self.model):
             raise Exception("Model not loaded")
         return llmodel.llmodel_threadCount(self.model)
@@ -322,6 +340,9 @@ class LLModel:
         if not text:
             raise ValueError("text must not be None or empty")
 
+        if self.model is None:
+            self._raise_closed()
+
         if (single_text := isinstance(text, str)):
             text = [text]
 
@@ -387,6 +408,9 @@ class LLModel:
         None
         """
 
+        if self.model is None:
+            self._raise_closed()
+
         self.buffer.clear()
         self.buff_expecting_cont_bytes = 0
 
@@ -419,6 +443,9 @@ class LLModel:
     def prompt_model_streaming(
         self, prompt: str, prompt_template: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
     ) -> Iterable[str]:
+        if self.model is None:
+            self._raise_closed()
+
         output_queue: Queue[str | Sentinel] = Queue()
 
         # Put response tokens into an output queue

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -11,6 +11,7 @@ import time
 import warnings
 from contextlib import contextmanager
 from pathlib import Path
+from types import TracebackType
 from typing import TYPE_CHECKING, Any, Iterable, Literal, Protocol, overload
 
 import requests
@@ -18,11 +19,10 @@ from requests.exceptions import ChunkedEncodingError
 from tqdm import tqdm
 from urllib3.exceptions import IncompleteRead, ProtocolError
 
-from . import _pyllmodel
-from ._pyllmodel import EmbedResult as EmbedResult
+from ._pyllmodel import EmbedResult as EmbedResult, LLModel, ResponseCallbackType, empty_response_callback
 
 if TYPE_CHECKING:
-    from typing_extensions import TypeAlias
+    from typing_extensions import Self, TypeAlias
 
 if sys.platform == 'darwin':
     import fcntl
@@ -43,16 +43,30 @@ class Embed4All:
 
     MIN_DIMENSIONALITY = 64
 
-    def __init__(self, model_name: str | None = None, n_threads: int | None = None, **kwargs):
+    def __init__(self, model_name: str | None = None, *, n_threads: int | None = None, device: str | None = "cpu", **kwargs: Any):
         """
         Constructor
 
         Args:
             n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
+            device: The processing unit on which the embedding model will run. See the `GPT4All` constructor for more info.
+            kwargs: Remaining keyword arguments are passed to the `GPT4All` constructor.
         """
         if model_name is None:
             model_name = 'all-MiniLM-L6-v2.gguf2.f16.gguf'
-        self.gpt4all = GPT4All(model_name, n_threads=n_threads, **kwargs)
+        self.gpt4all = GPT4All(model_name, n_threads=n_threads, device=device, **kwargs)
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self, typ: type[BaseException] | None, value: BaseException | None, tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def close(self) -> None:
+        """Delete the model instance and free associated system resources."""
+        self.gpt4all.close()
 
     # return_dict=False
     @overload
@@ -144,6 +158,7 @@ class GPT4All:
     def __init__(
         self,
         model_name: str,
+        *,
         model_path: str | os.PathLike[str] | None = None,
         model_type: str | None = None,
         allow_download: bool = True,
@@ -168,7 +183,7 @@ class GPT4All:
                 - "cpu": Model will run on the central processing unit.
                 - "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
                 - "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
-                Alternatively, a specific GPU name can also be provided, and the model will run on the GPU that matches the name if it's available.
+                - A specific device name from the list returned by `GPT4All.list_gpus()`.
                 Default is "cpu".
 
                 Note: If a selected GPU device does not have sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the model.
@@ -179,7 +194,7 @@ class GPT4All:
         self.model_type = model_type
         # Retrieve model and download if allowed
         self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download, verbose=verbose)
-        self.model = _pyllmodel.LLModel(self.config["path"], n_ctx, ngl)
+        self.model = LLModel(self.config["path"], n_ctx, ngl)
         if device is not None and device != "cpu":
             self.model.init_gpu(device)
         self.model.load_model()
@@ -190,6 +205,18 @@ class GPT4All:
         self._history: list[MessageType] | None = None
         self._current_prompt_template: str = "{0}"
 
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self, typ: type[BaseException] | None, value: BaseException | None, tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def close(self) -> None:
+        """Delete the model instance and free associated system resources."""
+        self.model.close()
+
     @property
     def current_chat_session(self) -> list[MessageType] | None:
         return None if self._history is None else list(self._history)
@@ -394,19 +421,19 @@ class GPT4All:
     def generate(
         self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
         min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
-        n_predict: int | None = ..., streaming: Literal[False] = ..., callback: _pyllmodel.ResponseCallbackType = ...,
+        n_predict: int | None = ..., streaming: Literal[False] = ..., callback: ResponseCallbackType = ...,
     ) -> str: ...
     @overload
     def generate(
         self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
         min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
-        n_predict: int | None = ..., streaming: Literal[True], callback: _pyllmodel.ResponseCallbackType = ...,
+        n_predict: int | None = ..., streaming: Literal[True], callback: ResponseCallbackType = ...,
     ) -> Iterable[str]: ...
     @overload
     def generate(
         self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
         min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
-        n_predict: int | None = ..., streaming: bool, callback: _pyllmodel.ResponseCallbackType = ...,
+        n_predict: int | None = ..., streaming: bool, callback: ResponseCallbackType = ...,
     ) -> Any: ...
 
     def generate(
@@ -423,7 +450,7 @@ class GPT4All:
         n_batch: int = 8,
         n_predict: int | None = None,
         streaming: bool = False,
-        callback: _pyllmodel.ResponseCallbackType = _pyllmodel.empty_response_callback,
+        callback: ResponseCallbackType = empty_response_callback,
     ) -> Any:
         """
         Generate outputs from any GPT4All model.
@@ -469,7 +496,7 @@ class GPT4All:
                 if reset:
                     # ingest system prompt
                     self.model.prompt_model(self._history[0]["content"], "%1",
-                                            _pyllmodel.empty_response_callback,
+                                            empty_response_callback,
                                             n_batch=n_batch, n_predict=0, special=True)
                 prompt_template = self._current_prompt_template.format("%1", "%2")
             else:
@@ -498,9 +525,9 @@ class GPT4All:
             output_collector = self._history
 
         def _callback_wrapper(
-            callback: _pyllmodel.ResponseCallbackType,
+            callback: ResponseCallbackType,
             output_collector: list[MessageType],
-        ) -> _pyllmodel.ResponseCallbackType:
+        ) -> ResponseCallbackType:
             def _callback(token_id: int, response: str) -> bool:
                 nonlocal callback, output_collector
 
@@ -564,6 +591,16 @@ class GPT4All:
             self._history = None
             self._current_prompt_template = "{0}"
 
+    @staticmethod
+    def list_gpus() -> list[str]:
+        """
+        List the names of the available GPU devices.
+
+        Returns:
+            A list of strings representing the names of the available GPU devices.
+        """
+        return LLModel.list_gpus()
+
     def _format_chat_prompt_template(
         self,
         messages: list[MessageType],
@@ -573,6 +610,9 @@ class GPT4All:
         """
         Helper method for building a prompt from list of messages using the self._current_prompt_template as a template for each message.
 
+        Warning:
+            This function was deprecated in version 2.3.0, and will be removed in a future release.
+
         Args:
             messages:  List of dictionaries. Each dictionary should have a "role" key
                 with value of "system", "assistant", or "user" and a "content" key with a

gpt4all-bindings/python/setup.py

@@ -68,7 +68,7 @@ def get_long_description():
 
 setup(
     name=package_name,
-    version="2.3.1",
+    version="2.4.0",
     description="Python bindings for GPT4All",
     long_description=get_long_description(),
     long_description_content_type="text/markdown",

gpt4all-bindings/typescript/.clang-format

@@ -0,0 +1,4 @@
+---
+Language: Cpp
+BasedOnStyle: Microsoft
+ColumnLimit: 120

gpt4all-bindings/typescript/README.md

@@ -10,45 +10,170 @@ npm install gpt4all@latest
 pnpm install gpt4all@latest
 
 ```
-
-The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
-
-*   New bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
-*   The nodejs api has made strides to mirror the python api. It is not 100% mirrored, but many pieces of the api resemble its python counterpart.
-*   Everything should work out the box.
+## Breaking changes in version 4!!
+*   See [Transition](#changes)
+## Contents
 *   See [API Reference](#api-reference)
-
+*   See [Examples](#api-example)
+*   See [Developing](#develop)
+*   GPT4ALL nodejs bindings created by [jacoobes](https://github.com/jacoobes), [limez](https://github.com/iimez) and the [nomic ai community](https://home.nomic.ai), for all to use.
+*   [spare change](https://github.com/sponsors/jacoobes) for a college student? 🤑
+## Api Examples
 ### Chat Completion
 
+Use a chat session to keep context between completions. This is useful for efficient back and forth conversations.
+
 ```js
-import { createCompletion, loadModel } from '../src/gpt4all.js'
+import { createCompletion, loadModel } from "../src/gpt4all.js";
 
-const model = await loadModel('mistral-7b-openorca.Q4_0.gguf', { verbose: true });
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true, // logs loaded model configuration
+    device: "gpu", // defaults to 'cpu'
+    nCtx: 2048, // the maximum sessions context window size.
+});
 
-const response = await createCompletion(model, [
-    { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
-    { role : 'user', content: 'What is 1 + 1?'  } 
+// initialize a chat session on the model. a model instance can have only one chat session at a time.
+const chat = await model.createChatSession({
+    // any completion options set here will be used as default for all completions in this chat session
+    temperature: 0.8,
+    // a custom systemPrompt can be set here. note that the template depends on the model.
+    // if unset, the systemPrompt that comes with the model will be used.
+    systemPrompt: "### System:\nYou are an advanced mathematician.\n\n",
+});
+
+// create a completion using a string as input
+const res1 = await createCompletion(chat, "What is 1 + 1?");
+console.debug(res1.choices[0].message);
+
+// multiple messages can be input to the conversation at once.
+// note that if the last message is not of role 'user', an empty message will be returned.
+await createCompletion(chat, [
+    {
+        role: "user",
+        content: "What is 2 + 2?",
+    },
+    {
+        role: "assistant",
+        content: "It's 5.",
+    },
 ]);
 
+const res3 = await createCompletion(chat, "Could you recalculate that?");
+console.debug(res3.choices[0].message);
+
+model.dispose();
+```
+
+### Stateless usage
+You can use the model without a chat session. This is useful for one-off completions.
+
+```js
+import { createCompletion, loadModel } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf");
+
+// createCompletion methods can also be used on the model directly.
+// context is not maintained between completions.
+const res1 = await createCompletion(model, "What is 1 + 1?");
+console.debug(res1.choices[0].message);
+
+// a whole conversation can be input as well.
+// note that if the last message is not of role 'user', an error will be thrown.
+const res2 = await createCompletion(model, [
+    {
+        role: "user",
+        content: "What is 2 + 2?",
+    },
+    {
+        role: "assistant",
+        content: "It's 5.",
+    },
+    {
+        role: "user",
+        content: "Could you recalculate that?",
+    },
+]);
+console.debug(res2.choices[0].message);
+
 ```
 
 ### Embedding
 
 ```js
-import { createEmbedding, loadModel } from '../src/gpt4all.js'
+import { loadModel, createEmbedding } from '../src/gpt4all.js'
 
-const model = await loadModel('ggml-all-MiniLM-L6-v2-f16', { verbose: true });
+const embedder = await loadModel("nomic-embed-text-v1.5.f16.gguf", { verbose: true, type: 'embedding'})
 
-const fltArray = createEmbedding(model, "Pain is inevitable, suffering optional");
+console.log(createEmbedding(embedder, "Maybe Minecraft was the friends we made along the way"));
 ```
 
+### Streaming responses
+```js
+import { loadModel, createCompletionStream } from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("Output: ");
+const stream = createCompletionStream(model, "How are you?");
+stream.tokens.on("data", (data) => {
+    process.stdout.write(data);
+});
+//wait till stream finishes. We cannot continue until this one is done.
+await stream.result;
+process.stdout.write("\n");
+model.dispose();
+
+```
+
+### Async Generators
+```js
+import { loadModel, createCompletionGenerator } from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf");
+
+process.stdout.write("Output: ");
+const gen = createCompletionGenerator(
+    model,
+    "Redstone in Minecraft is Turing Complete. Let that sink in. (let it in!)"
+);
+for await (const chunk of gen) {
+    process.stdout.write(chunk);
+}
+
+process.stdout.write("\n");
+model.dispose();
+
+```
+### Offline usage
+do this b4 going offline
+```sh
+curl -L https://gpt4all.io/models/models3.json -o ./models3.json
+```
+```js
+import { createCompletion, loadModel } from 'gpt4all'
+
+//make sure u downloaded the models before going offline!
+const model = await loadModel('mistral-7b-openorca.gguf2.Q4_0.gguf', {
+    verbose: true,
+    device: 'gpu',
+    modelConfigFile: "./models3.json"
+});
+
+await createCompletion(model, 'What is 1 + 1?', { verbose: true })
+
+model.dispose();
+```
+
+## Develop
 ### Build Instructions
 
-*   binding.gyp is compile config
+*   `binding.gyp` is compile config
 *   Tested on Ubuntu. Everything seems to work fine
 *   Tested on Windows. Everything works fine.
 *   Sparse testing on mac os.
-*   MingW works as well to build the gpt4all-backend. **HOWEVER**, this package works only with MSVC built dlls.
+*   MingW script works to build the gpt4all-backend. We left it there just in case. **HOWEVER**, this package works only with MSVC built dlls.
 
 ### Requirements
 
@@ -76,23 +201,18 @@ cd gpt4all-bindings/typescript
 *   To Build and Rebuild:
 
 ```sh
-yarn
+node scripts/prebuild.js
 ```
 *   llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
 
 ```sh
-git submodule update --init --depth 1 --recursive
+git submodule update --init --recursive
 ```
 
 ```sh
 yarn build:backend
 ```
-
-This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native The only current way to use them is to put them in the current working directory of your application. That is, **WHEREVER YOU RUN YOUR NODE APPLICATION**
-
-*   llama-xxxx.dll is required.
-*   According to whatever model you are using, you'll need to select the proper model loader.
-    *   For example, if you running an Mosaic MPT model, you will need to select the mpt-(buildvariant).(dynamiclibrary)
+This will build platform-dependent dynamic libraries, and will be located in runtimes/(platform)/native
 
 ### Test
 
@@ -130,17 +250,20 @@ yarn test
 
 *   why your model may be spewing bull 💩
     *   The downloaded model is broken (just reinstall or download from official site)
-    *   That's it so far
+*   Your model is hanging after a call to generate tokens.
+    * Is `nPast` set too high? This may cause your model to hang (03/16/2024), Linux Mint, Ubuntu 22.04
+*  Your GPU usage is still high after node.js exits.
+    * Make sure to call `model.dispose()`!!!
 
 ### Roadmap
 
-This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
+This package has been stabilizing over time development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
 
 *   \[ ] Purely offline. Per the gui, which can be run completely offline, the bindings should be as well.
 *   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
     *   Should include prebuilds to avoid painful node-gyp errors
-*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
-*   \[x] generateTokens, the new name for createTokenStream. As of 3.2.0, this is released but not 100% tested. Check spec/generator.mjs! 
+*   \[x] createChatSession ( the python equivalent to create\_chat\_session )
+*   \[x] generateTokens, the new name for createTokenStream. As of 3.2.0, this is released but not 100% tested. Check spec/generator.mjs!
 *   \[x] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
 *   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
 *   \[x] generateTokens is the new name for this^
@@ -149,5 +272,13 @@ This package is in active development, and breaking changes may happen until the
 *   \[x] have more people test on other platforms (mac tester needed)
 *   \[x] switch to new pluggable backend
 
+## Changes
+This repository serves as the new bindings for nodejs users.
+- If you were a user of [these bindings](https://github.com/nomic-ai/gpt4all-ts), they are outdated.
+- Version 4 includes the follow breaking changes
+    * `createEmbedding` & `EmbeddingModel.embed()` returns an object, `EmbeddingResult`, instead of a float32array.
+    * Removed deprecated types `ModelType` and `ModelFile`
+    * Removed deprecated initiation of model by string path only
+
 
 ### API Reference

gpt4all-bindings/typescript/binding.ci.gyp

@@ -6,12 +6,12 @@
         "<!@(node -p \"require('node-addon-api').include\")",
         "gpt4all-backend",
       ],
-      "sources": [ 
+      "sources": [
         # PREVIOUS VERSION: had to required the sources, but with newest changes do not need to
         #"../../gpt4all-backend/llama.cpp/examples/common.cpp",
         #"../../gpt4all-backend/llama.cpp/ggml.c",
         #"../../gpt4all-backend/llama.cpp/llama.cpp",
-        # "../../gpt4all-backend/utils.cpp", 
+        # "../../gpt4all-backend/utils.cpp",
         "gpt4all-backend/llmodel_c.cpp",
         "gpt4all-backend/llmodel.cpp",
         "prompt.cc",
@@ -40,7 +40,7 @@
                     "AdditionalOptions": [
                         "/std:c++20",
                         "/EHsc",
-                  ], 
+                  ],
                 },
             },
         }],

gpt4all-bindings/typescript/binding.gyp

@@ -6,12 +6,12 @@
         "<!@(node -p \"require('node-addon-api').include\")",
         "../../gpt4all-backend",
       ],
-      "sources": [ 
+      "sources": [
         # PREVIOUS VERSION: had to required the sources, but with newest changes do not need to
         #"../../gpt4all-backend/llama.cpp/examples/common.cpp",
         #"../../gpt4all-backend/llama.cpp/ggml.c",
         #"../../gpt4all-backend/llama.cpp/llama.cpp",
-        # "../../gpt4all-backend/utils.cpp", 
+        # "../../gpt4all-backend/utils.cpp",
         "../../gpt4all-backend/llmodel_c.cpp",
         "../../gpt4all-backend/llmodel.cpp",
         "prompt.cc",
@@ -40,7 +40,7 @@
                     "AdditionalOptions": [
                         "/std:c++20",
                         "/EHsc",
-                  ], 
+                  ],
                 },
             },
         }],

gpt4all-bindings/typescript/index.cc

@@ -1,175 +1,171 @@
 #include "index.h"
+#include "napi.h"
 
-
-Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
-    Napi::Function self = DefineClass(env, "LLModel", {
-       InstanceMethod("type",  &NodeModelWrapper::GetType),
-       InstanceMethod("isModelLoaded", &NodeModelWrapper::IsModelLoaded),
-       InstanceMethod("name", &NodeModelWrapper::GetName),
-       InstanceMethod("stateSize", &NodeModelWrapper::StateSize),
-       InstanceMethod("raw_prompt", &NodeModelWrapper::Prompt),
-       InstanceMethod("setThreadCount", &NodeModelWrapper::SetThreadCount),
-       InstanceMethod("embed", &NodeModelWrapper::GenerateEmbedding),
-       InstanceMethod("threadCount", &NodeModelWrapper::ThreadCount),
-       InstanceMethod("getLibraryPath", &NodeModelWrapper::GetLibraryPath),
-       InstanceMethod("initGpuByString", &NodeModelWrapper::InitGpuByString),
-       InstanceMethod("hasGpuDevice", &NodeModelWrapper::HasGpuDevice),
-       InstanceMethod("listGpu", &NodeModelWrapper::GetGpuDevices),
-       InstanceMethod("memoryNeeded", &NodeModelWrapper::GetRequiredMemory),
-       InstanceMethod("dispose", &NodeModelWrapper::Dispose)
-    });
+Napi::Function NodeModelWrapper::GetClass(Napi::Env env)
+{
+    Napi::Function self = DefineClass(env, "LLModel",
+                                      {InstanceMethod("type", &NodeModelWrapper::GetType),
+                                       InstanceMethod("isModelLoaded", &NodeModelWrapper::IsModelLoaded),
+                                       InstanceMethod("name", &NodeModelWrapper::GetName),
+                                       InstanceMethod("stateSize", &NodeModelWrapper::StateSize),
+                                       InstanceMethod("infer", &NodeModelWrapper::Infer),
+                                       InstanceMethod("setThreadCount", &NodeModelWrapper::SetThreadCount),
+                                       InstanceMethod("embed", &NodeModelWrapper::GenerateEmbedding),
+                                       InstanceMethod("threadCount", &NodeModelWrapper::ThreadCount),
+                                       InstanceMethod("getLibraryPath", &NodeModelWrapper::GetLibraryPath),
+                                       InstanceMethod("initGpuByString", &NodeModelWrapper::InitGpuByString),
+                                       InstanceMethod("hasGpuDevice", &NodeModelWrapper::HasGpuDevice),
+                                       InstanceMethod("listGpu", &NodeModelWrapper::GetGpuDevices),
+                                       InstanceMethod("memoryNeeded", &NodeModelWrapper::GetRequiredMemory),
+                                       InstanceMethod("dispose", &NodeModelWrapper::Dispose)});
     // Keep a static reference to the constructor
     //
-    Napi::FunctionReference* constructor = new Napi::FunctionReference();
+    Napi::FunctionReference *constructor = new Napi::FunctionReference();
     *constructor = Napi::Persistent(self);
     env.SetInstanceData(constructor);
     return self;
 }
-Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info) 
+Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo &info)
 {
     auto env = info.Env();
-    return Napi::Number::New(env, static_cast<uint32_t>(llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers) ));
-
+    return Napi::Number::New(
+        env, static_cast<uint32_t>(llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers)));
 }
-  Napi::Value NodeModelWrapper::GetGpuDevices(const Napi::CallbackInfo& info) 
-  {
+Napi::Value NodeModelWrapper::GetGpuDevices(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
     int num_devices = 0;
     auto mem_size = llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers);
-    llmodel_gpu_device* all_devices = llmodel_available_gpu_devices(GetInference(), mem_size, &num_devices);
-    if(all_devices == nullptr) {
-        Napi::Error::New(
-            env, 
-            "Unable to retrieve list of all GPU devices"
-        ).ThrowAsJavaScriptException(); 
+    llmodel_gpu_device *all_devices = llmodel_available_gpu_devices(mem_size, &num_devices);
+    if (all_devices == nullptr)
+    {
+        Napi::Error::New(env, "Unable to retrieve list of all GPU devices").ThrowAsJavaScriptException();
         return env.Undefined();
     }
     auto js_array = Napi::Array::New(env, num_devices);
-    for(int i = 0; i < num_devices; ++i) {
-       auto gpu_device = all_devices[i];
-       /* 
-        *
-        * struct llmodel_gpu_device {
-            int index = 0;
-            int type = 0;           // same as VkPhysicalDeviceType
-            size_t heapSize = 0; 
-            const char * name;
-            const char * vendor;
-          };
-        *
-        */
-       Napi::Object js_gpu_device = Napi::Object::New(env);
+    for (int i = 0; i < num_devices; ++i)
+    {
+        auto gpu_device = all_devices[i];
+        /*
+         *
+         * struct llmodel_gpu_device {
+             int index = 0;
+             int type = 0;           // same as VkPhysicalDeviceType
+             size_t heapSize = 0;
+             const char * name;
+             const char * vendor;
+           };
+         *
+         */
+        Napi::Object js_gpu_device = Napi::Object::New(env);
         js_gpu_device["index"] = uint32_t(gpu_device.index);
         js_gpu_device["type"] = uint32_t(gpu_device.type);
-        js_gpu_device["heapSize"] = static_cast<uint32_t>( gpu_device.heapSize );
-        js_gpu_device["name"]= gpu_device.name;
+        js_gpu_device["heapSize"] = static_cast<uint32_t>(gpu_device.heapSize);
+        js_gpu_device["name"] = gpu_device.name;
         js_gpu_device["vendor"] = gpu_device.vendor;
 
         js_array[i] = js_gpu_device;
     }
     return js_array;
-  }
+}
 
-  Napi::Value NodeModelWrapper::GetType(const Napi::CallbackInfo& info) 
-  {
-    if(type.empty()) {
+Napi::Value NodeModelWrapper::GetType(const Napi::CallbackInfo &info)
+{
+    if (type.empty())
+    {
         return info.Env().Undefined();
-    } 
+    }
     return Napi::String::New(info.Env(), type);
-  }
+}
 
-  Napi::Value NodeModelWrapper::InitGpuByString(const Napi::CallbackInfo& info) 
-  {
+Napi::Value NodeModelWrapper::InitGpuByString(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
     size_t memory_required = static_cast<size_t>(info[0].As<Napi::Number>().Uint32Value());
-    
-    std::string gpu_device_identifier = info[1].As<Napi::String>();   
+
+    std::string gpu_device_identifier = info[1].As<Napi::String>();
 
     size_t converted_value;
-    if(memory_required <= std::numeric_limits<size_t>::max()) {
+    if (memory_required <= std::numeric_limits<size_t>::max())
+    {
         converted_value = static_cast<size_t>(memory_required);
-    } else {
-         Napi::Error::New(
-            env, 
-            "invalid number for memory size. Exceeded bounds for memory."
-        ).ThrowAsJavaScriptException(); 
+    }
+    else
+    {
+        Napi::Error::New(env, "invalid number for memory size. Exceeded bounds for memory.")
+            .ThrowAsJavaScriptException();
         return env.Undefined();
     }
-    
+
     auto result = llmodel_gpu_init_gpu_device_by_string(GetInference(), converted_value, gpu_device_identifier.c_str());
     return Napi::Boolean::New(env, result);
-  }
-  Napi::Value NodeModelWrapper::HasGpuDevice(const Napi::CallbackInfo& info) 
-  {
+}
+Napi::Value NodeModelWrapper::HasGpuDevice(const Napi::CallbackInfo &info)
+{
     return Napi::Boolean::New(info.Env(), llmodel_has_gpu_device(GetInference()));
-  }
+}
 
-  NodeModelWrapper::NodeModelWrapper(const Napi::CallbackInfo& info) : Napi::ObjectWrap<NodeModelWrapper>(info) 
-  {
+NodeModelWrapper::NodeModelWrapper(const Napi::CallbackInfo &info) : Napi::ObjectWrap<NodeModelWrapper>(info)
+{
     auto env = info.Env();
-    fs::path model_path;
+    auto config_object = info[0].As<Napi::Object>();
 
-    std::string full_weight_path,
-                library_path = ".",
-                model_name, 
-                device;
-    if(info[0].IsString()) {
-        model_path = info[0].As<Napi::String>().Utf8Value();
-        full_weight_path = model_path.string();
-        std::cout << "DEPRECATION: constructor accepts object now. Check docs for more.\n";
-    } else {
-        auto config_object = info[0].As<Napi::Object>();
-        model_name = config_object.Get("model_name").As<Napi::String>();
-        model_path = config_object.Get("model_path").As<Napi::String>().Utf8Value(); 
-        if(config_object.Has("model_type")) {
-            type = config_object.Get("model_type").As<Napi::String>(); 
-        }
-        full_weight_path = (model_path / fs::path(model_name)).string();
-        
-        if(config_object.Has("library_path")) {
-            library_path = config_object.Get("library_path").As<Napi::String>(); 
-        } else {
-            library_path = ".";
-        }
-        device = config_object.Get("device").As<Napi::String>();
+    // sets the directory where models (gguf files) are to be searched
+    llmodel_set_implementation_search_path(
+        config_object.Has("library_path") ? config_object.Get("library_path").As<Napi::String>().Utf8Value().c_str()
+                                          : ".");
 
-        nCtx = config_object.Get("nCtx").As<Napi::Number>().Int32Value();
-        nGpuLayers = config_object.Get("ngl").As<Napi::Number>().Int32Value();
-    }
-    llmodel_set_implementation_search_path(library_path.c_str());
-    const char* e;
+    std::string model_name = config_object.Get("model_name").As<Napi::String>();
+    fs::path model_path = config_object.Get("model_path").As<Napi::String>().Utf8Value();
+    std::string full_weight_path = (model_path / fs::path(model_name)).string();
+
+    name = model_name.empty() ? model_path.filename().string() : model_name;
+    full_model_path = full_weight_path;
+    nCtx = config_object.Get("nCtx").As<Napi::Number>().Int32Value();
+    nGpuLayers = config_object.Get("ngl").As<Napi::Number>().Int32Value();
+
+    const char *e;
     inference_ = llmodel_model_create2(full_weight_path.c_str(), "auto", &e);
-    if(!inference_) {
-       Napi::Error::New(env, e).ThrowAsJavaScriptException();
-       return;
+    if (!inference_)
+    {
+        Napi::Error::New(env, e).ThrowAsJavaScriptException();
+        return;
     }
-    if(GetInference() == nullptr) {
-       std::cerr << "Tried searching libraries in \"" << library_path << "\"" <<  std::endl;
-       std::cerr << "Tried searching for model weight in \"" << full_weight_path << "\"" << std::endl;
-       std::cerr << "Do you have runtime libraries installed?" << std::endl;
-       Napi::Error::New(env, "Had an issue creating llmodel object, inference is null").ThrowAsJavaScriptException(); 
-       return;
+    if (GetInference() == nullptr)
+    {
+        std::cerr << "Tried searching libraries in \"" << llmodel_get_implementation_search_path() << "\"" << std::endl;
+        std::cerr << "Tried searching for model weight in \"" << full_weight_path << "\"" << std::endl;
+        std::cerr << "Do you have runtime libraries installed?" << std::endl;
+        Napi::Error::New(env, "Had an issue creating llmodel object, inference is null").ThrowAsJavaScriptException();
+        return;
     }
-    if(device != "cpu") {
-        size_t mem = llmodel_required_mem(GetInference(), full_weight_path.c_str(),nCtx, nGpuLayers);
+
+    std::string device = config_object.Get("device").As<Napi::String>();
+    if (device != "cpu")
+    {
+        size_t mem = llmodel_required_mem(GetInference(), full_weight_path.c_str(), nCtx, nGpuLayers);
 
         auto success = llmodel_gpu_init_gpu_device_by_string(GetInference(), mem, device.c_str());
-        if(!success) {
-            //https://github.com/nomic-ai/gpt4all/blob/3acbef14b7c2436fe033cae9036e695d77461a16/gpt4all-bindings/python/gpt4all/pyllmodel.py#L215
-            //Haven't implemented this but it is still open to contribution
+        if (!success)
+        {
+            // https://github.com/nomic-ai/gpt4all/blob/3acbef14b7c2436fe033cae9036e695d77461a16/gpt4all-bindings/python/gpt4all/pyllmodel.py#L215
+            // Haven't implemented this but it is still open to contribution
             std::cout << "WARNING: Failed to init GPU\n";
         }
     }
 
     auto success = llmodel_loadModel(GetInference(), full_weight_path.c_str(), nCtx, nGpuLayers);
-    if(!success) {
-        Napi::Error::New(env, "Failed to load model at given path").ThrowAsJavaScriptException(); 
+    if (!success)
+    {
+        Napi::Error::New(env, "Failed to load model at given path").ThrowAsJavaScriptException();
         return;
     }
-
-    name = model_name.empty() ? model_path.filename().string() : model_name;
-    full_model_path = full_weight_path;
-  };
+    // optional
+    if (config_object.Has("model_type"))
+    {
+        type = config_object.Get("model_type").As<Napi::String>();
+    }
+};
 
 //  NodeModelWrapper::~NodeModelWrapper() {
 //    if(GetInference() != nullptr) {
@@ -182,177 +178,275 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
 //    if(inference_ != nullptr) {
 //        std::cout << "Debug: deleting model\n";
 //
-//    } 
+//    }
 //  }
-  Napi::Value NodeModelWrapper::IsModelLoaded(const Napi::CallbackInfo& info) {
+Napi::Value NodeModelWrapper::IsModelLoaded(const Napi::CallbackInfo &info)
+{
     return Napi::Boolean::New(info.Env(), llmodel_isModelLoaded(GetInference()));
-  }
+}
 
-  Napi::Value NodeModelWrapper::StateSize(const Napi::CallbackInfo& info) {
+Napi::Value NodeModelWrapper::StateSize(const Napi::CallbackInfo &info)
+{
     // Implement the binding for the stateSize method
     return Napi::Number::New(info.Env(), static_cast<int64_t>(llmodel_get_state_size(GetInference())));
-  }
-  
-  Napi::Value NodeModelWrapper::GenerateEmbedding(const Napi::CallbackInfo& info) {
+}
+
+Napi::Array ChunkedFloatPtr(float *embedding_ptr, int embedding_size, int text_len, Napi::Env const &env)
+{
+    auto n_embd = embedding_size / text_len;
+    // std::cout << "Embedding size: " << embedding_size << std::endl;
+    // std::cout << "Text length: " << text_len << std::endl;
+    // std::cout << "Chunk size (n_embd): " << n_embd << std::endl;
+    Napi::Array result = Napi::Array::New(env, text_len);
+    auto count = 0;
+    for (int i = 0; i < embedding_size; i += n_embd)
+    {
+        int end = std::min(i + n_embd, embedding_size);
+        // possible bounds error?
+        // Constructs a container with as many elements as the range [first,last), with each element emplace-constructed
+        // from its corresponding element in that range, in the same order.
+        std::vector<float> chunk(embedding_ptr + i, embedding_ptr + end);
+        Napi::Float32Array fltarr = Napi::Float32Array::New(env, chunk.size());
+        // I know there's a way to emplace the raw float ptr into a Napi::Float32Array but idk how and
+        //  im too scared to cause memory issues
+        //  this is goodenough
+        for (int j = 0; j < chunk.size(); j++)
+        {
+
+            fltarr.Set(j, chunk[j]);
+        }
+        result.Set(count++, fltarr);
+    }
+    return result;
+}
+
+Napi::Value NodeModelWrapper::GenerateEmbedding(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
-    std::string text = info[0].As<Napi::String>().Utf8Value();
-    size_t embedding_size = 0;
-    float* arr = llmodel_embedding(GetInference(), text.c_str(), &embedding_size);
-    if(arr == nullptr) {
-        Napi::Error::New(
-            env, 
-            "Cannot embed. native embedder returned 'nullptr'"
-        ).ThrowAsJavaScriptException(); 
+
+    auto prefix = info[1];
+    auto dimensionality = info[2].As<Napi::Number>().Int32Value();
+    auto do_mean = info[3].As<Napi::Boolean>().Value();
+    auto atlas = info[4].As<Napi::Boolean>().Value();
+    size_t embedding_size;
+    size_t token_count = 0;
+
+    // This procedure can maybe be optimized but its whatever, i have too many intermediary structures
+    std::vector<std::string> text_arr;
+    bool is_single_text = false;
+    if (info[0].IsString())
+    {
+        is_single_text = true;
+        text_arr.push_back(info[0].As<Napi::String>().Utf8Value());
+    }
+    else
+    {
+        auto jsarr = info[0].As<Napi::Array>();
+        size_t len = jsarr.Length();
+        text_arr.reserve(len);
+        for (size_t i = 0; i < len; ++i)
+        {
+            std::string str = jsarr.Get(i).As<Napi::String>().Utf8Value();
+            text_arr.push_back(str);
+        }
+    }
+    std::vector<const char *> str_ptrs;
+    str_ptrs.reserve(text_arr.size() + 1);
+    for (size_t i = 0; i < text_arr.size(); ++i)
+        str_ptrs.push_back(text_arr[i].c_str());
+    str_ptrs.push_back(nullptr);
+    const char *_err = nullptr;
+    float *embeds = llmodel_embed(GetInference(), str_ptrs.data(),  &embedding_size,
+                                  prefix.IsUndefined() ? nullptr : prefix.As<Napi::String>().Utf8Value().c_str(),
+                                  dimensionality, &token_count, do_mean, atlas, &_err);
+    if (!embeds)
+    {
+        // i dont wanna deal with c strings lol
+        std::string err(_err);
+        Napi::Error::New(env, err == "(unknown error)" ? "Unknown error: sorry bud" : err).ThrowAsJavaScriptException();
         return env.Undefined();
     }
+    auto embedmat = ChunkedFloatPtr(embeds, embedding_size, text_arr.size(), env);
 
-    if(embedding_size == 0 && text.size() != 0 ) {
-        std::cout << "Warning: embedding length 0 but input text length > 0" << std::endl;
-    }
-    Napi::Float32Array js_array = Napi::Float32Array::New(env, embedding_size);
-    
-    for (size_t i = 0; i < embedding_size; ++i) {
-        float element = *(arr + i);
-        js_array[i] = element;
+    llmodel_free_embedding(embeds);
+    auto res = Napi::Object::New(env);
+    res.Set("n_prompt_tokens", token_count);
+    if(is_single_text) {
+        res.Set("embeddings", embedmat.Get(static_cast<uint32_t>(0)));
+    } else {
+        res.Set("embeddings", embedmat);
     }
 
-    llmodel_free_embedding(arr);
-
-    return js_array;
-  }
+    return res;
+}
 
 /**
  * Generate a response using the model.
- * @param model A pointer to the llmodel_model instance.
  * @param prompt A string representing the input prompt.
- * @param prompt_callback A callback function for handling the processing of prompt.
- * @param response_callback A callback function for handling the generated response.
- * @param recalculate_callback A callback function for handling recalculation requests.
- * @param ctx A pointer to the llmodel_prompt_context structure.
+ * @param options Inference options.
  */
-  Napi::Value NodeModelWrapper::Prompt(const Napi::CallbackInfo& info) {
+Napi::Value NodeModelWrapper::Infer(const Napi::CallbackInfo &info)
+{
     auto env = info.Env();
-    std::string question;
-    if(info[0].IsString()) {
-        question = info[0].As<Napi::String>().Utf8Value();
-    } else {
+    std::string prompt;
+    if (info[0].IsString())
+    {
+        prompt = info[0].As<Napi::String>().Utf8Value();
+    }
+    else
+    {
         Napi::Error::New(info.Env(), "invalid string argument").ThrowAsJavaScriptException();
         return info.Env().Undefined();
     }
-    //defaults copied from python bindings
-    llmodel_prompt_context promptContext = {
-             .logits = nullptr,
-             .tokens = nullptr,
-             .n_past = 0,
-             .n_ctx = 1024,
-             .n_predict = 128,
-             .top_k = 40,
-             .top_p = 0.9f,
-             .min_p = 0.0f,
-             .temp = 0.72f,
-             .n_batch = 8,
-             .repeat_penalty = 1.0f,
-             .repeat_last_n = 10,
-             .context_erase = 0.5
-         };
-    
-    PromptWorkerConfig promptWorkerConfig;
 
-    if(info[1].IsObject())
-    {
-       auto inputObject = info[1].As<Napi::Object>();
-             
-        // Extract and assign the properties
-       if (inputObject.Has("logits") || inputObject.Has("tokens")) {
-           Napi::Error::New(info.Env(), "Invalid input: 'logits' or 'tokens' properties are not allowed").ThrowAsJavaScriptException();
-           return info.Env().Undefined();
-       }
-             // Assign the remaining properties
-       if(inputObject.Has("n_past")) 
-            promptContext.n_past = inputObject.Get("n_past").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("n_ctx")) 
-            promptContext.n_ctx = inputObject.Get("n_ctx").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("n_predict"))
-            promptContext.n_predict = inputObject.Get("n_predict").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("top_k"))
-            promptContext.top_k = inputObject.Get("top_k").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("top_p")) 
-            promptContext.top_p = inputObject.Get("top_p").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("min_p")) 
-            promptContext.min_p = inputObject.Get("min_p").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("temp")) 
-            promptContext.temp = inputObject.Get("temp").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("n_batch")) 
-            promptContext.n_batch = inputObject.Get("n_batch").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("repeat_penalty")) 
-            promptContext.repeat_penalty = inputObject.Get("repeat_penalty").As<Napi::Number>().FloatValue();
-       if(inputObject.Has("repeat_last_n")) 
-            promptContext.repeat_last_n = inputObject.Get("repeat_last_n").As<Napi::Number>().Int32Value();
-       if(inputObject.Has("context_erase")) 
-            promptContext.context_erase = inputObject.Get("context_erase").As<Napi::Number>().FloatValue();
-    }
-    else
+    if (!info[1].IsObject())
     {
         Napi::Error::New(info.Env(), "Missing Prompt Options").ThrowAsJavaScriptException();
         return info.Env().Undefined();
     }
+    // defaults copied from python bindings
+    llmodel_prompt_context promptContext = {.logits = nullptr,
+                                            .tokens = nullptr,
+                                            .n_past = 0,
+                                            .n_ctx = nCtx,
+                                            .n_predict = 4096,
+                                            .top_k = 40,
+                                            .top_p = 0.9f,
+                                            .min_p = 0.0f,
+                                            .temp = 0.1f,
+                                            .n_batch = 8,
+                                            .repeat_penalty = 1.2f,
+                                            .repeat_last_n = 10,
+                                            .context_erase = 0.75};
 
-    if(info.Length() >= 3 && info[2].IsFunction()){
-        promptWorkerConfig.bHasTokenCallback = true;
-        promptWorkerConfig.tokenCallback = info[2].As<Napi::Function>();
+    PromptWorkerConfig promptWorkerConfig;
+
+    auto inputObject = info[1].As<Napi::Object>();
+
+    if (inputObject.Has("logits") || inputObject.Has("tokens"))
+    {
+        Napi::Error::New(info.Env(), "Invalid input: 'logits' or 'tokens' properties are not allowed")
+            .ThrowAsJavaScriptException();
+        return info.Env().Undefined();
     }
 
-    
+    // Assign the remaining properties
+    if (inputObject.Has("nPast") && inputObject.Get("nPast").IsNumber())
+    {
+        promptContext.n_past = inputObject.Get("nPast").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("nPredict") && inputObject.Get("nPredict").IsNumber())
+    {
+        promptContext.n_predict = inputObject.Get("nPredict").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("topK") && inputObject.Get("topK").IsNumber())
+    {
+        promptContext.top_k = inputObject.Get("topK").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("topP") && inputObject.Get("topP").IsNumber())
+    {
+        promptContext.top_p = inputObject.Get("topP").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("minP") && inputObject.Get("minP").IsNumber())
+    {
+        promptContext.min_p = inputObject.Get("minP").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("temp") && inputObject.Get("temp").IsNumber())
+    {
+        promptContext.temp = inputObject.Get("temp").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("nBatch") && inputObject.Get("nBatch").IsNumber())
+    {
+        promptContext.n_batch = inputObject.Get("nBatch").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("repeatPenalty") && inputObject.Get("repeatPenalty").IsNumber())
+    {
+        promptContext.repeat_penalty = inputObject.Get("repeatPenalty").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("repeatLastN") && inputObject.Get("repeatLastN").IsNumber())
+    {
+        promptContext.repeat_last_n = inputObject.Get("repeatLastN").As<Napi::Number>().Int32Value();
+    }
+    if (inputObject.Has("contextErase") && inputObject.Get("contextErase").IsNumber())
+    {
+        promptContext.context_erase = inputObject.Get("contextErase").As<Napi::Number>().FloatValue();
+    }
+    if (inputObject.Has("onPromptToken") && inputObject.Get("onPromptToken").IsFunction())
+    {
+        promptWorkerConfig.promptCallback = inputObject.Get("onPromptToken").As<Napi::Function>();
+        promptWorkerConfig.hasPromptCallback = true;
+    }
+    if (inputObject.Has("onResponseToken") && inputObject.Get("onResponseToken").IsFunction())
+    {
+        promptWorkerConfig.responseCallback = inputObject.Get("onResponseToken").As<Napi::Function>();
+        promptWorkerConfig.hasResponseCallback = true;
+    }
 
-    //copy to protect llmodel resources when splitting to new thread
-    // llmodel_prompt_context copiedPrompt = promptContext;
+    // copy to protect llmodel resources when splitting to new thread
+    //  llmodel_prompt_context copiedPrompt = promptContext;
     promptWorkerConfig.context = promptContext;
     promptWorkerConfig.model = GetInference();
     promptWorkerConfig.mutex = &inference_mutex;
-    promptWorkerConfig.prompt = question;
+    promptWorkerConfig.prompt = prompt;
     promptWorkerConfig.result = "";
 
-    
+    promptWorkerConfig.promptTemplate = inputObject.Get("promptTemplate").As<Napi::String>();
+    if (inputObject.Has("special"))
+    {
+        promptWorkerConfig.special = inputObject.Get("special").As<Napi::Boolean>();
+    }
+    if (inputObject.Has("fakeReply"))
+    {
+        // this will be deleted in the worker
+        promptWorkerConfig.fakeReply = new std::string(inputObject.Get("fakeReply").As<Napi::String>().Utf8Value());
+    }
     auto worker = new PromptWorker(env, promptWorkerConfig);
 
     worker->Queue();
 
     return worker->GetPromise();
-  }
-  void NodeModelWrapper::Dispose(const Napi::CallbackInfo& info) {
+}
+void NodeModelWrapper::Dispose(const Napi::CallbackInfo &info)
+{
     llmodel_model_destroy(inference_);
-  }
-  void NodeModelWrapper::SetThreadCount(const Napi::CallbackInfo& info) {
-    if(info[0].IsNumber()) {
+}
+void NodeModelWrapper::SetThreadCount(const Napi::CallbackInfo &info)
+{
+    if (info[0].IsNumber())
+    {
         llmodel_setThreadCount(GetInference(), info[0].As<Napi::Number>().Int64Value());
-    } else {
-        Napi::Error::New(info.Env(), "Could not set thread count: argument 1 is NaN").ThrowAsJavaScriptException(); 
+    }
+    else
+    {
+        Napi::Error::New(info.Env(), "Could not set thread count: argument 1 is NaN").ThrowAsJavaScriptException();
         return;
     }
-  }
-
-  Napi::Value NodeModelWrapper::GetName(const Napi::CallbackInfo& info) {
-    return Napi::String::New(info.Env(), name);
-  }
-  Napi::Value NodeModelWrapper::ThreadCount(const Napi::CallbackInfo& info) {
-    return Napi::Number::New(info.Env(), llmodel_threadCount(GetInference()));
-  }
-
-  Napi::Value NodeModelWrapper::GetLibraryPath(const Napi::CallbackInfo& info) {
-      return Napi::String::New(info.Env(),
-        llmodel_get_implementation_search_path());
-  }
-
-  llmodel_model NodeModelWrapper::GetInference() {
-    return inference_;
-  }
-
-//Exports Bindings
-Napi::Object Init(Napi::Env env, Napi::Object exports) {
-  exports["LLModel"] = NodeModelWrapper::GetClass(env);
-  return exports;
 }
 
+Napi::Value NodeModelWrapper::GetName(const Napi::CallbackInfo &info)
+{
+    return Napi::String::New(info.Env(), name);
+}
+Napi::Value NodeModelWrapper::ThreadCount(const Napi::CallbackInfo &info)
+{
+    return Napi::Number::New(info.Env(), llmodel_threadCount(GetInference()));
+}
 
+Napi::Value NodeModelWrapper::GetLibraryPath(const Napi::CallbackInfo &info)
+{
+    return Napi::String::New(info.Env(), llmodel_get_implementation_search_path());
+}
+
+llmodel_model NodeModelWrapper::GetInference()
+{
+    return inference_;
+}
+
+// Exports Bindings
+Napi::Object Init(Napi::Env env, Napi::Object exports)
+{
+    exports["LLModel"] = NodeModelWrapper::GetClass(env);
+    return exports;
+}
 
 NODE_API_MODULE(NODE_GYP_MODULE_NAME, Init)

gpt4all-bindings/typescript/index.h

@@ -1,62 +1,63 @@
-#include <napi.h>
 #include "llmodel.h"
-#include <iostream>
-#include "llmodel_c.h" 
+#include "llmodel_c.h"
 #include "prompt.h"
 #include <atomic>
-#include <memory>
 #include <filesystem>
-#include <set>
+#include <iostream>
+#include <memory>
 #include <mutex>
+#include <napi.h>
+#include <set>
 
 namespace fs = std::filesystem;
 
+class NodeModelWrapper : public Napi::ObjectWrap<NodeModelWrapper>
+{
 
-class NodeModelWrapper: public Napi::ObjectWrap<NodeModelWrapper> {
-  
-public:
-  NodeModelWrapper(const Napi::CallbackInfo &);
-  //virtual ~NodeModelWrapper();
-  Napi::Value GetType(const Napi::CallbackInfo& info);
-  Napi::Value IsModelLoaded(const Napi::CallbackInfo& info);
-  Napi::Value StateSize(const Napi::CallbackInfo& info);
-  //void Finalize(Napi::Env env) override;
-  /**
-   * Prompting the model. This entails spawning a new thread and adding the response tokens
-   * into a thread local string variable.
-   */
-  Napi::Value Prompt(const Napi::CallbackInfo& info);
-  void SetThreadCount(const Napi::CallbackInfo& info);
-  void Dispose(const Napi::CallbackInfo& info);
-  Napi::Value GetName(const Napi::CallbackInfo& info);
-  Napi::Value ThreadCount(const Napi::CallbackInfo& info);
-  Napi::Value GenerateEmbedding(const Napi::CallbackInfo& info);
-  Napi::Value HasGpuDevice(const Napi::CallbackInfo& info);
-  Napi::Value ListGpus(const Napi::CallbackInfo& info);
-  Napi::Value InitGpuByString(const Napi::CallbackInfo& info);
-  Napi::Value GetRequiredMemory(const Napi::CallbackInfo& info);
-  Napi::Value GetGpuDevices(const Napi::CallbackInfo& info);
-  /*
-   * The path that is used to search for the dynamic libraries
-   */
-  Napi::Value GetLibraryPath(const Napi::CallbackInfo& info);
-  /**
-   * Creates the LLModel class
-   */
-  static Napi::Function GetClass(Napi::Env);
-  llmodel_model GetInference();
-private:
-  /**
-   * The underlying inference that interfaces with the C interface
-   */
-  llmodel_model inference_;
+  public:
+    NodeModelWrapper(const Napi::CallbackInfo &);
+    // virtual ~NodeModelWrapper();
+    Napi::Value GetType(const Napi::CallbackInfo &info);
+    Napi::Value IsModelLoaded(const Napi::CallbackInfo &info);
+    Napi::Value StateSize(const Napi::CallbackInfo &info);
+    // void Finalize(Napi::Env env) override;
+    /**
+     * Prompting the model. This entails spawning a new thread and adding the response tokens
+     * into a thread local string variable.
+     */
+    Napi::Value Infer(const Napi::CallbackInfo &info);
+    void SetThreadCount(const Napi::CallbackInfo &info);
+    void Dispose(const Napi::CallbackInfo &info);
+    Napi::Value GetName(const Napi::CallbackInfo &info);
+    Napi::Value ThreadCount(const Napi::CallbackInfo &info);
+    Napi::Value GenerateEmbedding(const Napi::CallbackInfo &info);
+    Napi::Value HasGpuDevice(const Napi::CallbackInfo &info);
+    Napi::Value ListGpus(const Napi::CallbackInfo &info);
+    Napi::Value InitGpuByString(const Napi::CallbackInfo &info);
+    Napi::Value GetRequiredMemory(const Napi::CallbackInfo &info);
+    Napi::Value GetGpuDevices(const Napi::CallbackInfo &info);
+    /*
+     * The path that is used to search for the dynamic libraries
+     */
+    Napi::Value GetLibraryPath(const Napi::CallbackInfo &info);
+    /**
+     * Creates the LLModel class
+     */
+    static Napi::Function GetClass(Napi::Env);
+    llmodel_model GetInference();
 
-  std::mutex inference_mutex;
+  private:
+    /**
+     * The underlying inference that interfaces with the C interface
+     */
+    llmodel_model inference_;
 
-  std::string type;
-  // corresponds to LLModel::name() in typescript
-  std::string name;
-  int nCtx{};
-  int nGpuLayers{};
-  std::string full_model_path;
+    std::mutex inference_mutex;
+
+    std::string type;
+    // corresponds to LLModel::name() in typescript
+    std::string name;
+    int nCtx{};
+    int nGpuLayers{};
+    std::string full_model_path;
 };

gpt4all-bindings/typescript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt4all",
-  "version": "3.2.0",
+  "version": "4.0.0",
   "packageManager": "yarn@3.6.1",
   "main": "src/gpt4all.js",
   "repository": "nomic-ai/gpt4all",
@@ -22,7 +22,6 @@
   ],
   "dependencies": {
     "md5-file": "^5.0.0",
-    "mkdirp": "^3.0.1",
     "node-addon-api": "^6.1.0",
     "node-gyp-build": "^4.6.0"
   },

gpt4all-bindings/typescript/prompt.cc

@@ -2,145 +2,195 @@
 #include <future>
 
 PromptWorker::PromptWorker(Napi::Env env, PromptWorkerConfig config)
-        : promise(Napi::Promise::Deferred::New(env)), _config(config), AsyncWorker(env) {
-            if(_config.bHasTokenCallback){
-                _tsfn = Napi::ThreadSafeFunction::New(config.tokenCallback.Env(),config.tokenCallback,"PromptWorker",0,1,this);
-            }
-        }
-
-    PromptWorker::~PromptWorker()
+    : promise(Napi::Promise::Deferred::New(env)), _config(config), AsyncWorker(env)
+{
+    if (_config.hasResponseCallback)
     {
-        if(_config.bHasTokenCallback){
-            _tsfn.Release();
-        }
+        _responseCallbackFn = Napi::ThreadSafeFunction::New(config.responseCallback.Env(), config.responseCallback,
+                                                            "PromptWorker", 0, 1, this);
     }
 
-    void PromptWorker::Execute()
+    if (_config.hasPromptCallback)
     {
-        _config.mutex->lock();
+        _promptCallbackFn = Napi::ThreadSafeFunction::New(config.promptCallback.Env(), config.promptCallback,
+                                                          "PromptWorker", 0, 1, this);
+    }
+}
 
-        LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper *>(_config.model);
+PromptWorker::~PromptWorker()
+{
+    if (_config.hasResponseCallback)
+    {
+        _responseCallbackFn.Release();
+    }
+    if (_config.hasPromptCallback)
+    {
+        _promptCallbackFn.Release();
+    }
+}
 
-        auto ctx = &_config.context;
+void PromptWorker::Execute()
+{
+    _config.mutex->lock();
 
-        if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
-            wrapper->promptContext.tokens.resize(ctx->n_past);
+    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper *>(_config.model);
 
-        // Copy the C prompt context
-        wrapper->promptContext.n_past = ctx->n_past;
-        wrapper->promptContext.n_ctx = ctx->n_ctx;
-        wrapper->promptContext.n_predict = ctx->n_predict;
-        wrapper->promptContext.top_k = ctx->top_k;
-        wrapper->promptContext.top_p = ctx->top_p;
-        wrapper->promptContext.temp = ctx->temp;
-        wrapper->promptContext.n_batch = ctx->n_batch;
-        wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
-        wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
-        wrapper->promptContext.contextErase = ctx->context_erase;
+    auto ctx = &_config.context;
 
-        // Napi::Error::Fatal(
-        //         "SUPRA",
-        //         "About to prompt");
-        // Call the C++ prompt method
-        wrapper->llModel->prompt(
-            _config.prompt, 
-            [](int32_t tid) { return true; },
-            [this](int32_t token_id, const std::string tok)
-            {
-                return ResponseCallback(token_id, tok);
-            },
-            [](bool isRecalculating)
-            {
-                return isRecalculating;
-            },
-            wrapper->promptContext);
+    if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
+        wrapper->promptContext.tokens.resize(ctx->n_past);
 
-        // Update the C context by giving access to the wrappers raw pointers to std::vector data
-        // which involves no copies
-        ctx->logits = wrapper->promptContext.logits.data();
-        ctx->logits_size = wrapper->promptContext.logits.size();
-        ctx->tokens = wrapper->promptContext.tokens.data();
-        ctx->tokens_size = wrapper->promptContext.tokens.size();
+    // Copy the C prompt context
+    wrapper->promptContext.n_past = ctx->n_past;
+    wrapper->promptContext.n_ctx = ctx->n_ctx;
+    wrapper->promptContext.n_predict = ctx->n_predict;
+    wrapper->promptContext.top_k = ctx->top_k;
+    wrapper->promptContext.top_p = ctx->top_p;
+    wrapper->promptContext.temp = ctx->temp;
+    wrapper->promptContext.n_batch = ctx->n_batch;
+    wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
+    wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
+    wrapper->promptContext.contextErase = ctx->context_erase;
 
-        // Update the rest of the C prompt context
-        ctx->n_past = wrapper->promptContext.n_past;
-        ctx->n_ctx = wrapper->promptContext.n_ctx;
-        ctx->n_predict = wrapper->promptContext.n_predict;
-        ctx->top_k = wrapper->promptContext.top_k;
-        ctx->top_p = wrapper->promptContext.top_p;
-        ctx->temp = wrapper->promptContext.temp;
-        ctx->n_batch = wrapper->promptContext.n_batch;
-        ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
-        ctx->repeat_last_n = wrapper->promptContext.repeat_last_n;
-        ctx->context_erase = wrapper->promptContext.contextErase;
+    // Call the C++ prompt method
 
-        _config.mutex->unlock();
+    wrapper->llModel->prompt(
+        _config.prompt, _config.promptTemplate, [this](int32_t token_id) { return PromptCallback(token_id); },
+        [this](int32_t token_id, const std::string token) { return ResponseCallback(token_id, token); },
+        [](bool isRecalculating) { return isRecalculating; }, wrapper->promptContext, _config.special,
+        _config.fakeReply);
+
+    // Update the C context by giving access to the wrappers raw pointers to std::vector data
+    // which involves no copies
+    ctx->logits = wrapper->promptContext.logits.data();
+    ctx->logits_size = wrapper->promptContext.logits.size();
+    ctx->tokens = wrapper->promptContext.tokens.data();
+    ctx->tokens_size = wrapper->promptContext.tokens.size();
+
+    // Update the rest of the C prompt context
+    ctx->n_past = wrapper->promptContext.n_past;
+    ctx->n_ctx = wrapper->promptContext.n_ctx;
+    ctx->n_predict = wrapper->promptContext.n_predict;
+    ctx->top_k = wrapper->promptContext.top_k;
+    ctx->top_p = wrapper->promptContext.top_p;
+    ctx->temp = wrapper->promptContext.temp;
+    ctx->n_batch = wrapper->promptContext.n_batch;
+    ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
+    ctx->repeat_last_n = wrapper->promptContext.repeat_last_n;
+    ctx->context_erase = wrapper->promptContext.contextErase;
+
+    _config.mutex->unlock();
+}
+
+void PromptWorker::OnOK()
+{
+    Napi::Object returnValue = Napi::Object::New(Env());
+    returnValue.Set("text", result);
+    returnValue.Set("nPast", _config.context.n_past);
+    promise.Resolve(returnValue);
+    delete _config.fakeReply;
+}
+
+void PromptWorker::OnError(const Napi::Error &e)
+{
+    delete _config.fakeReply;
+    promise.Reject(e.Value());
+}
+
+Napi::Promise PromptWorker::GetPromise()
+{
+    return promise.Promise();
+}
+
+bool PromptWorker::ResponseCallback(int32_t token_id, const std::string token)
+{
+    if (token_id == -1)
+    {
+        return false;
     }
 
-    void PromptWorker::OnOK()
-    {
-        promise.Resolve(Napi::String::New(Env(), result));
-    }
-
-    void PromptWorker::OnError(const Napi::Error &e)
-    {
-        promise.Reject(e.Value());
-    }
-
-    Napi::Promise PromptWorker::GetPromise()
-    {
-        return promise.Promise();
-    }
-
-    bool PromptWorker::ResponseCallback(int32_t token_id, const std::string token)
-    {
-        if (token_id == -1)
-        {
-            return false;
-        }
-
-        if(!_config.bHasTokenCallback){
-            return true;
-        }
-
-        result += token;
-
-       std::promise<bool> promise;
-
-       auto info = new TokenCallbackInfo();
-       info->tokenId = token_id;
-       info->token = token;
-       info->total = result;
-
-       auto future = promise.get_future();
-
-       auto status = _tsfn.BlockingCall(info, [&promise](Napi::Env env, Napi::Function jsCallback, TokenCallbackInfo *value)
-       {
-           // Transform native data into JS data, passing it to the provided
-           // `jsCallback` -- the TSFN's JavaScript function.
-           auto token_id = Napi::Number::New(env, value->tokenId);
-           auto token = Napi::String::New(env, value->token);
-           auto total = Napi::String::New(env,value->total);
-           auto jsResult = jsCallback.Call({ token_id, token, total}).ToBoolean();
-           promise.set_value(jsResult);
-           // We're finished with the data.
-           delete value;
-       });
-       if (status != napi_ok) {
-            Napi::Error::Fatal(
-                "PromptWorkerResponseCallback",
-                "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
-        }
-
-       return future.get();
-    }
-
-    bool PromptWorker::RecalculateCallback(bool isRecalculating)
-    {
-        return isRecalculating;
-    }
-
-    bool PromptWorker::PromptCallback(int32_t tid)
+    if (!_config.hasResponseCallback)
     {
         return true;
     }
+
+    result += token;
+
+    std::promise<bool> promise;
+
+    auto info = new ResponseCallbackData();
+    info->tokenId = token_id;
+    info->token = token;
+
+    auto future = promise.get_future();
+
+    auto status = _responseCallbackFn.BlockingCall(
+        info, [&promise](Napi::Env env, Napi::Function jsCallback, ResponseCallbackData *value) {
+            try
+            {
+                // Transform native data into JS data, passing it to the provided
+                // `jsCallback` -- the TSFN's JavaScript function.
+                auto token_id = Napi::Number::New(env, value->tokenId);
+                auto token = Napi::String::New(env, value->token);
+                auto jsResult = jsCallback.Call({token_id, token}).ToBoolean();
+                promise.set_value(jsResult);
+            }
+            catch (const Napi::Error &e)
+            {
+                std::cerr << "Error in onResponseToken callback: " << e.what() << std::endl;
+                promise.set_value(false);
+            }
+
+            delete value;
+        });
+    if (status != napi_ok)
+    {
+        Napi::Error::Fatal("PromptWorkerResponseCallback", "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
+    }
+
+    return future.get();
+}
+
+bool PromptWorker::RecalculateCallback(bool isRecalculating)
+{
+    return isRecalculating;
+}
+
+bool PromptWorker::PromptCallback(int32_t token_id)
+{
+    if (!_config.hasPromptCallback)
+    {
+        return true;
+    }
+
+    std::promise<bool> promise;
+
+    auto info = new PromptCallbackData();
+    info->tokenId = token_id;
+
+    auto future = promise.get_future();
+
+    auto status = _promptCallbackFn.BlockingCall(
+        info, [&promise](Napi::Env env, Napi::Function jsCallback, PromptCallbackData *value) {
+            try
+            {
+                // Transform native data into JS data, passing it to the provided
+                // `jsCallback` -- the TSFN's JavaScript function.
+                auto token_id = Napi::Number::New(env, value->tokenId);
+                auto jsResult = jsCallback.Call({token_id}).ToBoolean();
+                promise.set_value(jsResult);
+            }
+            catch (const Napi::Error &e)
+            {
+                std::cerr << "Error in onPromptToken callback: " << e.what() << std::endl;
+                promise.set_value(false);
+            }
+            delete value;
+        });
+    if (status != napi_ok)
+    {
+        Napi::Error::Fatal("PromptWorkerPromptCallback", "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
+    }
+
+    return future.get();
+}

gpt4all-bindings/typescript/prompt.h

@@ -1,59 +1,72 @@
 #ifndef PREDICT_WORKER_H
 #define PREDICT_WORKER_H
 
-#include "napi.h"
-#include "llmodel_c.h"
 #include "llmodel.h"
-#include <thread>
-#include <mutex>
-#include <iostream>
+#include "llmodel_c.h"
+#include "napi.h"
 #include <atomic>
+#include <iostream>
 #include <memory>
+#include <mutex>
+#include <thread>
 
-struct TokenCallbackInfo
+struct ResponseCallbackData
+{
+    int32_t tokenId;
+    std::string token;
+};
+
+struct PromptCallbackData
+{
+    int32_t tokenId;
+};
+
+struct LLModelWrapper
+{
+    LLModel *llModel = nullptr;
+    LLModel::PromptContext promptContext;
+    ~LLModelWrapper()
     {
-        int32_t tokenId;
-        std::string total;
-        std::string token;
-    };
+        delete llModel;
+    }
+};
 
-    struct LLModelWrapper
-    {
-        LLModel *llModel = nullptr;
-        LLModel::PromptContext promptContext;
-        ~LLModelWrapper() { delete llModel; }
-    };
+struct PromptWorkerConfig
+{
+    Napi::Function responseCallback;
+    bool hasResponseCallback = false;
+    Napi::Function promptCallback;
+    bool hasPromptCallback = false;
+    llmodel_model model;
+    std::mutex *mutex;
+    std::string prompt;
+    std::string promptTemplate;
+    llmodel_prompt_context context;
+    std::string result;
+    bool special = false;
+    std::string *fakeReply = nullptr;
+};
 
-    struct PromptWorkerConfig
-    {
-        Napi::Function tokenCallback;
-        bool bHasTokenCallback = false;
-        llmodel_model model;
-        std::mutex * mutex;
-        std::string prompt;
-        llmodel_prompt_context context;
-        std::string result;
-    };
+class PromptWorker : public Napi::AsyncWorker
+{
+  public:
+    PromptWorker(Napi::Env env, PromptWorkerConfig config);
+    ~PromptWorker();
+    void Execute() override;
+    void OnOK() override;
+    void OnError(const Napi::Error &e) override;
+    Napi::Promise GetPromise();
 
-    class PromptWorker : public Napi::AsyncWorker
-    {
-    public:
-        PromptWorker(Napi::Env env, PromptWorkerConfig config);
-        ~PromptWorker();
-        void Execute() override;
-        void OnOK() override;
-        void OnError(const Napi::Error &e) override;
-        Napi::Promise GetPromise();
+    bool ResponseCallback(int32_t token_id, const std::string token);
+    bool RecalculateCallback(bool isrecalculating);
+    bool PromptCallback(int32_t token_id);
 
-        bool ResponseCallback(int32_t token_id, const std::string token);
-        bool RecalculateCallback(bool isrecalculating);
-        bool PromptCallback(int32_t tid);
+  private:
+    Napi::Promise::Deferred promise;
+    std::string result;
+    PromptWorkerConfig _config;
+    Napi::ThreadSafeFunction _responseCallbackFn;
+    Napi::ThreadSafeFunction _promptCallbackFn;
+};
 
-    private:
-        Napi::Promise::Deferred promise;
-        std::string result;
-        PromptWorkerConfig _config;
-        Napi::ThreadSafeFunction _tsfn;
-    };
-
-#endif  // PREDICT_WORKER_H
+#endif // PREDICT_WORKER_H

gpt4all-bindings/typescript/scripts/build_unix.sh

@@ -24,7 +24,6 @@ mkdir -p "$NATIVE_DIR" "$BUILD_DIR"
 
 cmake -S ../../gpt4all-backend -B "$BUILD_DIR" &&
 cmake --build "$BUILD_DIR" -j --config Release && {
-  cp "$BUILD_DIR"/libbert*.$LIB_EXT   "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libgptj*.$LIB_EXT   "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libllama*.$LIB_EXT  "$NATIVE_DIR"/
 }

gpt4all-bindings/typescript/spec/callbacks.mjs

@@ -0,0 +1,31 @@
+import { promises as fs } from "node:fs";
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const res = await createCompletion(
+    model,
+    "I've got three 🍣 - What shall I name them?",
+    {
+        onPromptToken: (tokenId) => {
+            console.debug("onPromptToken", { tokenId });
+            // throwing an error will cancel
+            throw new Error("This is an error");
+            // const foo = thisMethodDoesNotExist();
+            // returning false will cancel as well
+            // return false;
+        },
+        onResponseToken: (tokenId, token) => {
+            console.debug("onResponseToken", { tokenId, token });
+            // same applies here
+        },
+    }
+);
+
+console.debug("Output:", {
+    usage: res.usage,
+    message: res.choices[0].message,
+});

gpt4all-bindings/typescript/spec/chat-memory.mjs

@@ -0,0 +1,65 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession({
+    messages: [
+        {
+            role: "user",
+            content: "I'll tell you a secret password: It's 63445.",
+        },
+        {
+            role: "assistant",
+            content: "I will do my best to remember that.",
+        },
+        {
+            role: "user",
+            content:
+                "And here another fun fact: Bananas may be bluer than bread at night.",
+        },
+        {
+            role: "assistant",
+            content: "Yes, that makes sense.",
+        },
+    ],
+});
+
+const turn1 = await createCompletion(
+    chat,
+    "Please tell me the secret password."
+);
+console.debug(turn1.choices[0].message);
+// "The secret password you shared earlier is 63445.""
+
+const turn2 = await createCompletion(
+    chat,
+    "Thanks! Have your heard about the bananas?"
+);
+console.debug(turn2.choices[0].message);
+
+for (let i = 0; i < 32; i++) {
+    // gpu go brr
+    const turn = await createCompletion(
+        chat,
+        i % 2 === 0 ? "Tell me a fun fact." : "And a boring one?"
+    );
+    console.debug({
+        message: turn.choices[0].message,
+        n_past_tokens: turn.usage.n_past_tokens,
+    });
+}
+
+const finalTurn = await createCompletion(
+    chat,
+    "Now I forgot the secret password. Can you remind me?"
+);
+console.debug(finalTurn.choices[0].message);
+
+// result of finalTurn may vary depending on whether the generated facts pushed the secret out of the context window.
+// "Of course! The secret password you shared earlier is 63445."
+// "I apologize for any confusion. As an AI language model, ..."
+
+model.dispose();

gpt4all-bindings/typescript/spec/chat-minimal.mjs

@@ -0,0 +1,19 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession();
+
+await createCompletion(
+    chat,
+    "Why are bananas rather blue than bread at night sometimes?",
+    {
+        verbose: true,
+    }
+);
+await createCompletion(chat, "Are you sure?", {
+    verbose: true,
+});

gpt4all-bindings/typescript/spec/chat.mjs

@@ -1,70 +0,0 @@
-import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY, loadModel } from '../src/gpt4all.js'
-
-const model = await loadModel(
-    'mistral-7b-openorca.Q4_0.gguf',
-    { verbose: true, device: 'gpu' }
-);
-const ll = model.llm;
-
-try {
-   class Extended extends LLModel {
-   }
-
-} catch(e) {
-    console.log("Extending from native class gone wrong " + e)
-}
-
-console.log("state size " + ll.stateSize())
-
-console.log("thread count " + ll.threadCount());
-ll.setThreadCount(5);
-
-console.log("thread count " + ll.threadCount());
-ll.setThreadCount(4);
-console.log("thread count " + ll.threadCount());
-console.log("name " + ll.name());
-console.log("type: " + ll.type());
-console.log("Default directory for models", DEFAULT_DIRECTORY);
-console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY);
-console.log("Has GPU", ll.hasGpuDevice());
-console.log("gpu devices", ll.listGpu())
-console.log("Required Mem in bytes", ll.memoryNeeded())
-const completion1 = await createCompletion(model, [ 
-    { role : 'system', content: 'You are an advanced mathematician.'  },
-    { role : 'user', content: 'What is 1 + 1?'  }, 
-], { verbose: true })
-console.log(completion1.choices[0].message)
-
-const completion2 = await createCompletion(model, [
-    { role : 'system', content: 'You are an advanced mathematician.'  },
-    { role : 'user', content: 'What is two plus two?'  }, 
-], {  verbose: true })
-
-console.log(completion2.choices[0].message)
-
-//CALLING DISPOSE WILL INVALID THE NATIVE MODEL. USE THIS TO CLEANUP
-model.dispose()
-// At the moment, from testing this code, concurrent model prompting is not possible. 
-// Behavior: The last prompt gets answered, but the rest are cancelled
-// my experience with threading is not the best, so if anyone who is good is willing to give this a shot,
-// maybe this is possible
-// INFO: threading with llama.cpp is not the best maybe not even possible, so this will be left here as reference
-
-//const responses = await Promise.all([
-//    createCompletion(model, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//    ], { verbose: true }),
-//    createCompletion(model, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//    ], { verbose: true }),
-//
-//createCompletion(model, [ 
-//    { role : 'system', content: 'You are an advanced mathematician.'  },
-//    { role : 'user', content: 'What is 1 + 1?'  }, 
-//], { verbose: true })
-//
-//])
-//console.log(responses.map(s => s.choices[0].message))
-

gpt4all-bindings/typescript/spec/concurrency.mjs

@@ -0,0 +1,29 @@
+import {
+    loadModel,
+    createCompletion,
+} from "../src/gpt4all.js";
+
+const modelOptions = {
+    verbose: true,
+};
+
+const model1 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    ...modelOptions,
+    device: "gpu", // only one model can be on gpu
+});
+const model2 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", modelOptions);
+const model3 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", modelOptions);
+
+const promptContext = {
+    verbose: true,
+}
+
+const responses = await Promise.all([
+    createCompletion(model1, "What is 1 + 1?", promptContext),
+    // generating with the same model instance will wait for the previous completion to finish
+    createCompletion(model1, "What is 1 + 1?", promptContext),
+    // generating with different model instances will run in parallel
+    createCompletion(model2, "What is 1 + 2?", promptContext),
+    createCompletion(model3, "What is 1 + 3?", promptContext),
+]);
+console.log(responses.map((res) => res.choices[0].message));

gpt4all-bindings/typescript/spec/embed-jsonl.mjs

@@ -0,0 +1,26 @@
+import { loadModel, createEmbedding } from '../src/gpt4all.js'
+import { createGunzip, createGzip, createUnzip } from 'node:zlib';
+import { Readable } from 'stream'
+import readline  from 'readline'
+const embedder = await loadModel("nomic-embed-text-v1.5.f16.gguf", { verbose: true, type: 'embedding', device: 'gpu' })
+console.log("Running with", embedder.llm.threadCount(), "threads");
+
+
+const unzip = createGunzip();
+const url = "https://huggingface.co/datasets/sentence-transformers/embedding-training-data/resolve/main/squad_pairs.jsonl.gz"
+const stream = await fetch(url)
+        .then(res => Readable.fromWeb(res.body));
+
+const lineReader = readline.createInterface({
+    input: stream.pipe(unzip),
+    crlfDelay: Infinity
+})
+
+lineReader.on('line', line => {
+    //pairs of questions and answers
+    const question_answer = JSON.parse(line)
+    console.log(createEmbedding(embedder, question_answer))
+})
+
+lineReader.on('close', () => embedder.dispose())
+

gpt4all-bindings/typescript/spec/embed.mjs

@@ -1,6 +1,12 @@
 import { loadModel, createEmbedding } from '../src/gpt4all.js'
 
-const embedder = await loadModel("ggml-all-MiniLM-L6-v2-f16.bin", { verbose: true, type: 'embedding'})
+const embedder = await loadModel("nomic-embed-text-v1.5.f16.gguf", { verbose: true, type: 'embedding' , device: 'gpu' })
 
-console.log(createEmbedding(embedder, "Accept your current situation"))
+try {
+console.log(createEmbedding(embedder, ["Accept your current situation", "12312"], { prefix: "search_document"  }))
 
+} catch(e) {
+console.log(e)
+}
+
+embedder.dispose()

gpt4all-bindings/typescript/spec/generator.mjs

@@ -1,41 +0,0 @@
-import gpt from '../src/gpt4all.js'
-
-const model = await gpt.loadModel("mistral-7b-openorca.Q4_0.gguf", { device: 'gpu' })  
-
-process.stdout.write('Response: ')
-
-
-const tokens = gpt.generateTokens(model, [{ 
-    role: 'user',
-    content: "How are you ?"
-}], { nPredict: 2048 })
-for await (const token of tokens){
-    process.stdout.write(token);
-}
-
-
-const result = await gpt.createCompletion(model, [{ 
-    role: 'user',
-    content: "You sure?"
-}])
-
-console.log(result)
-
-const result2 = await gpt.createCompletion(model, [{ 
-    role: 'user',
-    content: "You sure you sure?"
-}])
-
-console.log(result2)
-
-
-const tokens2 = gpt.generateTokens(model, [{ 
-    role: 'user',
-    content: "If 3 + 3 is 5, what is 2 + 2?"
-}], { nPredict: 2048 })
-for await (const token of tokens2){
-    process.stdout.write(token);
-}
-console.log("done")
-model.dispose();
-

gpt4all-bindings/typescript/spec/llmodel.mjs

@@ -0,0 +1,61 @@
+import {
+    LLModel,
+    createCompletion,
+    DEFAULT_DIRECTORY,
+    DEFAULT_LIBRARIES_DIRECTORY,
+    loadModel,
+} from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+const ll = model.llm;
+
+try {
+    class Extended extends LLModel {}
+} catch (e) {
+    console.log("Extending from native class gone wrong " + e);
+}
+
+console.log("state size " + ll.stateSize());
+
+console.log("thread count " + ll.threadCount());
+ll.setThreadCount(5);
+
+console.log("thread count " + ll.threadCount());
+ll.setThreadCount(4);
+console.log("thread count " + ll.threadCount());
+console.log("name " + ll.name());
+console.log("type: " + ll.type());
+console.log("Default directory for models", DEFAULT_DIRECTORY);
+console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY);
+console.log("Has GPU", ll.hasGpuDevice());
+console.log("gpu devices", ll.listGpu());
+console.log("Required Mem in bytes", ll.memoryNeeded());
+
+// to ingest a custom system prompt without using a chat session.
+await createCompletion(
+    model,
+    "<|im_start|>system\nYou are an advanced mathematician.\n<|im_end|>\n",
+    {
+        promptTemplate: "%1",
+        nPredict: 0,
+        special: true,
+    }
+);
+const completion1 = await createCompletion(model, "What is 1 + 1?", {
+    verbose: true,
+});
+console.log(`🤖 > ${completion1.choices[0].message.content}`);
+//Very specific:
+// tested on Ubuntu 22.0, Linux Mint, if I set nPast to 100, the app hangs.
+const completion2 = await createCompletion(model, "And if we add two?", {
+    verbose: true,
+});
+console.log(`🤖 > ${completion2.choices[0].message.content}`);
+
+//CALLING DISPOSE WILL INVALID THE NATIVE MODEL. USE THIS TO CLEANUP
+model.dispose();
+
+console.log("model disposed, exiting...");

gpt4all-bindings/typescript/spec/long-context.mjs

@@ -0,0 +1,21 @@
+import { promises as fs } from "node:fs";
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+    nCtx: 32768,
+});
+
+const typeDefSource = await fs.readFile("./src/gpt4all.d.ts", "utf-8");
+
+const res = await createCompletion(
+    model,
+    "Here are the type definitions for the GPT4All API:\n\n" +
+        typeDefSource +
+        "\n\nHow do I create a completion with a really large context window?",
+    {
+        verbose: true,
+    }
+);
+console.debug(res.choices[0].message);

gpt4all-bindings/typescript/spec/model-switching.mjs

@@ -0,0 +1,60 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model1 = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    device: "gpu",
+    nCtx: 4096,
+});
+
+const chat1 = await model1.createChatSession({
+    temperature: 0.8,
+    topP: 0.7,
+    topK: 60,
+});
+
+const chat1turn1 = await createCompletion(
+    chat1,
+    "Outline a short story concept for adults. About why bananas are rather blue than bread is green at night sometimes. Not too long."
+);
+console.debug(chat1turn1.choices[0].message);
+
+const chat1turn2 = await createCompletion(
+    chat1,
+    "Lets sprinkle some plot twists. And a cliffhanger at the end."
+);
+console.debug(chat1turn2.choices[0].message);
+
+const chat1turn3 = await createCompletion(
+    chat1,
+    "Analyze your plot. Find the weak points."
+);
+console.debug(chat1turn3.choices[0].message);
+
+const chat1turn4 = await createCompletion(
+    chat1,
+    "Rewrite it based on the analysis."
+);
+console.debug(chat1turn4.choices[0].message);
+
+model1.dispose();
+
+const model2 = await loadModel("gpt4all-falcon-newbpe-q4_0.gguf", {
+    device: "gpu",
+});
+
+const chat2 = await model2.createChatSession({
+    messages: chat1.messages,
+});
+
+const chat2turn1 = await createCompletion(
+    chat2,
+    "Give three ideas how this plot could be improved."
+);
+console.debug(chat2turn1.choices[0].message);
+
+const chat2turn2 = await createCompletion(
+    chat2,
+    "Revise the plot, applying your ideas."
+);
+console.debug(chat2turn2.choices[0].message);
+
+model2.dispose();

gpt4all-bindings/typescript/spec/stateless.mjs

@@ -0,0 +1,50 @@
+import { loadModel, createCompletion } from "../src/gpt4all.js";
+
+const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const messages = [
+    {
+        role: "system",
+        content: "<|im_start|>system\nYou are an advanced mathematician.\n<|im_end|>\n",
+    },
+    {
+        role: "user",
+        content: "What's 2+2?",
+    },
+    {
+        role: "assistant",
+        content: "5",
+    },
+    {
+        role: "user",
+        content: "Are you sure?",
+    },
+];
+
+
+const res1 = await createCompletion(model, messages);
+console.debug(res1.choices[0].message);
+messages.push(res1.choices[0].message);
+
+messages.push({
+    role: "user",
+    content: "Could you double check that?",
+});
+
+const res2 = await createCompletion(model, messages);
+console.debug(res2.choices[0].message);
+messages.push(res2.choices[0].message);
+
+messages.push({
+    role: "user",
+    content: "Let's bring out the big calculators.",
+});
+
+const res3 = await createCompletion(model, messages);
+console.debug(res3.choices[0].message);
+messages.push(res3.choices[0].message);
+
+// console.debug(messages);

gpt4all-bindings/typescript/spec/streaming.mjs

@@ -0,0 +1,57 @@
+import {
+    loadModel,
+    createCompletion,
+    createCompletionStream,
+    createCompletionGenerator,
+} from "../src/gpt4all.js";
+
+const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", {
+    device: "gpu",
+});
+
+process.stdout.write("### Stream:");
+const stream = createCompletionStream(model, "How are you?");
+stream.tokens.on("data", (data) => {
+    process.stdout.write(data);
+});
+await stream.result;
+process.stdout.write("\n");
+
+process.stdout.write("### Stream with pipe:");
+const stream2 = createCompletionStream(
+    model,
+    "Please say something nice about node streams."
+);
+stream2.tokens.pipe(process.stdout);
+const stream2Res = await stream2.result;
+process.stdout.write("\n");
+
+process.stdout.write("### Generator:");
+const gen = createCompletionGenerator(model, "generators instead?", {
+    nPast: stream2Res.usage.n_past_tokens,
+});
+for await (const chunk of gen) {
+    process.stdout.write(chunk);
+}
+
+process.stdout.write("\n");
+
+process.stdout.write("### Callback:");
+await createCompletion(model, "Why not just callbacks?", {
+    onResponseToken: (tokenId, token) => {
+        process.stdout.write(token);
+    },
+});
+process.stdout.write("\n");
+
+process.stdout.write("### 2nd Generator:");
+const gen2 = createCompletionGenerator(model, "If 3 + 3 is 5, what is 2 + 2?");
+
+let chunk = await gen2.next();
+while (!chunk.done) {
+    process.stdout.write(chunk.value);
+    chunk = await gen2.next();
+}
+process.stdout.write("\n");
+console.debug("generator finished", chunk);
+model.dispose();

gpt4all-bindings/typescript/spec/system.mjs

@@ -0,0 +1,19 @@
+import {
+    loadModel,
+    createCompletion,
+} from "../src/gpt4all.js";
+
+const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", {
+    verbose: true,
+    device: "gpu",
+});
+
+const chat = await model.createChatSession({
+    verbose: true,
+    systemPrompt: "<|im_start|>system\nRoleplay as Batman. Answer as if you are Batman, never say you're an Assistant.\n<|im_end|>",
+});
+const turn1 = await createCompletion(chat, "You have any plans tonight?");
+console.log(turn1.choices[0].message);
+// "I'm afraid I must decline any personal invitations tonight. As Batman, I have a responsibility to protect Gotham City."
+
+model.dispose();

gpt4all-bindings/typescript/src/chat-session.js

@@ -0,0 +1,169 @@
+const { DEFAULT_PROMPT_CONTEXT } = require("./config");
+const { prepareMessagesForIngest } = require("./util");
+
+class ChatSession {
+    model;
+    modelName;
+    /**
+     * @type {import('./gpt4all').ChatMessage[]}
+     */
+    messages;
+    /**
+     * @type {string}
+     */
+    systemPrompt;
+    /**
+     * @type {import('./gpt4all').LLModelPromptContext}
+     */
+    promptContext;
+    /**
+     * @type {boolean}
+     */
+    initialized;
+
+    constructor(model, chatSessionOpts = {}) {
+        const { messages, systemPrompt, ...sessionDefaultPromptContext } =
+            chatSessionOpts;
+        this.model = model;
+        this.modelName = model.llm.name();
+        this.messages = messages ?? [];
+        this.systemPrompt = systemPrompt ?? model.config.systemPrompt;
+        this.initialized = false;
+        this.promptContext = {
+            ...DEFAULT_PROMPT_CONTEXT,
+            ...sessionDefaultPromptContext,
+            nPast: 0,
+        };
+    }
+
+    async initialize(completionOpts = {}) {
+        if (this.model.activeChatSession !== this) {
+            this.model.activeChatSession = this;
+        }
+
+        let tokensIngested = 0;
+
+        // ingest system prompt
+
+        if (this.systemPrompt) {
+            const systemRes = await this.model.generate(this.systemPrompt, {
+                promptTemplate: "%1",
+                nPredict: 0,
+                special: true,
+                nBatch: this.promptContext.nBatch,
+                // verbose: true,
+            });
+            tokensIngested += systemRes.tokensIngested;
+            this.promptContext.nPast = systemRes.nPast;
+        }
+
+        // ingest initial messages
+        if (this.messages.length > 0) {
+            tokensIngested += await this.ingestMessages(
+                this.messages,
+                completionOpts
+            );
+        }
+
+        this.initialized = true;
+
+        return tokensIngested;
+    }
+
+    async ingestMessages(messages, completionOpts = {}) {
+        const turns = prepareMessagesForIngest(messages);
+
+        // send the message pairs to the model
+        let tokensIngested = 0;
+
+        for (const turn of turns) {
+            const turnRes = await this.model.generate(turn.user, {
+                ...this.promptContext,
+                ...completionOpts,
+                fakeReply: turn.assistant,
+            });
+            tokensIngested += turnRes.tokensIngested;
+            this.promptContext.nPast = turnRes.nPast;
+        }
+        return tokensIngested;
+    }
+
+    async generate(input, completionOpts = {}) {
+        if (this.model.activeChatSession !== this) {
+            throw new Error(
+                "Chat session is not active. Create a new chat session or call initialize to continue."
+            );
+        }
+        if (completionOpts.nPast > this.promptContext.nPast) {
+            throw new Error(
+                `nPast cannot be greater than ${this.promptContext.nPast}.`
+            );
+        }
+        let tokensIngested = 0;
+
+        if (!this.initialized) {
+            tokensIngested += await this.initialize(completionOpts);
+        }
+
+        let prompt = input;
+
+        if (Array.isArray(input)) {
+            // assuming input is a messages array
+            // -> tailing user message will be used as the final prompt. its optional.
+            // -> all system messages will be ignored.
+            // -> all other messages will be ingested with fakeReply
+            // -> user/assistant messages will be pushed into the messages array
+
+            let tailingUserMessage = "";
+            let messagesToIngest = input;
+
+            const lastMessage = input[input.length - 1];
+            if (lastMessage.role === "user") {
+                tailingUserMessage = lastMessage.content;
+                messagesToIngest = input.slice(0, input.length - 1);
+            }
+
+            if (messagesToIngest.length > 0) {
+                tokensIngested += await this.ingestMessages(
+                    messagesToIngest,
+                    completionOpts
+                );
+                this.messages.push(...messagesToIngest);
+            }
+
+            if (tailingUserMessage) {
+                prompt = tailingUserMessage;
+            } else {
+                return {
+                    text: "",
+                    nPast: this.promptContext.nPast,
+                    tokensIngested,
+                    tokensGenerated: 0,
+                };
+            }
+        }
+
+        const result = await this.model.generate(prompt, {
+            ...this.promptContext,
+            ...completionOpts,
+        });
+
+        this.promptContext.nPast = result.nPast;
+        result.tokensIngested += tokensIngested;
+
+        this.messages.push({
+            role: "user",
+            content: prompt,
+        });
+        this.messages.push({
+            role: "assistant",
+            content: result.text,
+        });
+
+        return result;
+    }
+}
+
+module.exports = {
+    ChatSession,
+};

gpt4all-bindings/typescript/src/config.js

@@ -27,15 +27,16 @@ const DEFAULT_MODEL_CONFIG = {
     promptTemplate: "### Human:\n%1\n\n### Assistant:\n",
 }
 
-const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models2.json";
+const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models3.json";
 
 const DEFAULT_PROMPT_CONTEXT = {
-    temp: 0.7,
+    temp: 0.1,
     topK: 40,
-    topP: 0.4,
+    topP: 0.9,
+    minP: 0.0,
     repeatPenalty: 1.18,
-    repeatLastN: 64,
-    nBatch: 8,
+    repeatLastN: 10,
+    nBatch: 100,
 }
 
 module.exports = {

gpt4all-bindings/typescript/src/gpt4all.d.ts

@@ -1,43 +1,11 @@
 /// <reference types="node" />
 declare module "gpt4all";
 
-type ModelType = "gptj" | "llama" | "mpt" | "replit";
-
-// NOTE: "deprecated" tag in below comment breaks the doc generator https://github.com/documentationjs/documentation/issues/1596
-/**
- * Full list of models available
- * DEPRECATED!! These model names are outdated and this type will not be maintained, please use a string literal instead
- */
-interface ModelFile {
-    /** List of GPT-J Models */
-    gptj:
-        | "ggml-gpt4all-j-v1.3-groovy.bin"
-        | "ggml-gpt4all-j-v1.2-jazzy.bin"
-        | "ggml-gpt4all-j-v1.1-breezy.bin"
-        | "ggml-gpt4all-j.bin";
-    /** List Llama Models */
-    llama:
-        | "ggml-gpt4all-l13b-snoozy.bin"
-        | "ggml-vicuna-7b-1.1-q4_2.bin"
-        | "ggml-vicuna-13b-1.1-q4_2.bin"
-        | "ggml-wizardLM-7B.q4_2.bin"
-        | "ggml-stable-vicuna-13B.q4_2.bin"
-        | "ggml-nous-gpt4-vicuna-13b.bin"
-        | "ggml-v3-13b-hermes-q5_1.bin";
-    /** List of MPT Models */
-    mpt:
-        | "ggml-mpt-7b-base.bin"
-        | "ggml-mpt-7b-chat.bin"
-        | "ggml-mpt-7b-instruct.bin";
-    /** List of Replit Models */
-    replit: "ggml-replit-code-v1-3b.bin";
-}
-
 interface LLModelOptions {
     /**
      * Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
      */
-    type?: ModelType;
+    type?: string;
     model_name: string;
     model_path: string;
     library_path?: string;
@@ -51,47 +19,259 @@ interface ModelConfig {
 }
 
 /**
- * Callback for controlling token generation
+ * Options for the chat session.
  */
-type TokenCallback = (tokenId: number, token: string, total: string) => boolean
+interface ChatSessionOptions extends Partial<LLModelPromptContext> {
+    /**
+     * System prompt to ingest on initialization.
+     */
+    systemPrompt?: string;
 
-/**
- *
- * InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
- *
- */
-declare class InferenceModel {
-    constructor(llm: LLModel, config: ModelConfig);
-    llm: LLModel;
-    config: ModelConfig;
-
-    generate(
-        prompt: string,
-        options?: Partial<LLModelPromptContext>,
-        callback?: TokenCallback
-    ): Promise<string>;
-
-   /**
-     * delete and cleanup the native model
-    */
-    dispose(): void
+    /**
+     * Messages to ingest on initialization.
+     */
+    messages?: ChatMessage[];
 }
 
+/**
+ * ChatSession utilizes an InferenceModel for efficient processing of chat conversations.
+ */
+declare class ChatSession implements CompletionProvider {
+    /**
+     * Constructs a new ChatSession using the provided InferenceModel and options.
+     * Does not set the chat session as the active chat session until initialize is called.
+     * @param {InferenceModel} model An InferenceModel instance.
+     * @param {ChatSessionOptions} [options] Options for the chat session including default completion options.
+     */
+    constructor(model: InferenceModel, options?: ChatSessionOptions);
+    /**
+     * The underlying InferenceModel used for generating completions.
+     */
+    model: InferenceModel;
+    /**
+     * The name of the model.
+     */
+    modelName: string;
+    /**
+     * The messages that have been exchanged in this chat session.
+     */
+    messages: ChatMessage[];
+    /**
+     * The system prompt that has been ingested at the beginning of the chat session.
+     */
+    systemPrompt: string;
+    /**
+     * The current prompt context of the chat session.
+     */
+    promptContext: LLModelPromptContext;
+
+    /**
+     * Ingests system prompt and initial messages.
+     * Sets this chat session as the active chat session of the model.
+     * @param {CompletionOptions} [options] Set completion options for initialization.
+     * @returns {Promise<number>} The number of tokens ingested during initialization. systemPrompt + messages.
+     */
+    initialize(completionOpts?: CompletionOptions): Promise<number>;
+
+    /**
+     * Prompts the model in chat-session context.
+     * @param {CompletionInput} input Input string or message array.
+     * @param {CompletionOptions} [options] Set completion options for this generation.
+     * @returns {Promise<InferenceResult>} The inference result.
+     * @throws {Error} If the chat session is not the active chat session of the model.
+     * @throws {Error} If nPast is set to a value higher than what has been ingested in the session.
+     */
+    generate(
+        input: CompletionInput,
+        options?: CompletionOptions
+    ): Promise<InferenceResult>;
+}
+
+/**
+ * Shape of InferenceModel generations.
+ */
+interface InferenceResult extends LLModelInferenceResult {
+    tokensIngested: number;
+    tokensGenerated: number;
+}
+
+/**
+ * InferenceModel represents an LLM which can make next-token predictions.
+ */
+declare class InferenceModel implements CompletionProvider {
+    constructor(llm: LLModel, config: ModelConfig);
+    /** The native LLModel */
+    llm: LLModel;
+    /** The configuration the instance was constructed with. */
+    config: ModelConfig;
+    /** The active chat session of the model. */
+    activeChatSession?: ChatSession;
+    /** The name of the model. */
+    modelName: string;
+
+    /**
+     * Create a chat session with the model and set it as the active chat session of this model.
+     * A model instance can only have one active chat session at a time.
+     * @param {ChatSessionOptions} options The options for the chat session.
+     * @returns {Promise<ChatSession>} The chat session.
+     */
+    createChatSession(options?: ChatSessionOptions): Promise<ChatSession>;
+
+    /**
+     * Prompts the model with a given input and optional parameters.
+     * @param {CompletionInput} input The prompt input.
+     * @param {CompletionOptions} options Prompt context and other options.
+     * @returns {Promise<InferenceResult>} The model's response to the prompt.
+     * @throws {Error} If nPast is set to a value smaller than 0.
+     * @throws {Error} If a messages array without a tailing user message is provided.
+     */
+    generate(
+        prompt: string,
+        options?: CompletionOptions
+    ): Promise<InferenceResult>;
+
+    /**
+     * delete and cleanup the native model
+     */
+    dispose(): void;
+}
+
+/**
+ * Options for generating one or more embeddings.
+ */
+interface EmbedddingOptions {
+    /**
+     * The model-specific prefix representing the embedding task, without the trailing colon. For Nomic Embed
+     * this can be `search_query`, `search_document`, `classification`, or `clustering`.
+     */
+    prefix?: string;
+    /**
+     *The embedding dimension, for use with Matryoshka-capable models. Defaults to full-size.
+     * @default determines on the model being used.
+     */
+    dimensionality?: number;
+    /**
+     * How to handle texts longer than the model can accept. One of `mean` or `truncate`.
+     * @default "mean"
+     */
+    longTextMode?: "mean" | "truncate";
+    /**
+     * Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens
+     * with long_text_mode="mean" will raise an error. Disabled by default.
+     * @default false
+     */
+    atlas?: boolean;
+}
+
+/**
+ * The nodejs moral equivalent to python binding's Embed4All().embed()
+ * meow
+ * @param {EmbeddingModel} model The embedding model instance.
+ * @param {string} text Text to embed.
+ * @param {EmbeddingOptions} options Optional parameters for the embedding.
+ * @returns {EmbeddingResult} The embedding result.
+ * @throws {Error} If dimensionality is set to a value smaller than 1.
+ */
+declare function createEmbedding(
+    model: EmbeddingModel,
+    text: string,
+    options?: EmbedddingOptions
+): EmbeddingResult<Float32Array>;
+
+/**
+ * Overload that takes multiple strings to embed.
+ * @param {EmbeddingModel} model The embedding model instance.
+ * @param {string[]} texts Texts to embed.
+ * @param {EmbeddingOptions} options Optional parameters for the embedding.
+ * @returns {EmbeddingResult<Float32Array[]>} The embedding result.
+ * @throws {Error} If dimensionality is set to a value smaller than 1.
+ */
+declare function createEmbedding(
+    model: EmbeddingModel,
+    text: string[],
+    options?: EmbedddingOptions
+): EmbeddingResult<Float32Array[]>;
+
+/**
+ * The resulting embedding.
+ */
+interface EmbeddingResult<T> {
+    /**
+     * Encoded token count. Includes overlap but specifically excludes tokens used for the prefix/task_type, BOS/CLS token, and EOS/SEP token
+     **/
+    n_prompt_tokens: number;
+
+    embeddings: T;
+}
 /**
  * EmbeddingModel represents an LLM which can create embeddings, which are float arrays
  */
 declare class EmbeddingModel {
     constructor(llm: LLModel, config: ModelConfig);
+    /** The native LLModel */
     llm: LLModel;
+    /** The configuration the instance was constructed with. */
     config: ModelConfig;
 
-    embed(text: string): Float32Array;
+    /**
+     * Create an embedding from a given input string. See EmbeddingOptions.
+     * @param {string} text
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {EmbeddingResult<Float32Array>} The embedding result.
+     */
+    embed(
+        text: string,
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): EmbeddingResult<Float32Array>;
+    /**
+     * Create an embedding from a given input text array. See EmbeddingOptions.
+     * @param {string[]} text
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {EmbeddingResult<Float32Array[]>} The embedding result.
+     */
+    embed(
+        text: string[],
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): EmbeddingResult<Float32Array[]>;
 
     /**
-      * delete and cleanup the native model
+     * delete and cleanup the native model
      */
-    dispose(): void
+    dispose(): void;
+}
 
+/**
+ * Shape of LLModel's inference result.
+ */
+interface LLModelInferenceResult {
+    text: string;
+    nPast: number;
+}
+
+interface LLModelInferenceOptions extends Partial<LLModelPromptContext> {
+    /** Callback for response tokens, called for each generated token.
+     * @param {number} tokenId The token id.
+     * @param {string} token The token.
+     * @returns {boolean | undefined} Whether to continue generating tokens.
+     * */
+    onResponseToken?: (tokenId: number, token: string) => boolean | void;
+    /** Callback for prompt tokens, called for each input token in the prompt.
+     * @param {number} tokenId The token id.
+     * @returns {boolean | undefined} Whether to continue ingesting the prompt.
+     * */
+    onPromptToken?: (tokenId: number) => boolean | void;
 }
 
 /**
@@ -101,14 +281,13 @@ declare class EmbeddingModel {
 declare class LLModel {
     /**
      * Initialize a new LLModel.
-     * @param path Absolute path to the model file.
+     * @param {string} path Absolute path to the model file.
      * @throws {Error} If the model file does not exist.
      */
-    constructor(path: string);
     constructor(options: LLModelOptions);
 
-    /** either 'gpt', mpt', or 'llama' or undefined */
-    type(): ModelType | undefined;
+    /** undefined or user supplied */
+    type(): string | undefined;
 
     /** The name of the model. */
     name(): string;
@@ -134,29 +313,53 @@ declare class LLModel {
     setThreadCount(newNumber: number): void;
 
     /**
-     * Prompt the model with a given input and optional parameters.
-     * This is the raw output from model.
-     * Use the prompt function exported for a value
-     * @param q The prompt input.
-     * @param params Optional parameters for the prompt context.
-     * @param callback - optional callback to control token generation.
-     * @returns The result of the model prompt.
+     * Prompt the model directly with a given input string and optional parameters.
+     * Use the higher level createCompletion methods for a more user-friendly interface.
+     * @param {string} prompt The prompt input.
+     * @param {LLModelInferenceOptions} options Optional parameters for the generation.
+     * @returns {LLModelInferenceResult} The response text and final context size.
      */
-    raw_prompt(
-        q: string,
-        params: Partial<LLModelPromptContext>,
-        callback?: TokenCallback
-    ): Promise<string>
+    infer(
+        prompt: string,
+        options: LLModelInferenceOptions
+    ): Promise<LLModelInferenceResult>;
 
     /**
-     * Embed text with the model. Keep in mind that
-     * not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
-     * Use the prompt function exported for a value
-     * @param q The prompt input.
-     * @param params Optional parameters for the prompt context.
-     * @returns The result of the model prompt.
+     * Embed text with the model. See EmbeddingOptions for more information.
+     * Use the higher level createEmbedding methods for a more user-friendly interface.
+     * @param {string} text
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {Float32Array} The embedding of the text.
      */
-    embed(text: string): Float32Array;
+    embed(
+        text: string,
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): Float32Array;
+
+    /**
+     * Embed multiple texts with the model. See EmbeddingOptions for more information.
+     * Use the higher level createEmbedding methods for a more user-friendly interface.
+     * @param {string[]} texts
+     * @param {string} prefix
+     * @param {number} dimensionality
+     * @param {boolean} doMean
+     * @param {boolean} atlas
+     * @returns {Float32Array[]} The embeddings of the texts.
+     */
+    embed(
+        texts: string,
+        prefix: string,
+        dimensionality: number,
+        doMean: boolean,
+        atlas: boolean
+    ): Float32Array[];
+
     /**
      * Whether the model is loaded or not.
      */
@@ -166,81 +369,97 @@ declare class LLModel {
      * Where to search for the pluggable backend libraries
      */
     setLibraryPath(s: string): void;
+
     /**
      * Where to get the pluggable backend libraries
      */
     getLibraryPath(): string;
+
     /**
-     * Initiate a GPU by a string identifier. 
-     * @param {number} memory_required Should be in the range size_t or will throw 
+     * Initiate a GPU by a string identifier.
+     * @param {number} memory_required Should be in the range size_t or will throw
      * @param {string} device_name  'amd' | 'nvidia' | 'intel' | 'gpu' | gpu name.
      * read LoadModelOptions.device for more information
      */
-    initGpuByString(memory_required: number, device_name: string): boolean
+    initGpuByString(memory_required: number, device_name: string): boolean;
+
     /**
      * From C documentation
      * @returns True if a GPU device is successfully initialized, false otherwise.
      */
-    hasGpuDevice(): boolean
-    /**
-      * GPUs that are usable for this LLModel
-      * @param nCtx Maximum size of context window
-      * @throws if hasGpuDevice returns false (i think)
-      * @returns 
-      */
-    listGpu(nCtx: number) : GpuDevice[]
+    hasGpuDevice(): boolean;
 
     /**
-      * delete and cleanup the native model
+     * GPUs that are usable for this LLModel
+     * @param {number} nCtx Maximum size of context window
+     * @throws if hasGpuDevice returns false (i think)
+     * @returns
      */
-    dispose(): void
+    listGpu(nCtx: number): GpuDevice[];
+
+    /**
+     * delete and cleanup the native model
+     */
+    dispose(): void;
 }
-/** 
+/**
  * an object that contains gpu data on this machine.
  */
 interface GpuDevice {
     index: number;
     /**
-      * same as VkPhysicalDeviceType
+     * same as VkPhysicalDeviceType
      */
-    type: number;            
-    heapSize : number; 
+    type: number;
+    heapSize: number;
     name: string;
     vendor: string;
 }
 
 /**
-  * Options that configure a model's behavior.
-  */
+ * Options that configure a model's behavior.
+ */
 interface LoadModelOptions {
+    /**
+     * Where to look for model files.
+     */
     modelPath?: string;
+    /**
+     * Where to look for the backend libraries.
+     */
     librariesPath?: string;
+    /**
+     * The path to the model configuration file, useful for offline usage or custom model configurations.
+     */
     modelConfigFile?: string;
+    /**
+     * Whether to allow downloading the model if it is not present at the specified path.
+     */
     allowDownload?: boolean;
+    /**
+     * Enable verbose logging.
+     */
     verbose?: boolean;
-    /* The processing unit on which the model will run. It can be set to
+    /**
+     * The processing unit on which the model will run. It can be set to
      * - "cpu": Model will run on the central processing unit.
      * - "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
      * - "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
-
-	   Alternatively, a specific GPU name can also be provided, and the model will run on the GPU that matches the name
-       if it's available.
-
-       Default is "cpu".
-
-	   Note: If a GPU device lacks sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All
-       instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the
-       model.
-    */ 
+     * - "gpu name": Model will run on the GPU that matches the name if it's available.
+     * Note: If a GPU device lacks sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All
+     * instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the
+     * model.
+     * @default "cpu"
+     */
     device?: string;
-    /* 
+    /**
      * The Maximum window size of this model
-     * Default of 2048
+     * @default 2048
      */
     nCtx?: number;
-    /* 
+    /**
      * Number of gpu layers needed
-     * Default of 100
+     * @default 100
      */
     ngl?: number;
 }
@@ -277,66 +496,84 @@ declare function loadModel(
 ): Promise<InferenceModel | EmbeddingModel>;
 
 /**
- * The nodejs equivalent to python binding's chat_completion
- * @param {InferenceModel} model - The language model object.
- * @param {PromptMessage[]} messages - The array of messages for the conversation.
- * @param {CompletionOptions} options - The options for creating the completion.
- * @returns {CompletionReturn} The completion result.
+ * Interface for createCompletion methods, implemented by InferenceModel and ChatSession.
+ * Implement your own CompletionProvider or extend ChatSession to generate completions with custom logic.
  */
-declare function createCompletion(
-    model: InferenceModel,
-    messages: PromptMessage[],
-    options?: CompletionOptions
-): Promise<CompletionReturn>;
-
-/**
- * The nodejs moral equivalent to python binding's Embed4All().embed()
- * meow
- * @param {EmbeddingModel} model - The language model object.
- * @param {string} text - text to embed
- * @returns {Float32Array} The completion result.
- */
-declare function createEmbedding(
-    model: EmbeddingModel,
-    text: string
-): Float32Array;
-
-/**
- * The options for creating the completion.
- */
-interface CompletionOptions extends Partial<LLModelPromptContext> {
-    /**
-     * Indicates if verbose logging is enabled.
-     * @default true
-     */
-    verbose?: boolean;
-
-    /**
-     * Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
-     * Note that if this is not defined, system messages will not be included in the prompt.
-     */
-    systemPromptTemplate?: string;
-
-    /**
-     * Template for user messages, with %1 being replaced by the message.
-     */
-    promptTemplate?: boolean;
-
-    /**
-     * The initial instruction for the model, on top of the prompt
-     */
-    promptHeader?: string;
-
-    /**
-     * The last instruction for the model, appended to the end of the prompt.
-     */
-    promptFooter?: string;
+interface CompletionProvider {
+    modelName: string;
+    generate(
+        input: CompletionInput,
+        options?: CompletionOptions
+    ): Promise<InferenceResult>;
 }
 
 /**
- * A message in the conversation, identical to OpenAI's chat message.
+ * Options for creating a completion.
  */
-interface PromptMessage {
+interface CompletionOptions extends LLModelInferenceOptions {
+    /**
+     * Indicates if verbose logging is enabled.
+     * @default false
+     */
+    verbose?: boolean;
+}
+
+/**
+ * The input for creating a completion. May be a string or an array of messages.
+ */
+type CompletionInput = string | ChatMessage[];
+
+/**
+ * The nodejs equivalent to python binding's chat_completion
+ * @param {CompletionProvider} provider - The inference model object or chat session
+ * @param {CompletionInput} input - The input string or message array
+ * @param {CompletionOptions} options - The options for creating the completion.
+ * @returns {CompletionResult} The completion result.
+ */
+declare function createCompletion(
+    provider: CompletionProvider,
+    input: CompletionInput,
+    options?: CompletionOptions
+): Promise<CompletionResult>;
+
+/**
+ * Streaming variant of createCompletion, returns a stream of tokens and a promise that resolves to the completion result.
+ * @param {CompletionProvider} provider - The inference model object or chat session
+ * @param {CompletionInput} input - The input string or message array
+ * @param {CompletionOptions} options - The options for creating the completion.
+ * @returns {CompletionStreamReturn} An object of token stream and the completion result promise.
+ */
+declare function createCompletionStream(
+    provider: CompletionProvider,
+    input: CompletionInput,
+    options?: CompletionOptions
+): CompletionStreamReturn;
+
+/**
+ * The result of a streamed completion, containing a stream of tokens and a promise that resolves to the completion result.
+ */
+interface CompletionStreamReturn {
+    tokens: NodeJS.ReadableStream;
+    result: Promise<CompletionResult>;
+}
+
+/**
+ * Async generator variant of createCompletion, yields tokens as they are generated and returns the completion result.
+ * @param {CompletionProvider} provider - The inference model object or chat session
+ * @param {CompletionInput} input - The input string or message array
+ * @param {CompletionOptions} options - The options for creating the completion.
+ * @returns {AsyncGenerator<string>} The stream of generated tokens
+ */
+declare function createCompletionGenerator(
+    provider: CompletionProvider,
+    input: CompletionInput,
+    options: CompletionOptions
+): AsyncGenerator<string, CompletionResult>;
+
+/**
+ * A message in the conversation.
+ */
+interface ChatMessage {
     /** The role of the message. */
     role: "system" | "assistant" | "user";
 
@@ -345,34 +582,31 @@ interface PromptMessage {
 }
 
 /**
- * The result of the completion, similar to OpenAI's format.
+ * The result of a completion.
  */
-interface CompletionReturn {
+interface CompletionResult {
     /** The model used for the completion. */
     model: string;
 
     /** Token usage report. */
     usage: {
-        /** The number of tokens used in the prompt. */
+        /** The number of tokens ingested during the completion. */
         prompt_tokens: number;
 
-        /** The number of tokens used in the completion. */
+        /** The number of tokens generated in the completion. */
         completion_tokens: number;
 
         /** The total number of tokens used. */
         total_tokens: number;
+
+        /** Number of tokens used in the conversation. */
+        n_past_tokens: number;
     };
 
-    /** The generated completions. */
-    choices: CompletionChoice[];
-}
-
-/**
- * A completion choice, similar to OpenAI's format.
- */
-interface CompletionChoice {
-    /** Response message */
-    message: PromptMessage;
+    /** The generated completion. */
+    choices: Array<{
+        message: ChatMessage;
+    }>;
 }
 
 /**
@@ -385,19 +619,33 @@ interface LLModelPromptContext {
     /** The size of the raw tokens vector. */
     tokensSize: number;
 
-    /** The number of tokens in the past conversation. */
+    /** The number of tokens in the past conversation.
+     * This may be used to "roll back" the conversation to a previous state.
+     * Note that for most use cases the default value should be sufficient and this should not be set.
+     * @default 0 For completions using InferenceModel, meaning the model will only consider the input prompt.
+     * @default nPast For completions using ChatSession. This means the context window will be automatically determined
+     * and possibly resized (see contextErase) to keep the conversation performant.
+     * */
     nPast: number;
 
-    /** The number of tokens possible in the context window.
-     * @default 1024
-     */
-    nCtx: number;
-
-    /** The number of tokens to predict.
-     * @default 128
+    /** The maximum number of tokens to predict.
+     * @default 4096
      * */
     nPredict: number;
 
+    /** Template for user / assistant message pairs.
+     * %1 is required and will be replaced by the user input.
+     * %2 is optional and will be replaced by the assistant response. If not present, the assistant response will be appended.
+     */
+    promptTemplate?: string;
+
+    /** The context window size. Do not use, it has no effect. See loadModel options.
+     * THIS IS DEPRECATED!!!
+     * Use loadModel's nCtx option instead.
+     * @default 2048
+     */
+    nCtx: number;
+
     /** The top-k logits to sample from.
      * Top-K sampling selects the next token only from the top K most likely tokens predicted by the model.
      * It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit
@@ -409,26 +657,33 @@ interface LLModelPromptContext {
     topK: number;
 
     /** The nucleus sampling probability threshold.
-     * Top-P limits the selection of the next token to a subset of tokens with a cumulative probability 
+     * Top-P limits the selection of the next token to a subset of tokens with a cumulative probability
      * above a threshold P. This method, also known as nucleus sampling, finds a balance between diversity
      * and quality by considering both token probabilities and the number of tokens available for sampling.
      * When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
      * On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
-     * The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
-     * for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
-     * @default 0.4
+     * @default 0.9
+     *
      * */
     topP: number;
 
+    /**
+     * The minimum probability of a token to be considered.
+     * @default 0.0
+     */
+    minP: number;
+
     /** The temperature to adjust the model's output distribution.
      * Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
      * (eg., 1.2) increase randomness, resulting in more imaginative and diverse text. Lower temperatures (eg., 0.5)
      * make the output more focused, predictable, and conservative. When the temperature is set to 0, the output
      * becomes completely deterministic, always selecting the most probable next token and producing identical results
-     * each time. A safe range would be around 0.6 - 0.85, but you are free to search what value fits best for you.
-     * @default 0.7
+     * each time. Try what value fits best for your use case and model.
+     * @default 0.1
+     * @alias temperature
      * */
     temp: number;
+    temperature: number;
 
     /** The number of predictions to generate in parallel.
      * By splitting the prompt every N tokens, prompt-batch-size reduces RAM usage during processing. However,
@@ -451,31 +706,17 @@ interface LLModelPromptContext {
      * The repeat-penalty-tokens N option controls the number of tokens in the history to consider for penalizing repetition.
      * A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only
      * consider recent tokens.
-     * @default 64
+     * @default 10
      * */
     repeatLastN: number;
 
     /** The percentage of context to erase if the context window is exceeded.
-     * @default 0.5
+     * Set it to a lower value to keep context for longer at the cost of performance.
+     * @default 0.75
      * */
     contextErase: number;
 }
 
-
-/**
- * Creates an async generator of tokens
- * @param {InferenceModel} llmodel - The language model object.
- * @param {PromptMessage[]} messages - The array of messages for the conversation.
- * @param {CompletionOptions} options - The options for creating the completion.
- * @param {TokenCallback} callback - optional callback to control token generation.
- * @returns {AsyncGenerator<string>} The stream of generated tokens
- */
-declare function generateTokens(
-    llmodel: InferenceModel,
-    messages: PromptMessage[],
-    options: CompletionOptions,
-    callback?: TokenCallback
-): AsyncGenerator<string>;
 /**
  * From python api:
  * models will be stored in (homedir)/.cache/gpt4all/`
@@ -508,7 +749,7 @@ declare const DEFAULT_MODEL_LIST_URL: string;
  * Initiates the download of a model file.
  * By default this downloads without waiting. use the controller returned to alter this behavior.
  * @param {string} modelName - The model to be downloaded.
- * @param {DownloadOptions} options - to pass into the downloader. Default is { location: (cwd), verbose: false }.
+ * @param {DownloadModelOptions} options - to pass into the downloader. Default is { location: (cwd), verbose: false }.
  * @returns {DownloadController} object that allows controlling the download process.
  *
  * @throws {Error} If the model already exists in the specified location.
@@ -556,7 +797,9 @@ interface ListModelsOptions {
     file?: string;
 }
 
-declare function listModels(options?: ListModelsOptions): Promise<ModelConfig[]>;
+declare function listModels(
+    options?: ListModelsOptions
+): Promise<ModelConfig[]>;
 
 interface RetrieveModelOptions {
     allowDownload?: boolean;
@@ -581,30 +824,35 @@ interface DownloadController {
 }
 
 export {
-    ModelType,
-    ModelFile,
-    ModelConfig,
-    InferenceModel,
-    EmbeddingModel,
     LLModel,
     LLModelPromptContext,
-    PromptMessage,
+    ModelConfig,
+    InferenceModel,
+    InferenceResult,
+    EmbeddingModel,
+    EmbeddingResult,
+    ChatSession,
+    ChatMessage,
+    CompletionInput,
+    CompletionProvider,
     CompletionOptions,
+    CompletionResult,
     LoadModelOptions,
+    DownloadController,
+    RetrieveModelOptions,
+    DownloadModelOptions,
+    GpuDevice,
     loadModel,
+    downloadModel,
+    retrieveModel,
+    listModels,
     createCompletion,
+    createCompletionStream,
+    createCompletionGenerator,
     createEmbedding,
-    generateTokens,
     DEFAULT_DIRECTORY,
     DEFAULT_LIBRARIES_DIRECTORY,
     DEFAULT_MODEL_CONFIG,
     DEFAULT_PROMPT_CONTEXT,
     DEFAULT_MODEL_LIST_URL,
-    downloadModel,
-    retrieveModel,
-    listModels,
-    DownloadController,
-    RetrieveModelOptions,
-    DownloadModelOptions,
-    GpuDevice
 };

gpt4all-bindings/typescript/src/gpt4all.js

@@ -2,8 +2,10 @@
 
 /// This file implements the gpt4all.d.ts file endings.
 /// Written in commonjs to support both ESM and CJS projects.
-const { existsSync } = require("fs");
+const { existsSync } = require("node:fs");
 const path = require("node:path");
+const Stream = require("node:stream");
+const assert = require("node:assert");
 const { LLModel } = require("node-gyp-build")(path.resolve(__dirname, ".."));
 const {
     retrieveModel,
@@ -18,15 +20,14 @@ const {
     DEFAULT_MODEL_LIST_URL,
 } = require("./config.js");
 const { InferenceModel, EmbeddingModel } = require("./models.js");
-const Stream = require('stream')
-const assert = require("assert");
+const { ChatSession } = require("./chat-session.js");
 
 /**
  * Loads a machine learning model with the specified name. The defacto way to create a model.
  * By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
  *
  * @param {string} modelName - The name of the model to load.
- * @param {LoadModelOptions|undefined} [options] - (Optional) Additional options for loading the model.
+ * @param {import('./gpt4all').LoadModelOptions|undefined} [options] - (Optional) Additional options for loading the model.
  * @returns {Promise<InferenceModel | EmbeddingModel>} A promise that resolves to an instance of the loaded LLModel.
  */
 async function loadModel(modelName, options = {}) {
@@ -35,10 +36,10 @@ async function loadModel(modelName, options = {}) {
         librariesPath: DEFAULT_LIBRARIES_DIRECTORY,
         type: "inference",
         allowDownload: true,
-        verbose: true,
-        device: 'cpu',
+        verbose: false,
+        device: "cpu",
         nCtx: 2048,
-        ngl : 100,
+        ngl: 100,
         ...options,
     };
 
@@ -49,12 +50,14 @@ async function loadModel(modelName, options = {}) {
         verbose: loadOptions.verbose,
     });
 
-    assert.ok(typeof loadOptions.librariesPath === 'string');
+    assert.ok(
+        typeof loadOptions.librariesPath === "string",
+        "Libraries path should be a string"
+    );
     const existingPaths = loadOptions.librariesPath
         .split(";")
         .filter(existsSync)
-        .join(';');
-    console.log("Passing these paths into runtime library search:", existingPaths)
+        .join(";");
 
     const llmOptions = {
         model_name: appendBinSuffixIfMissing(modelName),
@@ -62,13 +65,15 @@ async function loadModel(modelName, options = {}) {
         library_path: existingPaths,
         device: loadOptions.device,
         nCtx: loadOptions.nCtx,
-        ngl: loadOptions.ngl
+        ngl: loadOptions.ngl,
     };
 
     if (loadOptions.verbose) {
-        console.debug("Creating LLModel with options:", llmOptions);
+        console.debug("Creating LLModel:", {
+            llmOptions,
+            modelConfig,
+        });
     }
-    console.log(modelConfig)
     const llmodel = new LLModel(llmOptions);
     if (loadOptions.type === "embedding") {
         return new EmbeddingModel(llmodel, modelConfig);
@@ -79,75 +84,43 @@ async function loadModel(modelName, options = {}) {
     }
 }
 
-/**
- * Formats a list of messages into a single prompt string.
- */
-function formatChatPrompt(
-    messages,
-    {
-        systemPromptTemplate,
-        defaultSystemPrompt,
-        promptTemplate,
-        promptFooter,
-        promptHeader,
-    }
-) {
-    const systemMessages = messages
-        .filter((message) => message.role === "system")
-        .map((message) => message.content);
+function createEmbedding(model, text, options={}) {
+    let {
+        dimensionality = undefined,
+        longTextMode = "mean",
+        atlas = false,
+    } = options;
 
-    let fullPrompt = "";
-
-    if (promptHeader) {
-        fullPrompt += promptHeader + "\n\n";
-    }
-
-    if (systemPromptTemplate) {
-        // if user specified a template for the system prompt, put all system messages in the template
-        let systemPrompt = "";
-
-        if (systemMessages.length > 0) {
-            systemPrompt += systemMessages.join("\n");
-        }
-
-        if (systemPrompt) {
-            fullPrompt +=
-                systemPromptTemplate.replace("%1", systemPrompt) + "\n";
-        }
-    } else if (defaultSystemPrompt) {
-        // otherwise, use the system prompt from the model config and ignore system messages
-        fullPrompt += defaultSystemPrompt + "\n\n";
-    }
-
-    if (systemMessages.length > 0 && !systemPromptTemplate) {
-        console.warn(
-            "System messages were provided, but no systemPromptTemplate was specified. System messages will be ignored."
-        );
-    }
-
-    for (const message of messages) {
-        if (message.role === "user") {
-            const userMessage = promptTemplate.replace(
-                "%1",
-                message["content"]
+    if (dimensionality === undefined) {
+        dimensionality = -1;
+    } else {
+        if (dimensionality <= 0) {
+            throw new Error(
+                `Dimensionality must be undefined or a positive integer, got ${dimensionality}`
             );
-            fullPrompt += userMessage;
         }
-        if (message["role"] == "assistant") {
-            const assistantMessage = message["content"] + "\n";
-            fullPrompt += assistantMessage;
+        if (dimensionality < model.MIN_DIMENSIONALITY) {
+            console.warn(
+                `Dimensionality ${dimensionality} is less than the suggested minimum of ${model.MIN_DIMENSIONALITY}. Performance may be degraded.`
+            );
         }
     }
 
-    if (promptFooter) {
-        fullPrompt += "\n\n" + promptFooter;
+    let doMean;
+    switch (longTextMode) {
+        case "mean":
+            doMean = true;
+            break;
+        case "truncate":
+            doMean = false;
+            break;
+        default:
+            throw new Error(
+                `Long text mode must be one of 'mean' or 'truncate', got ${longTextMode}`
+            );
     }
 
-    return fullPrompt;
-}
-
-function createEmbedding(model, text) {
-    return model.embed(text);
+    return model.embed(text, options?.prefix, dimensionality, doMean, atlas);
 }
 
 const defaultCompletionOptions = {
@@ -155,162 +128,76 @@ const defaultCompletionOptions = {
     ...DEFAULT_PROMPT_CONTEXT,
 };
 
-function preparePromptAndContext(model,messages,options){
-    if (options.hasDefaultHeader !== undefined) {
-        console.warn(
-            "hasDefaultHeader (bool) is deprecated and has no effect, use promptHeader (string) instead"
-        );
-    }
-
-    if (options.hasDefaultFooter !== undefined) {
-        console.warn(
-            "hasDefaultFooter (bool) is deprecated and has no effect, use promptFooter (string) instead"
-        );
-    }
-
-    const optionsWithDefaults = {
+async function createCompletion(
+    provider,
+    input,
+    options = defaultCompletionOptions
+) {
+    const completionOptions = {
         ...defaultCompletionOptions,
         ...options,
     };
 
-    const {
-        verbose,
-        systemPromptTemplate,
-        promptTemplate,
-        promptHeader,
-        promptFooter,
-        ...promptContext
-    } = optionsWithDefaults;
-
-    
-    const prompt = formatChatPrompt(messages, {
-        systemPromptTemplate,
-        defaultSystemPrompt: model.config.systemPrompt,
-        promptTemplate: promptTemplate || model.config.promptTemplate || "%1",
-        promptHeader: promptHeader || "",
-        promptFooter: promptFooter || "",
-        // These were the default header/footer prompts used for non-chat single turn completions.
-        // both seem to be working well still with some models, so keeping them here for reference.
-        // promptHeader: '### Instruction: The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.',
-        // promptFooter: '### Response:',
-    });
+    const result = await provider.generate(
+        input,
+        completionOptions,
+    );
 
     return {
-        prompt, promptContext, verbose
-    }
-}
-
-async function createCompletion(
-    model,
-    messages,
-    options = defaultCompletionOptions
-) {
-    const { prompt, promptContext, verbose } = preparePromptAndContext(model,messages,options);
-
-    if (verbose) {
-        console.debug("Sending Prompt:\n" + prompt);
-    }
-
-    let tokensGenerated = 0
-
-    const response = await model.generate(prompt, promptContext,() => {
-        tokensGenerated++;
-        return true;
-    });
-
-    if (verbose) {
-        console.debug("Received Response:\n" + response);
-    }
-
-    return {
-        llmodel: model.llm.name(),
+        model: provider.modelName,
         usage: {
-            prompt_tokens: prompt.length,
-            completion_tokens: tokensGenerated,
-            total_tokens: prompt.length + tokensGenerated, //TODO Not sure how to get tokens in prompt
+            prompt_tokens: result.tokensIngested,
+            total_tokens: result.tokensIngested + result.tokensGenerated,
+            completion_tokens: result.tokensGenerated,
+            n_past_tokens: result.nPast,
         },
         choices: [
             {
                 message: {
                     role: "assistant",
-                    content: response,
+                    content: result.text,
                 },
+                // TODO some completion APIs also provide logprobs and finish_reason, could look into adding those
             },
         ],
     };
 }
 
-function _internal_createTokenStream(stream,model,
-    messages,
-    options = defaultCompletionOptions,callback = undefined) {
-    const { prompt, promptContext, verbose } = preparePromptAndContext(model,messages,options);
+function createCompletionStream(
+    provider,
+    input,
+    options = defaultCompletionOptions
+) {
+    const completionStream = new Stream.PassThrough({
+        encoding: "utf-8",
+    });
 
-
-    if (verbose) {
-        console.debug("Sending Prompt:\n" + prompt);
-    }
-
-    model.generate(prompt, promptContext,(tokenId, token, total) => {
-        stream.push(token);
-        
-        if(callback !== undefined){
-            return callback(tokenId,token,total);
-        }
-
-        return true;
-    }).then(() => {
-        stream.end()
-    })
-
-    return stream;
-}
-
-function _createTokenStream(model,
-    messages,
-    options = defaultCompletionOptions,callback = undefined) {
-   
-   // Silent crash if we dont do this here
-   const stream = new Stream.PassThrough({
-     encoding: 'utf-8'
-   });
-   return _internal_createTokenStream(stream,model,messages,options,callback);
-}
-
-async function* generateTokens(model,
-    messages,
-    options = defaultCompletionOptions, callback = undefined) {
-    const stream = _createTokenStream(model,messages,options,callback)
-
-    let bHasFinished = false;
-    let activeDataCallback = undefined;
-    const finishCallback = () => {
-        bHasFinished = true;
-        if(activeDataCallback !== undefined){
-            activeDataCallback(undefined);
-        }
-    }
-
-    stream.on("finish",finishCallback)
-
-    while (!bHasFinished) {
-        const token = await new Promise((res) => {
-
-            activeDataCallback = (d) => {
-                stream.off("data",activeDataCallback)
-                activeDataCallback = undefined
-                res(d);
+    const completionPromise = createCompletion(provider, input, {
+        ...options,
+        onResponseToken: (tokenId, token) => {
+            completionStream.push(token);
+            if (options.onResponseToken) {
+                return options.onResponseToken(tokenId, token);
             }
-            stream.on('data', activeDataCallback)
-        })
+        },
+    }).then((result) => {
+        completionStream.push(null);
+        completionStream.emit("end");
+        return result;
+    });
 
-        if (token == undefined) {
-            break;
-        }
+    return {
+        tokens: completionStream,
+        result: completionPromise,
+    };
+}
 
-        yield token;
+async function* createCompletionGenerator(provider, input, options) {
+    const completion = createCompletionStream(provider, input, options);
+    for await (const chunk of completion.tokens) {
+        yield chunk;
     }
-
-    stream.off("finish",finishCallback);
+    return await completion.result;
 }
 
 module.exports = {
@@ -322,10 +209,12 @@ module.exports = {
     LLModel,
     InferenceModel,
     EmbeddingModel,
+    ChatSession,
     createCompletion,
+    createCompletionStream,
+    createCompletionGenerator,
     createEmbedding,
     downloadModel,
     retrieveModel,
     loadModel,
-    generateTokens
 };

gpt4all-bindings/typescript/src/models.js

@@ -1,18 +1,138 @@
-const { normalizePromptContext, warnOnSnakeCaseKeys } = require('./util');
+const { DEFAULT_PROMPT_CONTEXT } = require("./config");
+const { ChatSession } = require("./chat-session");
+const { prepareMessagesForIngest } = require("./util");
 
 class InferenceModel {
     llm;
+    modelName;
     config;
+    activeChatSession;
 
     constructor(llmodel, config) {
         this.llm = llmodel;
         this.config = config;
+        this.modelName = this.llm.name();
     }
 
-    async generate(prompt, promptContext,callback) {
-        warnOnSnakeCaseKeys(promptContext);
-        const normalizedPromptContext = normalizePromptContext(promptContext);
-        const result = this.llm.raw_prompt(prompt, normalizedPromptContext,callback);
+    async createChatSession(options) {
+        const chatSession = new ChatSession(this, options);
+        await chatSession.initialize();
+        this.activeChatSession = chatSession;
+        return this.activeChatSession;
+    }
+
+    async generate(input, options = DEFAULT_PROMPT_CONTEXT) {
+        const { verbose, ...otherOptions } = options;
+        const promptContext = {
+            promptTemplate: this.config.promptTemplate,
+            temp:
+                otherOptions.temp ??
+                otherOptions.temperature ??
+                DEFAULT_PROMPT_CONTEXT.temp,
+            ...otherOptions,
+        };
+        
+        if (promptContext.nPast < 0) {
+            throw new Error("nPast must be a non-negative integer.");
+        }
+
+        if (verbose) {
+            console.debug("Generating completion", {
+                input,
+                promptContext,
+            });
+        }
+
+        let prompt = input;
+        let nPast = promptContext.nPast;
+        let tokensIngested = 0;
+
+        if (Array.isArray(input)) {
+            // assuming input is a messages array
+            // -> tailing user message will be used as the final prompt. its required.
+            // -> leading system message will be ingested as systemPrompt, further system messages will be ignored
+            // -> all other messages will be ingested with fakeReply
+            // -> model/context will only be kept for this completion; "stateless"
+            nPast = 0;
+            const messages = [...input];
+            const lastMessage = input[input.length - 1];
+            if (lastMessage.role !== "user") {
+                // this is most likely a user error
+                throw new Error("The final message must be of role 'user'.");
+            }
+            if (input[0].role === "system") {
+                // needs to be a pre-templated prompt ala '<|im_start|>system\nYou are an advanced mathematician.\n<|im_end|>\n'
+                const systemPrompt = input[0].content;
+                const systemRes = await this.llm.infer(systemPrompt, {
+                    promptTemplate: "%1",
+                    nPredict: 0,
+                    special: true,
+                });
+                nPast = systemRes.nPast;
+                tokensIngested += systemRes.tokensIngested;
+                messages.shift();
+            }
+
+            prompt = lastMessage.content;
+            const messagesToIngest = messages.slice(0, input.length - 1);
+            const turns = prepareMessagesForIngest(messagesToIngest);
+
+            for (const turn of turns) {
+                const turnRes = await this.llm.infer(turn.user, {
+                    ...promptContext,
+                    nPast,
+                    fakeReply: turn.assistant,
+                });
+                tokensIngested += turnRes.tokensIngested;
+                nPast = turnRes.nPast;
+            }
+        }
+
+        let tokensGenerated = 0;
+
+        const result = await this.llm.infer(prompt, {
+            ...promptContext,
+            nPast,
+            onPromptToken: (tokenId) => {
+                let continueIngestion = true;
+                tokensIngested++;
+                if (options.onPromptToken) {
+                    // catch errors because if they go through cpp they will loose stacktraces
+                    try {
+                        // don't cancel ingestion unless user explicitly returns false
+                        continueIngestion =
+                            options.onPromptToken(tokenId) !== false;
+                    } catch (e) {
+                        console.error("Error in onPromptToken callback", e);
+                        continueIngestion = false;
+                    }
+                }
+                return continueIngestion;
+            },
+            onResponseToken: (tokenId, token) => {
+                let continueGeneration = true;
+                tokensGenerated++;
+                if (options.onResponseToken) {
+                    try {
+                        // don't cancel the generation unless user explicitly returns false
+                        continueGeneration =
+                            options.onResponseToken(tokenId, token) !== false;
+                    } catch (err) {
+                        console.error("Error in onResponseToken callback", err);
+                        continueGeneration = false;
+                    }
+                }
+                return continueGeneration;
+            },
+        });
+
+        result.tokensGenerated = tokensGenerated;
+        result.tokensIngested = tokensIngested;
+
+        if (verbose) {
+            console.debug("Finished completion:\n", result);
+        }
+
         return result;
     }
 
@@ -24,14 +144,14 @@ class InferenceModel {
 class EmbeddingModel {
     llm;
     config;
-
+    MIN_DIMENSIONALITY = 64;
     constructor(llmodel, config) {
         this.llm = llmodel;
         this.config = config;
     }
 
-    embed(text) {
-        return this.llm.embed(text)
+    embed(text, prefix, dimensionality, do_mean, atlas) {
+        return this.llm.embed(text, prefix, dimensionality, do_mean, atlas);
     }
 
     dispose() {
@@ -39,7 +159,6 @@ class EmbeddingModel {
     }
 }
 
-
 module.exports = {
     InferenceModel,
     EmbeddingModel,

gpt4all-bindings/typescript/src/util.js

@@ -1,8 +1,7 @@
-const { createWriteStream, existsSync, statSync } = require("node:fs");
+const { createWriteStream, existsSync, statSync, mkdirSync } = require("node:fs");
 const fsp = require("node:fs/promises");
 const { performance } = require("node:perf_hooks");
 const path = require("node:path");
-const { mkdirp } = require("mkdirp");
 const md5File = require("md5-file");
 const {
     DEFAULT_DIRECTORY,
@@ -50,6 +49,63 @@ function appendBinSuffixIfMissing(name) {
     return name;
 }
 
+function prepareMessagesForIngest(messages) {
+    const systemMessages = messages.filter(
+        (message) => message.role === "system"
+    );
+    if (systemMessages.length > 0) {
+        console.warn(
+            "System messages are currently not supported and will be ignored. Use the systemPrompt option instead."
+        );
+    }
+
+    const userAssistantMessages = messages.filter(
+        (message) => message.role !== "system"
+    );
+
+    // make sure the first message is a user message
+    // if its not, the turns will be out of order
+    if (userAssistantMessages[0].role !== "user") {
+        userAssistantMessages.unshift({
+            role: "user",
+            content: "",
+        });
+    }
+
+    // create turns of user input + assistant reply
+    const turns = [];
+    let userMessage = null;
+    let assistantMessage = null;
+
+    for (const message of userAssistantMessages) {
+        // consecutive messages of the same role are concatenated into one message
+        if (message.role === "user") {
+            if (!userMessage) {
+                userMessage = message.content;
+            } else {
+                userMessage += "\n" + message.content;
+            }
+        } else if (message.role === "assistant") {
+            if (!assistantMessage) {
+                assistantMessage = message.content;
+            } else {
+                assistantMessage += "\n" + message.content;
+            }
+        }
+
+        if (userMessage && assistantMessage) {
+            turns.push({
+                user: userMessage,
+                assistant: assistantMessage,
+            });
+            userMessage = null;
+            assistantMessage = null;
+        }
+    }
+
+    return turns;
+}
+
 // readChunks() reads from the provided reader and yields the results into an async iterable
 // https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
 function readChunks(reader) {
@@ -64,49 +120,13 @@ function readChunks(reader) {
     };
 }
 
-/**
- * Prints a warning if any keys in the prompt context are snake_case.
- */
-function warnOnSnakeCaseKeys(promptContext) {
-    const snakeCaseKeys = Object.keys(promptContext).filter((key) =>
-        key.includes("_")
-    );
-
-    if (snakeCaseKeys.length > 0) {
-        console.warn(
-            "Prompt context keys should be camelCase. Support for snake_case might be removed in the future. Found keys: " +
-                snakeCaseKeys.join(", ")
-        );
-    }
-}
-
-/**
- * Converts all keys in the prompt context to snake_case
- * For duplicate definitions, the value of the last occurrence will be used.
- */
-function normalizePromptContext(promptContext) {
-    const normalizedPromptContext = {};
-
-    for (const key in promptContext) {
-        if (promptContext.hasOwnProperty(key)) {
-            const snakeKey = key.replace(
-                /[A-Z]/g,
-                (match) => `_${match.toLowerCase()}`
-            );
-            normalizedPromptContext[snakeKey] = promptContext[key];
-        }
-    }
-
-    return normalizedPromptContext;
-}
-
 function downloadModel(modelName, options = {}) {
     const downloadOptions = {
         modelPath: DEFAULT_DIRECTORY,
         verbose: false,
         ...options,
     };
-    
+
     const modelFileName = appendBinSuffixIfMissing(modelName);
     const partialModelPath = path.join(
         downloadOptions.modelPath,
@@ -114,16 +134,17 @@ function downloadModel(modelName, options = {}) {
     );
     const finalModelPath = path.join(downloadOptions.modelPath, modelFileName);
     const modelUrl =
-        downloadOptions.url ?? `https://gpt4all.io/models/gguf/${modelFileName}`;
+        downloadOptions.url ??
+        `https://gpt4all.io/models/gguf/${modelFileName}`;
 
-    mkdirp.sync(downloadOptions.modelPath)
+    mkdirSync(downloadOptions.modelPath, { recursive: true });
 
     if (existsSync(finalModelPath)) {
         throw Error(`Model already exists at ${finalModelPath}`);
     }
-    
+
     if (downloadOptions.verbose) {
-        console.log(`Downloading ${modelName} from ${modelUrl}`);
+        console.debug(`Downloading ${modelName} from ${modelUrl}`);
     }
 
     const headers = {
@@ -134,7 +155,9 @@ function downloadModel(modelName, options = {}) {
     const writeStreamOpts = {};
 
     if (existsSync(partialModelPath)) {
-        console.log("Partial model exists, resuming download...");
+        if (downloadOptions.verbose) {
+            console.debug("Partial model exists, resuming download...");
+        }
         const startRange = statSync(partialModelPath).size;
         headers["Range"] = `bytes=${startRange}-`;
         writeStreamOpts.flags = "a";
@@ -144,15 +167,15 @@ function downloadModel(modelName, options = {}) {
     const signal = abortController.signal;
 
     const finalizeDownload = async () => {
-        if (options.md5sum) {
+        if (downloadOptions.md5sum) {
             const fileHash = await md5File(partialModelPath);
-            if (fileHash !== options.md5sum) {
+            if (fileHash !== downloadOptions.md5sum) {
                 await fsp.unlink(partialModelPath);
-                const message = `Model "${modelName}" failed verification: Hashes mismatch. Expected ${options.md5sum}, got ${fileHash}`;
+                const message = `Model "${modelName}" failed verification: Hashes mismatch. Expected ${downloadOptions.md5sum}, got ${fileHash}`;
                 throw Error(message);
             }
-            if (options.verbose) {
-                console.log(`MD5 hash verified: ${fileHash}`);
+            if (downloadOptions.verbose) {
+                console.debug(`MD5 hash verified: ${fileHash}`);
             }
         }
 
@@ -163,8 +186,8 @@ function downloadModel(modelName, options = {}) {
     const downloadPromise = new Promise((resolve, reject) => {
         let timestampStart;
 
-        if (options.verbose) {
-            console.log(`Downloading @ ${partialModelPath} ...`);
+        if (downloadOptions.verbose) {
+            console.debug(`Downloading @ ${partialModelPath} ...`);
             timestampStart = performance.now();
         }
 
@@ -179,7 +202,7 @@ function downloadModel(modelName, options = {}) {
         });
 
         writeStream.on("finish", () => {
-            if (options.verbose) {
+            if (downloadOptions.verbose) {
                 const elapsed = performance.now() - timestampStart;
                 console.log(`Finished. Download took ${elapsed.toFixed(2)} ms`);
             }
@@ -221,10 +244,10 @@ async function retrieveModel(modelName, options = {}) {
     const retrieveOptions = {
         modelPath: DEFAULT_DIRECTORY,
         allowDownload: true,
-        verbose: true,
+        verbose: false,
         ...options,
     };
-    await mkdirp(retrieveOptions.modelPath);
+    mkdirSync(retrieveOptions.modelPath, { recursive: true });
 
     const modelFileName = appendBinSuffixIfMissing(modelName);
     const fullModelPath = path.join(retrieveOptions.modelPath, modelFileName);
@@ -236,7 +259,7 @@ async function retrieveModel(modelName, options = {}) {
         file: retrieveOptions.modelConfigFile,
         url:
             retrieveOptions.allowDownload &&
-            "https://gpt4all.io/models/models2.json",
+            "https://gpt4all.io/models/models3.json",
     });
 
     const loadedModelConfig = availableModels.find(
@@ -262,10 +285,9 @@ async function retrieveModel(modelName, options = {}) {
         config.path = fullModelPath;
 
         if (retrieveOptions.verbose) {
-            console.log(`Found ${modelName} at ${fullModelPath}`);
+            console.debug(`Found ${modelName} at ${fullModelPath}`);
         }
     } else if (retrieveOptions.allowDownload) {
-
         const downloadController = downloadModel(modelName, {
             modelPath: retrieveOptions.modelPath,
             verbose: retrieveOptions.verbose,
@@ -278,7 +300,7 @@ async function retrieveModel(modelName, options = {}) {
         config.path = downloadPath;
 
         if (retrieveOptions.verbose) {
-            console.log(`Model downloaded to ${downloadPath}`);
+            console.debug(`Model downloaded to ${downloadPath}`);
         }
     } else {
         throw Error("Failed to retrieve model.");
@@ -288,9 +310,8 @@ async function retrieveModel(modelName, options = {}) {
 
 module.exports = {
     appendBinSuffixIfMissing,
+    prepareMessagesForIngest,
     downloadModel,
     retrieveModel,
     listModels,
-    normalizePromptContext,
-    warnOnSnakeCaseKeys,
 };

gpt4all-bindings/typescript/test/gpt4all.test.js

@@ -7,7 +7,6 @@ const {
     listModels,
     downloadModel,
     appendBinSuffixIfMissing,
-    normalizePromptContext,
 } = require("../src/util.js");
 const {
     DEFAULT_DIRECTORY,
@@ -19,8 +18,6 @@ const {
     createPrompt,
     createCompletion,
 } = require("../src/gpt4all.js");
-const { mock } = require("node:test");
-const { mkdirp } = require("mkdirp");
 
 describe("config", () => {
     test("default paths constants are available and correct", () => {
@@ -87,7 +84,7 @@ describe("listModels", () => {
         expect(fetch).toHaveBeenCalledTimes(0);
         expect(models[0]).toEqual(fakeModel);
     });
-    
+
     it("should throw an error if neither url nor file is specified", async () => {
         await expect(listModels(null)).rejects.toThrow(
             "No model list source specified. Please specify either a url or a file."
@@ -141,10 +138,10 @@ describe("downloadModel", () => {
         mockAbortController.mockReset();
         mockFetch.mockClear();
         global.fetch.mockRestore();
-        
+
         const rootDefaultPath = path.resolve(DEFAULT_DIRECTORY),
               partialPath = path.resolve(rootDefaultPath, fakeModelName+'.part'),
-              fullPath = path.resolve(rootDefaultPath, fakeModelName+'.bin')  
+              fullPath = path.resolve(rootDefaultPath, fakeModelName+'.bin')
 
         //if tests fail, remove the created files
         // acts as cleanup if tests fail
@@ -206,46 +203,3 @@ describe("downloadModel", () => {
     // test("should be able to cancel and resume a download", async () => {
     // });
 });
-
-describe("normalizePromptContext", () => {
-    it("should convert a dict with camelCased keys to snake_case", () => {
-        const camelCased = {
-            topK: 20,
-            repeatLastN: 10,
-        };
-
-        const expectedSnakeCased = {
-            top_k: 20,
-            repeat_last_n: 10,
-        };
-
-        const result = normalizePromptContext(camelCased);
-        expect(result).toEqual(expectedSnakeCased);
-    });
-
-    it("should convert a mixed case dict to snake_case, last value taking precedence", () => {
-        const mixedCased = {
-            topK: 20,
-            top_k: 10,
-            repeatLastN: 10,
-        };
-
-        const expectedSnakeCased = {
-            top_k: 10,
-            repeat_last_n: 10,
-        };
-
-        const result = normalizePromptContext(mixedCased);
-        expect(result).toEqual(expectedSnakeCased);
-    });
-
-    it("should not modify already snake cased dict", () => {
-        const snakeCased = {
-            top_k: 10,
-            repeast_last_n: 10,
-        };
-
-        const result = normalizePromptContext(snakeCased);
-        expect(result).toEqual(snakeCased);
-    });
-});

gpt4all-bindings/typescript/yarn.lock

@@ -2300,7 +2300,6 @@ __metadata:
     documentation: ^14.0.2
     jest: ^29.5.0
     md5-file: ^5.0.0
-    mkdirp: ^3.0.1
     node-addon-api: ^6.1.0
     node-gyp: 9.x.x
     node-gyp-build: ^4.6.0
@@ -2631,9 +2630,9 @@ __metadata:
   linkType: hard
 
 "ip@npm:^2.0.0":
-  version: 2.0.0
-  resolution: "ip@npm:2.0.0"
-  checksum: cfcfac6b873b701996d71ec82a7dd27ba92450afdb421e356f44044ed688df04567344c36cbacea7d01b1c39a4c732dc012570ebe9bebfb06f27314bca625349
+  version: 2.0.1
+  resolution: "ip@npm:2.0.1"
+  checksum: d765c9fd212b8a99023a4cde6a558a054c298d640fec1020567494d257afd78ca77e37126b1a3ef0e053646ced79a816bf50621d38d5e768cdde0431fa3b0d35
   languageName: node
   linkType: hard
 
@@ -4258,15 +4257,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"mkdirp@npm:^3.0.1":
-  version: 3.0.1
-  resolution: "mkdirp@npm:3.0.1"
-  bin:
-    mkdirp: dist/cjs/src/bin.js
-  checksum: 972deb188e8fb55547f1e58d66bd6b4a3623bf0c7137802582602d73e6480c1c2268dcbafbfb1be466e00cc7e56ac514d7fd9334b7cf33e3e2ab547c16f83a8d
-  languageName: node
-  linkType: hard
-
 "mri@npm:^1.1.0":
   version: 1.2.0
   resolution: "mri@npm:1.2.0"

gpt4all-chat/CMakeLists.txt

@@ -68,6 +68,10 @@ if(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
   set(METAL_SHADER_FILE ../gpt4all-backend/llama.cpp-mainline/ggml-metal.metal)
 endif()
 
+set(APP_ICON_FILE "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns")
+set_source_files_properties(${APP_ICON_FILE} PROPERTIES
+    MACOSX_PACKAGE_LOCATION "Resources")
+
 qt_add_executable(chat
     main.cpp
     chat.h chat.cpp
@@ -87,6 +91,7 @@ qt_add_executable(chat
     logger.h logger.cpp
     responsetext.h responsetext.cpp
     ${METAL_SHADER_FILE}
+    ${APP_ICON_FILE}
 )
 
 qt_add_qml_module(chat
@@ -98,11 +103,11 @@ qt_add_qml_module(chat
       qml/ChatDrawer.qml
       qml/ChatView.qml
       qml/CollectionsDialog.qml
-      qml/ModelDownloaderDialog.qml
+      qml/ModelDownloaderView.qml
       qml/NetworkDialog.qml
       qml/NewVersionDialog.qml
       qml/ThumbsDownDialog.qml
-      qml/SettingsDialog.qml
+      qml/SettingsView.qml
       qml/StartupDialog.qml
       qml/PopupDialog.qml
       qml/AboutDialog.qml
@@ -141,6 +146,8 @@ qt_add_qml_module(chat
       icons/edit.svg
       icons/image.svg
       icons/info.svg
+      icons/local-docs.svg
+      icons/models.svg
       icons/search.svg
       icons/trash.svg
       icons/network.svg

gpt4all-chat/icons/chat.svg

@@ -1,6 +1,3 @@
-
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M144 208c-17.7 0-32 14.3-32 32s14.3 32 32 32 32-14.3 32-32-14.3-32-32-32zm112 0c-17.7 0-32 14.3-32 32s14.3 32 32 32 32-14.3 32-32-14.3-32-32-32zm112 0c-17.7 0-32 14.3-32 32s14.3 32 32 32 32-14.3 32-32-14.3-32-32-32zM256 32C114.6 32 0 125.1 0 240c0 47.6 19.9 91.2 52.9 126.3C38 405.7 7 439.1 6.5 439.5c-6.6 7-8.4 17.2-4.6 26S14.4 480 24 480c61.5 0 110-25.7 139.1-46.3C192 442.8 223.2 448 256 448c141.4 0 256-93.1 256-208S397.4 32 256 32zm0 368c-26.7 0-53.1-4.1-78.4-12.1l-22.7-7.2-19.5 13.8c-14.3 10.1-33.9 21.4-57.5 29 7.3-12.1 14.4-25.7 19.9-40.2l10.6-28.1-20.6-21.8C69.7 314.1 48 282.2 48 240c0-88.2 93.3-160 208-160s208 71.8 208 160-93.3 160-208 160z"/></svg>
-<!--
-Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
-License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
--->
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M26.1538 9.76H22.4615V5.92C22.4615 5.41078 22.267 4.92242 21.9208 4.56235C21.5746 4.20229 21.105 4 20.6154 4H5.84615C5.35652 4 4.88695 4.20229 4.54073 4.56235C4.1945 4.92242 4 5.41078 4 5.92V21.28C4.00054 21.4606 4.05006 21.6374 4.14287 21.79C4.23568 21.9427 4.36801 22.065 4.52467 22.143C4.68132 22.2209 4.85595 22.2513 5.02847 22.2307C5.201 22.2101 5.36444 22.1393 5.5 22.0264L9.53846 18.64V22.24C9.53846 22.7492 9.73297 23.2376 10.0792 23.5976C10.4254 23.9577 10.895 24.16 11.3846 24.16H22.1835L26.5 27.7864C26.6633 27.9238 26.8668 27.9991 27.0769 28C27.3217 28 27.5565 27.8989 27.7296 27.7188C27.9027 27.5388 28 27.2946 28 27.04V11.68C28 11.1708 27.8055 10.6824 27.4593 10.3224C27.1131 9.96229 26.6435 9.76 26.1538 9.76ZM8.90962 16.6936L5.84615 19.27V5.92H20.6154V16.48H9.49C9.27874 16.48 9.07387 16.5554 8.90962 16.6936ZM26.1538 25.03L23.0904 22.4536C22.9271 22.3162 22.7235 22.2408 22.5135 22.24H11.3846V18.4H20.6154C21.105 18.4 21.5746 18.1977 21.9208 17.8376C22.267 17.4776 22.4615 16.9892 22.4615 16.48V11.68H26.1538V25.03Z" fill="black"/>
+</svg>

gpt4all-chat/icons/db.svg

@@ -1,5 +1,3 @@
-<svg xmlns="http://www.w3.org/2000/svg" fill="#7d7d8e" viewBox="0 0 448 512"><path d="M448 80v48c0 44.2-100.3 80-224 80S0 172.2 0 128V80C0 35.8 100.3 0 224 0S448 35.8 448 80zM393.2 214.7c20.8-7.4 39.9-16.9 54.8-28.6V288c0 44.2-100.3 80-224 80S0 332.2 0 288V186.1c14.9 11.8 34 21.2 54.8 28.6C99.7 230.7 159.5 240 224 240s124.3-9.3 169.2-25.3zM0 346.1c14.9 11.8 34 21.2 54.8 28.6C99.7 390.7 159.5 400 224 400s124.3-9.3 169.2-25.3c20.8-7.4 39.9-16.9 54.8-28.6V432c0 44.2-100.3 80-224 80S0 476.2 0 432V346.1z"/></svg>
-<!--
-Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
-License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
--->
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M16 3C9.27125 3 4 6.075 4 10V22C4 25.925 9.27125 29 16 29C22.7288 29 28 25.925 28 22V10C28 6.075 22.7288 3 16 3ZM26 16C26 17.2025 25.015 18.4287 23.2987 19.365C21.3662 20.4187 18.7738 21 16 21C13.2262 21 10.6338 20.4187 8.70125 19.365C6.985 18.4287 6 17.2025 6 16V13.92C8.1325 15.795 11.7787 17 16 17C20.2213 17 23.8675 15.79 26 13.92V16ZM8.70125 6.635C10.6338 5.58125 13.2262 5 16 5C18.7738 5 21.3662 5.58125 23.2987 6.635C25.015 7.57125 26 8.7975 26 10C26 11.2025 25.015 12.4287 23.2987 13.365C21.3662 14.4187 18.7738 15 16 15C13.2262 15 10.6338 14.4187 8.70125 13.365C6.985 12.4287 6 11.2025 6 10C6 8.7975 6.985 7.57125 8.70125 6.635ZM23.2987 25.365C21.3662 26.4187 18.7738 27 16 27C13.2262 27 10.6338 26.4187 8.70125 25.365C6.985 24.4287 6 23.2025 6 22V19.92C8.1325 21.795 11.7787 23 16 23C20.2213 23 23.8675 21.79 26 19.92V22C26 23.2025 25.015 24.4287 23.2987 25.365Z" fill="black"/>
+</svg>

gpt4all-chat/icons/edit.svg

@@ -1,5 +1,3 @@
-<svg xmlns="http://www.w3.org/2000/svg" fill="#7d7d8e" viewBox="0 0 576 512"><path d="M402.6 83.2l90.2 90.2c3.8 3.8 3.8 10 0 13.8L274.4 405.6l-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8L388.8 83.2c3.8-3.8 10-3.8 13.8 0zm162-22.9l-48.8-48.8c-15.2-15.2-39.9-15.2-55.2 0l-35.4 35.4c-3.8 3.8-3.8 10 0 13.8l90.2 90.2c3.8 3.8 10 3.8 13.8 0l35.4-35.4c15.2-15.3 15.2-40 0-55.2zM384 346.2V448H64V128h229.8c3.2 0 6.2-1.3 8.5-3.5l40-40c7.6-7.6 2.2-20.5-8.5-20.5H48C21.5 64 0 85.5 0 112v352c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48V306.2c0-10.7-12.9-16-20.5-8.5l-40 40c-2.2 2.3-3.5 5.3-3.5 8.5z"/></svg>
-<!--
-Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
-License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
--->
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M28.4138 9.17125L22.8288 3.585C22.643 3.39924 22.4225 3.25188 22.1799 3.15134C21.9372 3.0508 21.6771 2.99905 21.4144 2.99905C21.1517 2.99905 20.8916 3.0508 20.6489 3.15134C20.4062 3.25188 20.1857 3.39924 20 3.585L4.58626 19C4.39973 19.185 4.25185 19.4053 4.15121 19.648C4.05057 19.8907 3.99917 20.151 4.00001 20.4138V26C4.00001 26.5304 4.21072 27.0391 4.5858 27.4142C4.96087 27.7893 5.46958 28 6.00001 28H11.5863C11.849 28.0008 12.1093 27.9494 12.352 27.8488C12.5947 27.7482 12.815 27.6003 13 27.4138L28.4138 12C28.5995 11.8143 28.7469 11.5938 28.8474 11.3511C28.948 11.1084 28.9997 10.8483 28.9997 10.5856C28.9997 10.3229 28.948 10.0628 28.8474 9.82015C28.7469 9.57747 28.5995 9.35698 28.4138 9.17125ZM6.41376 20L17 9.41375L19.0863 11.5L8.50001 22.085L6.41376 20ZM6.00001 22.4138L9.58626 26H6.00001V22.4138ZM12 25.5863L9.91376 23.5L20.5 12.9138L22.5863 15L12 25.5863ZM24 13.5863L18.4138 8L21.4138 5L27 10.585L24 13.5863Z" fill="black"/>
+</svg>

gpt4all-chat/icons/info.svg

@@ -1,6 +1,3 @@
-
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M256 8C119.043 8 8 119.083 8 256c0 136.997 111.043 248 248 248s248-111.003 248-248C504 119.083 392.957 8 256 8zm0 110c23.196 0 42 18.804 42 42s-18.804 42-42 42-42-18.804-42-42 18.804-42 42-42zm56 254c0 6.627-5.373 12-12 12h-88c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h12v-64h-12c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h64c6.627 0 12 5.373 12 12v100h12c6.627 0 12 5.373 12 12v24z"/></svg>
-<!--
-Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
-License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
--->
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M16 3C13.4288 3 10.9154 3.76244 8.77759 5.1909C6.63975 6.61935 4.97351 8.64968 3.98957 11.0251C3.00563 13.4006 2.74819 16.0144 3.2498 18.5362C3.75141 21.0579 4.98953 23.3743 6.80762 25.1924C8.6257 27.0105 10.9421 28.2486 13.4638 28.7502C15.9856 29.2518 18.5995 28.9944 20.9749 28.0104C23.3503 27.0265 25.3807 25.3603 26.8091 23.2224C28.2376 21.0846 29 18.5712 29 16C28.9964 12.5533 27.6256 9.24882 25.1884 6.81163C22.7512 4.37445 19.4467 3.00364 16 3ZM16 27C13.8244 27 11.6977 26.3549 9.88873 25.1462C8.07979 23.9375 6.66989 22.2195 5.83733 20.2095C5.00477 18.1995 4.78693 15.9878 5.21137 13.854C5.63581 11.7202 6.68345 9.7602 8.22183 8.22183C9.76021 6.68345 11.7202 5.6358 13.854 5.21136C15.9878 4.78692 18.1995 5.00476 20.2095 5.83733C22.2195 6.66989 23.9375 8.07979 25.1462 9.88873C26.3549 11.6977 27 13.8244 27 16C26.9967 18.9164 25.8367 21.7123 23.7745 23.7745C21.7123 25.8367 18.9164 26.9967 16 27ZM18 22C18 22.2652 17.8946 22.5196 17.7071 22.7071C17.5196 22.8946 17.2652 23 17 23C16.4696 23 15.9609 22.7893 15.5858 22.4142C15.2107 22.0391 15 21.5304 15 21V16C14.7348 16 14.4804 15.8946 14.2929 15.7071C14.1054 15.5196 14 15.2652 14 15C14 14.7348 14.1054 14.4804 14.2929 14.2929C14.4804 14.1054 14.7348 14 15 14C15.5304 14 16.0391 14.2107 16.4142 14.5858C16.7893 14.9609 17 15.4696 17 16V21C17.2652 21 17.5196 21.1054 17.7071 21.2929C17.8946 21.4804 18 21.7348 18 22ZM14 10.5C14 10.2033 14.088 9.91332 14.2528 9.66665C14.4176 9.41997 14.6519 9.22771 14.926 9.11418C15.2001 9.00065 15.5017 8.97094 15.7926 9.02882C16.0836 9.0867 16.3509 9.22956 16.5607 9.43934C16.7704 9.64912 16.9133 9.91639 16.9712 10.2074C17.0291 10.4983 16.9994 10.7999 16.8858 11.074C16.7723 11.3481 16.58 11.5824 16.3334 11.7472C16.0867 11.912 15.7967 12 15.5 12C15.1022 12 14.7206 11.842 14.4393 11.5607C14.158 11.2794 14 10.8978 14 10.5Z" fill="black"/>
+</svg>

gpt4all-chat/icons/local-docs.svg

@@ -0,0 +1,3 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M26.7075 8.2925L21.7075 3.2925C21.6146 3.19967 21.5042 3.12605 21.3829 3.07586C21.2615 3.02568 21.1314 2.9999 21 3H11C10.4696 3 9.96086 3.21071 9.58579 3.58579C9.21071 3.96086 9 4.46957 9 5V7H7C6.46957 7 5.96086 7.21071 5.58579 7.58579C5.21071 7.96086 5 8.46957 5 9V27C5 27.5304 5.21071 28.0391 5.58579 28.4142C5.96086 28.7893 6.46957 29 7 29H21C21.5304 29 22.0391 28.7893 22.4142 28.4142C22.7893 28.0391 23 27.5304 23 27V25H25C25.5304 25 26.0391 24.7893 26.4142 24.4142C26.7893 24.0391 27 23.5304 27 23V9C27.0001 8.86864 26.9743 8.73855 26.9241 8.61715C26.8739 8.49576 26.8003 8.38544 26.7075 8.2925ZM21 27H7V9H16.5863L21 13.4137V23.98C21 23.9875 21 23.9937 21 24C21 24.0063 21 24.0125 21 24.02V27ZM25 23H23V13C23.0001 12.8686 22.9743 12.7385 22.9241 12.6172C22.8739 12.4958 22.8003 12.3854 22.7075 12.2925L17.7075 7.2925C17.6146 7.19967 17.5042 7.12605 17.3829 7.07586C17.2615 7.02568 17.1314 6.9999 17 7H11V5H20.5863L25 9.41375V23ZM18 19C18 19.2652 17.8946 19.5196 17.7071 19.7071C17.5196 19.8946 17.2652 20 17 20H11C10.7348 20 10.4804 19.8946 10.2929 19.7071C10.1054 19.5196 10 19.2652 10 19C10 18.7348 10.1054 18.4804 10.2929 18.2929C10.4804 18.1054 10.7348 18 11 18H17C17.2652 18 17.5196 18.1054 17.7071 18.2929C17.8946 18.4804 18 18.7348 18 19ZM18 23C18 23.2652 17.8946 23.5196 17.7071 23.7071C17.5196 23.8946 17.2652 24 17 24H11C10.7348 24 10.4804 23.8946 10.2929 23.7071C10.1054 23.5196 10 23.2652 10 23C10 22.7348 10.1054 22.4804 10.2929 22.2929C10.4804 22.1054 10.7348 22 11 22H17C17.2652 22 17.5196 22.1054 17.7071 22.2929C17.8946 22.4804 18 22.7348 18 23Z" fill="black"/>
+</svg>

gpt4all-chat/icons/models.svg

@@ -0,0 +1,3 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M27.96 8.26876L16.96 2.25001C16.6661 2.08761 16.3358 2.00243 16 2.00243C15.6642 2.00243 15.3339 2.08761 15.04 2.25001L4.04 8.27126C3.72586 8.44314 3.46363 8.69621 3.28069 9.00404C3.09775 9.31187 3.00081 9.66317 3 10.0213V21.9763C3.00081 22.3343 3.09775 22.6856 3.28069 22.9935C3.46363 23.3013 3.72586 23.5544 4.04 23.7263L15.04 29.7475C15.3339 29.9099 15.6642 29.9951 16 29.9951C16.3358 29.9951 16.6661 29.9099 16.96 29.7475L27.96 23.7263C28.2741 23.5544 28.5364 23.3013 28.7193 22.9935C28.9023 22.6856 28.9992 22.3343 29 21.9763V10.0225C28.9999 9.66378 28.9032 9.3117 28.7203 9.00315C28.5373 8.6946 28.2747 8.44095 27.96 8.26876ZM16 4.00001L26.0425 9.50001L16 15L5.9575 9.50001L16 4.00001ZM5 11.25L15 16.7225V27.4463L5 21.9775V11.25ZM17 27.4463V16.7275L27 11.25V21.9725L17 27.4463Z" fill="black"/>
+</svg>

gpt4all-chat/icons/trash.svg

@@ -1,5 +1,3 @@
-<svg xmlns="http://www.w3.org/2000/svg" fill="#7d7d8e" viewBox="0 0 448 512"><path d="M0 84V56c0-13.3 10.7-24 24-24h112l9.4-18.7c4-8.2 12.3-13.3 21.4-13.3h114.3c9.1 0 17.4 5.1 21.5 13.3L312 32h112c13.3 0 24 10.7 24 24v28c0 6.6-5.4 12-12 12H12C5.4 96 0 90.6 0 84zm416 56v324c0 26.5-21.5 48-48 48H80c-26.5 0-48-21.5-48-48V140c0-6.6 5.4-12 12-12h360c6.6 0 12 5.4 12 12zm-272 68c0-8.8-7.2-16-16-16s-16 7.2-16 16v224c0 8.8 7.2 16 16 16s16-7.2 16-16V208zm96 0c0-8.8-7.2-16-16-16s-16 7.2-16 16v224c0 8.8 7.2 16 16 16s16-7.2 16-16V208zm96 0c0-8.8-7.2-16-16-16s-16 7.2-16 16v224c0 8.8 7.2 16 16 16s16-7.2 16-16V208z"/></svg>
-<!--
-Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
-License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
--->
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M27 6H5C4.73478 6 4.48043 6.10536 4.29289 6.29289C4.10536 6.48043 4 6.73478 4 7C4 7.26522 4.10536 7.51957 4.29289 7.70711C4.48043 7.89464 4.73478 8 5 8H6V26C6 26.5304 6.21071 27.0391 6.58579 27.4142C6.96086 27.7893 7.46957 28 8 28H24C24.5304 28 25.0391 27.7893 25.4142 27.4142C25.7893 27.0391 26 26.5304 26 26V8H27C27.2652 8 27.5196 7.89464 27.7071 7.70711C27.8946 7.51957 28 7.26522 28 7C28 6.73478 27.8946 6.48043 27.7071 6.29289C27.5196 6.10536 27.2652 6 27 6ZM24 26H8V8H24V26ZM10 3C10 2.73478 10.1054 2.48043 10.2929 2.29289C10.4804 2.10536 10.7348 2 11 2H21C21.2652 2 21.5196 2.10536 21.7071 2.29289C21.8946 2.48043 22 2.73478 22 3C22 3.26522 21.8946 3.51957 21.7071 3.70711C21.5196 3.89464 21.2652 4 21 4H11C10.7348 4 10.4804 3.89464 10.2929 3.70711C10.1054 3.51957 10 3.26522 10 3Z" fill="black"/>
+</svg>

gpt4all-chat/main.qml

@@ -43,29 +43,6 @@ Window {
         font.pixelSize: theme.fontSizeLarge
     }
 
-    SettingsDialog {
-        id: settingsDialog
-        anchors.centerIn: parent
-        width: Math.min(1920, window.width - (window.width * .1))
-        height: window.height - (window.height * .1)
-        onDownloadClicked: {
-            downloadNewModels.showEmbeddingModels = true
-            downloadNewModels.open()
-        }
-    }
-
-    ModelDownloaderDialog {
-        id: downloadNewModels
-        anchors.centerIn: parent
-        width: Math.min(1920, window.width - (window.width * .1))
-        height: window.height - (window.height * .1)
-        Item {
-            Accessible.role: Accessible.Dialog
-            Accessible.name: qsTr("Download new models")
-            Accessible.description: qsTr("Dialog for downloading new models")
-        }
-    }
-
     NetworkDialog {
         id: networkDialog
         anchors.centerIn: parent
@@ -124,48 +101,56 @@ Window {
             MyToolButton {
                 id: chatButton
                 backgroundColor: toggled ? theme.iconBackgroundViewBarToggled : theme.iconBackgroundViewBar
-                backgroundColorHovered: theme.iconBackgroundViewBarHovered
+                backgroundColorHovered: toggled ? backgroundColor : theme.iconBackgroundViewBarHovered
+                Layout.preferredWidth: 40
+                Layout.preferredHeight: 40
+                Layout.alignment: Qt.AlignCenter
                 toggledWidth: 0
-                toggled: true
-                Layout.preferredWidth: 50
-                Layout.preferredHeight: 50
+                toggled: stackLayout.currentIndex === 0
+                toggledColor: theme.iconBackgroundViewBarToggled
                 scale: 1.5
                 source: "qrc:/gpt4all/icons/chat.svg"
                 Accessible.name: qsTr("Chat view")
                 Accessible.description: qsTr("Chat view to interact with models")
                 onClicked: {
+                    stackLayout.currentIndex = 0
                 }
             }
 
             MyToolButton {
                 id: searchButton
-                backgroundColor: theme.iconBackgroundViewBar
-                backgroundColorHovered: theme.iconBackgroundViewBarHovered
-                Layout.preferredWidth: 50
-                Layout.preferredHeight: 50
+                backgroundColor: toggled ? theme.iconBackgroundViewBarToggled : theme.iconBackgroundViewBar
+                backgroundColorHovered: toggled ? backgroundColor : theme.iconBackgroundViewBarHovered
+                Layout.preferredWidth: 40
+                Layout.preferredHeight: 40
+                toggledWidth: 0
+                toggled: stackLayout.currentIndex === 1
+                toggledColor: theme.iconBackgroundViewBarToggled
                 scale: 1.5
-                source: "qrc:/gpt4all/icons/search.svg"
+                source: "qrc:/gpt4all/icons/models.svg"
                 Accessible.name: qsTr("Search")
                 Accessible.description: qsTr("Launch a dialog to download new models")
                 onClicked: {
-                    downloadNewModels.showEmbeddingModels = false
-                    downloadNewModels.open()
+                    stackLayout.currentIndex = 1
                 }
             }
 
             MyToolButton {
                 id: settingsButton
-                backgroundColor: theme.iconBackgroundViewBar
-                backgroundColorHovered: theme.iconBackgroundViewBarHovered
-                Layout.preferredWidth: 50
-                Layout.preferredHeight: 50
+                backgroundColor: toggled ? theme.iconBackgroundViewBarToggled : theme.iconBackgroundViewBar
+                backgroundColorHovered: toggled ? backgroundColor : theme.iconBackgroundViewBarHovered
+                Layout.preferredWidth: 40
+                Layout.preferredHeight: 40
+                toggledWidth: 0
+                toggledColor: theme.iconBackgroundViewBarToggled
+                toggled: stackLayout.currentIndex === 2
                 scale: 1.5
                 source: "qrc:/gpt4all/icons/settings.svg"
                 Accessible.name: qsTr("Settings")
                 Accessible.description: qsTr("Reveals a dialogue with settings")
 
                 onClicked: {
-                    settingsDialog.open()
+                    stackLayout.currentIndex = 2
                 }
             }
         }
@@ -181,8 +166,8 @@ Window {
 
             MyToolButton {
                 id: networkButton
-                backgroundColor: theme.iconBackgroundViewBar
-                backgroundColorHovered: theme.iconBackgroundViewBarHovered
+                backgroundColor: toggled ? theme.iconBackgroundViewBarToggled : theme.iconBackgroundViewBar
+                backgroundColorHovered: toggled ? backgroundColor : theme.iconBackgroundViewBarHovered
                 toggledColor: theme.iconBackgroundViewBar
                 Layout.preferredWidth: 40
                 Layout.preferredHeight: 40
@@ -218,10 +203,52 @@ Window {
         }
     }
 
-    ChatView {
+    StackLayout {
+        id: stackLayout
         anchors.top: parent.top
         anchors.bottom: parent.bottom
         anchors.left: viewBar.right
         anchors.right: parent.right
+
+        ChatView {
+            id: chatView
+            Layout.fillWidth: true
+            Layout.fillHeight: true
+
+            Connections {
+                target: chatView
+                function onDownloadViewRequested(showEmbeddingModels) {
+                    console.log("onDownloadViewRequested")
+                    stackLayout.currentIndex = 1;
+                    if (showEmbeddingModels)
+                        downloadView.showEmbeddingModels();
+                }
+                function onSettingsViewRequested(page) {
+                    settingsDialog.pageToDisplay = page;
+                    stackLayout.currentIndex = 2;
+                }
+            }
+        }
+
+        ModelDownloaderView {
+            id: downloadView
+            Layout.fillWidth: true
+            Layout.fillHeight: true
+            Item {
+                Accessible.role: Accessible.Dialog
+                Accessible.name: qsTr("Download new models")
+                Accessible.description: qsTr("View for downloading new models")
+            }
+        }
+
+        SettingsView {
+            id: settingsDialog
+            Layout.fillWidth: true
+            Layout.fillHeight: true
+            onDownloadClicked: {
+                stackLayout.currentIndex = 1
+                downloadView.showEmbeddingModels()
+            }
+        }
     }
 }

gpt4all-chat/qml/ChatView.qml

@@ -21,6 +21,8 @@ Rectangle {
 
     property var currentChat: ChatListModel.currentChat
     property var chatModel: currentChat.chatModel
+    signal settingsViewRequested(int page)
+    signal downloadViewRequested(bool showEmbeddingModels)
 
     color: theme.black
 
@@ -36,13 +38,6 @@ Rectangle {
         }
     }
 
-    Connections {
-        target: downloadNewModels
-        function onClosed() {
-            startupDialogs();
-        }
-    }
-
     Connections {
         target: Download
         function onHasNewerReleaseChanged() {
@@ -90,15 +85,15 @@ Rectangle {
             return;
         }
 
-        // check for any current models and if not, open download dialog once
+        // check for any current models and if not, open download view once
         if (!hasShownModelDownload && ModelList.installedModels.count === 0 && !firstStartDialog.opened) {
-            downloadNewModels.open();
+            downloadViewRequested();
             hasShownModelDownload = true;
             return;
         }
 
         // check for new version
-        if (Download.hasNewerRelease && !firstStartDialog.opened && !downloadNewModels.opened) {
+        if (Download.hasNewerRelease && !firstStartDialog.opened) {
             newVersionDialog.open();
             return;
         }
@@ -296,22 +291,29 @@ Rectangle {
             anchors.verticalCenter: parent.verticalCenter
             spacing: 20
 
-            MyToolButton {
-                id: drawerButton
+            Rectangle {
                 Layout.alignment: Qt.AlignLeft
                 Layout.leftMargin: 30
-                backgroundColor: theme.iconBackgroundLight
-                width: 40
-                height: 40
-                scale: 1.5
-                z: 200
-                padding: 15
-                source: conversation.state === "expanded" ? "qrc:/gpt4all/icons/left_panel_open.svg" : "qrc:/gpt4all/icons/left_panel_closed.svg"
-                Accessible.role: Accessible.ButtonMenu
-                Accessible.name: qsTr("Chat panel")
-                Accessible.description: qsTr("Chat panel with options")
-                onClicked: {
-                    conversation.toggleLeftPanel()
+                Layout.fillWidth: true
+                Layout.preferredWidth: 100
+                Layout.topMargin: 20
+                color: "transparent"
+                Layout.preferredHeight: childrenRect.height
+                MyToolButton {
+                    id: drawerButton
+                    anchors.left: parent.left
+                    backgroundColor: theme.iconBackgroundLight
+                    width: 40
+                    height: 40
+                    scale: 1.5
+                    padding: 15
+                    source: conversation.state === "expanded" ? "qrc:/gpt4all/icons/left_panel_open.svg" : "qrc:/gpt4all/icons/left_panel_closed.svg"
+                    Accessible.role: Accessible.ButtonMenu
+                    Accessible.name: qsTr("Chat panel")
+                    Accessible.description: qsTr("Chat panel with options")
+                    onClicked: {
+                        conversation.toggleLeftPanel()
+                    }
                 }
             }
 
@@ -320,7 +322,7 @@ Rectangle {
                 Layout.alignment: Qt.AlignHCenter
                 Layout.fillHeight: true
                 Layout.fillWidth: true
-                Layout.minimumWidth: 375
+                Layout.preferredWidth: 100
                 Layout.maximumWidth: 675
                 enabled: !currentChat.isServer
                     && !window.trySwitchContextInProgress
@@ -478,86 +480,67 @@ Rectangle {
                 }
             }
 
-            RowLayout {
+            Rectangle {
+                color: "transparent"
                 Layout.alignment: Qt.AlignRight
                 Layout.rightMargin: 30
-                spacing: 20
+                Layout.fillWidth: true
+                Layout.preferredWidth: 100
+                Layout.preferredHeight: childrenRect.height
+                Layout.topMargin: 20
 
-                MyToolButton {
-                    id: resetContextButton
-                    backgroundColor: theme.iconBackgroundLight
-                    width: 40
-                    height: 40
-                    z: 200
-                    padding: 15
-                    source: "qrc:/gpt4all/icons/regenerate.svg"
+                RowLayout {
+                    spacing: 20
+                    anchors.right: parent.right
+                    MyButton {
+                        id: collectionsButton
+                        Image {
+                            id: collectionsImage
+                            anchors.verticalCenter: parent.verticalCenter
+                            anchors.left: parent.left
+                            anchors.leftMargin: 15
+                            width: 24
+                            height: 24
+                            mipmap: true
+                            source: "qrc:/gpt4all/icons/db.svg"
+                        }
 
-                    Accessible.name: text
-                    Accessible.description: qsTr("Reset the context and erase current conversation")
+                        ColorOverlay {
+                            anchors.fill: collectionsImage
+                            source: collectionsImage
+                            color: collectionsButton.hovered || collectionsImage.toggled ? theme.iconBackgroundHovered : theme.iconBackgroundLight
+                        }
 
-                    onClicked: {
-                        Network.sendResetContext(chatModel.count)
-                        currentChat.reset();
-                        currentChat.processSystemPrompt();
-                    }
-                }
+                        leftPadding: 50
+                        borderWidth: 0
+                        backgroundColor: theme.mainComboBackground
+                        backgroundColorHovered: theme.conversationButtonBackgroundHovered
+                        backgroundRadius: 5
+                        padding: 15
+                        topPadding: 8
+                        bottomPadding: 8
+                        textColor: hovered || toggled ? theme.iconBackgroundHovered : theme.iconBackgroundLight
+                        text: qsTr("LocalDocs")
+                        fontPixelSize: theme.fontSizeSmall
 
-                MyToolButton {
-                    id: copyButton
-                    backgroundColor: theme.iconBackgroundLight
-                    width: 40
-                    height: 40
-                    z: 200
-                    padding: 15
-                    source: "qrc:/gpt4all/icons/copy.svg"
-                    Accessible.name: qsTr("Copy")
-                    Accessible.description: qsTr("Copy the conversation to the clipboard")
+                        property bool toggled: currentChat.collectionList.length
+                        background: Rectangle {
+                            radius: collectionsButton.backgroundRadius
+                            color: collectionsButton.toggled ? collectionsButton.backgroundColorHovered : collectionsButton.backgroundColor
+                        }
 
-                    TextEdit{
-                        id: copyEdit
-                        visible: false
-                    }
+                        Accessible.name: qsTr("Add documents")
+                        Accessible.description: qsTr("add collections of documents to the chat")
 
-                    onClicked: {
-                        var conversation = getConversation()
-                        copyEdit.text = conversation
-                        copyEdit.selectAll()
-                        copyEdit.copy()
-                        copyMessage.open()
-                    }
-                }
-
-                MyToolButton {
-                    id: collectionsButton
-                    backgroundColor: theme.iconBackgroundLight
-                    width: 40
-                    height: 42.5
-                    z: 200
-                    padding: 15
-                    toggled: currentChat.collectionList.length
-                    source: "qrc:/gpt4all/icons/db.svg"
-                    Accessible.name: qsTr("Add documents")
-                    Accessible.description: qsTr("add collections of documents to the chat")
-
-                    onClicked: {
-                        collectionsDialog.open()
+                        onClicked: {
+                            collectionsDialog.open()
+                        }
                     }
                 }
             }
         }
     }
 
-    SettingsDialog {
-        id: settingsDialog
-        anchors.centerIn: parent
-        width: Math.min(1920, window.width - (window.width * .1))
-        height: window.height - (window.height * .1)
-        onDownloadClicked: {
-            downloadNewModels.showEmbeddingModels = true
-            downloadNewModels.open()
-        }
-    }
-
     Connections {
         target: Network
         function onHealthCheckFailed(code) {
@@ -569,8 +552,7 @@ Rectangle {
         id: collectionsDialog
         anchors.centerIn: parent
         onAddRemoveClicked: {
-            settingsDialog.pageToDisplay = 2;
-            settingsDialog.open();
+            settingsViewRequested(2 /*page 2*/)
         }
     }
 
@@ -680,18 +662,6 @@ Rectangle {
         }
     }
 
-    ModelDownloaderDialog {
-        id: downloadNewModels
-        anchors.centerIn: parent
-        width: Math.min(1920, window.width - (window.width * .1))
-        height: window.height - (window.height * .1)
-        Item {
-            Accessible.role: Accessible.Dialog
-            Accessible.name: qsTr("Download new models")
-            Accessible.description: qsTr("Dialog for downloading new models")
-        }
-    }
-
     ChatDrawer {
         id: drawer
         anchors.left: parent.left
@@ -841,6 +811,32 @@ Rectangle {
                             }
                         }
 
+                        Text {
+                            Layout.alignment: Qt.AlignHCenter
+                            textFormat: Text.StyledText
+                            text: qsTr(
+                            "<p></p><a href=\"https://docs.gpt4all.io/gpt4all_chat.html\">Documentation
+                            ")
+                            onLinkActivated: { Qt.openUrlExternally("https://docs.gpt4all.io/gpt4all_chat.html") }
+                            color: theme.textColor
+                            linkColor: theme.linkColor
+                            font.pixelSize: theme.fontSizeLarge
+                            wrapMode: Text.WordWrap
+                        }
+
+                        Text {
+                            Layout.alignment: Qt.AlignHCenter
+                            textFormat: Text.StyledText
+                            text: qsTr(
+                            "<a href=\"https://docs.gpt4all.io/gpt4all_faq.html\">Frequently Asked Questions
+                            ")
+                            onLinkActivated: { Qt.openUrlExternally("https://docs.gpt4all.io/gpt4all_faq.html") }
+                            color: theme.textColor
+                            linkColor: theme.linkColor
+                            font.pixelSize: theme.fontSizeLarge
+                            wrapMode: Text.WordWrap
+                        }
+
                         MyButton {
                             id: downloadButton
                             visible: LLM.isNetworkOnline
@@ -851,7 +847,7 @@ Rectangle {
                             padding: 18
                             leftPadding: 50
                             Image {
-                                id: image
+                                id: downloadImage
                                 anchors.verticalCenter: parent.verticalCenter
                                 anchors.left: parent.left
                                 anchors.leftMargin: 15
@@ -861,12 +857,13 @@ Rectangle {
                                 source: "qrc:/gpt4all/icons/download.svg"
                             }
                             ColorOverlay {
-                                anchors.fill: image
-                                source: image
+                                anchors.fill: downloadImage
+                                source: downloadImage
                                 color: theme.accentColor
                             }
                             onClicked: {
-                                downloadNewModels.open();
+                                console.log("download button")
+                                downloadViewRequested(false /*showEmbeddingModels*/);
                             }
                         }
                     }

gpt4all-chat/qml/ModelDownloaderView.qml

@@ -11,21 +11,15 @@ import modellist
 import network
 import mysettings
 
-MyDialog {
+Rectangle {
     id: modelDownloaderDialog
-    modal: true
-    closePolicy: Popup.CloseOnEscape | Popup.CloseOnPressOutside
-    padding: 10
-    property bool showEmbeddingModels: false
+    color: theme.containerBackground
 
-    onOpened: {
+    function showEmbeddingModels() {
         Network.sendModelDownloaderDialog();
-
-        if (showEmbeddingModels) {
-            ModelList.downloadableModels.expanded = true
-            var targetModelIndex = ModelList.defaultEmbeddingModelIndex
-            modelListView.positionViewAtIndex(targetModelIndex, ListView.Beginning)
-        }
+        ModelList.downloadableModels.expanded = true
+        var targetModelIndex = ModelList.defaultEmbeddingModelIndex
+        modelListView.positionViewAtIndex(targetModelIndex, ListView.Beginning)
     }
 
     PopupDialog {
@@ -36,7 +30,7 @@ MyDialog {
 
     ColumnLayout {
         anchors.fill: parent
-        anchors.margins: 10
+        anchors.margins: 20
         spacing: 30
 
         Label {

gpt4all-chat/qml/MyToolButton.qml

@@ -27,11 +27,12 @@ Button {
         anchors.fill: parent
         Rectangle {
             anchors.fill: parent
-            color: "transparent"
+            color: myButton.toggledColor
             visible: myButton.toggled
             border.color: myButton.toggledColor
             border.width: myButton.toggledWidth
-            radius: 10
+            radius: 6
+            opacity: .2
         }
         Image {
             id: image

gpt4all-chat/qml/SettingsDialog.qml

@@ -1,132 +0,0 @@
-import QtCore
-import QtQuick
-import QtQuick.Controls
-import QtQuick.Controls.Basic
-import QtQuick.Dialogs
-import QtQuick.Layouts
-import Qt.labs.folderlistmodel
-import download
-import modellist
-import network
-import llm
-import mysettings
-
-MyDialog {
-    id: settingsDialog
-    modal: true
-    padding: 20
-    onOpened: {
-        Network.sendSettingsDialog();
-    }
-
-    signal downloadClicked
-    property alias pageToDisplay: listView.currentIndex
-
-    Item {
-        Accessible.role: Accessible.Dialog
-        Accessible.name: qsTr("Settings")
-        Accessible.description: qsTr("Contains various application settings")
-    }
-
-    ListModel {
-        id: stacksModel
-        ListElement {
-            title: qsTr("Models")
-        }
-        ListElement {
-            title: qsTr("Application")
-        }
-        ListElement {
-            title: qsTr("LocalDocs")
-        }
-    }
-
-    Rectangle {
-        id: stackList
-        anchors.top: parent.top
-        anchors.bottom: parent.bottom
-        anchors.left: parent.left
-        width: 220
-        color: theme.controlBackground
-        radius: 10
-
-        ScrollView {
-            anchors.top: parent.top
-            anchors.bottom: parent.bottom
-            anchors.left: parent.left
-            anchors.right: parent.right
-            anchors.topMargin: 10
-            ScrollBar.vertical.policy: ScrollBar.AsNeeded
-            clip: true
-
-            ListView {
-                id: listView
-                anchors.fill: parent
-                model: stacksModel
-
-                delegate: Rectangle {
-                    id: item
-                    width: listView.width
-                    height: titleLabel.height + 10
-                    color: "transparent"
-
-                    MyButton {
-                        id: titleLabel
-                        backgroundColor: index === listView.currentIndex ? theme.buttonBackground : theme.controlBackground
-                        backgroundColorHovered: index === listView.currentIndex ? backgroundColor : theme.containerBackground
-                        borderColor: index === listView.currentIndex ? theme.accentColor : "transparent"
-                        borderWidth: index === listView.currentIndex ? 1 : 0
-                        textColor: index === listView.currentIndex ? theme.oppositeTextColor : theme.titleTextColor
-                        anchors.verticalCenter: parent.verticalCenter
-                        anchors.left: parent.left
-                        anchors.right: parent.right
-                        anchors.margins: 10
-                        font.bold: index === listView.currentIndex
-                        text: title
-                        font.pixelSize: theme.fontSizeLarge
-                        onClicked: {
-                            listView.currentIndex = index
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    StackLayout {
-        id: stackLayout
-        anchors.top: parent.top
-        anchors.bottom: parent.bottom
-        anchors.left: stackList.right
-        anchors.right: parent.right
-        currentIndex: listView.currentIndex
-
-        MySettingsStack {
-            title: qsTr("Model/Character Settings")
-            tabs: [
-                Component { ModelSettings { } }
-            ]
-        }
-
-        MySettingsStack {
-            title: qsTr("Application General Settings")
-            tabs: [
-                Component { ApplicationSettings { } }
-            ]
-        }
-
-        MySettingsStack {
-            title: qsTr("Local Document Collections")
-            tabs: [
-                Component {
-                    LocalDocsSettings {
-                        id: localDocsSettings
-                        Component.onCompleted: {
-                             localDocsSettings.downloadClicked.connect(settingsDialog.downloadClicked);
-                        }
-                    }
-                }
-            ]
-        }
-    }
-}

gpt4all-chat/qml/SettingsView.qml

@@ -0,0 +1,132 @@
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import QtQuick.Dialogs
+import QtQuick.Layouts
+import Qt.labs.folderlistmodel
+import download
+import modellist
+import network
+import llm
+import mysettings
+
+Rectangle {
+    id: settingsDialog
+    color: theme.containerBackground
+
+    signal downloadClicked
+    property alias pageToDisplay: listView.currentIndex
+
+    Item {
+        Accessible.role: Accessible.Dialog
+        Accessible.name: qsTr("Settings")
+        Accessible.description: qsTr("Contains various application settings")
+    }
+
+    ListModel {
+        id: stacksModel
+        ListElement {
+            title: qsTr("Models")
+        }
+        ListElement {
+            title: qsTr("Application")
+        }
+        ListElement {
+            title: qsTr("LocalDocs")
+        }
+    }
+
+    Item {
+        anchors.fill: parent
+        anchors.margins: 20
+        Rectangle {
+            id: stackList
+            anchors.top: parent.top
+            anchors.bottom: parent.bottom
+            anchors.left: parent.left
+            width: 220
+            color: theme.controlBackground
+            radius: 10
+
+            ScrollView {
+                anchors.top: parent.top
+                anchors.bottom: parent.bottom
+                anchors.left: parent.left
+                anchors.right: parent.right
+                anchors.topMargin: 10
+                ScrollBar.vertical.policy: ScrollBar.AsNeeded
+                clip: true
+
+                ListView {
+                    id: listView
+                    anchors.fill: parent
+                    model: stacksModel
+
+                    delegate: Rectangle {
+                        id: item
+                        width: listView.width
+                        height: titleLabel.height + 10
+                        color: "transparent"
+
+                        MyButton {
+                            id: titleLabel
+                            backgroundColor: index === listView.currentIndex ? theme.buttonBackground : theme.controlBackground
+                            backgroundColorHovered: index === listView.currentIndex ? backgroundColor : theme.containerBackground
+                            borderColor: index === listView.currentIndex ? theme.accentColor : "transparent"
+                            borderWidth: index === listView.currentIndex ? 1 : 0
+                            textColor: index === listView.currentIndex ? theme.oppositeTextColor : theme.titleTextColor
+                            anchors.verticalCenter: parent.verticalCenter
+                            anchors.left: parent.left
+                            anchors.right: parent.right
+                            anchors.margins: 10
+                            font.bold: index === listView.currentIndex
+                            text: title
+                            font.pixelSize: theme.fontSizeLarge
+                            onClicked: {
+                                listView.currentIndex = index
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        StackLayout {
+            id: stackLayout
+            anchors.top: parent.top
+            anchors.bottom: parent.bottom
+            anchors.left: stackList.right
+            anchors.right: parent.right
+            currentIndex: listView.currentIndex
+
+            MySettingsStack {
+                title: qsTr("Model/Character Settings")
+                tabs: [
+                    Component { ModelSettings { } }
+                ]
+            }
+
+            MySettingsStack {
+                title: qsTr("Application General Settings")
+                tabs: [
+                    Component { ApplicationSettings { } }
+                ]
+            }
+
+            MySettingsStack {
+                title: qsTr("Local Document Collections")
+                tabs: [
+                    Component {
+                        LocalDocsSettings {
+                            id: localDocsSettings
+                            Component.onCompleted: {
+                                 localDocsSettings.downloadClicked.connect(settingsDialog.downloadClicked);
+                            }
+                        }
+                    }
+                ]
+            }
+        }
+    }
+}

gpt4all-chat/qml/Theme.qml

@@ -390,33 +390,33 @@ QtObject {
     property color iconBackgroundViewBar: {
         switch (MySettings.chatTheme) {
             case "LegacyDark":
-                return blue200;
+                return iconBackgroundLight;
             case "Dark":
-                return green400;
+                return iconBackgroundLight;
             default:
-                return green700;
+                return green600;
         }
     }
 
     property color iconBackgroundViewBarToggled: {
         switch (MySettings.chatTheme) {
             case "LegacyDark":
-                return purple400;
+                return iconBackgroundHovered;
             case "Dark":
-                return accentColor;
+                return green400;
             default:
-                return black;
+                return green950;
         }
     }
 
     property color iconBackgroundViewBarHovered: {
         switch (MySettings.chatTheme) {
             case "LegacyDark":
-                return blue400;
+                return iconBackgroundHovered;
             case "Dark":
-                return green600;
+                return iconBackgroundHovered;
             default:
-                return green500;
+                return green950;
         }
     }