python: bump version to 2.3.2 to include *all* of the bugfixes (#2171)

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/continue_config.yml

@@ -343,19 +343,18 @@ jobs:
     steps:
       - checkout
       - node/install:
-          install-yarn: true
           node-version: "18.16"
       - run: node --version
       - run: corepack enable
       - node/install-packages:
-          pkg-manager: yarn
+          pkg-manager: npm
           app-dir: gpt4all-bindings/typescript
-          override-ci-command: yarn install
+          override-ci-command: npm install --ignore-scripts
       - run:
           name: build docs ts yo
           command: |
             cd gpt4all-bindings/typescript
-            yarn docs:build
+            npm run docs:build
   build-py-docs:
     docker:
       - image: circleci/python:3.8

README.md

@@ -22,6 +22,10 @@
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
 </p>
 
+<p align="center">
+ <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg?&logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNSIgaGVpZ2h0PSI0IiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxwYXRoIGQ9Ik00LjQzIDEuODgyYTEuNDQgMS40NCAwIDAgMS0uMDk4LjQyNmMtLjA1LjEyMy0uMTE1LjIzLS4xOTIuMzIyLS4wNzUuMDktLjE2LjE2NS0uMjU1LjIyNmExLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxMmMtLjA5OS4wMTItLjE5Mi4wMTQtLjI3OS4wMDZsLTEuNTkzLS4xNHYtLjQwNmgxLjY1OGMuMDkuMDAxLjE3LS4xNjkuMjQ2LS4xOTFhLjYwMy42MDMgMCAwIDAgLjItLjEwNi41MjkuNTI5IDAgMCAwIC4xMzgtLjE3LjY1NC42NTQgMCAwIDAgLjA2NS0uMjRsLjAyOC0uMzJhLjkzLjkzIDAgMCAwLS4wMzYtLjI0OS41NjcuNTY3IDAgMCAwLS4xMDMtLjIuNTAyLjUwMiAwIDAgMC0uMTY4LS4xMzguNjA4LjYwOCAwIDAgMC0uMjQtLjA2N0wyLjQzNy43MjkgMS42MjUuNjcxYS4zMjIuMzIyIDAgMCAwLS4yMzIuMDU4LjM3NS4zNzUgMCAwIDAtLjExNi4yMzJsLS4xMTYgMS40NS0uMDU4LjY5Ny0uMDU4Ljc1NEwuNzA1IDRsLS4zNTctLjA3OUwuNjAyLjkwNkMuNjE3LjcyNi42NjMuNTc0LjczOS40NTRhLjk1OC45NTggMCAwIDEgLjI3NC0uMjg1Ljk3MS45NzEgMCAwIDEgLjMzNy0uMTRjLjExOS0uMDI2LjIyNy0uMDM0LjMyNS0uMDI2TDMuMjMyLjE2Yy4xNTkuMDE0LjMzNi4wMy40NTkuMDgyYTEuMTczIDEuMTczIDAgMCAxIC41NDUuNDQ3Yy4wNi4wOTQuMTA5LjE5Mi4xNDQuMjkzYTEuMzkyIDEuMzkyIDAgMCAxIC4wNzguNThsLS4wMjkuMzJaIiBmaWxsPSIjRjI3NzdBIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+Cjwvc3ZnPgo=" alt="phorm.ai"></a>
+</p>
+
 <p align="center">
   <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
 </p>
@@ -43,7 +47,7 @@ A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4
 ### What's New ([Issue Tracker](https://github.com/orgs/nomic-ai/projects/2))
 - **October 19th, 2023**: GGUF Support Launches with Support for:
     - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
-    - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4_0, Q6 quantizations in GGUF.
+    - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4\_0 and Q4\_1 quantizations in GGUF.
     - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
 - **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on AMD, Intel, Samsung, Qualcomm and NVIDIA GPUs.
 - **August 15th, 2023**: GPT4All API launches allowing inference of local LLMs from docker containers.

gpt4all-backend/CMakeLists.txt

@@ -97,11 +97,6 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         add_library(gptj-${BUILD_VARIANT} SHARED
             gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
         prepare_target(gptj llama-mainline)
-
-        add_library(bert-${BUILD_VARIANT} SHARED
-            bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
-        prepare_target(bert llama-mainline)
     endif()
 endforeach()
 

gpt4all-backend/bert.cpp

@@ -1,910 +0,0 @@
-#define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "bert_impl.h"
-#include "llmodel_shared.h"
-#include "ggml.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <map>
-#include <string>
-#include <vector>
-#include <iostream>
-#include <regex>
-#include <thread>
-#include <algorithm>
-#include <numeric>
-
-//#define DEBUG_BERT
-
-namespace {
-const char *modelType_ = "Bert";
-}
-
-typedef int32_t bert_vocab_id;
-
-// default hparams (all-MiniLM-L6-v2)
-struct bert_hparams
-{
-    int32_t n_vocab = 30522;
-    int32_t n_max_tokens = 512;
-    int32_t n_embd = 256;
-    int32_t n_intermediate = 1536;
-    int32_t n_head = 12;
-    int32_t n_layer = 6;
-};
-
-struct bert_layer
-{
-    // normalization
-    struct ggml_tensor *ln_att_w;
-    struct ggml_tensor *ln_att_b;
-
-    struct ggml_tensor *ln_out_w;
-    struct ggml_tensor *ln_out_b;
-
-    // attention
-    struct ggml_tensor *q_w;
-    struct ggml_tensor *q_b;
-    struct ggml_tensor *k_w;
-    struct ggml_tensor *k_b;
-    struct ggml_tensor *v_w;
-    struct ggml_tensor *v_b;
-
-    struct ggml_tensor *o_w;
-    struct ggml_tensor *o_b;
-
-    // ff
-    struct ggml_tensor *ff_i_w;
-    struct ggml_tensor *ff_i_b;
-
-    struct ggml_tensor *ff_o_w;
-    struct ggml_tensor *ff_o_b;
-};
-
-struct bert_vocab
-{
-    std::map<std::string, bert_vocab_id> token_to_id;
-    std::map<std::string, bert_vocab_id> subword_token_to_id;
-
-    std::map<bert_vocab_id, std::string> _id_to_token;
-    std::map<bert_vocab_id, std::string> _id_to_subword_token;
-};
-
-struct bert_model
-{
-    bert_hparams hparams;
-
-    // embeddings weights
-    struct ggml_tensor *word_embeddings;
-    struct ggml_tensor *token_type_embeddings;
-    struct ggml_tensor *position_embeddings;
-    struct ggml_tensor *ln_e_w;
-    struct ggml_tensor *ln_e_b;
-
-    std::vector<bert_layer> layers;
-
-    struct ggml_context *ctx;
-};
-
-// Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
-struct bert_ctx
-{
-    bert_model model;
-    bert_vocab vocab;
-
-    size_t mem_per_token;
-    int64_t mem_per_input;
-    int32_t max_batch_n;
-    llm_buffer buf_compute;
-    llm_buffer work_buf;
-};
-
-int32_t bert_n_embd(bert_ctx * ctx)
-{
-    return ctx->model.hparams.n_embd;
-}
-
-int32_t bert_n_max_tokens(bert_ctx * ctx)
-{
-    return ctx->model.hparams.n_max_tokens;
-}
-
-const char* bert_vocab_id_to_token(bert_ctx * ctx, bert_vocab_id id) {
-    bert_vocab & vocab = ctx->vocab;
-    auto it = vocab._id_to_token.find(id);
-    if (it != vocab._id_to_token.end())
-    {
-        return it->second.c_str();
-    }
-    it = vocab._id_to_subword_token.find(id);
-    if (it != vocab._id_to_subword_token.end())
-    {
-        return it->second.c_str();
-    }
-    return "[UNK TOKEN from bert_vocab]";
-}
-
-//
-// Tokenizing
-//
-
-static size_t utf8_len(char src)
-{
-    const size_t lookup[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4};
-    uint8_t highbits = static_cast<uint8_t>(src) >> 4;
-    return lookup[highbits];
-}
-
-std::string stripAccents(const std::string &inputString)
-{
-    std::string resultString;
-    std::map<std::string, char> accentMap = {{"À", 'A'},{"Á", 'A'},
-        {"Â", 'A'},{"Ã", 'A'},{"Ä", 'A'},{"Å", 'A'},{"à", 'a'},{"á", 'a'},
-        {"â", 'a'},{"ã", 'a'},{"ä", 'a'},{"å", 'a'},{"È", 'E'},{"É", 'E'},
-        {"Ê", 'E'},{"Ë", 'E'},{"è", 'e'},{"é", 'e'},{"ê", 'e'},{"ë", 'e'},
-        {"Ì", 'I'},{"Í", 'I'},{"Î", 'I'},{"Ï", 'I'},{"ì", 'i'},{"í", 'i'},
-        {"î", 'i'},{"ï", 'i'},{"Ò", 'O'},{"Ó", 'O'},{"Ô", 'O'},{"Õ", 'O'},
-        {"Ö", 'O'},{"ò", 'o'},{"ó", 'o'},{"ô", 'o'},{"õ", 'o'},{"ö", 'o'},
-        {"Ù", 'U'},{"Ú", 'U'},{"Û", 'U'},{"Ü", 'U'},{"ù", 'u'},{"ú", 'u'},
-        {"û", 'u'},{"ü", 'u'},{"Ý", 'Y'},{"ý", 'y'},{"Ç", 'C'},{"ç", 'c'},
-        {"Ñ", 'N'},{"ñ", 'n'},
-    };
-
-    for (size_t i = 0; i < inputString.length();)
-    {
-        int len = utf8_len(inputString[i]);
-        std::string curChar = inputString.substr(i, len);
-        auto iter = accentMap.find(curChar);
-        if (iter != accentMap.end())
-        {
-            resultString += iter->second;
-        }
-        else
-        {
-            resultString += curChar;
-        }
-        i += len;
-    }
-
-    return resultString;
-}
-
-std::string bert_normalize_prompt(const std::string &text)
-{
-    // TODO: handle chinese characters? https://github.com/huggingface/tokenizers/blob/ef5f50605ddf9f8caef1598c0e4853862b9707a7/tokenizers/src/normalizers/bert.rs#L98
-    std::string text2 = stripAccents(text);
-    for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i]))
-    {
-        char c = text2[i];
-        if (c >= 'A' && c <= 'Z')
-            text2[i] = c - 'A' + 'a';
-    }
-    return text2;
-}
-
-std::vector<bert_vocab_id> bert_tokenize(
-    struct bert_ctx * ctx,
-    const char * text)
-{
-    const bert_vocab &vocab = ctx->vocab;
-
-    std::string str = text;
-
-    std::vector<std::string> words;
-    // first split the text into words
-    {
-        str = bert_normalize_prompt(str);
-
-        std::string pat = R"([[:punct:]]|[[:alpha:]]+|[[:digit:]]+)";
-
-        std::regex re(pat);
-        std::smatch m;
-
-        while (std::regex_search(str, m, re))
-        {
-            for (std::string x : m)
-            {
-                words.push_back(x);
-            }
-            str = m.suffix();
-        }
-    }
-
-    // find the longest tokens that form the words:
-    std::vector<bert_vocab_id> tokens;
-    int cls_tok_id = 101;
-    tokens.push_back(cls_tok_id);
-    for (const auto &word : words)
-    {
-        if (word.size() == 0)
-            continue;
-
-        int i = 0;
-        int n = word.size();
-        auto *token_map = &vocab.token_to_id;
-        while (i < n)
-        {
-            int j = n;
-            while (j > i)
-            {
-                auto it = token_map->find(word.substr(i, j - i));
-                if (it != token_map->end())
-                {
-                    tokens.push_back(it->second);
-                    i = j;
-                    token_map = &vocab.subword_token_to_id;
-                }
-                --j;
-            }
-            if (j == i)
-            {
-                fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data());
-                token_map = &vocab.subword_token_to_id;
-                ++i;
-            }
-        }
-    }
-
-    return tokens;
-}
-
-void bert_resize_ctx(bert_ctx * ctx, int32_t new_size) {
-    int64_t buf_size_new = ctx->mem_per_input * new_size;
-
-    // TODO: Max memory should be a param? Now just 1 GB
-    int64_t GB = 1 << 30;
-#if defined(DEBUG_BERT)
-    printf("%s: requested_buf_size %lldMB\n", __func__, buf_size_new / (1 << 20));
-#endif
-    if (buf_size_new > GB) {
-        int32_t adjusted_new_size = GB / ctx->mem_per_input;
-        if (adjusted_new_size < 1) adjusted_new_size = 1;
-#if defined(DEBUG_BERT)
-        printf("%s: requested batch size %d, actual new batch size %d\n", __func__, new_size, adjusted_new_size);
-#endif
-        new_size = adjusted_new_size;
-        buf_size_new = ctx->mem_per_input * new_size;
-    }
-    if (new_size > ctx->max_batch_n) {
-        ctx->buf_compute.resize(buf_size_new);
-        ctx->max_batch_n = new_size;
-    }
-}
-
-void bert_eval(
-    struct bert_ctx *ctx,
-    int32_t n_threads,
-    const bert_vocab_id *raw_tokens,
-    int32_t n_tokens,
-    float *embeddings)
-{
-    const bert_model& model = ctx->model;
-    bool mem_req_mode = !embeddings;
-
-    // batch_embeddings is nullptr for the initial memory requirements run
-    if (!mem_req_mode && 1 > ctx->max_batch_n)
-        bert_resize_ctx(ctx, 1);
-
-    const int N = n_tokens;
-    const auto &tokens = raw_tokens;
-
-    const auto &hparams = model.hparams;
-
-    const int n_embd = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_max_tokens = hparams.n_max_tokens;
-    const int n_head = hparams.n_head;
-
-    const int d_head = n_embd / n_head;
-
-    std::vector<float> result;
-    if (N > n_max_tokens)
-    {
-        fprintf(stderr, "Too many tokens, maximum is %d\n", n_max_tokens);
-        return;
-    }
-
-    auto & mem_per_token = ctx->mem_per_token;
-    auto & buf_compute   = ctx->buf_compute;
-
-    struct ggml_init_params params = {
-        .mem_size = buf_compute.size,
-        .mem_buffer = buf_compute.addr,
-        .no_alloc = false,
-    };
-
-    struct ggml_context *ctx0 = ggml_init(params);
-    struct ggml_cgraph *gf = ggml_new_graph(ctx0);
-
-    // Embeddings. word_embeddings + token_type_embeddings + position_embeddings
-    struct ggml_tensor *token_layer = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(token_layer->data, tokens, N * ggml_element_size(token_layer));
-
-    struct ggml_tensor *token_types = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    ggml_set_zero(token_types);
-
-    struct ggml_tensor *positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    for (int i = 0; i < N; i++)
-    {
-        ggml_set_i32_1d(positions, i, i);
-    }
-
-    struct ggml_tensor *inpL = ggml_get_rows(ctx0, model.word_embeddings, token_layer);
-
-    inpL = ggml_add(ctx0,
-                    ggml_get_rows(ctx0, model.token_type_embeddings, token_types),
-                    inpL);
-    inpL = ggml_add(ctx0,
-                    ggml_get_rows(ctx0, model.position_embeddings, positions),
-                    inpL);
-
-    // embd norm
-    {
-        inpL = ggml_norm(ctx0, inpL, 1e-12f);
-
-        inpL = ggml_add(ctx0,
-                        ggml_mul(ctx0,
-                                 ggml_repeat(ctx0, model.ln_e_w, inpL),
-                                 inpL),
-                        ggml_repeat(ctx0, model.ln_e_b, inpL));
-    }
-    // layers
-    for (int il = 0; il < n_layer; il++)
-    {
-        struct ggml_tensor *cur = inpL;
-
-        // self-attention
-        {
-            struct ggml_tensor *Qcur = cur;
-            Qcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].q_b, Qcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].q_w, Qcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *Kcur = cur;
-            Kcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].k_b, Kcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].k_w, Kcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *K = ggml_permute(ctx0, Kcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *Vcur = cur;
-            Vcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].v_b, Vcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].v_w, Vcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *V = ggml_permute(ctx0, Vcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *KQ = ggml_mul_mat(ctx0, K, Q);
-            // KQ = soft_max(KQ / sqrt(head width))
-            KQ = ggml_soft_max(
-                ctx0, ggml_scale(ctx0, KQ, 1.0f / sqrt((float)d_head))
-            );
-
-            V = ggml_cont(ctx0, ggml_transpose(ctx0, V));
-            struct ggml_tensor *KQV = ggml_mul_mat(ctx0, V, KQ);
-            KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            cur = ggml_cpy(ctx0,
-                           KQV,
-                           ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-        }
-        // attention output
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].o_b, cur),
-                       ggml_mul_mat(ctx0, model.layers[il].o_w, cur));
-
-        // re-add the layer input
-        cur = ggml_add(ctx0, cur, inpL);
-
-        // attention norm
-        {
-            cur = ggml_norm(ctx0, cur, 1e-12f);
-
-            cur = ggml_add(ctx0,
-                           ggml_mul(ctx0,
-                                    ggml_repeat(ctx0, model.layers[il].ln_att_w, cur),
-                                    cur),
-                           ggml_repeat(ctx0, model.layers[il].ln_att_b, cur));
-        }
-        struct ggml_tensor *att_output = cur;
-        // intermediate_output = self.intermediate(attention_output)
-        cur = ggml_mul_mat(ctx0, model.layers[il].ff_i_w, cur);
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].ff_i_b, cur),
-                       cur);
-        cur = ggml_gelu(ctx0, cur);
-
-        // layer_output = self.output(intermediate_output, attention_output)
-        cur = ggml_mul_mat(ctx0, model.layers[il].ff_o_w, cur);
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].ff_o_b, cur),
-                       cur);
-        // attentions bypass the intermediate layer
-        cur = ggml_add(ctx0, att_output, cur);
-
-        // output norm
-        {
-            cur = ggml_norm(ctx0, cur, 1e-12f);
-
-            cur = ggml_add(ctx0,
-                           ggml_mul(ctx0,
-                                    ggml_repeat(ctx0, model.layers[il].ln_out_w, cur),
-                                    cur),
-                           ggml_repeat(ctx0, model.layers[il].ln_out_b, cur));
-        }
-        inpL = cur;
-    }
-    inpL = ggml_cont(ctx0, ggml_transpose(ctx0, inpL));
-    // pooler
-    struct ggml_tensor *sum = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, N, 1);
-    ggml_set_f32(sum, 1.0f / N);
-    inpL = ggml_mul_mat(ctx0, inpL, sum);
-
-    ggml_tensor *output = inpL;
-    // run the computation
-    ggml_build_forward_expand(gf, output);
-    //ggml_graph_compute_g4a()
-    ggml_graph_compute_g4a(ctx->work_buf, gf, n_threads);
-    //ggml_graph_compute(ctx0, gf);
-
-
-    // float *dat = ggml_get_data_f32(output);
-    // pretty_print_tensor(dat, output->ne, output->nb, output->n_dims - 1, "");
-
-    #ifdef GGML_PERF
-        // print timing information per ggml operation (for debugging purposes)
-        // requires GGML_PERF to be defined
-        ggml_graph_print(gf);
-    #endif
-
-    if (!mem_req_mode) {
-        memcpy(embeddings, (float *)ggml_get_data(output), sizeof(float) * n_embd);
-    } else {
-        mem_per_token = ggml_used_mem(ctx0) / N;
-    }
-
-    // printf("used_mem = %zu KB \n", ggml_used_mem(ctx0) / 1024);
-    // printf("mem_per_token = %zu KB \n", mem_per_token / 1024);
-
-    ggml_free(ctx0);
-}
-
-//
-// Loading and setup
-//
-
-void bert_free(bert_ctx * ctx) {
-    delete ctx;
-}
-
-struct bert_ctx * bert_load_from_file(const char *fname)
-{
-#if defined(DEBUG_BERT)
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname);
-#endif
-
-    bert_ctx * new_bert = new bert_ctx;
-
-    bert_model & model = new_bert->model;
-    bert_vocab & vocab = new_bert->vocab;
-
-    struct gguf_init_params params = {
-        /*.no_alloc = */ false,
-        /*.ctx      = */ &model.ctx,
-    };
-    gguf_context *ggufctx = gguf_init_from_file(fname, params);
-    if (!ggufctx) {
-        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
-        return nullptr;
-    }
-
-    printf("%s: gguf version     = %d\n", __func__, gguf_get_version(ggufctx));
-    printf("%s: gguf alignment   = %zu\n", __func__, gguf_get_alignment(ggufctx));
-    printf("%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx));
-
-    // print some standard metadata
-    {
-        int keyidx;
-
-        keyidx = gguf_find_key(ggufctx, "general.name");
-        if (keyidx != -1) { printf("%s: model name           = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.description");
-        if (keyidx != -1) { printf("%s: model description    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.author");
-        if (keyidx != -1) { printf("%s: model author         = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.license");
-        if (keyidx != -1) { printf("%s: model license        = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx != -1) { printf("%s: model architecture   = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.file_type");
-        if (keyidx != -1) { printf("%s: model file type      = %" PRIu32 "\n", __func__, gguf_get_val_u32(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "gptneox.tensor_data_layout");
-        if (keyidx != -1) { printf("%s: model data layout    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.source.huggingface.repository");
-        if (keyidx != -1) { printf("%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-    }
-
-    // check required metadata
-    {
-        // check model architecture kv
-        int keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: gguf model architecture not found!\n", __func__);
-            return nullptr;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
-            fprintf(stderr, "%s: model architecture not supported!\n", __func__);
-            return nullptr;
-        }
-    }
-
-    // load hparams
-    {
-        auto &hparams = model.hparams;
-
-        bool ok = false;
-        int keyidx;
-
-        do {
-            keyidx = gguf_find_key(ggufctx, "bert.context_length");
-            if (keyidx == -1) { break; }
-            hparams.n_max_tokens = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.embedding_length");
-            if (keyidx == -1) { break; }
-            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.feed_forward_length");
-            if (keyidx == -1) { break; }
-            hparams.n_intermediate = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.attention.head_count");
-            if (keyidx == -1) { break; }
-            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.block_count");
-            if (keyidx == -1) { break; }
-            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
-
-            ok = true;
-        } while (false);
-
-        if (!ok) {
-            fprintf(stderr, "%s: required hparam missing!\n", __func__);
-            return nullptr;
-        }
-
-#if defined(DEBUG_BERT)
-        printf("%s: n_max_tokens   = %d\n", __func__, hparams.n_max_tokens);
-        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_intermediate = %d\n", __func__, hparams.n_intermediate);
-        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
-#endif
-    }
-
-    // load vocab
-    {
-        auto & hparams = model.hparams;
-
-        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
-            return nullptr;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
-            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
-            return nullptr;
-        }
-
-        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
-        if (tokens_keyidx == -1) {
-            fprintf(stderr, "%s: bert tokenizer vocab not found!\n", __func__);
-            return nullptr;
-        }
-
-        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
-        printf("%s: bert tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
-
-        for (int i = 0; i < hparams.n_vocab; i++) {
-            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
-
-            if (word[0] == '#' && word[1] == '#')
-            {
-                vocab.subword_token_to_id[word.substr(2)] = i;
-                vocab._id_to_subword_token[i] = word;
-            }
-
-            if (vocab.token_to_id.count(word) == 0)
-            {
-                vocab.token_to_id[word] = i;
-                vocab._id_to_token[i] = word;
-            }
-        }
-    }
-
-    auto &ctx = model.ctx;
-
-#if defined(DEBUG_BERT)
-    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ggml_get_mem_size(ctx) / (1024.0 * 1024.0));
-#endif
-
-    // prepare memory for the weights
-    {
-        const int n_layer = model.hparams.n_layer;
-        model.layers.resize(n_layer);
-
-        model.word_embeddings       = ggml_get_tensor(ctx, "token_embd.weight");
-        model.token_type_embeddings = ggml_get_tensor(ctx, "token_types.weight");
-        model.position_embeddings   = ggml_get_tensor(ctx, "position_embd.weight");
-        model.ln_e_w                = ggml_get_tensor(ctx, "output_norm.weight");
-        model.ln_e_b                = ggml_get_tensor(ctx, "output_norm.bias");
-
-        auto name = [](int i, std::string n) {
-            static std::string key;
-            key = "blk." + std::to_string(i) + "." + n;
-            return key.c_str();
-        };
-
-        for (int i = 0; i < n_layer; ++i)
-        {
-            auto &layer = model.layers[i];
-
-            layer.ln_att_w = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
-            layer.ln_att_b = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
-            layer.ln_out_w = ggml_get_tensor(ctx, name(i, "ffn_norm.weight"));
-            layer.ln_out_b = ggml_get_tensor(ctx, name(i, "ffn_norm.bias"));
-            layer.q_w      = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
-            layer.q_b      = ggml_get_tensor(ctx, name(i, "attn_q.bias"));
-            layer.k_w      = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
-            layer.k_b      = ggml_get_tensor(ctx, name(i, "attn_k.bias"));
-            layer.v_w      = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
-            layer.v_b      = ggml_get_tensor(ctx, name(i, "attn_v.bias"));
-            layer.o_w      = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
-            layer.o_b      = ggml_get_tensor(ctx, name(i, "attn_output.bias"));
-            layer.ff_i_w   = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
-            layer.ff_i_b   = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
-            layer.ff_o_w   = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
-            layer.ff_o_b   = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
-        }
-    }
-
-    // Calculate space requirements for setting up context buffers later
-    {
-        bert_vocab_id tokens[] = {0, 1, 2, 3};
-        // TODO: We set the initial buffer size to 16MB and hope it's enough. Maybe there is a better way to do this?
-        new_bert->buf_compute.resize(16 * 1024 * 1024);
-        bert_eval(new_bert, 1, tokens, 4, nullptr);
-        new_bert->max_batch_n = 0;
-
-        // TODO: Max tokens should be a param?
-        int32_t N = new_bert->model.hparams.n_max_tokens;
-        new_bert->mem_per_input = 2.2 * (new_bert->mem_per_token * N); // add 10% to account for ggml object overhead
-
-    }
-#if defined(DEBUG_BERT)
-    printf("%s: mem_per_token %ld KB, mem_per_input %ld MB\n", __func__, new_bert->mem_per_token / (1 << 10), new_bert->mem_per_input / (1 << 20));
-#endif
-
-    return new_bert;
-}
-
-struct BertPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    bert_ctx *ctx = nullptr;
-    int64_t n_threads = 0;
-};
-
-Bert::Bert() : d_ptr(new BertPrivate) {
-    d_ptr->modelLoaded = false;
-}
-
-Bert::~Bert() {
-    bert_free(d_ptr->ctx);
-}
-
-bool Bert::loadModel(const std::string &modelPath, int n_ctx, int ngl)
-{
-    (void)n_ctx;
-    (void)ngl;
-    d_ptr->modelLoaded = false;
-
-    auto * ctx = bert_load_from_file(modelPath.c_str());
-    fflush(stdout);
-    if (!ctx)
-        return false;
-
-    d_ptr->ctx = ctx;
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    return true;
-}
-
-bool Bert::isModelLoaded() const
-{
-    return d_ptr->modelLoaded;
-}
-
-size_t Bert::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
-{
-    (void)modelPath;
-    (void)n_ctx;
-    (void)ngl;
-    return 0;
-}
-
-size_t Bert::stateSize() const
-{
-    return 0;
-}
-
-size_t Bert::saveState(uint8_t */*dest*/) const
-{
-    return 0;
-}
-
-size_t Bert::restoreState(const uint8_t */*src*/)
-{
-    return 0;
-}
-
-void Bert::setThreadCount(int32_t n_threads)
-{
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t Bert::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-std::vector<float> Bert::embedding(const std::string &text)
-{
-    const int overlap = 32;
-    const LLModel::Token clsToken = 101;
-    const size_t contextLength = bert_n_max_tokens(d_ptr->ctx);
-    typedef std::vector<LLModel::Token> TokenString;
-    TokenString tokens = ::bert_tokenize(d_ptr->ctx, text.c_str());
-#if defined(DEBUG_BERT)
-    std::cerr << "embedding: " << tokens.size()
-              << " contextLength " << contextLength
-              << "\n";
-#endif
-    std::vector<double> embeddingsSum(bert_n_embd(d_ptr->ctx), 0);
-    int embeddingsSumTotal = 0;
-    size_t start_pos = 0;
-    bool isFirstChunk = true;
-    while (start_pos < tokens.size()) {
-        TokenString chunk;
-        if (!isFirstChunk)
-            chunk.push_back(clsToken);
-        const size_t l = isFirstChunk ? contextLength : contextLength - 1;
-        if (tokens.size() - start_pos > l) {
-            chunk.insert(chunk.end(), tokens.begin() + start_pos, tokens.begin() + start_pos + l);
-            start_pos = start_pos + contextLength - overlap;
-        } else {
-            chunk.insert(chunk.end(), tokens.begin() + start_pos, tokens.end());
-            start_pos = tokens.size();
-        }
-#if defined(DEBUG_BERT)
-        std::cerr << "chunk length: " << chunk.size()
-            << " embeddingsSumTotal " << embeddingsSumTotal
-            << " contextLength " << contextLength
-            << " start_pos " << start_pos
-            << "\n";
-#endif
-        embeddingsSumTotal++;
-        std::vector<float> embeddings(bert_n_embd(d_ptr->ctx));
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, chunk.data(), chunk.size(), embeddings.data());
-        std::transform(embeddingsSum.begin(), embeddingsSum.end(), embeddings.begin(), embeddingsSum.begin(), std::plus<float>());
-        isFirstChunk = false;
-    }
-
-    std::transform(embeddingsSum.begin(), embeddingsSum.end(), embeddingsSum.begin(), [embeddingsSumTotal](float num){ return num / embeddingsSumTotal; });
-    double magnitude = std::sqrt(std::inner_product(embeddingsSum.begin(), embeddingsSum.end(), embeddingsSum.begin(), 0.0));
-    for (auto &value : embeddingsSum)
-        value /= magnitude;
-    std::vector<float> finalEmbeddings(embeddingsSum.begin(), embeddingsSum.end());
-    return finalEmbeddings;
-}
-
-std::vector<LLModel::Token> Bert::tokenize(PromptContext &ctx, const std::string &str, bool special) const
-{
-    (void)ctx;
-    (void)special;
-    return ::bert_tokenize(d_ptr->ctx, str.c_str());
-}
-
-LLModel::Token Bert::sampleToken(PromptContext &/*promptCtx*/) const
-{
-    return 999 /*!*/;
-}
-
-std::string Bert::tokenToString(Token id) const
-{
-    return bert_vocab_id_to_token(d_ptr->ctx, id);
-}
-
-bool Bert::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    std::vector<float> embeddings(bert_n_embd(d_ptr->ctx));
-    int32_t cls = 101;
-    const bool useCLS = tokens.front() != cls;
-    if (useCLS) {
-        std::vector<int32_t> myTokens;
-        myTokens.push_back(cls);
-        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, myTokens.data(), myTokens.size(), embeddings.data());
-    } else
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, tokens.data(), tokens.size(), embeddings.data());
-    ctx.n_past = 0; // bert does not store any context
-    return true;
-}
-
-int32_t Bert::contextLength() const
-{
-    return bert_n_max_tokens(d_ptr->ctx);
-}
-
-const std::vector<LLModel::Token> &Bert::endTokens() const
-{
-    static const std::vector<LLModel::Token> out = { 102 /*sep*/};
-    return out;
-}
-
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(const char * fname) {
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
-
-    bool isValid = gguf_get_version(ctx_gguf) <= 3;
-    isValid = isValid && get_arch_name(ctx_gguf) == "bert";
-
-    gguf_free(ctx_gguf);
-    return isValid;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new Bert;
-}
-}

gpt4all-backend/bert_impl.h

@@ -1,45 +0,0 @@
-#ifndef BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of bert.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef BERT_H
-#define BERT_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include <memory>
-#include "llmodel.h"
-
-struct BertPrivate;
-class Bert : public LLModel {
-public:
-    Bert();
-    ~Bert();
-
-    bool supportsEmbedding() const override { return true; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-    std::vector<float> embedding(const std::string &text) override;
-
-private:
-    std::unique_ptr<BertPrivate> d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token id) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token> &endTokens() const override;
-    bool shouldAddBOS() const override { return true; }
-};
-
-#endif // BERT_H

gpt4all-backend/llamamodel.cpp

@@ -6,9 +6,11 @@
 #include <cstdio>
 #include <cstring>
 #include <fstream>
+#include <initializer_list>
 #include <iomanip>
 #include <iostream>
 #include <map>
+#include <numeric>
 #include <random>
 #include <sstream>
 #include <stdexcept>
@@ -30,6 +32,19 @@ static constexpr int GGUF_VER_MAX = 3;
 
 static const char * const modelType_ = "LLaMA";
 
+static const std::vector<const char *> KNOWN_ARCHES {
+    "baichuan", "bert", "bloom", "codeshell", "falcon", "gemma", "gpt2", "llama", "mpt", "nomic-bert", "orion",
+    "persimmon", "phi2", "plamo", "qwen", "qwen2", "refact", "stablelm", "starcoder"
+};
+
+static const std::vector<const char *> EMBEDDING_ARCHES {
+    "bert", "nomic-bert"
+};
+
+static bool is_embedding_arch(const std::string &arch) {
+    return std::find(EMBEDDING_ARCHES.begin(), EMBEDDING_ARCHES.end(), arch) < EMBEDDING_ARCHES.end();
+}
+
 static bool llama_verbose() {
     const char* var = getenv("GPT4ALL_VERBOSE_LLAMACPP");
     return var && *var;
@@ -124,7 +139,7 @@ static int32_t get_arch_key_u32(std::string const &modelPath, std::string const
     auto * ctx = load_gguf(modelPath.c_str());
     if (!ctx)
         return -1;
-    auto arch = get_arch_name(ctx);
+    std::string arch = get_arch_name(ctx);
 
     int32_t value = -1;
     if (ctx) {
@@ -193,7 +208,7 @@ size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
     return filesize + est_kvcache_size;
 }
 
-bool LLamaModel::isModelBlacklisted(const std::string &modelPath) {
+bool LLamaModel::isModelBlacklisted(const std::string &modelPath) const {
     auto * ctx = load_gguf(modelPath.c_str());
     if (!ctx) {
         std::cerr << __func__ << ": failed to load " << modelPath << "\n";
@@ -229,6 +244,18 @@ bool LLamaModel::isModelBlacklisted(const std::string &modelPath) {
     return res;
 }
 
+bool LLamaModel::isEmbeddingModel(const std::string &modelPath) const {
+    auto *ctx_gguf = load_gguf(modelPath.c_str());
+    if (!ctx_gguf) {
+        std::cerr << __func__ << ": failed to load GGUF from " <<  modelPath << "\n";
+        return false;
+    }
+
+    std::string arch = get_arch_name(ctx_gguf);
+    gguf_free(ctx_gguf);
+    return is_embedding_arch(arch);
+}
+
 bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
 {
     d_ptr->modelLoaded = false;
@@ -287,20 +314,25 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     if (!d_ptr->model) {
         fflush(stdout);
         d_ptr->device = -1;
-        std::cerr << "LLAMA ERROR: failed to load model from " <<  modelPath << std::endl;
+        std::cerr << "LLAMA ERROR: failed to load model from " << modelPath << std::endl;
         return false;
     }
 
-    const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
-    if (n_ctx > n_ctx_train) {
-        std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
-                  << n_ctx << " specified)\n";
-    }
-
     // -- initialize the context --
 
     d_ptr->ctx_params = llama_context_default_params();
 
+    bool isEmbedding = is_embedding_arch(llama_model_arch(d_ptr->model));
+    const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
+    if (isEmbedding) {
+        d_ptr->ctx_params.n_batch = n_ctx_train;
+    } else {
+        if (n_ctx > n_ctx_train) {
+            std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
+                      << n_ctx << " specified)\n";
+        }
+    }
+
     d_ptr->ctx_params.n_ctx   = n_ctx;
     d_ptr->ctx_params.seed    = params.seed;
     d_ptr->ctx_params.type_k  = params.kv_type;
@@ -314,6 +346,9 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     d_ptr->ctx_params.n_threads       = d_ptr->n_threads;
     d_ptr->ctx_params.n_threads_batch = d_ptr->n_threads;
 
+    if (isEmbedding)
+        d_ptr->ctx_params.embeddings = true;
+
     d_ptr->ctx = llama_new_context_with_model(d_ptr->model, d_ptr->ctx_params);
     if (!d_ptr->ctx) {
         fflush(stdout);
@@ -332,6 +367,9 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     }
 #endif
 
+    m_supportsEmbedding = isEmbedding;
+    m_supportsCompletion = !isEmbedding;
+
     fflush(stdout);
     d_ptr->modelLoaded = true;
     return true;
@@ -438,7 +476,9 @@ const std::vector<LLModel::Token> &LLamaModel::endTokens() const
 bool LLamaModel::shouldAddBOS() const
 {
     int add_bos = llama_add_bos_token(d_ptr->model);
-    return add_bos != -1 ? bool(add_bos) : llama_vocab_type(d_ptr->model) == LLAMA_VOCAB_TYPE_SPM;
+    if (add_bos != -1) { return add_bos; }
+    auto vocab_type = llama_vocab_type(d_ptr->model);
+    return vocab_type == LLAMA_VOCAB_TYPE_SPM || vocab_type == LLAMA_VOCAB_TYPE_WPM;
 }
 
 int32_t LLamaModel::maxContextLength(std::string const &modelPath) const
@@ -535,6 +575,326 @@ bool LLamaModel::usingGPUDevice()
 #endif
 }
 
+void llama_batch_add(
+                 struct llama_batch & batch,
+                        llama_token   id,
+                          llama_pos   pos,
+    const std::vector<llama_seq_id> & seq_ids,
+                               bool   logits) {
+    batch.token   [batch.n_tokens] = id;
+    batch.pos     [batch.n_tokens] = pos;
+    batch.n_seq_id[batch.n_tokens] = seq_ids.size();
+    for (size_t i = 0; i < seq_ids.size(); ++i) {
+        batch.seq_id[batch.n_tokens][i] = seq_ids[i];
+    }
+    batch.logits  [batch.n_tokens] = logits;
+
+    batch.n_tokens++;
+}
+
+static void batch_add_seq(llama_batch &batch, const std::vector<LLModel::Token> &tokens, int seq_id) {
+    for (unsigned i = 0; i < tokens.size(); i++) {
+        llama_batch_add(batch, tokens[i], i, { seq_id }, i == tokens.size() - 1);
+    }
+}
+
+size_t LLamaModel::embeddingSize() const {
+    return llama_n_embd(d_ptr->model);
+}
+
+struct EmbModelSpec {
+    const char *docPrefix;
+    const char *queryPrefix;
+    std::vector<const char *> otherPrefixes = {};
+    bool matryoshkaCapable = false;
+    const char *recommendedDims = nullptr;
+};
+
+struct EmbModelGroup {
+    EmbModelSpec spec;
+    std::vector<const char *> names;
+};
+
+static const EmbModelSpec NOPREFIX_SPEC {"", ""};
+static const EmbModelSpec NOMIC_SPEC    {"search_document", "search_query", {"clustering", "classification"}};
+static const EmbModelSpec E5_SPEC       {"passage", "query"};
+
+static const EmbModelSpec NOMIC_1_5_SPEC {
+    "search_document", "search_query", {"clustering", "classification"}, true, "[768, 512, 384, 256, 128]",
+};
+static const EmbModelSpec LLM_EMBEDDER_SPEC {
+    "Represent this document for retrieval",
+    "Represent this query for retrieving relevant documents",
+};
+static const EmbModelSpec BGE_SPEC {
+    "", "Represent this sentence for searching relevant passages",
+};
+static const EmbModelSpec E5_MISTRAL_SPEC {
+    "", "Instruct: Given a query, retrieve relevant passages that answer the query\nQuery",
+};
+
+static const EmbModelGroup EMBEDDING_MODEL_SPECS[] {
+    {NOPREFIX_SPEC,     {"all-MiniLM-L6-v1", "all-MiniLM-L12-v1", "all-MiniLM-L6-v2", "all-MiniLM-L12-v2"}},
+    {NOMIC_SPEC,        {"nomic-embed-text-v1", "nomic-embed-text-v1-ablated", "nomic-embed-text-v1-unsupervised"}},
+    {NOMIC_1_5_SPEC,    {"nomic-embed-text-v1.5"}},
+    {LLM_EMBEDDER_SPEC, {"llm-embedder"}},
+    {BGE_SPEC,          {"bge-small-en", "bge-base-en", "bge-large-en",
+                         "bge-small-en-v1.5", "bge-base-en-v1.5", "bge-large-en-v1.5"}},
+    // NOTE: E5 Mistral is not yet implemented in llama.cpp, so it's not in EMBEDDING_ARCHES
+    {E5_SPEC,           {"e5-small", "e5-base", "e5-large",
+                         "e5-small-unsupervised", "e5-base-unsupervised", "e5-large-unsupervised",
+                         "e5-small-v2", "e5-base-v2", "e5-large-v2"}},
+    {E5_MISTRAL_SPEC,   {"e5-mistral-7b-instruct",
+                         "multilingual-e5-small", "multilingual-e5-base", "multilingual-e5-large",
+                         "multilingual-e5-large-instruct"}},
+};
+
+static const EmbModelSpec *getEmbedSpec(const std::string &modelName) {
+    static const auto &specs = EMBEDDING_MODEL_SPECS;
+    auto it = std::find_if(specs, std::end(specs),
+        [&modelName](auto &spec) {
+            auto &names = spec.names;
+            return std::find(names.begin(), names.end(), modelName) < names.end();
+        }
+    );
+    return it < std::end(specs) ? &it->spec : nullptr;
+}
+
+void LLamaModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
+    bool doMean, bool atlas
+) {
+    const EmbModelSpec *spec;
+    std::optional<std::string> prefix;
+    if (d_ptr->model && (spec = getEmbedSpec(llama_model_name(d_ptr->model))))
+        prefix = isRetrieval ? spec->queryPrefix : spec->docPrefix;
+
+    embed(texts, embeddings, prefix, dimensionality, tokenCount, doMean, atlas);
+}
+
+void LLamaModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas
+) {
+    if (!d_ptr->model)
+        throw std::logic_error("no model is loaded");
+
+    const char *modelName = llama_model_name(d_ptr->model);
+    if (!m_supportsEmbedding)
+        throw std::logic_error("not an embedding model: "s + modelName);
+
+    auto *spec = getEmbedSpec(modelName);
+    if (!spec)
+        std::cerr << __func__ << ": warning: unknown model " << modelName << "\n";
+
+    const int32_t n_embd = llama_n_embd(d_ptr->model);
+    if (dimensionality < 0) {
+        dimensionality = n_embd;
+    } else if (spec && dimensionality != n_embd) {
+        auto msg = [dimensionality, modelName]() {
+            return "unsupported dimensionality " + std::to_string(dimensionality) + " for model " + modelName;
+        };
+        if (!spec->matryoshkaCapable)
+            throw std::out_of_range(msg() + " (supported: " + std::to_string(n_embd) + ")");
+        if (dimensionality == 0 || dimensionality > n_embd)
+            throw std::out_of_range(msg() + " (recommended: " + spec->recommendedDims + ")");
+    }
+
+    if (!prefix) {
+        if (!spec)
+            throw std::invalid_argument("unknown model "s + modelName + ", specify a prefix if applicable or an empty string");
+        prefix = spec->docPrefix;
+    } else if (spec && prefix != spec->docPrefix && prefix != spec->queryPrefix &&
+               std::find(spec->otherPrefixes.begin(), spec->otherPrefixes.end(), *prefix) == spec->otherPrefixes.end())
+    {
+        std::stringstream ss;
+        ss << std::quoted(*prefix) << " is not a valid task type for model " << modelName;
+        throw std::invalid_argument(ss.str());
+    }
+
+    embedInternal(texts, embeddings, *prefix, dimensionality, tokenCount, doMean, atlas, spec);
+}
+
+// MD5 hash of "nomic empty"
+static const char EMPTY_PLACEHOLDER[] = "24df574ea1c998de59d5be15e769658e";
+
+auto product(double a) -> std::function<double(double)> {
+    return [a](double b) { return a * b; };
+}
+
+template <typename T>
+double getL2NormScale(T *start, T *end) {
+    double magnitude = std::sqrt(std::inner_product(start, end, start, 0.0));
+    return 1.0 / std::max(magnitude, 1e-12);
+}
+
+void LLamaModel::embedInternal(
+    const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas, const EmbModelSpec *spec
+) {
+    typedef std::vector<LLModel::Token> TokenString;
+    static constexpr int32_t atlasMaxLength = 8192;
+    static constexpr int chunkOverlap = 8; // Atlas overlaps n_batch-sized chunks of input by 8 tokens
+
+    const llama_token bos_token = llama_token_bos(d_ptr->model);
+    const llama_token eos_token = llama_token_eos(d_ptr->model);
+
+    bool useBOS = shouldAddBOS();
+    bool useEOS = llama_vocab_type(d_ptr->model) == LLAMA_VOCAB_TYPE_WPM;
+
+    // no EOS, optional BOS
+    auto tokenize = [this, useBOS, useEOS, eos_token](std::string text, TokenString &tokens, bool wantBOS) {
+        if (!text.empty() && text[0] != ' ') {
+            text = ' ' + text; // normalize for SPM - our fork of llama.cpp doesn't add a space prefix
+        }
+        wantBOS &= useBOS;
+
+        tokens.resize(text.length()+4);
+        int32_t n_tokens = llama_tokenize(d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), wantBOS, false);
+        assert(useEOS == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
+        tokens.resize(n_tokens - useEOS); // erase EOS/SEP
+    };
+
+    // tokenize the texts
+    std::vector<TokenString> inputs;
+    for (unsigned i = 0; i < texts.size(); i++) {
+        auto &text = texts[i];
+        auto &inp = inputs.emplace_back();
+        tokenize(text, inp, false);
+        if (atlas && inp.size() > atlasMaxLength) {
+            if (doMean) {
+                throw std::length_error(
+                    "length of text at index " + std::to_string(i) + " is " + std::to_string(inp.size()) +
+                    " tokens which exceeds limit of " + std::to_string(atlasMaxLength)
+                );
+            }
+            inp.resize(atlasMaxLength);
+        } else if (inp.empty()) {
+            if (!atlas || !text.empty()) {
+                std::cerr << __func__ << ": warning: chunking tokenized text at index " << std::to_string(i)
+                          << " into zero tokens\n";
+            }
+            tokenize(EMPTY_PLACEHOLDER, inp, false);
+        }
+    }
+
+    // tokenize the prefix
+    TokenString prefixTokens;
+    if (prefix.empty()) {
+        prefixTokens.push_back(bos_token);
+    } else {
+        tokenize(prefix + ':', prefixTokens, true);
+    }
+
+    const uint32_t n_batch = llama_n_batch(d_ptr->ctx);
+    const uint32_t max_len = n_batch - (prefixTokens.size() + useEOS); // minus BOS/CLS and EOS/SEP
+    if (chunkOverlap >= max_len) {
+        throw std::logic_error("max chunk length of " + std::to_string(max_len) + " is smaller than overlap of " +
+                               std::to_string(chunkOverlap) + " tokens");
+    }
+
+    // split into max_len-sized chunks
+    struct split_batch { unsigned idx; TokenString batch; };
+    std::vector<split_batch> batches;
+    size_t totalTokens = 0;
+    for (unsigned i = 0; i < inputs.size(); i++) {
+        auto &input = inputs[i];
+        for (auto it = input.begin(); it < input.end(); it += max_len) {
+            if (it > input.begin()) { it -= chunkOverlap; }
+            auto end = std::min(it + max_len, input.end());
+            batches.push_back({ i, {} });
+            auto &batch = batches.back().batch;
+            batch = prefixTokens;
+            batch.insert(batch.end(), it, end);
+            totalTokens += end - it;
+            batch.push_back(eos_token);
+            if (!doMean) { break; /* limit text to one chunk */ }
+        }
+    }
+    inputs.clear();
+
+    // initialize batch
+    struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
+
+    // n_texts x n_embd matrix
+    const int32_t n_embd = llama_n_embd(d_ptr->model);
+    std::vector<double> embeddingsSum(texts.size() * n_embd);
+    std::vector<int> embeddingsSumTotal(texts.size());
+    std::vector<int> queued_indices; // text indices of batches to be processed
+
+    auto decode = [this, &queued_indices, n_embd, &batch, &embeddingsSum, &embeddingsSumTotal, spec, dimensionality]() {
+        if (llama_decode(d_ptr->ctx, batch) < 0)
+            throw std::runtime_error("llama_decode failed");
+
+        for (int i = 0; i < batch.n_tokens; ++i) {
+            if (!batch.logits[i]) { continue; }
+            int i_prompt = queued_indices[batch.seq_id[i][0]];
+            auto *out = &embeddingsSum[i_prompt * n_embd];
+
+            // sequence embeddings aren't available when pooling_type is NONE
+            auto *embd = llama_get_embeddings_seq(d_ptr->ctx, batch.seq_id[i][0]);
+            if (!embd) { embd = llama_get_embeddings_ith(d_ptr->ctx, i); }
+            assert(embd);
+
+            auto *embd_end = embd + n_embd;
+
+            // layer normalization for nomic-embed-text-v1.5
+            if (spec && spec->matryoshkaCapable) {
+                // normalize mean
+                double mean = std::accumulate(embd, embd_end, 0.0) / n_embd;
+                std::transform(embd, embd_end, embd, [mean](double f){ return f - mean; });
+
+                // unbiased sample variance, with Bessel's correction
+                double variance = std::inner_product(embd, embd_end, embd, 0.0) / (n_embd - 1);
+
+                // trim to matryoshka dim
+                embd_end = embd + dimensionality;
+
+                // normalize variance
+                std::transform(embd, embd_end, embd, product(1.0 / std::sqrt(variance + 1e-5)));
+            }
+
+            // L2 norm
+            auto scale = getL2NormScale(embd, embd_end);
+            std::transform(embd, embd_end, out, out, [scale](double e, double o){ return o + scale * e; });
+            embeddingsSumTotal[i_prompt]++;
+        }
+    };
+
+    // break into batches
+    for (auto &inp: batches) {
+        // encode if at capacity
+        if (batch.n_tokens + inp.batch.size() > n_batch) {
+            decode();
+            batch.n_tokens = 0;
+            queued_indices.clear();
+        }
+
+        // add to batch
+        batch_add_seq(batch, inp.batch, queued_indices.size());
+        queued_indices.push_back(inp.idx);
+    }
+
+    // final batch
+    decode();
+
+    for (unsigned i = 0; i < texts.size(); i++) {
+        auto *embd = &embeddingsSum[i * n_embd];
+        auto *embd_end = embd + dimensionality;
+        int total = embeddingsSumTotal[i];
+
+        // average over chunks
+        std::transform(embd, embd_end, embd, product(1.0 / total));
+
+        // L2 norm and copy
+        auto scale = getL2NormScale(embd, embd_end);
+        std::transform(embd, embd_end, embeddings, product(scale));
+        embeddings += dimensionality;
+    }
+
+    if (tokenCount) { *tokenCount = totalTokens; }
+}
+
 #if defined(_WIN32)
 #define DLL_EXPORT __declspec(dllexport)
 #else
@@ -556,23 +916,21 @@ DLL_EXPORT const char *get_build_variant() {
 
 DLL_EXPORT bool magic_match(const char *fname) {
     auto * ctx = load_gguf(fname);
-    auto arch = get_arch_name(ctx);
+    std::string arch = get_arch_name(ctx);
 
     bool valid = true;
 
-    static const std::vector<const char *> known_arches {
-        "baichuan", "bloom", "codeshell", "falcon", "gemma", "gpt2", "llama", "mpt", "orion", "persimmon", "phi2",
-        "plamo", "qwen", "qwen2", "refact", "stablelm", "starcoder"
-    };
-
-    if (std::find(known_arches.begin(), known_arches.end(), arch) == known_arches.end()) {
+    if (std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), arch) == KNOWN_ARCHES.end()) {
         // not supported by this version of llama.cpp
-        if (!(arch == "gptj" || arch == "bert")) { // we support these via other modules
+        if (arch != "gptj") { // we support this via another module
             std::cerr << __func__ << ": unsupported model architecture: " << arch << "\n";
         }
         valid = false;
     }
 
+    if (valid && is_embedding_arch(arch) && gguf_find_key(ctx, (arch + ".pooling_type").c_str()) < 0)
+        valid = false; // old pre-llama.cpp embedding model, e.g. all-MiniLM-L6-v2-f16.gguf
+
     gguf_free(ctx);
     return valid;
 }

gpt4all-backend/llamamodel_impl.h

@@ -11,15 +11,18 @@
 #include "llmodel.h"
 
 struct LLamaPrivate;
+struct EmbModelSpec;
+
 class LLamaModel : public LLModel {
 public:
     LLamaModel();
     ~LLamaModel();
 
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
+    bool supportsEmbedding() const override { return m_supportsEmbedding; }
+    bool supportsCompletion() const override { return m_supportsCompletion; }
     bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
-    bool isModelBlacklisted(const std::string &modelPath) override;
+    bool isModelBlacklisted(const std::string &modelPath) const override;
+    bool isEmbeddingModel(const std::string &modelPath) const override;
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
@@ -29,12 +32,22 @@ public:
     int32_t threadCount() const override;
     std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const override;
     bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const override;
-    bool initializeGPUDevice(int device, std::string *unavail_reason) const override;
+    bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const override;
     bool hasGPUDevice() override;
     bool usingGPUDevice() override;
 
+    size_t embeddingSize() const override;
+    // user-specified prefix
+    void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
+               int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
+    // automatic prefix
+    void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality = -1,
+               size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
+
 private:
     std::unique_ptr<LLamaPrivate> d_ptr;
+    bool m_supportsEmbedding = false;
+    bool m_supportsCompletion = false;
 
 protected:
     std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
@@ -44,9 +57,11 @@ protected:
     int32_t contextLength() const override;
     const std::vector<Token> &endTokens() const override;
     bool shouldAddBOS() const override;
-
     int32_t maxContextLength(std::string const &modelPath) const override;
     int32_t layerCount(std::string const &modelPath) const override;
+
+    void embedInternal(const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
+                       size_t *tokenCount, bool doMean, bool atlas, const EmbModelSpec *spec);
 };
 
 #endif // LLAMAMODEL_H

gpt4all-backend/llmodel.cpp

@@ -19,33 +19,27 @@
 
 std::string s_implementations_search_path = ".";
 
-static bool has_at_least_minimal_hardware() {
-#if defined(__x86_64__) || defined(_M_X64)
-    #ifndef _MSC_VER
-        return __builtin_cpu_supports("avx");
-    #else
-        int cpuInfo[4];
-        __cpuid(cpuInfo, 1);
-        return cpuInfo[2] & (1 << 28);
-    #endif
-#else
-    return true; // Don't know how to handle non-x86_64
-#endif
-}
+#if !(defined(__x86_64__) || defined(_M_X64))
+    // irrelevant on non-x86_64
+    #define cpu_supports_avx()  -1
+    #define cpu_supports_avx2() -1
+#elif defined(_MSC_VER)
+    // MSVC
+    static int get_cpu_info(int func_id, int reg_id) {
+        int info[4];
+        __cpuid(info, func_id);
+        return info[reg_id];
+    }
 
-static bool requires_avxonly() {
-#if defined(__x86_64__) || defined(_M_X64)
-    #ifndef _MSC_VER
-        return !__builtin_cpu_supports("avx2");
-    #else
-        int cpuInfo[4];
-        __cpuidex(cpuInfo, 7, 0);
-        return !(cpuInfo[1] & (1 << 5));
-    #endif
+    // AVX via EAX=1: Processor Info and Feature Bits, bit 28 of ECX
+    #define cpu_supports_avx()  (get_cpu_info(1, 2) & (1 << 28))
+    // AVX2 via EAX=7, ECX=0: Extended Features, bit 5 of EBX
+    #define cpu_supports_avx2() (get_cpu_info(7, 1) & (1 <<  5))
 #else
-    return false; // Don't know how to handle non-x86_64
+    // gcc/clang
+    #define cpu_supports_avx()  __builtin_cpu_supports("avx")
+    #define cpu_supports_avx2() __builtin_cpu_supports("avx2")
 #endif
-}
 
 LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     : m_dlhandle(new Dlhandle(std::move(dlhandle_))) {
@@ -71,21 +65,25 @@ LLModel::Implementation::Implementation(Implementation &&o)
 }
 
 LLModel::Implementation::~Implementation() {
-    if (m_dlhandle) delete m_dlhandle;
+    delete m_dlhandle;
 }
 
-bool LLModel::Implementation::isImplementation(const Dlhandle &dl) {
+static bool isImplementation(const Dlhandle &dl) {
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
 const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
+    if (cpu_supports_avx() == 0) {
+        throw std::runtime_error("CPU does not support AVX");
+    }
+
     // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
     // individual models without the cleanup of the static list interfering
     static auto* libs = new std::vector<Implementation>([] () {
         std::vector<Implementation> fres;
 
-        std::string impl_name_re = "(bert|gptj|llamamodel-mainline)";
-        if (requires_avxonly()) {
+        std::string impl_name_re = "(gptj|llamamodel-mainline)";
+        if (cpu_supports_avx2() == 0) {
             impl_name_re += "-avxonly";
         } else {
             impl_name_re += "-(default|metal)";
@@ -107,9 +105,8 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
                     // Add to list if model implementation
                     try {
                         Dlhandle dl(p.string());
-                        if (!Implementation::isImplementation(dl)) {
+                        if (!isImplementation(dl))
                             continue;
-                        }
                         fres.emplace_back(Implementation(std::move(dl)));
                     } catch (...) {}
                 }
@@ -134,18 +131,13 @@ const LLModel::Implementation* LLModel::Implementation::implementation(const cha
         return &i;
     }
 
-    if (!buildVariantMatched) {
-        std::cerr << "LLModel ERROR: Could not find any implementations for build variant: " << buildVariant << "\n";
-    }
-    return nullptr;
+    if (!buildVariantMatched)
+        throw std::runtime_error("Could not find any implementations for build variant: " + buildVariant);
+
+    return nullptr; // unsupported model format
 }
 
 LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant, int n_ctx) {
-    if (!has_at_least_minimal_hardware()) {
-        std::cerr << "LLModel ERROR: CPU does not support AVX\n";
-        return nullptr;
-    }
-
     // Get correct implementation
     const Implementation* impl = nullptr;
 
@@ -178,7 +170,7 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
     if (!impl) {
         //TODO: Auto-detect CUDA/OpenCL
         if (buildVariant == "auto") {
-            if (requires_avxonly()) {
+            if (cpu_supports_avx2() == 0) {
                 buildVariant = "avxonly";
             } else {
                 buildVariant = "default";
@@ -196,15 +188,24 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
 
 LLModel *LLModel::Implementation::constructDefaultLlama() {
     static std::unique_ptr<LLModel> llama([]() -> LLModel * {
+        const std::vector<LLModel::Implementation> *impls;
+        try {
+            impls = &implementationList();
+        } catch (const std::runtime_error &e) {
+            std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
+            return nullptr;
+        }
+
         const LLModel::Implementation *impl = nullptr;
-        for (const auto &i : implementationList()) {
+        for (const auto &i: *impls) {
             if (i.m_buildVariant == "metal" || i.m_modelType != "LLaMA") continue;
             impl = &i;
         }
         if (!impl) {
-            std::cerr << "LLModel ERROR: Could not find CPU LLaMA implementation\n";
+            std::cerr << __func__ << ": could not find llama.cpp implementation\n";
             return nullptr;
         }
+
         auto fres = impl->m_construct();
         fres->m_implementation = impl;
         return fres;
@@ -213,21 +214,26 @@ LLModel *LLModel::Implementation::constructDefaultLlama() {
 }
 
 std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices() {
-    auto * llama = constructDefaultLlama();
+    auto *llama = constructDefaultLlama();
     if (llama) { return llama->availableGPUDevices(0); }
     return {};
 }
 
 int32_t LLModel::Implementation::maxContextLength(const std::string &modelPath) {
-    auto * llama = constructDefaultLlama();
+    auto *llama = constructDefaultLlama();
     return llama ? llama->maxContextLength(modelPath) : -1;
 }
 
 int32_t LLModel::Implementation::layerCount(const std::string &modelPath) {
-    auto * llama = constructDefaultLlama();
+    auto *llama = constructDefaultLlama();
     return llama ? llama->layerCount(modelPath) : -1;
 }
 
+bool LLModel::Implementation::isEmbeddingModel(const std::string &modelPath) {
+    auto *llama = constructDefaultLlama();
+    return llama && llama->isEmbeddingModel(modelPath);
+}
+
 void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
     s_implementations_search_path = path;
 }
@@ -235,3 +241,7 @@ void LLModel::Implementation::setImplementationsSearchPath(const std::string& pa
 const std::string& LLModel::Implementation::implementationsSearchPath() {
     return s_implementations_search_path;
 }
+
+bool LLModel::Implementation::hasSupportedCPU() {
+    return cpu_supports_avx() != 0;
+}

gpt4all-backend/llmodel.h

@@ -1,13 +1,14 @@
 #ifndef LLMODEL_H
 #define LLMODEL_H
 
-#include <string>
-#include <functional>
-#include <vector>
-#include <string_view>
-#include <fstream>
 #include <cstdint>
+#include <fstream>
+#include <functional>
 #include <limits>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
 
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
@@ -29,7 +30,6 @@ public:
 
     class Implementation {
     public:
-        Implementation(Dlhandle &&);
         Implementation(const Implementation &) = delete;
         Implementation(Implementation &&);
         ~Implementation();
@@ -37,17 +37,20 @@ public:
         std::string_view modelType() const { return m_modelType; }
         std::string_view buildVariant() const { return m_buildVariant; }
 
-        static bool isImplementation(const Dlhandle &dl);
-        static const std::vector<Implementation> &implementationList();
-        static const Implementation *implementation(const char *fname, const std::string &buildVariant);
         static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto", int n_ctx = 2048);
         static std::vector<GPUDevice> availableGPUDevices();
         static int32_t maxContextLength(const std::string &modelPath);
         static int32_t layerCount(const std::string &modelPath);
+        static bool isEmbeddingModel(const std::string &modelPath);
         static void setImplementationsSearchPath(const std::string &path);
         static const std::string &implementationsSearchPath();
+        static bool hasSupportedCPU();
 
     private:
+        Implementation(Dlhandle &&);
+
+        static const std::vector<Implementation> &implementationList();
+        static const Implementation *implementation(const char *fname, const std::string &buildVariant);
         static LLModel *constructDefaultLlama();
 
         bool (*m_magicMatch)(const char *fname);
@@ -83,7 +86,8 @@ public:
     virtual bool supportsEmbedding() const = 0;
     virtual bool supportsCompletion() const = 0;
     virtual bool loadModel(const std::string &modelPath, int n_ctx, int ngl) = 0;
-    virtual bool isModelBlacklisted(const std::string &modelPath) { (void)modelPath; return false; };
+    virtual bool isModelBlacklisted(const std::string &modelPath) const { (void)modelPath; return false; };
+    virtual bool isEmbeddingModel(const std::string &modelPath) const { (void)modelPath; return false; }
     virtual bool isModelLoaded() const = 0;
     virtual size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) = 0;
     virtual size_t stateSize() const { return 0; }
@@ -101,7 +105,15 @@ public:
                         bool special = false,
                         std::string *fakeReply = nullptr);
 
-    virtual std::vector<float> embedding(const std::string &text);
+    virtual size_t embeddingSize() const {
+        throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+    }
+    // user-specified prefix
+    virtual void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
+                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false);
+    // automatic prefix
+    virtual void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval,
+                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false);
 
     virtual void setThreadCount(int32_t n_threads) { (void)n_threads; }
     virtual int32_t threadCount() const { return 1; }

gpt4all-backend/llmodel_c.cpp

@@ -4,6 +4,7 @@
 #include <cerrno>
 #include <cstring>
 #include <iostream>
+#include <optional>
 #include <utility>
 
 struct LLModelWrapper {
@@ -12,8 +13,6 @@ struct LLModelWrapper {
     ~LLModelWrapper() { delete llModel; }
 };
 
-thread_local static std::string last_error_message;
-
 llmodel_model llmodel_model_create(const char *model_path) {
     const char *error;
     auto fres = llmodel_model_create2(model_path, "auto", &error);
@@ -23,40 +22,46 @@ llmodel_model llmodel_model_create(const char *model_path) {
     return fres;
 }
 
+static void llmodel_set_error(const char **errptr, const char *message) {
+    thread_local static std::string last_error_message;
+    if (errptr) {
+        last_error_message = message;
+        *errptr = last_error_message.c_str();
+    }
+}
+
 llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error) {
-    auto wrapper = new LLModelWrapper;
-
+    LLModel *llModel;
     try {
-        wrapper->llModel = LLModel::Implementation::construct(model_path, build_variant);
-        if (!wrapper->llModel) {
-            last_error_message = "Model format not supported (no matching implementation found)";
-        }
+        llModel = LLModel::Implementation::construct(model_path, build_variant);
     } catch (const std::exception& e) {
-        last_error_message = e.what();
+        llmodel_set_error(error, e.what());
+        return nullptr;
     }
 
-    if (!wrapper->llModel) {
-        delete std::exchange(wrapper, nullptr);
-        if (error) {
-            *error = last_error_message.c_str();
-        }
+    if (!llModel) {
+        llmodel_set_error(error, "Model format not supported (no matching implementation found)");
+        return nullptr;
     }
-    return reinterpret_cast<llmodel_model*>(wrapper);
+
+    auto wrapper = new LLModelWrapper;
+    wrapper->llModel = llModel;
+    return wrapper;
 }
 
 void llmodel_model_destroy(llmodel_model model) {
-    delete reinterpret_cast<LLModelWrapper*>(model);
+    delete static_cast<LLModelWrapper *>(model);
 }
 
 size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->requiredMem(model_path, n_ctx, ngl);
 }
 
 bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
 
     std::string modelPath(model_path);
     if (wrapper->llModel->isModelBlacklisted(modelPath)) {
@@ -69,44 +74,28 @@ bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, i
 
 bool llmodel_isModelLoaded(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->isModelLoaded();
 }
 
 uint64_t llmodel_get_state_size(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->stateSize();
 }
 
 uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->saveState(dest);
 }
 
 uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->restoreState(src);
 }
 
-// Wrapper functions for the C callbacks
-bool prompt_wrapper(int32_t token_id, void *user_data) {
-    llmodel_prompt_callback callback = reinterpret_cast<llmodel_prompt_callback>(user_data);
-    return callback(token_id);
-}
-
-bool response_wrapper(int32_t token_id, const std::string &response, void *user_data) {
-    llmodel_response_callback callback = reinterpret_cast<llmodel_response_callback>(user_data);
-    return callback(token_id, response.c_str());
-}
-
-bool recalculate_wrapper(bool is_recalculating, void *user_data) {
-    llmodel_recalculate_callback callback = reinterpret_cast<llmodel_recalculate_callback>(user_data);
-    return callback(is_recalculating);
-}
-
 void llmodel_prompt(llmodel_model model, const char *prompt,
                     const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
@@ -116,15 +105,11 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
                     bool special,
                     const char *fake_reply)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
 
-    // Create std::function wrappers that call the C function pointers
-    std::function<bool(int32_t)> prompt_func =
-        std::bind(&prompt_wrapper, std::placeholders::_1, reinterpret_cast<void*>(prompt_callback));
-    std::function<bool(int32_t, const std::string&)> response_func =
-        std::bind(&response_wrapper, std::placeholders::_1, std::placeholders::_2, reinterpret_cast<void*>(response_callback));
-    std::function<bool(bool)> recalc_func =
-        std::bind(&recalculate_wrapper, std::placeholders::_1, reinterpret_cast<void*>(recalculate_callback));
+    auto response_func = [response_callback](int32_t token_id, const std::string &response) {
+        return response_callback(token_id, response.c_str());
+    };
 
     if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
         wrapper->promptContext.tokens.resize(ctx->n_past);
@@ -147,8 +132,8 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     auto *fake_reply_p = fake_reply ? &fake_reply_str : nullptr;
 
     // Call the C++ prompt method
-    wrapper->llModel->prompt(prompt, prompt_template, prompt_func, response_func, recalc_func, wrapper->promptContext,
-                             special, fake_reply_p);
+    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, recalculate_callback,
+                             wrapper->promptContext, special, fake_reply_p);
 
     // Update the C context by giving access to the wrappers raw pointers to std::vector data
     // which involves no copies
@@ -171,38 +156,58 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->context_erase = wrapper->promptContext.contextErase;
 }
 
-float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size)
-{
-    if (model == nullptr || text == nullptr || !strlen(text)) {
-        *embedding_size = 0;
+float *llmodel_embed(
+    llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix, int dimensionality,
+    size_t *token_count, bool do_mean, bool atlas, const char **error
+) {
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+
+    if (!texts || !*texts) {
+        llmodel_set_error(error, "'texts' is NULL or empty");
         return nullptr;
     }
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    std::vector<float> embeddingVector = wrapper->llModel->embedding(text);
-    float *embedding = (float *)malloc(embeddingVector.size() * sizeof(float));
-    if (embedding == nullptr) {
-        *embedding_size = 0;
+
+    std::vector<std::string> textsVec;
+    while (*texts) { textsVec.emplace_back(*texts++); }
+
+    size_t embd_size;
+    float *embedding;
+
+    try {
+        embd_size = wrapper->llModel->embeddingSize();
+        if (dimensionality > 0 && dimensionality < int(embd_size))
+            embd_size = dimensionality;
+
+        embd_size *= textsVec.size();
+
+        std::optional<std::string> prefixStr;
+        if (prefix) { prefixStr = prefix; }
+
+        embedding = new float[embd_size];
+        wrapper->llModel->embed(textsVec, embedding, prefixStr, dimensionality, token_count, do_mean, atlas);
+    } catch (std::exception const &e) {
+        llmodel_set_error(error, e.what());
         return nullptr;
     }
-    std::copy(embeddingVector.begin(), embeddingVector.end(), embedding);
-    *embedding_size = embeddingVector.size();
+
+    *embedding_size = embd_size;
     return embedding;
 }
 
 void llmodel_free_embedding(float *ptr)
 {
-    free(ptr);
+    delete[] ptr;
 }
 
 void llmodel_setThreadCount(llmodel_model model, int32_t n_threads)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     wrapper->llModel->setThreadCount(n_threads);
 }
 
 int32_t llmodel_threadCount(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->threadCount();
 }
 
@@ -218,7 +223,7 @@ const char *llmodel_get_implementation_search_path()
 
 struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     std::vector<LLModel::GPUDevice> devices = wrapper->llModel->availableGPUDevices(memoryRequired);
 
     // Set the num_devices
@@ -242,24 +247,24 @@ struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, si
 
 bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->initializeGPUDevice(memoryRequired, std::string(device));
 }
 
 bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->initializeGPUDevice(device->index);
 }
 
 bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->initializeGPUDevice(device);
 }
 
 bool llmodel_has_gpu_device(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->hasGPUDevice();
 }

gpt4all-backend/llmodel_c.h

@@ -186,13 +186,24 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
  * NOTE: If given NULL pointers for the model or text, or an empty text, a NULL pointer will be
  * returned. Bindings should signal an error when NULL is the return value.
  * @param model A pointer to the llmodel_model instance.
- * @param text A string representing the text to generate an embedding for.
+ * @param texts A pointer to a NULL-terminated array of strings representing the texts to generate an
+ * embedding for.
  * @param embedding_size A pointer to a size_t type that will be set by the call indicating the length
  * of the returned floating point array.
+ * @param prefix The model-specific prefix representing the embedding task, without the trailing colon. NULL for no
+ * prefix.
+ * @param dimensionality The embedding dimension, for use with Matryoshka-capable models. Set to -1 to for full-size.
+ * @param token_count Return location for the number of prompt tokens processed, or NULL.
+ * @param do_mean True to average multiple embeddings if the text is longer than the model can accept, False to
+ * truncate.
+ * @param atlas Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens with
+ * long_text_mode="mean" will raise an error. Disabled by default.
+ * @param error Return location for a malloc()ed string that will be set on error, or NULL.
  * @return A pointer to an array of floating point values passed to the calling method which then will
- * be responsible for lifetime of this memory.
+ * be responsible for lifetime of this memory. NULL if an error occurred.
  */
-float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size);
+float *llmodel_embed(llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix,
+                     int dimensionality, size_t *token_count, bool do_mean, bool atlas, const char **error);
 
 /**
  * Frees the memory allocated by the llmodel_embedding function.

gpt4all-backend/llmodel_shared.cpp

@@ -3,6 +3,7 @@
 #include <cassert>
 #include <iostream>
 #include <regex>
+#include <string>
 #include <unordered_set>
 
 // TODO(cebtenzzre): replace this with llama_kv_cache_seq_shift for llamamodel (GPT-J needs this as-is)
@@ -267,12 +268,30 @@ void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)>
     }
 }
 
-std::vector<float> LLModel::embedding(const std::string &text)
-{
-    (void)text;
-    if (!supportsCompletion()) {
-        std::string errorMessage = "ERROR: this model does not support generating embeddings!\n";
-        std::cerr << implementation().modelType() << errorMessage;
-    }
-    return std::vector<float>();
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)prefix;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+}
+
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
+    bool doMean, bool atlas
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)isRetrieval;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
 }

gpt4all-bindings/python/docs/gpt4all_chat.md

@@ -7,7 +7,7 @@ It is optimized to run 7-13B parameter LLMs on the CPU's of any computer running
 ## Running LLMs on CPU
 The GPT4All Chat UI supports models from all newer versions of `llama.cpp` with `GGUF` models including the `Mistral`, `LLaMA2`, `LLaMA`, `OpenLLaMa`, `Falcon`, `MPT`, `Replit`, `Starcoder`, and `Bert` architectures
 
-GPT4All maintains an official list of recommended models located in [models2.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
+GPT4All maintains an official list of recommended models located in [models3.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models3.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
 
 #### Sideloading any GGUF model
 If a model is compatible with the gpt4all-backend, you can sideload it into GPT4All Chat by:

gpt4all-bindings/python/docs/gpt4all_cli.md

@@ -5,7 +5,7 @@ The GPT4All command-line interface (CLI) is a Python script which is built on to
 package. The source code, README, and local build instructions can be found
 [here][repo-bindings-cli].
 
-[docs-bindings-python]: gpt4all_python.html
+[docs-bindings-python]: gpt4all_python.md
 [repo-bindings-python]: https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python
 [repo-bindings-cli]: https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/cli
 [typer]: https://typer.tiangolo.com/

gpt4all-bindings/python/docs/gpt4all_faq.md

@@ -61,12 +61,12 @@ or `allowDownload=true` (default), a model is automatically downloaded into `.ca
 unless it already exists.
 
 In case of connection issues or errors during the download, you might want to manually verify the model file's MD5
-checksum by comparing it with the one listed in [models2.json].
+checksum by comparing it with the one listed in [models3.json].
 
 As an alternative to the basic downloader built into the bindings, you can choose to download from the 
 <https://gpt4all.io/> website instead. Scroll down to 'Model Explorer' and pick your preferred model.
 
-[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
+[models3.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models3.json
 
 #### I need the chat GUI and bindings to behave the same
 
@@ -93,7 +93,7 @@ The chat GUI and bindings are based on the same backend. You can make them behav
 - Next you'll have to compare the templates, adjusting them as necessary, based on how you're using the bindings.
     - Specifically, in Python:
         - With simple `generate()` calls, the input has to be surrounded with system and prompt templates.
-        - When using a chat session, it depends on whether the bindings are allowed to download [models2.json]. If yes,
+        - When using a chat session, it depends on whether the bindings are allowed to download [models3.json]. If yes,
           and in the chat GUI the default templates are used, it'll be handled automatically. If no, use
           `chat_session()` template parameters to customize them.
 

gpt4all-bindings/python/docs/gpt4all_modal.md

@@ -1,34 +0,0 @@
-# GPT4All with Modal Labs
-
-You can easily query any GPT4All model on [Modal Labs](https://modal.com/) infrastructure!
-## Example
-
-```python
-import modal
-
-def download_model():
-    import gpt4all
-    #you can use any model from https://gpt4all.io/models/models2.json
-    return gpt4all.GPT4All("ggml-gpt4all-j-v1.3-groovy.bin")
-
-image=modal.Image.debian_slim().pip_install("gpt4all").run_function(download_model)
-stub = modal.Stub("gpt4all", image=image)
-@stub.cls(keep_warm=1)
-class GPT4All:
-    def __enter__(self):
-        print("Downloading model")
-        self.gptj = download_model()
-        print("Loaded model")
-
-    @modal.method()
-    def generate(self):
-        messages = [{"role": "user", "content": "Name 3 colors"}]
-        completion = self.gptj.chat_completion(messages)
-        print(f"Completion: {completion}")
-
-@stub.local_entrypoint()
-def main():
-    model = GPT4All()
-    for i in range(10):
-        model.generate.call()
-```

gpt4all-bindings/python/docs/gpt4all_python.md

@@ -8,30 +8,22 @@ The source code and local build instructions can be found [here](https://github.
 pip install gpt4all
 ```
 
-=== "GPT4All Example"
-    ``` py
-    from gpt4all import GPT4All
-    model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
-    output = model.generate("The capital of France is ", max_tokens=3)
-    print(output)
-    ```
-=== "Output"
-    ```
-    1. Paris
-    ```
+``` py
+from gpt4all import GPT4All
+model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
+```
 
 This will:
 
 - Instantiate `GPT4All`,  which is the primary public API to your large language model (LLM).
 - Automatically download the given model to `~/.cache/gpt4all/` if not already present.
-- Through `model.generate(...)` the model starts working on a response. There are various ways to
-  steer that process. Here, `max_tokens` sets an upper limit, i.e. a hard cut-off point to the output.
+
+Read further to see how to chat with this model.
 
 
 ### Chatting with GPT4All
-Local LLMs can be optimized for chat conversations by reusing previous computational history.
-
-Use the GPT4All `chat_session` context manager to hold chat conversations with the model.
+To start chatting with a local LLM, you will need to start a chat session. Within a chat session, the model will be
+prompted with the appropriate template, and history will be preserved between successive calls to `generate()`.
 
 === "GPT4All Example"
     ``` py
@@ -72,15 +64,19 @@ Use the GPT4All `chat_session` context manager to hold chat conversations with t
     ]
     ```
 
-When using GPT4All models in the `chat_session` context:
+When using GPT4All models in the `chat_session()` context:
 
 - Consecutive chat exchanges are taken into account and not discarded until the session ends; as long as the model has capacity.
-- Internal K/V caches are preserved from previous conversation history, speeding up inference.
-- The model is given a system and prompt template which make it chatty. Depending on `allow_download=True` (default),
-  it will obtain the latest version of [models2.json] from the repository, which contains specifically tailored templates
-  for models. Conversely, if it is not allowed to download, it falls back to default templates instead.
+- A system prompt is inserted into the beginning of the model's context.
+- Each prompt passed to `generate()` is wrapped in the appropriate prompt template. If you pass `allow_download=False`
+  to GPT4All or are using a model that is not from the official models list, you must pass a prompt template using the
+  `prompt_template` parameter of `chat_session()`.
 
-[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
+NOTE: If you do not use `chat_session()`, calls to `generate()` will not be wrapped in a prompt template. This will
+cause the model to *continue* the prompt instead of *answering* it. When in doubt, use a chat session, as many newer
+models are designed to be used exclusively with a prompt template.
+
+[models3.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models3.json
 
 
 ### Streaming Generations
@@ -91,13 +87,14 @@ To interact with GPT4All responses as the model generates, use the `streaming=Tr
     from gpt4all import GPT4All
     model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
     tokens = []
-    for token in model.generate("The capital of France is", max_tokens=20, streaming=True):
-        tokens.append(token)
+    with model.chat_session():
+        for token in model.generate("What is the capital of France?", streaming=True):
+            tokens.append(token)
     print(tokens)
     ```
 === "Output"
     ```
-    [' Paris', ' is', ' a', ' city', ' that', ' has', ' been', ' a', ' major', ' cultural', ' and', ' economic', ' center', ' for', ' over', ' ', '2', ',', '0', '0']
+    [' The', ' capital', ' of', ' France', ' is', ' Paris', '.']
     ```
 
 
@@ -131,20 +128,11 @@ generation; be sure to review all their descriptions.
 The model folder can be set with the `model_path` parameter when creating a `GPT4All` instance. The example below is
 is the same as if it weren't provided; that is, `~/.cache/gpt4all/` is the default folder.
 
-=== "GPT4All Model Folder Example"
-    ``` py
-    from pathlib import Path
-    from gpt4all import GPT4All
-    model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf',
-                    model_path=(Path.home() / '.cache' / 'gpt4all'),
-                    allow_download=False)
-    response = model.generate('my favorite 3 fruits are:', temp=0)
-    print(response)
-    ```
-=== "Output"
-    ```
-    My favorite three fruits are apples, bananas and oranges.
-    ```
+``` py
+from pathlib import Path
+from gpt4all import GPT4All
+model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf', model_path=Path.home() / '.cache' / 'gpt4all')
+```
 
 If you want to point it at the chat GUI's default folder, it should be:
 === "macOS"
@@ -179,22 +167,20 @@ Alternatively, you could also change the module's default model directory:
 
 ``` py
 from pathlib import Path
-import gpt4all.gpt4all
-gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = Path.home() / 'my' / 'models-directory'
-from gpt4all import GPT4All
+from gpt4all import GPT4All, gpt4all
+gpt4all.DEFAULT_MODEL_DIRECTORY = Path.home() / 'my' / 'models-directory'
 model = GPT4All('orca-mini-3b-gguf2-q4_0.gguf')
-...
 ```
 
 
 ### Managing Templates
-Session templates can be customized when starting a `chat_session` context:
+When using a `chat_session()`, you may customize the system prompt, and set the prompt template if necessary:
 
 === "GPT4All Custom Session Templates Example"
     ``` py
     from gpt4all import GPT4All
     model = GPT4All('wizardlm-13b-v1.2.Q4_0.gguf')
-    system_template = 'A chat between a curious user and an artificial intelligence assistant.'
+    system_template = 'A chat between a curious user and an artificial intelligence assistant.\n'
     # many models use triple hash '###' for keywords, Vicunas are simpler:
     prompt_template = 'USER: {0}\nASSISTANT: '
     with model.chat_session(system_template, prompt_template):
@@ -218,111 +204,38 @@ Session templates can be customized when starting a `chat_session` context:
     particles, making the sky appear blue to our eyes.
     ```
 
-To do the same outside a session, the input has to be formatted manually. For example:
-
-=== "GPT4All Templates Outside a Session Example"
-    ``` py
-    model = GPT4All('wizardlm-13b-v1.2.Q4_0.gguf')
-    system_template = 'A chat between a curious user and an artificial intelligence assistant.'
-    prompt_template = 'USER: {0}\nASSISTANT: '
-    prompts = ['name 3 colors', 'now name 3 fruits', 'what were the 3 colors in your earlier response?']
-    first_input = system_template + prompt_template.format(prompts[0])
-    response = model.generate(first_input, temp=0)
-    print(response)
-    for prompt in prompts[1:]:
-        response = model.generate(prompt_template.format(prompt), temp=0)
-        print(response)
-    ```
-=== "Output"
-    ```
-    1) Red
-    2) Blue
-    3) Green
-
-    1. Apple
-    2. Banana
-    3. Orange
-
-    The colors in my previous response are blue, green and red.
-    ```
-
-
-### Introspection
-A less apparent feature is the capacity to log the final prompt that gets sent to the model. It relies on
-[Python's logging facilities][py-logging] implemented in the `pyllmodel` module at the `INFO` level. You can activate it
-for example with a `basicConfig`, which displays it on the standard error stream. It's worth mentioning that Python's
-logging infrastructure offers [many more customization options][py-logging-cookbook].
-
-[py-logging]: https://docs.python.org/3/howto/logging.html
-[py-logging-cookbook]: https://docs.python.org/3/howto/logging-cookbook.html
-
-=== "GPT4All Prompt Logging Example"
-    ``` py
-    import logging
-    from gpt4all import GPT4All
-    logging.basicConfig(level=logging.INFO)
-    model = GPT4All('nous-hermes-llama2-13b.Q4_0.gguf')
-    with model.chat_session('You are a geography expert.\nBe terse.',
-                            '### Instruction:\n{0}\n\n### Response:\n'):
-        response = model.generate('who are you?', temp=0)
-        print(response)
-        response = model.generate('what are your favorite 3 mountains?', temp=0)
-        print(response)
-    ```
-=== "Output"
-    ```
-    INFO:gpt4all.pyllmodel:LLModel.prompt_model -- prompt:
-    You are a geography expert.
-    Be terse.
-
-    ### Instruction:
-    who are you?
-
-    ### Response:
-
-    ===/LLModel.prompt_model -- prompt/===
-    I am an AI-powered chatbot designed to assist users with their queries related to geographical information.
-    INFO:gpt4all.pyllmodel:LLModel.prompt_model -- prompt:
-    ### Instruction:
-    what are your favorite 3 mountains?
-
-    ### Response:
-
-    ===/LLModel.prompt_model -- prompt/===
-    1) Mount Everest - Located in the Himalayas, it is the highest mountain on Earth and a significant challenge for mountaineers.
-    2) Kangchenjunga - This mountain is located in the Himalayas and is the third-highest peak in the world after Mount Everest and K2.
-    3) Lhotse - Located in the Himalayas, it is the fourth highest mountain on Earth and offers a challenging climb for experienced mountaineers.
-    ```
-
 
 ### Without Online Connectivity
-To prevent GPT4All from accessing online resources, instantiate it with `allow_download=False`. This will disable both
-downloading missing models and [models2.json], which contains information about them. As a result, predefined templates
-are used instead of model-specific system and prompt templates:
+To prevent GPT4All from accessing online resources, instantiate it with `allow_download=False`. When using this flag,
+there will be no default system prompt by default, and you must specify the prompt template yourself.
 
-=== "GPT4All Default Templates Example"
+You can retrieve a model's default system prompt and prompt template with an online instance of GPT4All:
+
+=== "Prompt Template Retrieval"
     ``` py
     from gpt4all import GPT4All
-    model = GPT4All('ggml-mpt-7b-chat.bin', allow_download=False)
-    # when downloads are disabled, it will use the default templates:
-    print("default system template:", repr(model.config['systemPrompt']))
-    print("default prompt template:", repr(model.config['promptTemplate']))
-    print()
-    # even when inside a session:
-    with model.chat_session():
-        assert model.current_chat_session[0]['role'] == 'system'
-        print("session system template:", repr(model.current_chat_session[0]['content']))
-        print("session prompt template:", repr(model._current_prompt_template))
+    model = GPT4All('orca-mini-3b-gguf2-q4_0.gguf')
+    print(repr(model.config['systemPrompt']))
+    print(repr(model.config['promptTemplate']))
     ```
 === "Output"
-    ```
-    default system template: ''
-    default prompt template: '### Human: \n{0}\n\n### Assistant:\n'
-
-    session system template: ''
-    session prompt template: '### Human: \n{0}\n\n### Assistant:\n'
+    ```py
+    '### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n'
+    '### User:\n{0}\n### Response:\n'
     ```
 
+Then you can pass them explicitly when creating an offline instance:
+
+``` py
+from gpt4all import GPT4All
+model = GPT4All('orca-mini-3b-gguf2-q4_0.gguf', allow_download=False)
+
+system_prompt = '### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n'
+prompt_template = '### User:\n{0}\n\n### Response:\n'
+
+with model.chat_session(system_prompt=system_prompt, prompt_template=prompt_template):
+    ...
+```
 
 ### Interrupting Generation
 The simplest way to stop generation is to set a fixed upper limit with the `max_tokens` parameter.

gpt4all-bindings/python/docs/gpt4all_python_embedding.md

@@ -1,18 +1,41 @@
 # Embeddings
-GPT4All supports generating high quality embeddings of arbitrary length documents of text using a CPU optimized contrastively trained [Sentence Transformer](https://www.sbert.net/). These embeddings are comparable in quality for many tasks with OpenAI.
+GPT4All supports generating high quality embeddings of arbitrary length text using any embedding model supported by llama.cpp.
+
+An embedding is a vector representation of a piece of text. Embeddings are useful for tasks such as retrieval for
+question answering (including retrieval augmented generation or *RAG*), semantic similarity search, classification, and
+topic clustering.
+
+## Supported Embedding Models
+
+The following models have built-in support in Embed4All:
+
+| Name               | Embed4All `model_name`                               | Context Length | Embedding Length | File Size |
+|--------------------|------------------------------------------------------|---------------:|-----------------:|----------:|
+| [SBert]            | all‑MiniLM‑L6‑v2.gguf2.f16.gguf |            512 |              384 |    44 MiB |
+| [Nomic Embed v1]   | nomic‑embed‑text‑v1.f16.gguf    |           2048 |              768 |   262 MiB |
+| [Nomic Embed v1.5] | nomic‑embed‑text‑v1.5.f16.gguf  |           2048 |           64-768 |   262 MiB |
+
+The context length is the maximum number of word pieces, or *tokens*, that a model can embed at once. Embedding texts
+longer than a model's context length requires some kind of strategy; see [Embedding Longer Texts] for more information.
+
+The embedding length is the size of the vector returned by `Embed4All.embed`.
+
+[SBert]: https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2
+[Nomic Embed v1]: https://huggingface.co/nomic-ai/nomic-embed-text-v1
+[Nomic Embed v1.5]: https://huggingface.co/nomic-ai/nomic-embed-text-v1.5
+[Embedding Longer Texts]: #embedding-longer-texts
 
 ## Quickstart
-
 ```bash
 pip install gpt4all
 ```
 
-### Generating embeddings
-The embedding model will automatically be downloaded if not installed.
+### Generating Embeddings
+By default, embeddings will be generated on the CPU using all-MiniLM-L6-v2.
 
 === "Embed4All Example"
-    ``` py
-    from gpt4all import GPT4All, Embed4All
+    ```py
+    from gpt4all import Embed4All
     text = 'The quick brown fox jumps over the lazy dog'
     embedder = Embed4All()
     output = embedder.embed(text)
@@ -22,13 +45,131 @@ The embedding model will automatically be downloaded if not installed.
     ```
     [0.034696947783231735, -0.07192722707986832, 0.06923297047615051, ...]
     ```
-### Speed of embedding generation
-The following table lists the generation speed for text document captured on an Intel i913900HX CPU with DDR5 5600 running with 8 threads under stable load.
 
-| Tokens          | 128  | 512  | 2048 | 8129 | 16,384 |
-| --------------- | ---- | ---- | ---- | ---- | ---- |
-| Wall time (s)   | .02  | .08  | .24  | .96  | 1.9  |
-| Tokens / Second | 6508 | 6431 | 8622 | 8509 | 8369 |
+You can also use the GPU to accelerate the embedding model by specifying the `device` parameter. See the [GPT4All
+constructor] for more information.
+
+=== "GPU Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'The quick brown fox jumps over the lazy dog'
+    embedder = Embed4All(device='gpu')
+    output = embedder.embed(text)
+    print(output)
+    ```
+=== "Output"
+    ```
+    [0.034696947783231735, -0.07192722707986832, 0.06923297047615051, ...]
+    ```
+
+[GPT4All constructor]: gpt4all_python.md#gpt4all.gpt4all.GPT4All.__init__
+
+### Nomic Embed
+
+Embed4All has built-in support for Nomic's open-source embedding model, [Nomic Embed]. When using this model, you must
+specify the task type using the `prefix` argument. This may be one of `search_query`, `search_document`,
+`classification`, or `clustering`. For retrieval applications, you should prepend `search_document` for all of your
+documents and `search_query` for your queries. See the [Nomic Embedding Guide] for more info.
+
+=== "Nomic Embed Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'Who is Laurens van der Maaten?'
+    embedder = Embed4All('nomic-embed-text-v1.f16.gguf')
+    output = embedder.embed(text, prefix='search_query')
+    print(output)
+    ```
+=== "Output"
+    ```
+    [-0.013357644900679588, 0.027070969343185425, -0.0232995692640543, ...]
+    ```
+
+[Nomic Embed]: https://blog.nomic.ai/posts/nomic-embed-text-v1
+[Nomic Embedding Guide]: https://docs.nomic.ai/atlas/guides/embeddings#embedding-task-types
+
+### Embedding Longer Texts
+
+Embed4All accepts a parameter called `long_text_mode`. This controls the behavior of Embed4All for texts longer than the
+context length of the embedding model.
+
+In the default mode of "mean", Embed4All will break long inputs into chunks and average their embeddings to compute the
+final result.
+
+To change this behavior, you can set the `long_text_mode` parameter to "truncate", which will truncate the input to the
+sequence length of the model before generating a single embedding.
+
+=== "Truncation Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'The ' * 512 + 'The quick brown fox jumps over the lazy dog'
+    embedder = Embed4All()
+    output = embedder.embed(text, long_text_mode="mean")
+    print(output)
+    print()
+    output = embedder.embed(text, long_text_mode="truncate")
+    print(output)
+    ```
+=== "Output"
+    ```
+    [0.0039850445464253426, 0.04558328539133072, 0.0035536508075892925, ...]
+
+    [-0.009771130047738552, 0.034792833030223846, -0.013273917138576508, ...]
+    ```
+
+
+### Batching
+
+You can send multiple texts to Embed4All in a single call. This can give faster results when individual texts are
+significantly smaller than `n_ctx` tokens. (`n_ctx` defaults to 2048.)
+
+=== "Batching Example"
+    ```py
+    from gpt4all import Embed4All
+    texts = ['The quick brown fox jumps over the lazy dog', 'Foo bar baz']
+    embedder = Embed4All()
+    output = embedder.embed(texts)
+    print(output[0])
+    print()
+    print(output[1])
+    ```
+=== "Output"
+    ```
+    [0.03551332652568817, 0.06137588247656822, 0.05281158909201622, ...]
+
+    [-0.03879690542817116, 0.00013223080895841122, 0.023148687556385994, ...]
+    ```
+
+The number of texts that can be embedded in one pass of the model is proportional to the `n_ctx` parameter of Embed4All.
+Increasing it may increase batched embedding throughput if you have a fast GPU, at the cost of VRAM.
+```py
+embedder = Embed4All(n_ctx=4096, device='gpu')
+```
+
+
+### Resizable Dimensionality
+
+The embedding dimension of Nomic Embed v1.5 can be resized using the `dimensionality` parameter. This parameter supports
+any value between 64 and 768.
+
+Shorter embeddings use less storage, memory, and bandwidth with a small performance cost. See the [blog post] for more
+info.
+
+[blog post]: https://blog.nomic.ai/posts/nomic-embed-matryoshka
+
+=== "Matryoshka Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'The quick brown fox jumps over the lazy dog'
+    embedder = Embed4All('nomic-embed-text-v1.5.f16.gguf')
+    output = embedder.embed(text, dimensionality=64)
+    print(len(output))
+    print(output)
+    ```
+=== "Output"
+    ```
+    64
+    [-0.03567073494195938, 0.1301717758178711, -0.4333043396472931, ...]
+    ```
 
 
 ### API documentation

gpt4all-bindings/python/docs/index.md

@@ -38,7 +38,7 @@ The GPT4All software ecosystem is compatible with the following Transformer arch
 - `MPT` (including `Replit`)
 - `GPT-J`
 
-You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json)
+You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models3.json)
 
 
 GPT4All models are artifacts produced through a process known as neural network quantization.

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import ctypes
-import logging
 import os
 import platform
 import re
@@ -10,14 +9,20 @@ import sys
 import threading
 from enum import Enum
 from queue import Queue
-from typing import Callable, Iterable, List
+from typing import Any, Callable, Generic, Iterable, TypeVar, overload
 
 if sys.version_info >= (3, 9):
     import importlib.resources as importlib_resources
 else:
     import importlib_resources
 
-logger: logging.Logger = logging.getLogger(__name__)
+if (3, 9) <= sys.version_info < (3, 11):
+    # python 3.9 broke generic TypedDict, python 3.11 fixed it
+    from typing_extensions import TypedDict
+else:
+    from typing import TypedDict
+
+EmbeddingsType = TypeVar('EmbeddingsType', bound='list[Any]')
 
 
 # TODO: provide a config file to make this more robust
@@ -28,7 +33,7 @@ def load_llmodel_library():
     ext = {"Darwin": "dylib", "Linux": "so", "Windows": "dll"}[platform.system()]
 
     try:
-        # Linux, Windows, MinGW
+        # macOS, Linux, MinGW
         lib = ctypes.CDLL(str(MODEL_LIB_PATH / f"libllmodel.{ext}"))
     except FileNotFoundError:
         if ext != 'dll':
@@ -105,13 +110,19 @@ llmodel.llmodel_prompt.argtypes = [
 
 llmodel.llmodel_prompt.restype = None
 
-llmodel.llmodel_embedding.argtypes = [
+llmodel.llmodel_embed.argtypes = [
     ctypes.c_void_p,
-    ctypes.c_char_p,
+    ctypes.POINTER(ctypes.c_char_p),
     ctypes.POINTER(ctypes.c_size_t),
+    ctypes.c_char_p,
+    ctypes.c_int,
+    ctypes.POINTER(ctypes.c_size_t),
+    ctypes.c_bool,
+    ctypes.c_bool,
+    ctypes.POINTER(ctypes.c_char_p),
 ]
 
-llmodel.llmodel_embedding.restype = ctypes.POINTER(ctypes.c_float)
+llmodel.llmodel_embed.restype = ctypes.POINTER(ctypes.c_float)
 
 llmodel.llmodel_free_embedding.argtypes = [ctypes.POINTER(ctypes.c_float)]
 llmodel.llmodel_free_embedding.restype = None
@@ -125,7 +136,7 @@ llmodel.llmodel_set_implementation_search_path.restype = None
 llmodel.llmodel_threadCount.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_threadCount.restype = ctypes.c_int32
 
-llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).replace("\\", r"\\").encode())
+llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).encode())
 
 llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
 llmodel.llmodel_available_gpu_devices.restype = ctypes.POINTER(LLModelGPUDevice)
@@ -155,6 +166,11 @@ class Sentinel(Enum):
     TERMINATING_SYMBOL = 0
 
 
+class EmbedResult(Generic[EmbeddingsType], TypedDict):
+    embeddings: EmbeddingsType
+    n_prompt_tokens: int
+
+
 class LLModel:
     """
     Base class and universal wrapper for GPT4All language models
@@ -183,10 +199,10 @@ class LLModel:
         model = llmodel.llmodel_model_create2(self.model_path, b"auto", ctypes.byref(err))
         if model is None:
             s = err.value
-            raise ValueError(f"Unable to instantiate model: {'null' if s is None else s.decode()}")
+            raise RuntimeError(f"Unable to instantiate model: {'null' if s is None else s.decode()}")
         self.model = model
 
-    def __del__(self):
+    def __del__(self, llmodel=llmodel):
         if hasattr(self, 'model'):
             llmodel.llmodel_model_destroy(self.model)
 
@@ -287,16 +303,57 @@ class LLModel:
         self.context.repeat_last_n = repeat_last_n
         self.context.context_erase = context_erase
 
-    def generate_embedding(self, text: str) -> List[float]:
-        if not text:
-            raise ValueError("Text must not be None or empty")
+    @overload
+    def generate_embeddings(
+        self, text: str, prefix: str, dimensionality: int, do_mean: bool, atlas: bool,
+    ) -> EmbedResult[list[float]]: ...
+    @overload
+    def generate_embeddings(
+        self, text: list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+    ) -> EmbedResult[list[list[float]]]: ...
+    @overload
+    def generate_embeddings(
+        self, text: str | list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+    ) -> EmbedResult[list[Any]]: ...
 
+    def generate_embeddings(
+        self, text: str | list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+    ) -> EmbedResult[list[Any]]:
+        if not text:
+            raise ValueError("text must not be None or empty")
+
+        if (single_text := isinstance(text, str)):
+            text = [text]
+
+        # prepare input
         embedding_size = ctypes.c_size_t()
-        c_text = ctypes.c_char_p(text.encode())
-        embedding_ptr = llmodel.llmodel_embedding(self.model, c_text, ctypes.byref(embedding_size))
-        embedding_array = [embedding_ptr[i] for i in range(embedding_size.value)]
+        token_count = ctypes.c_size_t()
+        error = ctypes.c_char_p()
+        c_prefix = ctypes.c_char_p() if prefix is None else prefix.encode()
+        c_texts = (ctypes.c_char_p * (len(text) + 1))()
+        for i, t in enumerate(text):
+            c_texts[i] = t.encode()
+
+        # generate the embeddings
+        embedding_ptr = llmodel.llmodel_embed(
+            self.model, c_texts, ctypes.byref(embedding_size), c_prefix, dimensionality, ctypes.byref(token_count),
+            do_mean, atlas, ctypes.byref(error),
+        )
+
+        if not embedding_ptr:
+            msg = "(unknown error)" if error.value is None else error.value.decode()
+            raise RuntimeError(f'Failed to generate embeddings: {msg}')
+
+        # extract output
+        n_embd = embedding_size.value // len(text)
+        embedding_array = [
+            embedding_ptr[i:i + n_embd]
+            for i in range(0, embedding_size.value, n_embd)
+        ]
         llmodel.llmodel_free_embedding(embedding_ptr)
-        return list(embedding_array)
+
+        embeddings = embedding_array[0] if single_text else embedding_array
+        return {'embeddings': embeddings, 'n_prompt_tokens': token_count.value}
 
     def prompt_model(
         self,
@@ -333,13 +390,6 @@ class LLModel:
         self.buffer.clear()
         self.buff_expecting_cont_bytes = 0
 
-        logger.info(
-            "LLModel.prompt_model -- prompt:\n"
-            + "%s\n"
-            + "===/LLModel.prompt_model -- prompt/===",
-            prompt,
-        )
-
         self._set_context(
             n_predict=n_predict,
             top_k=top_k,

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -3,6 +3,7 @@ Python only API for running all GPT4All models.
 """
 from __future__ import annotations
 
+import hashlib
 import os
 import re
 import sys
@@ -10,7 +11,7 @@ import time
 import warnings
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Iterable, Literal, Protocol, overload
 
 import requests
 from requests.exceptions import ChunkedEncodingError
@@ -18,17 +19,21 @@ from tqdm import tqdm
 from urllib3.exceptions import IncompleteRead, ProtocolError
 
 from . import _pyllmodel
+from ._pyllmodel import EmbedResult as EmbedResult
+
+if TYPE_CHECKING:
+    from typing_extensions import TypeAlias
+
+if sys.platform == 'darwin':
+    import fcntl
 
 # TODO: move to config
-DEFAULT_MODEL_DIRECTORY = os.path.join(str(Path.home()), ".cache", "gpt4all").replace("\\", "\\\\")
+DEFAULT_MODEL_DIRECTORY = Path.home() / ".cache" / "gpt4all"
 
-DEFAULT_MODEL_CONFIG = {
-    "systemPrompt": "",
-    "promptTemplate": "### Human: \n{0}\n\n### Assistant:\n",
-}
+DEFAULT_PROMPT_TEMPLATE = "### Human:\n{0}\n\n### Assistant:\n"
 
-ConfigType = Dict[str, str]
-MessageType = Dict[str, str]
+ConfigType: TypeAlias = 'dict[str, Any]'
+MessageType: TypeAlias = 'dict[str, str]'
 
 
 class Embed4All:
@@ -36,26 +41,99 @@ class Embed4All:
     Python class that handles embeddings for GPT4All.
     """
 
-    def __init__(self, model_name: Optional[str] = None, n_threads: Optional[int] = None, **kwargs):
+    MIN_DIMENSIONALITY = 64
+
+    def __init__(self, model_name: str | None = None, n_threads: int | None = None, **kwargs):
         """
         Constructor
 
         Args:
             n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
         """
-        self.gpt4all = GPT4All(model_name or 'all-MiniLM-L6-v2-f16.gguf', n_threads=n_threads, **kwargs)
+        if model_name is None:
+            model_name = 'all-MiniLM-L6-v2.gguf2.f16.gguf'
+        self.gpt4all = GPT4All(model_name, n_threads=n_threads, **kwargs)
 
-    def embed(self, text: str) -> List[float]:
+    # return_dict=False
+    @overload
+    def embed(
+        self, text: str, *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[False] = ..., atlas: bool = ...,
+    ) -> list[float]: ...
+    @overload
+    def embed(
+        self, text: list[str], *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[False] = ..., atlas: bool = ...,
+    ) -> list[list[float]]: ...
+    @overload
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
+        long_text_mode: str = ..., return_dict: Literal[False] = ..., atlas: bool = ...,
+    ) -> list[Any]: ...
+
+    # return_dict=True
+    @overload
+    def embed(
+        self, text: str, *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[True], atlas: bool = ...,
+    ) -> EmbedResult[list[float]]: ...
+    @overload
+    def embed(
+        self, text: list[str], *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[True], atlas: bool = ...,
+    ) -> EmbedResult[list[list[float]]]: ...
+    @overload
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
+        long_text_mode: str = ..., return_dict: Literal[True], atlas: bool = ...,
+    ) -> EmbedResult[list[Any]]: ...
+
+    # return type unknown
+    @overload
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
+        long_text_mode: str = ..., return_dict: bool = ..., atlas: bool = ...,
+    ) -> Any: ...
+
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = None, dimensionality: int | None = None,
+        long_text_mode: str = "mean", return_dict: bool = False, atlas: bool = False,
+    ) -> Any:
         """
-        Generate an embedding.
+        Generate one or more embeddings.
 
         Args:
-            text: The text document to generate an embedding for.
+            text: A text or list of texts to generate embeddings for.
+            prefix: The model-specific prefix representing the embedding task, without the trailing colon. For Nomic
+                Embed, this can be `search_query`, `search_document`, `classification`, or `clustering`. Defaults to
+                `search_document` or equivalent if known; otherwise, you must explicitly pass a prefix or an empty
+                string if none applies.
+            dimensionality: The embedding dimension, for use with Matryoshka-capable models. Defaults to full-size.
+            long_text_mode: How to handle texts longer than the model can accept. One of `mean` or `truncate`.
+            return_dict: Return the result as a dict that includes the number of prompt tokens processed.
+            atlas: Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens
+                with long_text_mode="mean" will raise an error. Disabled by default.
 
         Returns:
-            An embedding of your document of text.
+            With return_dict=False, an embedding or list of embeddings of your text(s).
+            With return_dict=True, a dict with keys 'embeddings' and 'n_prompt_tokens'.
         """
-        return self.gpt4all.model.generate_embedding(text)
+        if dimensionality is None:
+            dimensionality = -1
+        else:
+            if dimensionality <= 0:
+                raise ValueError(f'Dimensionality must be None or a positive integer, got {dimensionality}')
+            if dimensionality < self.MIN_DIMENSIONALITY:
+                warnings.warn(
+                    f'Dimensionality {dimensionality} is less than the suggested minimum of {self.MIN_DIMENSIONALITY}.'
+                    ' Performance may be degraded.'
+                )
+        try:
+            do_mean = {"mean": True, "truncate": False}[long_text_mode]
+        except KeyError:
+            raise ValueError(f"Long text mode must be one of 'mean' or 'truncate', got {long_text_mode!r}")
+        result = self.gpt4all.model.generate_embeddings(text, prefix, dimensionality, do_mean, atlas)
+        return result if return_dict else result['embeddings']
 
 
 class GPT4All:
@@ -66,11 +144,11 @@ class GPT4All:
     def __init__(
         self,
         model_name: str,
-        model_path: Optional[Union[str, os.PathLike[str]]] = None,
-        model_type: Optional[str] = None,
+        model_path: str | os.PathLike[str] | None = None,
+        model_type: str | None = None,
         allow_download: bool = True,
-        n_threads: Optional[int] = None,
-        device: Optional[str] = "cpu",
+        n_threads: int | None = None,
+        device: str | None = "cpu",
         n_ctx: int = 2048,
         ngl: int = 100,
         verbose: bool = False,
@@ -109,27 +187,31 @@ class GPT4All:
         if n_threads is not None:
             self.model.set_thread_count(n_threads)
 
-        self._is_chat_session_activated: bool = False
-        self.current_chat_session: List[MessageType] = empty_chat_session()
+        self._history: list[MessageType] | None = None
         self._current_prompt_template: str = "{0}"
 
+    @property
+    def current_chat_session(self) -> list[MessageType] | None:
+        return None if self._history is None else list(self._history)
+
     @staticmethod
-    def list_models() -> List[ConfigType]:
+    def list_models() -> list[ConfigType]:
         """
-        Fetch model list from https://gpt4all.io/models/models2.json.
+        Fetch model list from https://gpt4all.io/models/models3.json.
 
         Returns:
             Model list in JSON format.
         """
-        resp = requests.get("https://gpt4all.io/models/models2.json")
+        resp = requests.get("https://gpt4all.io/models/models3.json")
         if resp.status_code != 200:
             raise ValueError(f'Request failed: HTTP {resp.status_code} {resp.reason}')
         return resp.json()
 
-    @staticmethod
+    @classmethod
     def retrieve_model(
+        cls,
         model_name: str,
-        model_path: Optional[Union[str, os.PathLike[str]]] = None,
+        model_path: str | os.PathLike[str] | None = None,
         allow_download: bool = True,
         verbose: bool = False,
     ) -> ConfigType:
@@ -150,58 +232,57 @@ class GPT4All:
         model_filename = append_extension_if_missing(model_name)
 
         # get the config for the model
-        config: ConfigType = DEFAULT_MODEL_CONFIG
+        config: ConfigType = {}
         if allow_download:
-            available_models = GPT4All.list_models()
+            available_models = cls.list_models()
 
             for m in available_models:
                 if model_filename == m["filename"]:
-                    config.update(m)
-                    config["systemPrompt"] = config["systemPrompt"].strip()
+                    tmpl = m.get("promptTemplate", DEFAULT_PROMPT_TEMPLATE)
                     # change to Python-style formatting
-                    config["promptTemplate"] = config["promptTemplate"].replace("%1", "{0}", 1).replace("%2", "{1}", 1)
+                    m["promptTemplate"] = tmpl.replace("%1", "{0}", 1).replace("%2", "{1}", 1)
+                    config.update(m)
                     break
 
         # Validate download directory
         if model_path is None:
             try:
                 os.makedirs(DEFAULT_MODEL_DIRECTORY, exist_ok=True)
-            except OSError as exc:
-                raise ValueError(
-                    f"Failed to create model download directory at {DEFAULT_MODEL_DIRECTORY}: {exc}. "
-                    "Please specify model_path."
-                )
+            except OSError as e:
+                raise RuntimeError("Failed to create model download directory") from e
             model_path = DEFAULT_MODEL_DIRECTORY
         else:
-            model_path = str(model_path).replace("\\", "\\\\")
+            model_path = Path(model_path)
 
-        if not os.path.exists(model_path):
-            raise ValueError(f"Invalid model directory: {model_path}")
+        if not model_path.exists():
+            raise FileNotFoundError(f"Model directory does not exist: {model_path!r}")
 
-        model_dest = os.path.join(model_path, model_filename).replace("\\", "\\\\")
-        if os.path.exists(model_dest):
-            config.pop("url", None)
-            config["path"] = model_dest
+        model_dest = model_path / model_filename
+        if model_dest.exists():
+            config["path"] = str(model_dest)
             if verbose:
-                print("Found model file at", model_dest, file=sys.stderr)
-
-        # If model file does not exist, download
+                print(f"Found model file at {str(model_dest)!r}", file=sys.stderr)
         elif allow_download:
-            url = config.pop("url", None)
-
-            config["path"] = GPT4All.download_model(model_filename, model_path, verbose=verbose, url=url)
+            # If model file does not exist, download
+            filesize = config.get("filesize")
+            config["path"] = str(cls.download_model(
+                model_filename, model_path, verbose=verbose, url=config.get("url"),
+                expected_size=None if filesize is None else int(filesize), expected_md5=config.get("md5sum"),
+            ))
         else:
-            raise ValueError("Failed to retrieve model")
+            raise FileNotFoundError(f"Model file does not exist: {model_dest!r}")
 
         return config
 
     @staticmethod
     def download_model(
         model_filename: str,
-        model_path: Union[str, os.PathLike[str]],
+        model_path: str | os.PathLike[str],
         verbose: bool = True,
-        url: Optional[str] = None,
-    ) -> str:
+        url: str | None = None,
+        expected_size: int | None = None,
+        expected_md5: str | None = None,
+    ) -> str | os.PathLike[str]:
         """
         Download model from https://gpt4all.io.
 
@@ -210,30 +291,30 @@ class GPT4All:
             model_path: Path to download model to.
             verbose: If True (default), print debug messages.
             url: the models remote url (e.g. may be hosted on HF)
+            expected_size: The expected size of the download.
+            expected_md5: The expected MD5 hash of the download.
 
         Returns:
             Model file destination.
         """
 
-        def get_download_url(model_filename):
-            if url:
-                return url
-            return f"https://gpt4all.io/models/gguf/{model_filename}"
-
         # Download model
-        download_path = os.path.join(model_path, model_filename).replace("\\", "\\\\")
-        download_url = get_download_url(model_filename)
+        if url is None:
+            url = f"https://gpt4all.io/models/gguf/{model_filename}"
 
         def make_request(offset=None):
             headers = {}
             if offset:
                 print(f"\nDownload interrupted, resuming from byte position {offset}", file=sys.stderr)
                 headers['Range'] = f'bytes={offset}-'  # resume incomplete response
-            response = requests.get(download_url, stream=True, headers=headers)
+                headers["Accept-Encoding"] = "identity"  # Content-Encoding changes meaning of ranges
+            response = requests.get(url, stream=True, headers=headers)
             if response.status_code not in (200, 206):
                 raise ValueError(f'Request failed: HTTP {response.status_code} {response.reason}')
             if offset and (response.status_code != 206 or str(offset) not in response.headers.get('Content-Range', '')):
                 raise ValueError('Connection was interrupted and server does not support range requests')
+            if (enc := response.headers.get("Content-Encoding")) is not None:
+                raise ValueError(f"Expected identity Content-Encoding, got {enc}")
             return response
 
         response = make_request()
@@ -241,49 +322,97 @@ class GPT4All:
         total_size_in_bytes = int(response.headers.get("content-length", 0))
         block_size = 2**20  # 1 MB
 
-        with open(download_path, "wb") as file, \
-                tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar:
+        partial_path = Path(model_path) / (model_filename + ".part")
+
+        with open(partial_path, "w+b") as partf:
             try:
-                while True:
-                    last_progress = progress_bar.n
-                    try:
-                        for data in response.iter_content(block_size):
-                            file.write(data)
-                            progress_bar.update(len(data))
-                    except ChunkedEncodingError as cee:
-                        if cee.args and isinstance(pe := cee.args[0], ProtocolError):
-                            if len(pe.args) >= 2 and isinstance(ir := pe.args[1], IncompleteRead):
-                                assert progress_bar.n <= ir.partial  # urllib3 may be ahead of us but never behind
-                                # the socket was closed during a read - retry
-                                response = make_request(progress_bar.n)
-                                continue
-                        raise
-                    if total_size_in_bytes != 0 and progress_bar.n < total_size_in_bytes:
-                        if progress_bar.n == last_progress:
-                            raise RuntimeError('Download not making progress, aborting.')
-                        # server closed connection prematurely - retry
-                        response = make_request(progress_bar.n)
-                        continue
-                    break
-            except Exception:
+                with tqdm(desc="Downloading", total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar:
+                    while True:
+                        last_progress = progress_bar.n
+                        try:
+                            for data in response.iter_content(block_size):
+                                partf.write(data)
+                                progress_bar.update(len(data))
+                        except ChunkedEncodingError as cee:
+                            if cee.args and isinstance(pe := cee.args[0], ProtocolError):
+                                if len(pe.args) >= 2 and isinstance(ir := pe.args[1], IncompleteRead):
+                                    assert progress_bar.n <= ir.partial  # urllib3 may be ahead of us but never behind
+                                    # the socket was closed during a read - retry
+                                    response = make_request(progress_bar.n)
+                                    continue
+                            raise
+                        if total_size_in_bytes != 0 and progress_bar.n < total_size_in_bytes:
+                            if progress_bar.n == last_progress:
+                                raise RuntimeError("Download not making progress, aborting.")
+                            # server closed connection prematurely - retry
+                            response = make_request(progress_bar.n)
+                            continue
+                        break
+
+                # verify file integrity
+                file_size = partf.tell()
+                if expected_size is not None and file_size != expected_size:
+                    raise ValueError(f"Expected file size of {expected_size} bytes, got {file_size}")
+                if expected_md5 is not None:
+                    partf.seek(0)
+                    hsh = hashlib.md5()
+                    with tqdm(desc="Verifying", total=file_size, unit="iB", unit_scale=True) as bar:
+                        while chunk := partf.read(block_size):
+                            hsh.update(chunk)
+                            bar.update(len(chunk))
+                    if hsh.hexdigest() != expected_md5.lower():
+                        raise ValueError(f"Expected MD5 hash of {expected_md5!r}, got {hsh.hexdigest()!r}")
+            except:
                 if verbose:
                     print("Cleaning up the interrupted download...", file=sys.stderr)
                 try:
-                    os.remove(download_path)
+                    os.remove(partial_path)
                 except OSError:
                     pass
                 raise
 
-        if os.name == 'nt':
-            time.sleep(2)  # Sleep for a little bit so Windows can remove file lock
+            # flush buffers and sync the inode
+            partf.flush()
+            _fsync(partf)
+
+        # move to final destination
+        download_path = Path(model_path) / model_filename
+        try:
+            os.rename(partial_path, download_path)
+        except FileExistsError:
+            try:
+                os.remove(partial_path)
+            except OSError:
+                pass
+            raise
 
         if verbose:
-            print("Model downloaded at:", download_path, file=sys.stderr)
+            print(f"Model downloaded to {str(download_path)!r}", file=sys.stderr)
         return download_path
 
+    @overload
+    def generate(
+        self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
+        min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
+        n_predict: int | None = ..., streaming: Literal[False] = ..., callback: _pyllmodel.ResponseCallbackType = ...,
+    ) -> str: ...
+    @overload
+    def generate(
+        self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
+        min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
+        n_predict: int | None = ..., streaming: Literal[True], callback: _pyllmodel.ResponseCallbackType = ...,
+    ) -> Iterable[str]: ...
+    @overload
+    def generate(
+        self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
+        min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
+        n_predict: int | None = ..., streaming: bool, callback: _pyllmodel.ResponseCallbackType = ...,
+    ) -> Any: ...
+
     def generate(
         self,
         prompt: str,
+        *,
         max_tokens: int = 200,
         temp: float = 0.7,
         top_k: int = 40,
@@ -292,10 +421,10 @@ class GPT4All:
         repeat_penalty: float = 1.18,
         repeat_last_n: int = 64,
         n_batch: int = 8,
-        n_predict: Optional[int] = None,
+        n_predict: int | None = None,
         streaming: bool = False,
         callback: _pyllmodel.ResponseCallbackType = _pyllmodel.empty_response_callback,
-    ) -> Union[str, Iterable[str]]:
+    ) -> Any:
         """
         Generate outputs from any GPT4All model.
 
@@ -317,12 +446,8 @@ class GPT4All:
             Either the entire completion or a generator that yields the completion token by token.
         """
 
-        if re.search(r"%1(?![0-9])", self._current_prompt_template):
-            raise ValueError("Prompt template containing a literal '%1' is not supported. For a prompt "
-                             "placeholder, please use '{0}' instead.")
-
         # Preparing the model request
-        generate_kwargs: Dict[str, Any] = dict(
+        generate_kwargs: dict[str, Any] = dict(
             temp=temp,
             top_k=top_k,
             top_p=top_p,
@@ -333,17 +458,17 @@ class GPT4All:
             n_predict=n_predict if n_predict is not None else max_tokens,
         )
 
-        if self._is_chat_session_activated:
+        if self._history is not None:
             # check if there is only one message, i.e. system prompt:
-            reset = len(self.current_chat_session) == 1
+            reset = len(self._history) == 1
             generate_kwargs["reset_context"] = reset
-            self.current_chat_session.append({"role": "user", "content": prompt})
+            self._history.append({"role": "user", "content": prompt})
 
             fct_func = self._format_chat_prompt_template.__func__  # type: ignore[attr-defined]
             if fct_func is GPT4All._format_chat_prompt_template:
                 if reset:
                     # ingest system prompt
-                    self.model.prompt_model(self.current_chat_session[0]["content"], "%1",
+                    self.model.prompt_model(self._history[0]["content"], "%1",
                                             _pyllmodel.empty_response_callback,
                                             n_batch=n_batch, n_predict=0, special=True)
                 prompt_template = self._current_prompt_template.format("%1", "%2")
@@ -354,8 +479,8 @@ class GPT4All:
                 )
                 # special tokens won't be processed
                 prompt = self._format_chat_prompt_template(
-                    self.current_chat_session[-1:],
-                    self.current_chat_session[0]["content"] if reset else "",
+                    self._history[-1:],
+                    self._history[0]["content"] if reset else "",
                 )
                 prompt_template = "%1"
         else:
@@ -363,18 +488,18 @@ class GPT4All:
             generate_kwargs["reset_context"] = True
 
         # Prepare the callback, process the model response
-        output_collector: List[MessageType]
+        output_collector: list[MessageType]
         output_collector = [
             {"content": ""}
-        ]  # placeholder for the self.current_chat_session if chat session is not activated
+        ]  # placeholder for the self._history if chat session is not activated
 
-        if self._is_chat_session_activated:
-            self.current_chat_session.append({"role": "assistant", "content": ""})
-            output_collector = self.current_chat_session
+        if self._history is not None:
+            self._history.append({"role": "assistant", "content": ""})
+            output_collector = self._history
 
         def _callback_wrapper(
             callback: _pyllmodel.ResponseCallbackType,
-            output_collector: List[MessageType],
+            output_collector: list[MessageType],
         ) -> _pyllmodel.ResponseCallbackType:
             def _callback(token_id: int, response: str) -> bool:
                 nonlocal callback, output_collector
@@ -406,8 +531,8 @@ class GPT4All:
     @contextmanager
     def chat_session(
         self,
-        system_prompt: str = "",
-        prompt_template: str = "",
+        system_prompt: str | None = None,
+        prompt_template: str | None = None,
     ):
         """
         Context manager to hold an inference optimized chat session with a GPT4All model.
@@ -416,21 +541,32 @@ class GPT4All:
             system_prompt: An initial instruction for the model.
             prompt_template: Template for the prompts with {0} being replaced by the user message.
         """
-        # Code to acquire resource, e.g.:
-        self._is_chat_session_activated = True
-        self.current_chat_session = empty_chat_session(system_prompt or self.config["systemPrompt"])
-        self._current_prompt_template = prompt_template or self.config["promptTemplate"]
+
+        if system_prompt is None:
+            system_prompt = self.config.get("systemPrompt", "")
+
+        if prompt_template is None:
+            if (tmpl := self.config.get("promptTemplate")) is None:
+                warnings.warn("Use of a sideloaded model or allow_download=False without specifying a prompt template "
+                              "is deprecated. Defaulting to Alpaca.", DeprecationWarning)
+                tmpl = DEFAULT_PROMPT_TEMPLATE
+            prompt_template = tmpl
+
+        if re.search(r"%1(?![0-9])", prompt_template):
+            raise ValueError("Prompt template containing a literal '%1' is not supported. For a prompt "
+                             "placeholder, please use '{0}' instead.")
+
+        self._history = [{"role": "system", "content": system_prompt}]
+        self._current_prompt_template = prompt_template
         try:
             yield self
         finally:
-            # Code to release resource, e.g.:
-            self._is_chat_session_activated = False
-            self.current_chat_session = empty_chat_session()
+            self._history = None
             self._current_prompt_template = "{0}"
 
     def _format_chat_prompt_template(
         self,
-        messages: List[MessageType],
+        messages: list[MessageType],
         default_prompt_header: str = "",
         default_prompt_footer: str = "",
     ) -> str:
@@ -463,11 +599,23 @@ class GPT4All:
         return full_prompt
 
 
-def empty_chat_session(system_prompt: str = "") -> List[MessageType]:
-    return [{"role": "system", "content": system_prompt}]
-
-
 def append_extension_if_missing(model_name):
     if not model_name.endswith((".bin", ".gguf")):
         model_name += ".gguf"
     return model_name
+
+
+class _HasFileno(Protocol):
+    def fileno(self) -> int: ...
+
+
+def _fsync(fd: int | _HasFileno) -> None:
+    if sys.platform == 'darwin':
+        # Apple's fsync does not flush the drive write cache
+        try:
+            fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
+        except OSError:
+            pass  # fall back to fsync
+        else:
+            return
+    os.fsync(fd)

gpt4all-bindings/python/gpt4all/tests/test_gpt4all.py

@@ -28,12 +28,8 @@ def test_inference():
     assert len(tokens) > 0
 
     with model.chat_session():
-        tokens = list(model.generate(prompt='hello', top_k=1, streaming=True))
-        model.current_chat_session.append({'role': 'assistant', 'content': ''.join(tokens)})
-
-        tokens = list(model.generate(prompt='write me a poem about dogs', top_k=1, streaming=True))
-        model.current_chat_session.append({'role': 'assistant', 'content': ''.join(tokens)})
-
+        model.generate(prompt='hello', top_k=1, streaming=True)
+        model.generate(prompt='write me a poem about dogs', top_k=1, streaming=True)
         print(model.current_chat_session)
 
 
@@ -115,13 +111,13 @@ def test_empty_embedding():
         output = embedder.embed(text)
 
 def test_download_model(tmp_path: Path):
-    import gpt4all.gpt4all
-    old_default_dir = gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY
-    gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = str(tmp_path)  # temporary pytest directory to ensure a download happens
+    from gpt4all import gpt4all
+    old_default_dir = gpt4all.DEFAULT_MODEL_DIRECTORY
+    gpt4all.DEFAULT_MODEL_DIRECTORY = tmp_path  # temporary pytest directory to ensure a download happens
     try:
         model = GPT4All(model_name='ggml-all-MiniLM-L6-v2-f16.bin')
         model_path = tmp_path / model.config['filename']
         assert model_path.absolute() == Path(model.config['path']).absolute()
         assert model_path.stat().st_size == int(model.config['filesize'])
     finally:
-        gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = old_default_dir
+        gpt4all.DEFAULT_MODEL_DIRECTORY = old_default_dir

gpt4all-bindings/python/mkdocs.yml

@@ -16,8 +16,6 @@ nav:
         - 'Embedding': 'gpt4all_python_embedding.md'
       - 'GPT4ALL in NodeJs': 'gpt4all_nodejs.md'
       - 'gpt4all_cli.md'
-#    - 'Tutorials':
-#      - 'gpt4all_modal.md'
     - 'Wiki':
       - 'gpt4all_faq.md'
 
@@ -44,8 +42,8 @@ markdown_extensions:
   - pymdownx.tabbed:
       alternate_style: true
   - pymdownx.emoji:
-      emoji_index: !!python/name:materialx.emoji.twemoji
-      emoji_generator: !!python/name:materialx.emoji.to_svg
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
       options:
         custom_icons:
           - docs/overrides/.icons

gpt4all-bindings/python/setup.py

@@ -68,7 +68,7 @@ def get_long_description():
 
 setup(
     name=package_name,
-    version="2.3.0",
+    version="2.3.2",
     description="Python bindings for GPT4All",
     long_description=get_long_description(),
     long_description_content_type="text/markdown",
@@ -90,6 +90,7 @@ setup(
         'requests',
         'tqdm',
         'importlib_resources; python_version < "3.9"',
+        'typing-extensions>=4.3.0; python_version >= "3.9" and python_version < "3.11"',
     ],
     extras_require={
         'dev': [
@@ -102,7 +103,8 @@ setup(
             'mkdocstrings[python]',
             'mkdocs-jupyter',
             'black',
-            'isort'
+            'isort',
+            'typing-extensions>=3.10',
         ]
     },
     package_data={'llmodel': [os.path.join(DEST_CLIB_DIRECTORY, "*")]},

gpt4all-bindings/typescript/src/config.js

@@ -24,7 +24,7 @@ const DEFAULT_LIBRARIES_DIRECTORY = librarySearchPaths.join(";");
 
 const DEFAULT_MODEL_CONFIG = {
     systemPrompt: "",
-    promptTemplate: "### Human: \n%1\n### Assistant:\n",
+    promptTemplate: "### Human:\n%1\n\n### Assistant:\n",
 }
 
 const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models2.json";

gpt4all-chat/CMakeLists.txt

@@ -18,7 +18,7 @@ endif()
 
 set(APP_VERSION_MAJOR 2)
 set(APP_VERSION_MINOR 7)
-set(APP_VERSION_PATCH 3)
+set(APP_VERSION_PATCH 4)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file
@@ -40,9 +40,9 @@ configure_file(
 )
 
 if(LINUX)
-  find_package(Qt6 6.5 COMPONENTS Core Quick WaylandCompositor QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
+  find_package(Qt6 6.4 COMPONENTS Core Quick WaylandCompositor QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
 else()
-  find_package(Qt6 6.5 COMPONENTS Core Quick QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
+  find_package(Qt6 6.4 COMPONENTS Core Quick QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
 endif()
 
 # Get the Qt6Core target properties
@@ -73,7 +73,7 @@ qt_add_executable(chat
     chat.h chat.cpp
     chatllm.h chatllm.cpp
     chatmodel.h chatlistmodel.h chatlistmodel.cpp
-    chatgpt.h chatgpt.cpp
+    chatapi.h chatapi.cpp
     database.h database.cpp
     embeddings.h embeddings.cpp
     download.h download.cpp
@@ -96,6 +96,7 @@ qt_add_qml_module(chat
     QML_FILES
       main.qml
       qml/ChatDrawer.qml
+      qml/ChatView.qml
       qml/CollectionsDialog.qml
       qml/ModelDownloaderDialog.qml
       qml/NetworkDialog.qml
@@ -130,6 +131,7 @@ qt_add_qml_module(chat
       icons/send_message.svg
       icons/stop_generating.svg
       icons/regenerate.svg
+      icons/chat.svg
       icons/close.svg
       icons/copy.svg
       icons/db.svg
@@ -138,10 +140,14 @@ qt_add_qml_module(chat
       icons/eject.svg
       icons/edit.svg
       icons/image.svg
+      icons/info.svg
+      icons/search.svg
       icons/trash.svg
       icons/network.svg
       icons/thumbs_up.svg
       icons/thumbs_down.svg
+      icons/left_panel_closed.svg
+      icons/left_panel_open.svg
       icons/logo.svg
       icons/logo-32.png
       icons/logo-48.png
@@ -184,7 +190,10 @@ target_link_libraries(chat
     PRIVATE llmodel)
 
 set(COMPONENT_NAME_MAIN ${PROJECT_NAME})
-set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/install)
+
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/install CACHE PATH "..." FORCE)
+endif()
 
 install(TARGETS chat DESTINATION bin COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llmodel DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
@@ -200,8 +209,6 @@ install(TARGETS llamamodel-mainline-default DESTINATION lib COMPONENT ${COMPONEN
 if(APPLE)
 install(TARGETS llamamodel-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 endif()
-install(TARGETS bert-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS bert-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 
 set(CPACK_GENERATOR "IFW")
 set(CPACK_VERBATIM_VARIABLES YES)

gpt4all-chat/chatapi.cpp

@@ -1,4 +1,4 @@
-#include "chatgpt.h"
+#include "chatapi.h"
 
 #include <string>
 #include <vector>
@@ -13,14 +13,15 @@
 
 //#define DEBUG
 
-ChatGPT::ChatGPT()
+ChatAPI::ChatAPI()
     : QObject(nullptr)
     , m_modelName("gpt-3.5-turbo")
+    , m_requestURL("")
     , m_responseCallback(nullptr)
 {
 }
 
-size_t ChatGPT::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
+size_t ChatAPI::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
 {
     Q_UNUSED(modelPath);
     Q_UNUSED(n_ctx);
@@ -28,7 +29,7 @@ size_t ChatGPT::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
     return 0;
 }
 
-bool ChatGPT::loadModel(const std::string &modelPath, int n_ctx, int ngl)
+bool ChatAPI::loadModel(const std::string &modelPath, int n_ctx, int ngl)
 {
     Q_UNUSED(modelPath);
     Q_UNUSED(n_ctx);
@@ -36,59 +37,59 @@ bool ChatGPT::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     return true;
 }
 
-void ChatGPT::setThreadCount(int32_t n_threads)
+void ChatAPI::setThreadCount(int32_t n_threads)
 {
     Q_UNUSED(n_threads);
     qt_noop();
 }
 
-int32_t ChatGPT::threadCount() const
+int32_t ChatAPI::threadCount() const
 {
     return 1;
 }
 
-ChatGPT::~ChatGPT()
+ChatAPI::~ChatAPI()
 {
 }
 
-bool ChatGPT::isModelLoaded() const
+bool ChatAPI::isModelLoaded() const
 {
     return true;
 }
 
 // All three of the state virtual functions are handled custom inside of chatllm save/restore
-size_t ChatGPT::stateSize() const
+size_t ChatAPI::stateSize() const
 {
     return 0;
 }
 
-size_t ChatGPT::saveState(uint8_t *dest) const
+size_t ChatAPI::saveState(uint8_t *dest) const
 {
     Q_UNUSED(dest);
     return 0;
 }
 
-size_t ChatGPT::restoreState(const uint8_t *src)
+size_t ChatAPI::restoreState(const uint8_t *src)
 {
     Q_UNUSED(src);
     return 0;
 }
 
-void ChatGPT::prompt(const std::string &prompt,
-        const std::string &promptTemplate,
-        std::function<bool(int32_t)> promptCallback,
-        std::function<bool(int32_t, const std::string&)> responseCallback,
-        std::function<bool(bool)> recalculateCallback,
-        PromptContext &promptCtx,
-        bool special,
-        std::string *fakeReply) {
+void ChatAPI::prompt(const std::string &prompt,
+                     const std::string &promptTemplate,
+                     std::function<bool(int32_t)> promptCallback,
+                     std::function<bool(int32_t, const std::string&)> responseCallback,
+                     std::function<bool(bool)> recalculateCallback,
+                     PromptContext &promptCtx,
+                     bool special,
+                     std::string *fakeReply) {
 
     Q_UNUSED(promptCallback);
     Q_UNUSED(recalculateCallback);
     Q_UNUSED(special);
 
     if (!isModelLoaded()) {
-        std::cerr << "ChatGPT ERROR: prompt won't work with an unloaded model!\n";
+        std::cerr << "ChatAPI ERROR: prompt won't work with an unloaded model!\n";
         return;
     }
 
@@ -128,7 +129,7 @@ void ChatGPT::prompt(const std::string &prompt,
     QJsonArray messages;
     for (int i = 0; i < m_context.count(); ++i) {
         QJsonObject message;
-        message.insert("role", i % 2 == 0 ? "assistant" : "user");
+        message.insert("role", i % 2 == 0 ? "user" : "assistant");
         message.insert("content", m_context.at(i));
         messages.append(message);
     }
@@ -142,7 +143,7 @@ void ChatGPT::prompt(const std::string &prompt,
     QJsonDocument doc(root);
 
 #if defined(DEBUG)
-    qDebug().noquote() << "ChatGPT::prompt begin network request" << doc.toJson();
+    qDebug().noquote() << "ChatAPI::prompt begin network request" << doc.toJson();
 #endif
 
     m_responseCallback = responseCallback;
@@ -150,10 +151,10 @@ void ChatGPT::prompt(const std::string &prompt,
     // The following code sets up a worker thread and object to perform the actual api request to
     // chatgpt and then blocks until it is finished
     QThread workerThread;
-    ChatGPTWorker worker(this);
+    ChatAPIWorker worker(this);
     worker.moveToThread(&workerThread);
-    connect(&worker, &ChatGPTWorker::finished, &workerThread, &QThread::quit, Qt::DirectConnection);
-    connect(this, &ChatGPT::request, &worker, &ChatGPTWorker::request, Qt::QueuedConnection);
+    connect(&worker, &ChatAPIWorker::finished, &workerThread, &QThread::quit, Qt::DirectConnection);
+    connect(this, &ChatAPI::request, &worker, &ChatAPIWorker::request, Qt::QueuedConnection);
     workerThread.start();
     emit request(m_apiKey, &promptCtx, doc.toJson(QJsonDocument::Compact));
     workerThread.wait();
@@ -164,40 +165,40 @@ void ChatGPT::prompt(const std::string &prompt,
     m_responseCallback = nullptr;
 
 #if defined(DEBUG)
-    qDebug() << "ChatGPT::prompt end network request";
+    qDebug() << "ChatAPI::prompt end network request";
 #endif
 }
 
-bool ChatGPT::callResponse(int32_t token, const std::string& string)
+bool ChatAPI::callResponse(int32_t token, const std::string& string)
 {
     Q_ASSERT(m_responseCallback);
     if (!m_responseCallback) {
-        std::cerr << "ChatGPT ERROR: no response callback!\n";
+        std::cerr << "ChatAPI ERROR: no response callback!\n";
         return false;
     }
     return m_responseCallback(token, string);
 }
 
-void ChatGPTWorker::request(const QString &apiKey,
-        LLModel::PromptContext *promptCtx,
-        const QByteArray &array)
+void ChatAPIWorker::request(const QString &apiKey,
+                            LLModel::PromptContext *promptCtx,
+                            const QByteArray &array)
 {
     m_ctx = promptCtx;
 
-    QUrl openaiUrl("https://api.openai.com/v1/chat/completions");
+    QUrl apiUrl(m_chat->url());
     const QString authorization = QString("Bearer %1").arg(apiKey).trimmed();
-    QNetworkRequest request(openaiUrl);
+    QNetworkRequest request(apiUrl);
     request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
     request.setRawHeader("Authorization", authorization.toUtf8());
     m_networkManager = new QNetworkAccessManager(this);
     QNetworkReply *reply = m_networkManager->post(request, array);
     connect(qApp, &QCoreApplication::aboutToQuit, reply, &QNetworkReply::abort);
-    connect(reply, &QNetworkReply::finished, this, &ChatGPTWorker::handleFinished);
-    connect(reply, &QNetworkReply::readyRead, this, &ChatGPTWorker::handleReadyRead);
-    connect(reply, &QNetworkReply::errorOccurred, this, &ChatGPTWorker::handleErrorOccurred);
+    connect(reply, &QNetworkReply::finished, this, &ChatAPIWorker::handleFinished);
+    connect(reply, &QNetworkReply::readyRead, this, &ChatAPIWorker::handleReadyRead);
+    connect(reply, &QNetworkReply::errorOccurred, this, &ChatAPIWorker::handleErrorOccurred);
 }
 
-void ChatGPTWorker::handleFinished()
+void ChatAPIWorker::handleFinished()
 {
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply) {
@@ -210,14 +211,14 @@ void ChatGPTWorker::handleFinished()
     bool ok;
     int code = response.toInt(&ok);
     if (!ok || code != 200) {
-        qWarning() << QString("ERROR: ChatGPT responded with error code \"%1-%2\"")
-            .arg(code).arg(reply->errorString()).toStdString();
+        qWarning().noquote() << "ERROR: ChatAPIWorker::handleFinished got HTTP Error" << code << "response:"
+                             << reply->errorString();
     }
     reply->deleteLater();
     emit finished();
 }
 
-void ChatGPTWorker::handleReadyRead()
+void ChatAPIWorker::handleReadyRead()
 {
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply) {
@@ -230,8 +231,11 @@ void ChatGPTWorker::handleReadyRead()
     bool ok;
     int code = response.toInt(&ok);
     if (!ok || code != 200) {
-        m_chat->callResponse(-1, QString("\nERROR: 2 ChatGPT responded with error code \"%1-%2\" %3\n")
-            .arg(code).arg(reply->errorString()).arg(reply->readAll()).toStdString());
+        m_chat->callResponse(
+            -1,
+            QString("ERROR: ChatAPIWorker::handleReadyRead got HTTP Error %1 %2: %3")
+                .arg(code).arg(reply->errorString()).arg(reply->readAll()).toStdString()
+        );
         emit finished();
         return;
     }
@@ -251,8 +255,8 @@ void ChatGPTWorker::handleReadyRead()
         QJsonParseError err;
         const QJsonDocument document = QJsonDocument::fromJson(jsonData.toUtf8(), &err);
         if (err.error != QJsonParseError::NoError) {
-            m_chat->callResponse(-1, QString("\nERROR: ChatGPT responded with invalid json \"%1\"\n")
-                .arg(err.errorString()).toStdString());
+            m_chat->callResponse(-1, QString("ERROR: ChatAPI responded with invalid json \"%1\"")
+                                         .arg(err.errorString()).toStdString());
             continue;
         }
 
@@ -271,7 +275,7 @@ void ChatGPTWorker::handleReadyRead()
     }
 }
 
-void ChatGPTWorker::handleErrorOccurred(QNetworkReply::NetworkError code)
+void ChatAPIWorker::handleErrorOccurred(QNetworkReply::NetworkError code)
 {
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply || reply->error() == QNetworkReply::OperationCanceledError /*when we call abort on purpose*/) {
@@ -279,7 +283,7 @@ void ChatGPTWorker::handleErrorOccurred(QNetworkReply::NetworkError code)
         return;
     }
 
-    qWarning() << QString("ERROR: ChatGPT responded with error code \"%1-%2\"")
-                      .arg(code).arg(reply->errorString()).toStdString();
+    qWarning().noquote() << "ERROR: ChatAPIWorker::handleErrorOccurred got HTTP Error" << code << "response:"
+                         << reply->errorString();
     emit finished();
 }

gpt4all-chat/chatapi.h

@@ -1,5 +1,5 @@
-#ifndef CHATGPT_H
-#define CHATGPT_H
+#ifndef CHATAPI_H
+#define CHATAPI_H
 
 #include <stdexcept>
 
@@ -13,22 +13,22 @@
 
 #include "../gpt4all-backend/llmodel.h"
 
-class ChatGPT;
-class ChatGPTWorker : public QObject {
+class ChatAPI;
+class ChatAPIWorker : public QObject {
     Q_OBJECT
 public:
-    ChatGPTWorker(ChatGPT *chatGPT)
+    ChatAPIWorker(ChatAPI *chatAPI)
         : QObject(nullptr)
         , m_ctx(nullptr)
         , m_networkManager(nullptr)
-        , m_chat(chatGPT) {}
-    virtual ~ChatGPTWorker() {}
+        , m_chat(chatAPI) {}
+    virtual ~ChatAPIWorker() {}
 
     QString currentResponse() const { return m_currentResponse; }
 
     void request(const QString &apiKey,
-        LLModel::PromptContext *promptCtx,
-        const QByteArray &array);
+                 LLModel::PromptContext *promptCtx,
+                 const QByteArray &array);
 
 Q_SIGNALS:
     void finished();
@@ -39,17 +39,17 @@ private Q_SLOTS:
     void handleErrorOccurred(QNetworkReply::NetworkError code);
 
 private:
-    ChatGPT *m_chat;
+    ChatAPI *m_chat;
     LLModel::PromptContext *m_ctx;
     QNetworkAccessManager *m_networkManager;
     QString m_currentResponse;
 };
 
-class ChatGPT : public QObject, public LLModel {
+class ChatAPI : public QObject, public LLModel {
     Q_OBJECT
 public:
-    ChatGPT();
-    virtual ~ChatGPT();
+    ChatAPI();
+    virtual ~ChatAPI();
 
     bool supportsEmbedding() const override { return false; }
     bool supportsCompletion() const override { return true; }
@@ -60,19 +60,21 @@ public:
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
     void prompt(const std::string &prompt,
-        const std::string &promptTemplate,
-        std::function<bool(int32_t)> promptCallback,
-        std::function<bool(int32_t, const std::string&)> responseCallback,
-        std::function<bool(bool)> recalculateCallback,
-        PromptContext &ctx,
-        bool special,
-        std::string *fakeReply) override;
+                const std::string &promptTemplate,
+                std::function<bool(int32_t)> promptCallback,
+                std::function<bool(int32_t, const std::string&)> responseCallback,
+                std::function<bool(bool)> recalculateCallback,
+                PromptContext &ctx,
+                bool special,
+                std::string *fakeReply) override;
 
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
 
     void setModelName(const QString &modelName) { m_modelName = modelName; }
     void setAPIKey(const QString &apiKey) { m_apiKey = apiKey; }
+    void setRequestURL(const QString &requestURL) { m_requestURL = requestURL; }
+    QString url() const { return m_requestURL; }
 
     QList<QString> context() const { return m_context; }
     void setContext(const QList<QString> &context) { m_context = context; }
@@ -81,8 +83,8 @@ public:
 
 Q_SIGNALS:
     void request(const QString &apiKey,
-        LLModel::PromptContext *ctx,
-        const QByteArray &array);
+                 LLModel::PromptContext *ctx,
+                 const QByteArray &array);
 
 protected:
     // We have to implement these as they are pure virtual in base class, but we don't actually use
@@ -128,8 +130,9 @@ private:
     std::function<bool(int32_t, const std::string&)> m_responseCallback;
     QString m_modelName;
     QString m_apiKey;
+    QString m_requestURL;
     QList<QString> m_context;
     QStringList m_queuedPrompts;
 };
 
-#endif // CHATGPT_H
+#endif // CHATAPI_H

gpt4all-chat/chatllm.cpp

@@ -1,6 +1,6 @@
 #include "chatllm.h"
 #include "chat.h"
-#include "chatgpt.h"
+#include "chatapi.h"
 #include "localdocs.h"
 #include "modellist.h"
 #include "network.h"
@@ -12,7 +12,6 @@
 
 #define GPTJ_INTERNAL_STATE_VERSION 0
 #define LLAMA_INTERNAL_STATE_VERSION 0
-#define BERT_INTERNAL_STATE_VERSION 0
 
 class LLModelStore {
 public:
@@ -214,7 +213,6 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     if (isModelLoaded() && this->modelInfo() == modelInfo)
         return true;
 
-    bool isChatGPT = modelInfo.isOnline; // right now only chatgpt is offered for online chat models...
     QString filePath = modelInfo.dirpath + modelInfo.filename();
     QFileInfo fileInfo(filePath);
 
@@ -280,19 +278,23 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     m_llModelInfo.fileInfo = fileInfo;
 
     if (fileInfo.exists()) {
-        if (isChatGPT) {
+        if (modelInfo.isOnline) {
             QString apiKey;
-            QString chatGPTModel = fileInfo.completeBaseName().remove(0, 8); // remove the chatgpt- prefix
+            QString modelName;
             {
                 QFile file(filePath);
                 file.open(QIODeviceBase::ReadOnly | QIODeviceBase::Text);
                 QTextStream stream(&file);
-                apiKey = stream.readAll();
-                file.close();
+                QString text = stream.readAll();
+                QJsonDocument doc = QJsonDocument::fromJson(text.toUtf8());
+                QJsonObject obj = doc.object();
+                apiKey = obj["apiKey"].toString();
+                modelName = obj["modelName"].toString();
             }
-            m_llModelType = LLModelType::CHATGPT_;
-            ChatGPT *model = new ChatGPT();
-            model->setModelName(chatGPTModel);
+            m_llModelType = LLModelType::API_;
+            ChatAPI *model = new ChatAPI();
+            model->setModelName(modelName);
+            model->setRequestURL(modelInfo.url());
             model->setAPIKey(apiKey);
             m_llModelInfo.model = model;
         } else {
@@ -386,7 +388,6 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     switch (m_llModelInfo.model->implementation().modelType()[0]) {
                     case 'L': m_llModelType = LLModelType::LLAMA_; break;
                     case 'G': m_llModelType = LLModelType::GPTJ_; break;
-                    case 'B': m_llModelType = LLModelType::BERT_; break;
                     default:
                         {
                             delete m_llModelInfo.model;
@@ -470,7 +471,7 @@ void ChatLLM::regenerateResponse()
 {
     // ChatGPT uses a different semantic meaning for n_past than local models. For ChatGPT, the meaning
     // of n_past is of the number of prompt/response pairs, rather than for total tokens.
-    if (m_llModelType == LLModelType::CHATGPT_)
+    if (m_llModelType == LLModelType::API_)
         m_ctx.n_past -= 1;
     else
         m_ctx.n_past -= m_promptResponseTokens;
@@ -840,7 +841,6 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
         switch (m_llModelType) {
         case GPTJ_: stream << GPTJ_INTERNAL_STATE_VERSION; break;
         case LLAMA_: stream << LLAMA_INTERNAL_STATE_VERSION; break;
-        case BERT_: stream << BERT_INTERNAL_STATE_VERSION; break;
         default: Q_UNREACHABLE();
         }
     }
@@ -961,12 +961,12 @@ void ChatLLM::saveState()
     if (!isModelLoaded())
         return;
 
-    if (m_llModelType == LLModelType::CHATGPT_) {
+    if (m_llModelType == LLModelType::API_) {
         m_state.clear();
         QDataStream stream(&m_state, QIODeviceBase::WriteOnly);
-        stream.setVersion(QDataStream::Qt_6_5);
-        ChatGPT *chatGPT = static_cast<ChatGPT*>(m_llModelInfo.model);
-        stream << chatGPT->context();
+        stream.setVersion(QDataStream::Qt_6_4);
+        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model);
+        stream << chatAPI->context();
         return;
     }
 
@@ -983,13 +983,13 @@ void ChatLLM::restoreState()
     if (!isModelLoaded())
         return;
 
-    if (m_llModelType == LLModelType::CHATGPT_) {
+    if (m_llModelType == LLModelType::API_) {
         QDataStream stream(&m_state, QIODeviceBase::ReadOnly);
-        stream.setVersion(QDataStream::Qt_6_5);
-        ChatGPT *chatGPT = static_cast<ChatGPT*>(m_llModelInfo.model);
+        stream.setVersion(QDataStream::Qt_6_4);
+        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model);
         QList<QString> context;
         stream >> context;
-        chatGPT->setContext(context);
+        chatAPI->setContext(context);
         m_state.clear();
         m_state.squeeze();
         return;

gpt4all-chat/chatllm.h

@@ -12,8 +12,7 @@
 enum LLModelType {
     GPTJ_,
     LLAMA_,
-    CHATGPT_,
-    BERT_,
+    API_,
 };
 
 struct LLModelInfo {

gpt4all-chat/download.cpp

@@ -182,8 +182,17 @@ void Download::installModel(const QString &modelFile, const QString &apiKey)
     QString filePath = MySettings::globalInstance()->modelPath() + modelFile;
     QFile file(filePath);
     if (file.open(QIODeviceBase::WriteOnly | QIODeviceBase::Text)) {
+
+        QJsonObject obj;
+        QString modelName(modelFile);
+        modelName.remove(0, 8); // strip "gpt4all-" prefix
+        modelName.chop(7); // strip ".rmodel" extension
+        obj.insert("apiKey", apiKey);
+        obj.insert("modelName", modelName);
+        QJsonDocument doc(obj);
+
         QTextStream stream(&file);
-        stream << apiKey;
+        stream << doc.toJson();
         file.close();
         ModelList::globalInstance()->updateModelsFromDirectory();
     }

gpt4all-chat/embllm.cpp

@@ -27,7 +27,7 @@ void EmbeddingLLMWorker::wait()
 
 bool EmbeddingLLMWorker::loadModel()
 {
-    const EmbeddingModels *embeddingModels = ModelList::globalInstance()->embeddingModels();
+    const EmbeddingModels *embeddingModels = ModelList::globalInstance()->installedEmbeddingModels();
     if (!embeddingModels->count())
         return false;
 
@@ -41,7 +41,8 @@ bool EmbeddingLLMWorker::loadModel()
         return false;
     }
 
-    bool isNomic = fileInfo.fileName().startsWith("nomic");
+    auto filename = fileInfo.fileName();
+    bool isNomic = filename.startsWith("nomic-") && filename.endsWith(".txt");
     if (isNomic) {
         QFile file(filePath);
         file.open(QIODeviceBase::ReadOnly | QIODeviceBase::Text);
@@ -52,16 +53,18 @@ bool EmbeddingLLMWorker::loadModel()
     }
 
     m_model = LLModel::Implementation::construct(filePath.toStdString());
+    // NOTE: explicitly loads model on CPU to avoid GPU OOM
+    // TODO(cebtenzzre): support GPU-accelerated embeddings
     bool success = m_model->loadModel(filePath.toStdString(), 2048, 0);
     if (!success) {
-        qWarning() << "WARNING: Could not load sbert";
+        qWarning() << "WARNING: Could not load embedding model";
         delete m_model;
         m_model = nullptr;
         return false;
     }
 
-    if (m_model->implementation().modelType() != "Bert") {
-        qWarning() << "WARNING: Model type is not sbert";
+    if (!m_model->supportsEmbedding()) {
+        qWarning() << "WARNING: Model type does not support embeddings";
         delete m_model;
         m_model = nullptr;
         return false;
@@ -79,21 +82,49 @@ bool EmbeddingLLMWorker::isNomic() const
     return !m_nomicAPIKey.isEmpty();
 }
 
+// this function is always called for retrieval tasks
 std::vector<float> EmbeddingLLMWorker::generateSyncEmbedding(const QString &text)
 {
     if (!hasModel() && !loadModel()) {
         qWarning() << "WARNING: Could not load model for embeddings";
-        return std::vector<float>();
+        return {};
     }
 
     if (isNomic()) {
         qWarning() << "WARNING: Request to generate sync embeddings for non-local model invalid";
-        return std::vector<float>();
+        return {};
     }
 
-    return m_model->embedding(text.toStdString());
+    std::vector<float> embedding(m_model->embeddingSize());
+    try {
+        m_model->embed({text.toStdString()}, embedding.data(), true);
+    } catch (const std::exception &e) {
+        qWarning() << "WARNING: LLModel::embed failed: " << e.what();
+        return {};
+    }
+    return embedding;
 }
 
+void EmbeddingLLMWorker::sendAtlasRequest(const QStringList &texts, const QString &taskType, QVariant userData) {
+    QJsonObject root;
+    root.insert("model", "nomic-embed-text-v1");
+    root.insert("texts", QJsonArray::fromStringList(texts));
+    root.insert("task_type", taskType);
+
+    QJsonDocument doc(root);
+
+    QUrl nomicUrl("https://api-atlas.nomic.ai/v1/embedding/text");
+    const QString authorization = QString("Bearer %1").arg(m_nomicAPIKey).trimmed();
+    QNetworkRequest request(nomicUrl);
+    request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
+    request.setRawHeader("Authorization", authorization.toUtf8());
+    request.setAttribute(QNetworkRequest::User, userData);
+    QNetworkReply *reply = m_networkManager->post(request, doc.toJson(QJsonDocument::Compact));
+    connect(qApp, &QCoreApplication::aboutToQuit, reply, &QNetworkReply::abort);
+    connect(reply, &QNetworkReply::finished, this, &EmbeddingLLMWorker::handleFinished);
+}
+
+// this function is always called for retrieval tasks
 void EmbeddingLLMWorker::requestSyncEmbedding(const QString &text)
 {
     if (!hasModel() && !loadModel()) {
@@ -108,25 +139,10 @@ void EmbeddingLLMWorker::requestSyncEmbedding(const QString &text)
 
     Q_ASSERT(hasModel());
 
-    QJsonObject root;
-    root.insert("model", "nomic-embed-text-v1");
-    QJsonArray texts;
-    texts.append(text);
-    root.insert("texts", texts);
-    root.insert("task_type", "search_query");
-
-    QJsonDocument doc(root);
-
-    QUrl nomicUrl("https://api-atlas.nomic.ai/v1/embedding/text");
-    const QString authorization = QString("Bearer %1").arg(m_nomicAPIKey).trimmed();
-    QNetworkRequest request(nomicUrl);
-    request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
-    request.setRawHeader("Authorization", authorization.toUtf8());
-    QNetworkReply *reply = m_networkManager->post(request, doc.toJson(QJsonDocument::Compact));
-    connect(qApp, &QCoreApplication::aboutToQuit, reply, &QNetworkReply::abort);
-    connect(reply, &QNetworkReply::finished, this, &EmbeddingLLMWorker::handleFinished);
+    sendAtlasRequest({text}, "search_query");
 }
 
+// this function is always called for storage into the database
 void EmbeddingLLMWorker::requestAsyncEmbedding(const QVector<EmbeddingChunk> &chunks)
 {
     if (!hasModel() && !loadModel()) {
@@ -141,33 +157,24 @@ void EmbeddingLLMWorker::requestAsyncEmbedding(const QVector<EmbeddingChunk> &ch
             EmbeddingResult result;
             result.folder_id = c.folder_id;
             result.chunk_id = c.chunk_id;
-            result.embedding = m_model->embedding(c.chunk.toStdString());
+            // TODO(cebtenzzre): take advantage of batched embeddings
+            result.embedding.resize(m_model->embeddingSize());
+            try {
+                m_model->embed({c.chunk.toStdString()}, result.embedding.data(), false);
+            } catch (const std::exception &e) {
+                qWarning() << "WARNING: LLModel::embed failed:" << e.what();
+                return;
+            }
             results << result;
         }
         emit embeddingsGenerated(results);
         return;
     };
 
-    QJsonObject root;
-    root.insert("model", "nomic-embed-text-v1");
-    QJsonArray texts;
-
-    for (auto c : chunks)
+    QStringList texts;
+    for (auto &c: chunks)
         texts.append(c.chunk);
-    root.insert("texts", texts);
-
-    QJsonDocument doc(root);
-
-    QUrl nomicUrl("https://api-atlas.nomic.ai/v1/embedding/text");
-    const QString authorization = QString("Bearer %1").arg(m_nomicAPIKey).trimmed();
-    QNetworkRequest request(nomicUrl);
-    request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
-    request.setRawHeader("Authorization", authorization.toUtf8());
-    request.setAttribute(QNetworkRequest::User, QVariant::fromValue(chunks));
-
-    QNetworkReply *reply = m_networkManager->post(request, doc.toJson(QJsonDocument::Compact));
-    connect(qApp, &QCoreApplication::aboutToQuit, reply, &QNetworkReply::abort);
-    connect(reply, &QNetworkReply::finished, this, &EmbeddingLLMWorker::handleFinished);
+    sendAtlasRequest(texts, "search_document", QVariant::fromValue(chunks));
 }
 
 std::vector<float> jsonArrayToVector(const QJsonArray &jsonArray) {

gpt4all-chat/embllm.h

@@ -1,10 +1,11 @@
 #ifndef EMBLLM_H
 #define EMBLLM_H
 
-#include <QObject>
-#include <QThread>
-#include <QNetworkReply>
 #include <QNetworkAccessManager>
+#include <QNetworkReply>
+#include <QObject>
+#include <QStringList>
+#include <QThread>
 
 #include "../gpt4all-backend/llmodel.h"
 
@@ -51,6 +52,8 @@ private Q_SLOTS:
     void handleFinished();
 
 private:
+    void sendAtlasRequest(const QStringList &texts, const QString &taskType, QVariant userData = {});
+
     QString m_nomicAPIKey;
     QNetworkAccessManager *m_networkManager;
     std::vector<float> m_lastResponse;

gpt4all-chat/icons/chat.svg

@@ -0,0 +1,6 @@
+
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M144 208c-17.7 0-32 14.3-32 32s14.3 32 32 32 32-14.3 32-32-14.3-32-32-32zm112 0c-17.7 0-32 14.3-32 32s14.3 32 32 32 32-14.3 32-32-14.3-32-32-32zm112 0c-17.7 0-32 14.3-32 32s14.3 32 32 32 32-14.3 32-32-14.3-32-32-32zM256 32C114.6 32 0 125.1 0 240c0 47.6 19.9 91.2 52.9 126.3C38 405.7 7 439.1 6.5 439.5c-6.6 7-8.4 17.2-4.6 26S14.4 480 24 480c61.5 0 110-25.7 139.1-46.3C192 442.8 223.2 448 256 448c141.4 0 256-93.1 256-208S397.4 32 256 32zm0 368c-26.7 0-53.1-4.1-78.4-12.1l-22.7-7.2-19.5 13.8c-14.3 10.1-33.9 21.4-57.5 29 7.3-12.1 14.4-25.7 19.9-40.2l10.6-28.1-20.6-21.8C69.7 314.1 48 282.2 48 240c0-88.2 93.3-160 208-160s208 71.8 208 160-93.3 160-208 160z"/></svg>
+<!--
+Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
+License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+-->

gpt4all-chat/icons/info.svg

@@ -0,0 +1,6 @@
+
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M256 8C119.043 8 8 119.083 8 256c0 136.997 111.043 248 248 248s248-111.003 248-248C504 119.083 392.957 8 256 8zm0 110c23.196 0 42 18.804 42 42s-18.804 42-42 42-42-18.804-42-42 18.804-42 42-42zm56 254c0 6.627-5.373 12-12 12h-88c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h12v-64h-12c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h64c6.627 0 12 5.373 12 12v100h12c6.627 0 12 5.373 12 12v24z"/></svg>
+<!--
+Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
+License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+-->

gpt4all-chat/icons/left_panel_closed.svg

@@ -0,0 +1,3 @@
+<svg width="64" height="64" viewBox="0 0 64 64" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M23 16H54C55.6569 16 57 17.3431 57 19V45C57 46.6569 55.6569 48 54 48H23V16ZM20 16H10C8.34315 16 7 17.3431 7 19V45C7 46.6569 8.34315 48 10 48H20V16ZM4 19C4 15.6863 6.68629 13 10 13H54C57.3137 13 60 15.6863 60 19V45C60 48.3137 57.3137 51 54 51H10C6.68629 51 4 48.3137 4 45V19Z" fill="black"/>
+</svg>

gpt4all-chat/icons/left_panel_open.svg

@@ -0,0 +1,3 @@
+<svg width="64" height="64" viewBox="0 0 64 64" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M23 16H54C55.6569 16 57 17.3431 57 19V45C57 46.6569 55.6569 48 54 48H23V16ZM4 19C4 15.6863 6.68629 13 10 13H54C57.3137 13 60 15.6863 60 19V45C60 48.3137 57.3137 51 54 51H10C6.68629 51 4 48.3137 4 45V19Z" fill="black"/>
+</svg>

gpt4all-chat/icons/search.svg

@@ -0,0 +1,6 @@
+
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M505 442.7L405.3 343c-4.5-4.5-10.6-7-17-7H372c27.6-35.3 44-79.7 44-128C416 93.1 322.9 0 208 0S0 93.1 0 208s93.1 208 208 208c48.3 0 92.7-16.4 128-44v16.3c0 6.4 2.5 12.5 7 17l99.7 99.7c9.4 9.4 24.6 9.4 33.9 0l28.3-28.3c9.4-9.4 9.4-24.6.1-34zM208 336c-70.7 0-128-57.2-128-128 0-70.7 57.2-128 128-128 70.7 0 128 57.2 128 128 0 70.7-57.2 128-128 128z"/></svg>
+<!--
+Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
+License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+-->

gpt4all-chat/llm.cpp

@@ -1,4 +1,5 @@
 #include "llm.h"
+#include "../gpt4all-backend/llmodel.h"
 #include "../gpt4all-backend/sysinfo.h"
 
 #include <QCoreApplication>
@@ -25,22 +26,8 @@ LLM *LLM::globalInstance()
 
 LLM::LLM()
     : QObject{nullptr}
-    , m_compatHardware(true)
+    , m_compatHardware(LLModel::Implementation::hasSupportedCPU())
 {
-#if defined(__x86_64__)
-    #ifndef _MSC_VER
-        const bool minimal(__builtin_cpu_supports("avx"));
-    #else
-        int cpuInfo[4];
-        __cpuid(cpuInfo, 1);
-        const bool minimal(cpuInfo[2] & (1 << 28));
-    #endif
-#else
-    const bool minimal = true; // Don't know how to handle non-x86_64
-#endif
-
-    m_compatHardware = minimal;
-
     QNetworkInformation::loadDefaultBackend();
     auto * netinfo = QNetworkInformation::instance();
     if (netinfo) {

gpt4all-chat/metadata/models3.json

@@ -29,7 +29,7 @@
     "description": "<strong>Strong overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
     "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.gguf2.Q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
-    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI. For multi-step problems, write out your reasoning for each step.\n<|im_end|>"
+    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI. For multi-step problems, write out your reasoning for each step.\n<|im_end|>\n"
   },
   {
     "order": "c",
@@ -42,7 +42,7 @@
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "Mistral",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<strong>Strong overall fast instruction following model</strong><br><ul><li>Fast responses</li><li>Trained by Mistral AI<li>Uncensored</li><li>Licensed for commercial use</li></ul>",
     "url": "https://gpt4all.io/models/gguf/mistral-7b-instruct-v0.1.Q4_0.gguf",
     "promptTemplate": "[INST] %1 [/INST]"
@@ -58,7 +58,7 @@
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "Falcon",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<strong>Very fast model with good quality</strong><br><ul><li>Fastest responses</li><li>Instruction based</li><li>Trained by TII<li>Finetuned by Nomic AI<li>Licensed for commercial use</ul>",
     "url": "https://gpt4all.io/models/gguf/gpt4all-falcon-newbpe-q4_0.gguf",
     "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
@@ -74,7 +74,7 @@
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "LLaMA2",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<ul><li>Instruction based<li>Trained by Microsoft<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/orca-2-7b.Q4_0.gguf"
   },
@@ -89,7 +89,7 @@
     "parameters": "13 billion",
     "quant": "q4_0",
     "type": "LLaMA2",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<ul><li>Instruction based<li>Trained by Microsoft<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/orca-2-13b.Q4_0.gguf"
   },
@@ -104,7 +104,7 @@
     "parameters": "13 billion",
     "quant": "q4_0",
     "type": "LLaMA2",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<strong>Strong overall larger model</strong><br><ul><li>Instruction based<li>Gives very long responses<li>Finetuned with only 1k of high-quality data<li>Trained by Microsoft and Peking University<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/wizardlm-13b-v1.2.Q4_0.gguf"
   },
@@ -119,7 +119,7 @@
     "parameters": "13 billion",
     "quant": "q4_0",
     "type": "LLaMA2",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<strong>Extremely good model</strong><br><ul><li>Instruction based<li>Gives long responses<li>Curated with 300,000 uncensored instructions<li>Trained by Nous Research<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/nous-hermes-llama2-13b.Q4_0.gguf",
     "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
@@ -135,7 +135,7 @@
     "parameters": "13 billion",
     "quant": "q4_0",
     "type": "LLaMA",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "description": "<strong>Very good overall model</strong><br><ul><li>Instruction based<li>Based on the same dataset as Groovy<li>Slower than Groovy, with higher quality responses<li>Trained by Nomic AI<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf"
   },
@@ -154,7 +154,7 @@
     "description": "<strong>Good model with novel architecture</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/mpt-7b-chat-newbpe-q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
-    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
+    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
   },
   {
     "order": "j",
@@ -170,7 +170,7 @@
     "description": "<strong>Good model with novel architecture</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/mpt-7b-chat.gguf4.Q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
-    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
+    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
   },
   {
     "order": "k",
@@ -200,7 +200,7 @@
     "parameters": "3 billion",
     "quant": "q4_0",
     "type": "Replit",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "promptTemplate": "%1",
     "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>Licensed for commercial use<li>WARNING: Not available for chat GUI</ul>",
     "url": "https://gpt4all.io/models/gguf/replit-code-v1_5-3b-newbpe-q4_0.gguf"
@@ -217,7 +217,7 @@
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "Starcoder",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "promptTemplate": "%1",
     "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>WARNING: Not available for chat GUI</ul>",
     "url": "https://gpt4all.io/models/gguf/starcoder-newbpe-q4_0.gguf"
@@ -234,7 +234,7 @@
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "LLaMA",
-    "systemPrompt": " ",
+    "systemPrompt": "",
     "promptTemplate": "%1",
     "description": "<strong>Trained on collection of Python and TypeScript</strong><br><ul><li>Code completion based<li>WARNING: Not available for chat GUI</li>",
     "url": "https://gpt4all.io/models/gguf/rift-coder-v0-7b-q4_0.gguf"
@@ -247,14 +247,31 @@
     "filename": "all-MiniLM-L6-v2-f16.gguf",
     "filesize": "45887744",
     "requires": "2.5.0",
+    "removedIn": "2.7.4",
     "ramrequired": "1",
     "parameters": "40 million",
     "quant": "f16",
     "type": "Bert",
-    "systemPrompt": " ",
+    "embeddingModel": true,
+    "systemPrompt": "",
     "description": "<strong>LocalDocs text embeddings model</strong><br><ul><li>For use with LocalDocs feature<li>Used for retrieval augmented generation (RAG)",
     "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf"
   },
+  {
+    "order": "o",
+    "md5sum": "dd90e2cb7f8e9316ac3796cece9883b5",
+    "name": "SBert",
+    "filename": "all-MiniLM-L6-v2.gguf2.f16.gguf",
+    "filesize": "45949216",
+    "requires": "2.7.4",
+    "ramrequired": "1",
+    "parameters": "40 million",
+    "quant": "f16",
+    "type": "Bert",
+    "embeddingModel": true,
+    "description": "<strong>LocalDocs text embeddings model</strong><br><ul><li>For use with LocalDocs feature<li>Used for retrieval augmented generation (RAG)",
+    "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2.gguf2.f16.gguf"
+  },
   {
     "order": "p",
     "md5sum": "919de4dd6f25351bcb0223790db1932d",
@@ -270,5 +287,55 @@
     "url": "https://huggingface.co/TheBloke/em_german_mistral_v01-GGUF/resolve/main/em_german_mistral_v01.Q4_0.gguf",
     "promptTemplate": "USER: %1 ASSISTANT: ",
     "systemPrompt": "Du bist ein hilfreicher Assistent. "
+  },
+  {
+    "order": "q",
+    "md5sum": "60ea031126f82db8ddbbfecc668315d2",
+    "disableGUI": "true",
+    "name": "Nomic Embed Text v1",
+    "filename": "nomic-embed-text-v1.f16.gguf",
+    "filesize": "274290560",
+    "requires": "2.7.4",
+    "ramrequired": "1",
+    "parameters": "137 million",
+    "quant": "f16",
+    "type": "Bert",
+    "embeddingModel": true,
+    "systemPrompt": "",
+    "description": "nomic-embed-text-v1",
+    "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.f16.gguf"
+  },
+  {
+    "order": "r",
+    "md5sum": "a5401e7f7e46ed9fcaed5b60a281d547",
+    "disableGUI": "true",
+    "name": "Nomic Embed Text v1.5",
+    "filename": "nomic-embed-text-v1.5.f16.gguf",
+    "filesize": "274290560",
+    "requires": "2.7.4",
+    "ramrequired": "1",
+    "parameters": "137 million",
+    "quant": "f16",
+    "type": "Bert",
+    "embeddingModel": true,
+    "systemPrompt": "",
+    "description": "nomic-embed-text-v1.5",
+    "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.5.f16.gguf"
+  },
+  {
+    "order": "g",
+    "md5sum": "31b47b4e8c1816b62684ac3ca373f9e1",
+    "name": "Ghost 7B v0.9.1",
+    "filename": "ghost-7b-v0.9.1-Q4_0.gguf",
+    "filesize": "4108916960",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "description": "<strong>Ghost 7B v0.9.1</strong> fast, powerful and smooth for Vietnamese and English languages.",
+    "url": "https://huggingface.co/lamhieu/ghost-7b-v0.9.1-gguf/resolve/main/ghost-7b-v0.9.1-Q4_0.gguf",
+    "promptTemplate": "<|user|>\n%1</s>\n<|assistant|>\n%2</s>\n",
+    "systemPrompt": "<|system|>\nYou are Ghost created by Lam Hieu. You are a helpful and knowledgeable assistant. You like to help and always give honest information, in its original language. In communication, you are always respectful, equal and promote positive behavior.\n</s>"
   }
 ]

gpt4all-chat/metadata/release.json

@@ -724,6 +724,34 @@
 * Jared Van Bortel (Nomic AI)
 * Adam Treat (Nomic AI)
 * Community (beta testers, bug reporters, bindings authors)
+"
+  },
+  {
+    "version": "2.7.3",
+    "notes":
+"
+* Fix for network reachability unknown
+* Fix undefined behavior with resetContext
+* Fix ChatGPT which was broken with previous release
+* Fix for clean up of chat llm thread destruction
+* Display of model loading warnings
+* Fix for issue 2080 where the GUI appears to hang when a chat is deleted
+* Fix for issue 2077 better responsiveness of model download dialog when download is taking place
+* Fix for issue 2092 don't include models that are disabled for GUI in application default model list
+* Fix for issue 2087 where cloned modelds were lost and listed in download dialog erroneously
+* Fix for MPT models without duplicated token embd weight
+* New feature with api server port setting
+* Fix for issue 2024 where combobox for model settings uses currently used model by default
+* Clean up settings properly for removed models and don't list stale model settings in download dialog
+* Fix for issue 2105 where the cancel button was not working for discovered model downloads
+",
+    "contributors":
+"
+* Christopher Barrera
+* Daniel Alencar
+* Jared Van Bortel (Nomic AI)
+* Adam Treat (Nomic AI)
+* Community (beta testers, bug reporters, bindings authors)
 "
   }
 ]

gpt4all-chat/modellist.cpp

@@ -10,8 +10,10 @@
 
 //#define USE_LOCAL_MODELSJSON
 
-#define DEFAULT_EMBEDDING_MODEL "all-MiniLM-L6-v2-f16.gguf"
-#define NOMIC_EMBEDDING_MODEL "nomic-embed-text-v1.txt"
+const char * const KNOWN_EMBEDDING_MODELS[] {
+    "all-MiniLM-L6-v2.gguf2.f16.gguf",
+    "nomic-embed-text-v1.txt",
+};
 
 QString ModelInfo::id() const
 {
@@ -223,13 +225,14 @@ void ModelInfo::setContextLength(int l)
 
 int ModelInfo::maxContextLength() const
 {
+    if (!installed || isOnline) return -1;
     if (m_maxContextLength != -1) return m_maxContextLength;
     auto path = (dirpath + filename()).toStdString();
-    int layers = LLModel::Implementation::maxContextLength(path);
-    if (layers < 0) {
-        layers = 4096; // fallback value
+    int n_ctx = LLModel::Implementation::maxContextLength(path);
+    if (n_ctx < 0) {
+        n_ctx = 4096; // fallback value
     }
-    m_maxContextLength = layers;
+    m_maxContextLength = n_ctx;
     return m_maxContextLength;
 }
 
@@ -306,9 +309,11 @@ bool ModelInfo::shouldSaveMetadata() const
     return installed && (isClone() || isDiscovered() || description() == "" /*indicates sideloaded*/);
 }
 
-EmbeddingModels::EmbeddingModels(QObject *parent)
+EmbeddingModels::EmbeddingModels(QObject *parent, bool requireInstalled)
     : QSortFilterProxyModel(parent)
 {
+    m_requireInstalled = requireInstalled;
+
     connect(this, &EmbeddingModels::rowsInserted, this, &EmbeddingModels::countChanged);
     connect(this, &EmbeddingModels::rowsRemoved, this, &EmbeddingModels::countChanged);
     connect(this, &EmbeddingModels::modelReset, this, &EmbeddingModels::countChanged);
@@ -319,36 +324,41 @@ bool EmbeddingModels::filterAcceptsRow(int sourceRow,
                                        const QModelIndex &sourceParent) const
 {
     QModelIndex index = sourceModel()->index(sourceRow, 0, sourceParent);
-    bool isInstalled = sourceModel()->data(index, ModelList::InstalledRole).toBool();
-    bool isEmbedding = sourceModel()->data(index, ModelList::FilenameRole).toString() == DEFAULT_EMBEDDING_MODEL ||
-        sourceModel()->data(index, ModelList::FilenameRole).toString() == NOMIC_EMBEDDING_MODEL;
-    return isInstalled && isEmbedding;
+    bool isEmbeddingModel = sourceModel()->data(index, ModelList::IsEmbeddingModelRole).toBool();
+    bool installed = sourceModel()->data(index, ModelList::InstalledRole).toBool();
+    QString filename = sourceModel()->data(index, ModelList::FilenameRole).toString();
+    auto &known = KNOWN_EMBEDDING_MODELS;
+    if (std::find(known, std::end(known), filename.toStdString()) == std::end(known))
+        return false; // we are currently not prepared to support other embedding models
+
+    return isEmbeddingModel && (!m_requireInstalled || installed);
 }
 
-int EmbeddingModels::count() const
+int EmbeddingModels::defaultModelIndex() const
 {
-    return rowCount();
+    auto *sourceListModel = qobject_cast<const ModelList*>(sourceModel());
+    if (!sourceListModel) return -1;
+
+    int rows = sourceListModel->rowCount();
+    for (int i = 0; i < rows; ++i) {
+        if (filterAcceptsRow(i, sourceListModel->index(i, 0).parent()))
+            return i;
+    }
+
+    return -1;
 }
 
 ModelInfo EmbeddingModels::defaultModelInfo() const
 {
-    if (!sourceModel())
-        return ModelInfo();
+    auto *sourceListModel = qobject_cast<const ModelList*>(sourceModel());
+    if (!sourceListModel) return ModelInfo();
 
-    const ModelList *sourceListModel = qobject_cast<const ModelList*>(sourceModel());
-    if (!sourceListModel)
-        return ModelInfo();
+    int i = defaultModelIndex();
+    if (i < 0) return ModelInfo();
 
-    const int rows = sourceListModel->rowCount();
-    for (int i = 0; i < rows; ++i) {
-        QModelIndex sourceIndex = sourceListModel->index(i, 0);
-        if (filterAcceptsRow(i, sourceIndex.parent())) {
-            const QString id = sourceListModel->data(sourceIndex, ModelList::IdRole).toString();
-            return sourceListModel->modelInfo(id);
-        }
-    }
-
-    return ModelInfo();
+    QModelIndex sourceIndex = sourceListModel->index(i, 0);
+    auto id = sourceListModel->data(sourceIndex, ModelList::IdRole).toString();
+    return sourceListModel->modelInfo(id);
 }
 
 InstalledModels::InstalledModels(QObject *parent)
@@ -365,13 +375,9 @@ bool InstalledModels::filterAcceptsRow(int sourceRow,
 {
     QModelIndex index = sourceModel()->index(sourceRow, 0, sourceParent);
     bool isInstalled = sourceModel()->data(index, ModelList::InstalledRole).toBool();
-    bool showInGUI = !sourceModel()->data(index, ModelList::DisableGUIRole).toBool();
-    return isInstalled && showInGUI;
-}
-
-int InstalledModels::count() const
-{
-    return rowCount();
+    bool isEmbeddingModel = sourceModel()->data(index, ModelList::IsEmbeddingModelRole).toBool();
+    // list installed chat models
+    return isInstalled && !isEmbeddingModel;
 }
 
 DownloadableModels::DownloadableModels(QObject *parent)
@@ -432,8 +438,9 @@ ModelList *ModelList::globalInstance()
 
 ModelList::ModelList()
     : QAbstractListModel(nullptr)
-    , m_embeddingModels(new EmbeddingModels(this))
+    , m_embeddingModels(new EmbeddingModels(this, false /* all models */))
     , m_installedModels(new InstalledModels(this))
+    , m_installedEmbeddingModels(new EmbeddingModels(this, true /* installed models */))
     , m_downloadableModels(new DownloadableModels(this))
     , m_asyncModelRequestOngoing(false)
     , m_discoverLimit(20)
@@ -445,6 +452,7 @@ ModelList::ModelList()
 {
     m_embeddingModels->setSourceModel(this);
     m_installedModels->setSourceModel(this);
+    m_installedEmbeddingModels->setSourceModel(this);
     m_downloadableModels->setSourceModel(this);
 
     connect(MySettings::globalInstance(), &MySettings::modelPathChanged, this, &ModelList::updateModelsFromDirectory);
@@ -494,8 +502,8 @@ const QList<QString> ModelList::userDefaultModelList() const
     bool foundUserDefault = false;
     for (ModelInfo *info : m_models) {
 
-        // Only installed models that are meant for GUI are suitable as a default
-        if (!info->installed || info->disableGUI)
+        // Only installed chat models are suitable as a default
+        if (!info->installed || info->isEmbeddingModel)
             continue;
 
         if (info->id() == userDefaultModelName) {
@@ -516,13 +524,7 @@ const QList<QString> ModelList::userDefaultModelList() const
 
 int ModelList::defaultEmbeddingModelIndex() const
 {
-    QMutexLocker locker(&m_mutex);
-    for (int i = 0; i < m_models.size(); ++i) {
-        const ModelInfo *info = m_models.at(i);
-        const bool isEmbedding = info->filename() == DEFAULT_EMBEDDING_MODEL;
-        if (isEmbedding) return i;
-    }
-    return -1;
+    return embeddingModels()->defaultModelIndex();
 }
 
 ModelInfo ModelList::defaultModelInfo() const
@@ -692,8 +694,6 @@ QVariant ModelList::dataInternal(const ModelInfo *info, int role) const
             return info->isDefault;
         case OnlineRole:
             return info->isOnline;
-        case DisableGUIRole:
-            return info->disableGUI;
         case DescriptionRole:
             return info->description();
         case RequiresVersionRole:
@@ -730,6 +730,8 @@ QVariant ModelList::dataInternal(const ModelInfo *info, int role) const
             return info->isClone();
         case IsDiscoveredRole:
             return info->isDiscovered();
+        case IsEmbeddingModelRole:
+            return info->isEmbeddingModel;
         case TemperatureRole:
             return info->temperature();
         case TopPRole:
@@ -844,8 +846,6 @@ void ModelList::updateData(const QString &id, const QVector<QPair<int, QVariant>
                 info->isDefault = value.toBool(); break;
             case OnlineRole:
                 info->isOnline = value.toBool(); break;
-            case DisableGUIRole:
-                info->disableGUI = value.toBool(); break;
             case DescriptionRole:
                 info->setDescription(value.toString()); break;
             case RequiresVersionRole:
@@ -900,6 +900,8 @@ void ModelList::updateData(const QString &id, const QVector<QPair<int, QVariant>
                     }
                     break;
                 }
+            case IsEmbeddingModelRole:
+                info->isEmbeddingModel = value.toBool(); break;
             case TemperatureRole:
                 info->setTemperature(value.toDouble()); break;
             case TopPRole:
@@ -952,11 +954,21 @@ void ModelList::updateData(const QString &id, const QVector<QPair<int, QVariant>
         }
 
         // Extra guarantee that these always remains in sync with filesystem
-        const QFileInfo fileInfo(info->dirpath + info->filename());
+        QString modelPath = info->dirpath + info->filename();
+        const QFileInfo fileInfo(modelPath);
         info->installed = fileInfo.exists();
         const QFileInfo incompleteInfo(incompleteDownloadPath(info->filename()));
         info->isIncomplete = incompleteInfo.exists();
 
+        // check installed, discovered/sideloaded models only (including clones)
+        if (!info->checkedEmbeddingModel && !info->isEmbeddingModel && info->installed
+            && (info->isDiscovered() || info->description().isEmpty()))
+        {
+            // read GGUF and decide based on model architecture
+            info->isEmbeddingModel = LLModel::Implementation::isEmbeddingModel(modelPath.toStdString());
+            info->checkedEmbeddingModel = true;
+        }
+
         if (shouldSort) {
             auto s = m_discoverSort;
             auto d = m_discoverSortDirection;
@@ -983,8 +995,11 @@ void ModelList::resortModel()
     emit layoutChanged();
 }
 
-void ModelList::updateDataByFilename(const QString &filename, const QVector<QPair<int, QVariant>> &data)
+void ModelList::updateDataByFilename(const QString &filename, QVector<QPair<int, QVariant>> data)
 {
+    if (data.isEmpty())
+        return; // no-op
+
     QVector<QString> modelsById;
     {
         QMutexLocker locker(&m_mutex);
@@ -1041,6 +1056,7 @@ QString ModelList::clone(const ModelInfo &model)
         { ModelList::FilenameRole, model.filename() },
         { ModelList::DirpathRole, model.dirpath },
         { ModelList::OnlineRole, model.isOnline },
+        { ModelList::IsEmbeddingModelRole, model.isEmbeddingModel },
         { ModelList::TemperatureRole, model.temperature() },
         { ModelList::TopPRole, model.topP() },
         { ModelList::MinPRole, model.minP() },
@@ -1156,6 +1172,44 @@ void ModelList::updateModelsFromDirectory()
     const QString exePath = QCoreApplication::applicationDirPath() + QDir::separator();
     const QString localPath = MySettings::globalInstance()->modelPath();
 
+    auto updateOldRemoteModels = [&](const QString& path) {
+        QDirIterator it(path, QDirIterator::Subdirectories);
+        while (it.hasNext()) {
+            it.next();
+            if (!it.fileInfo().isDir()) {
+                QString filename = it.fileName();
+                if (filename.startsWith("chatgpt-") && filename.endsWith(".txt")) {
+                    QString apikey;
+                    QString modelname(filename);
+                    modelname.chop(4); // strip ".txt" extension
+                    if (filename.startsWith("chatgpt-")) {
+                        modelname.remove(0, 8); // strip "chatgpt-" prefix
+                    }
+                    QFile file(path + filename);
+                    if (file.open(QIODevice::ReadWrite)) {
+                        QTextStream in(&file);
+                        apikey = in.readAll();
+                        file.close();
+                    }
+
+                    QJsonObject obj;
+                    obj.insert("apiKey", apikey);
+                    obj.insert("modelName", modelname);
+                    QJsonDocument doc(obj);
+
+                    auto newfilename = QString("gpt4all-%1.rmodel").arg(modelname);
+                    QFile newfile(path + newfilename);
+                    if (newfile.open(QIODevice::ReadWrite)) {
+                        QTextStream out(&newfile);
+                        out << doc.toJson();
+                        newfile.close();
+                    }
+                    file.remove();
+                }
+            }
+        }
+    };
+
     auto processDirectory = [&](const QString& path) {
         QDirIterator it(path, QDirIterator::Subdirectories);
         while (it.hasNext()) {
@@ -1164,9 +1218,7 @@ void ModelList::updateModelsFromDirectory()
             if (!it.fileInfo().isDir()) {
                 QString filename = it.fileName();
 
-                // All files that end with .bin and have 'ggml' somewhere in the name
-                if (((filename.endsWith(".bin") || filename.endsWith(".gguf")) && (/*filename.contains("ggml") ||*/ filename.contains("gguf")) && !filename.startsWith("incomplete"))
-                    || (filename.endsWith(".txt") && (filename.startsWith("chatgpt-") || filename.startsWith("nomic-")))) {
+                if ((filename.endsWith(".gguf") && !filename.startsWith("incomplete")) || filename.endsWith(".rmodel")) {
 
                     QString filePath = it.filePath();
                     QFileInfo info(filePath);
@@ -1192,8 +1244,7 @@ void ModelList::updateModelsFromDirectory()
                         QVector<QPair<int, QVariant>> data {
                             { InstalledRole, true },
                             { FilenameRole, filename },
-                            // FIXME: WE should change this to use a consistent filename for online models
-                            { OnlineRole, filename.startsWith("chatgpt-") || filename.startsWith("nomic-") },
+                            { OnlineRole, filename.endsWith(".rmodel") },
                             { DirpathRole, info.dir().absolutePath() + "/" },
                             { FilesizeRole, toFileSize(info.size()) },
                         };
@@ -1204,9 +1255,13 @@ void ModelList::updateModelsFromDirectory()
         }
     };
 
+
+    updateOldRemoteModels(exePath);
     processDirectory(exePath);
-    if (localPath != exePath)
+    if (localPath != exePath) {
+        updateOldRemoteModels(localPath);
         processDirectory(localPath);
+    }
 }
 
 #define MODELS_VERSION 3
@@ -1299,7 +1354,7 @@ void ModelList::handleModelsJsonDownloadErrorOccurred(QNetworkReply::NetworkErro
         return;
 
     qWarning() << QString("ERROR: Modellist download failed with error code \"%1-%2\"")
-                      .arg(code).arg(reply->errorString()).toStdString();
+                      .arg(code).arg(reply->errorString());
 }
 
 void ModelList::handleSslErrors(QNetworkReply *reply, const QList<QSslError> &errors)
@@ -1373,16 +1428,19 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
         QString parameters = obj["parameters"].toString();
         QString quant = obj["quant"].toString();
         QString type = obj["type"].toString();
+        bool isEmbeddingModel = obj["embeddingModel"].toBool();
+
+        // Some models aren't supported in the GUI at all
+        if (disableGUI)
+            continue;
 
         // If the current version is strictly less than required version, then skip
-        if (!requiresVersion.isEmpty() && compareVersions(currentVersion, requiresVersion) < 0) {
+        if (!requiresVersion.isEmpty() && compareVersions(currentVersion, requiresVersion) < 0)
             continue;
-        }
 
         // If the version removed is less than or equal to the current version, then skip
-        if (!versionRemoved.isEmpty() && compareVersions(versionRemoved, currentVersion) <= 0) {
+        if (!versionRemoved.isEmpty() && compareVersions(versionRemoved, currentVersion) <= 0)
             continue;
-        }
 
         modelFilesize = ModelList::toFileSize(modelFilesize.toULongLong());
 
@@ -1406,12 +1464,12 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             { ModelList::RequiresVersionRole, requiresVersion },
             { ModelList::VersionRemovedRole, versionRemoved },
             { ModelList::UrlRole, url },
-            { ModelList::DisableGUIRole, disableGUI },
             { ModelList::OrderRole, order },
             { ModelList::RamrequiredRole, ramrequired },
             { ModelList::ParametersRole, parameters },
             { ModelList::QuantRole, quant },
             { ModelList::TypeRole, type },
+            { ModelList::IsEmbeddingModelRole, isEmbeddingModel },
         };
         if (obj.contains("temperature"))
             data.append({ ModelList::TemperatureRole, obj["temperature"].toDouble() });
@@ -1448,7 +1506,7 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
     {
         const QString modelName = "ChatGPT-3.5 Turbo";
         const QString id = modelName;
-        const QString modelFilename = "chatgpt-gpt-3.5-turbo.txt";
+        const QString modelFilename = "gpt4all-gpt-3.5-turbo.rmodel";
         if (contains(modelFilename))
             changeId(modelFilename, id);
         if (!contains(id))
@@ -1460,12 +1518,13 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             { ModelList::OnlineRole, true },
             { ModelList::DescriptionRole,
              tr("<strong>OpenAI's ChatGPT model GPT-3.5 Turbo</strong><br>") + chatGPTDesc },
-            { ModelList::RequiresVersionRole, "2.4.2" },
+            { ModelList::RequiresVersionRole, "2.7.4" },
             { ModelList::OrderRole, "ca" },
             { ModelList::RamrequiredRole, 0 },
             { ModelList::ParametersRole, "?" },
             { ModelList::QuantRole, "NA" },
             { ModelList::TypeRole, "GPT" },
+            { ModelList::UrlRole, "https://api.openai.com/v1/chat/completions"},
         };
         updateData(id, data);
     }
@@ -1475,7 +1534,7 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
 
         const QString modelName = "ChatGPT-4";
         const QString id = modelName;
-        const QString modelFilename = "chatgpt-gpt-4.txt";
+        const QString modelFilename = "gpt4all-gpt-4.rmodel";
         if (contains(modelFilename))
             changeId(modelFilename, id);
         if (!contains(id))
@@ -1487,25 +1546,26 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             { ModelList::OnlineRole, true },
             { ModelList::DescriptionRole,
              tr("<strong>OpenAI's ChatGPT model GPT-4</strong><br>") + chatGPTDesc + chatGPT4Warn },
-            { ModelList::RequiresVersionRole, "2.4.2" },
+            { ModelList::RequiresVersionRole, "2.7.4" },
             { ModelList::OrderRole, "cb" },
             { ModelList::RamrequiredRole, 0 },
             { ModelList::ParametersRole, "?" },
             { ModelList::QuantRole, "NA" },
             { ModelList::TypeRole, "GPT" },
+            { ModelList::UrlRole, "https://api.openai.com/v1/chat/completions"},
         };
         updateData(id, data);
     }
+    
+	const QString mistralDesc = tr("<ul><li>Requires personal Mistral API key.</li><li>WARNING: Will send"
+                                   " your chats to Mistral!</li><li>Your API key will be stored on disk</li><li>Will only be used"
+                                   " to communicate with Mistral</li><li>You can apply for an API key"
+                                   " <a href=\"https://console.mistral.ai/user/api-keys\">here</a>.</li>");
 
     {
-        const QString nomicEmbedDesc = tr("<ul><li>For use with LocalDocs feature</li>"
-            "<li>Used for retrieval augmented generation (RAG)</li>"
-            "<li>Requires personal Nomic API key.</li>"
-            "<li>WARNING: Will send your localdocs to Nomic Atlas!</li>"
-            "<li>You can apply for an API key <a href=\"https://atlas.nomic.ai/\">with Nomic Atlas.</a></li>");
-        const QString modelName = "Nomic Embed";
+        const QString modelName = "Mistral Tiny API";
         const QString id = modelName;
-        const QString modelFilename = "nomic-embed-text-v1.txt";
+        const QString modelFilename = "gpt4all-mistral-tiny.rmodel";
         if (contains(modelFilename))
             changeId(modelFilename, id);
         if (!contains(id))
@@ -1515,7 +1575,90 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             { ModelList::FilenameRole, modelFilename },
             { ModelList::FilesizeRole, "minimal" },
             { ModelList::OnlineRole, true },
-            { ModelList::DisableGUIRole, true },
+            { ModelList::DescriptionRole,
+             tr("<strong>Mistral Tiny model</strong><br>") + mistralDesc },
+            { ModelList::RequiresVersionRole, "2.7.4" },
+            { ModelList::OrderRole, "cc" },
+            { ModelList::RamrequiredRole, 0 },
+            { ModelList::ParametersRole, "?" },
+            { ModelList::QuantRole, "NA" },
+            { ModelList::TypeRole, "Mistral" },
+            { ModelList::UrlRole, "https://api.mistral.ai/v1/chat/completions"},
+        };
+        updateData(id, data);
+    }
+    {
+        const QString modelName = "Mistral Small API";
+        const QString id = modelName;
+        const QString modelFilename = "gpt4all-mistral-small.rmodel";
+        if (contains(modelFilename))
+            changeId(modelFilename, id);
+        if (!contains(id))
+            addModel(id);
+        QVector<QPair<int, QVariant>> data {
+            { ModelList::NameRole, modelName },
+            { ModelList::FilenameRole, modelFilename },
+            { ModelList::FilesizeRole, "minimal" },
+            { ModelList::OnlineRole, true },
+            { ModelList::DescriptionRole,
+             tr("<strong>Mistral Small model</strong><br>") + mistralDesc },
+            { ModelList::RequiresVersionRole, "2.7.4" },
+            { ModelList::OrderRole, "cd" },
+            { ModelList::RamrequiredRole, 0 },
+            { ModelList::ParametersRole, "?" },
+            { ModelList::QuantRole, "NA" },
+            { ModelList::TypeRole, "Mistral" },
+            { ModelList::UrlRole, "https://api.mistral.ai/v1/chat/completions"},
+        };
+        updateData(id, data);
+    }
+    
+    {
+        const QString modelName = "Mistral Medium API";
+        const QString id = modelName;
+        const QString modelFilename = "gpt4all-mistral-medium.rmodel";
+        if (contains(modelFilename))
+            changeId(modelFilename, id);
+        if (!contains(id))
+            addModel(id);
+        QVector<QPair<int, QVariant>> data {
+            { ModelList::NameRole, modelName },
+            { ModelList::FilenameRole, modelFilename },
+            { ModelList::FilesizeRole, "minimal" },
+            { ModelList::OnlineRole, true },
+            { ModelList::DescriptionRole,
+             tr("<strong>Mistral Medium model</strong><br>") + mistralDesc },
+            { ModelList::RequiresVersionRole, "2.7.4" },
+            { ModelList::OrderRole, "ce" },
+            { ModelList::RamrequiredRole, 0 },
+            { ModelList::ParametersRole, "?" },
+            { ModelList::QuantRole, "NA" },
+            { ModelList::TypeRole, "Mistral" },
+            { ModelList::UrlRole, "https://api.mistral.ai/v1/chat/completions"},
+        };
+        updateData(id, data);
+    }
+
+
+    {
+        const QString nomicEmbedDesc = tr("<ul><li>For use with LocalDocs feature</li>"
+            "<li>Used for retrieval augmented generation (RAG)</li>"
+            "<li>Requires personal Nomic API key.</li>"
+            "<li>WARNING: Will send your localdocs to Nomic Atlas!</li>"
+            "<li>You can apply for an API key <a href=\"https://atlas.nomic.ai/\">with Nomic Atlas.</a></li>");
+        const QString modelName = "Nomic Embed";
+        const QString id = modelName;
+        const QString modelFilename = "nomic-embed-text-v1.txt"; // FIXME: This should be made to use '.rmodel' as well
+        if (contains(modelFilename))
+            changeId(modelFilename, id);
+        if (!contains(id))
+            addModel(id);
+        QVector<QPair<int, QVariant>> data {
+            { ModelList::NameRole, modelName },
+            { ModelList::FilenameRole, modelFilename },
+            { ModelList::FilesizeRole, "minimal" },
+            { ModelList::OnlineRole, true },
+            { ModelList::IsEmbeddingModelRole, true },
             { ModelList::DescriptionRole,
              tr("<strong>LocalDocs Nomic Atlas Embed</strong><br>") + nomicEmbedDesc },
             { ModelList::RequiresVersionRole, "2.6.3" },

gpt4all-chat/modellist.h

@@ -16,7 +16,6 @@ struct ModelInfo {
     Q_PROPERTY(bool calcHash MEMBER calcHash)
     Q_PROPERTY(bool installed MEMBER installed)
     Q_PROPERTY(bool isDefault MEMBER isDefault)
-    Q_PROPERTY(bool disableGUI MEMBER disableGUI)
     Q_PROPERTY(bool isOnline MEMBER isOnline)
     Q_PROPERTY(QString description READ description WRITE setDescription)
     Q_PROPERTY(QString requiresVersion MEMBER requiresVersion)
@@ -36,6 +35,7 @@ struct ModelInfo {
     Q_PROPERTY(QString type READ type WRITE setType)
     Q_PROPERTY(bool isClone READ isClone WRITE setIsClone)
     Q_PROPERTY(bool isDiscovered READ isDiscovered WRITE setIsDiscovered)
+    Q_PROPERTY(bool isEmbeddingModel MEMBER isEmbeddingModel)
     Q_PROPERTY(double temperature READ temperature WRITE setTemperature)
     Q_PROPERTY(double topP READ topP WRITE setTopP)
     Q_PROPERTY(double minP READ minP WRITE setMinP)
@@ -104,7 +104,6 @@ public:
     bool installed = false;
     bool isDefault = false;
     bool isOnline = false;
-    bool disableGUI = false;
     QString requiresVersion;
     QString versionRemoved;
     qint64 bytesReceived = 0;
@@ -117,6 +116,8 @@ public:
     QString order;
     int ramrequired = -1;
     QString parameters;
+    bool isEmbeddingModel = false;
+    bool checkedEmbeddingModel = false;
 
     bool operator==(const ModelInfo &other) const {
         return  m_id == other.m_id;
@@ -187,9 +188,10 @@ class EmbeddingModels : public QSortFilterProxyModel
     Q_OBJECT
     Q_PROPERTY(int count READ count NOTIFY countChanged)
 public:
-    explicit EmbeddingModels(QObject *parent);
-    int count() const;
+    EmbeddingModels(QObject *parent, bool requireInstalled);
+    int count() const { return rowCount(); }
 
+    int defaultModelIndex() const;
     ModelInfo defaultModelInfo() const;
 
 Q_SIGNALS:
@@ -198,6 +200,9 @@ Q_SIGNALS:
 
 protected:
     bool filterAcceptsRow(int sourceRow, const QModelIndex &sourceParent) const override;
+
+private:
+    bool m_requireInstalled;
 };
 
 class InstalledModels : public QSortFilterProxyModel
@@ -206,7 +211,7 @@ class InstalledModels : public QSortFilterProxyModel
     Q_PROPERTY(int count READ count NOTIFY countChanged)
 public:
     explicit InstalledModels(QObject *parent);
-    int count() const;
+    int count() const { return rowCount(); }
 
 Q_SIGNALS:
     void countChanged();
@@ -248,8 +253,8 @@ class ModelList : public QAbstractListModel
 {
     Q_OBJECT
     Q_PROPERTY(int count READ count NOTIFY countChanged)
-    Q_PROPERTY(int defaultEmbeddingModelIndex READ defaultEmbeddingModelIndex NOTIFY defaultEmbeddingModelIndexChanged)
-    Q_PROPERTY(EmbeddingModels* embeddingModels READ embeddingModels NOTIFY embeddingModelsChanged)
+    Q_PROPERTY(int defaultEmbeddingModelIndex READ defaultEmbeddingModelIndex)
+    Q_PROPERTY(EmbeddingModels* installedEmbeddingModels READ installedEmbeddingModels NOTIFY installedEmbeddingModelsChanged)
     Q_PROPERTY(InstalledModels* installedModels READ installedModels NOTIFY installedModelsChanged)
     Q_PROPERTY(DownloadableModels* downloadableModels READ downloadableModels NOTIFY downloadableModelsChanged)
     Q_PROPERTY(QList<QString> userDefaultModelList READ userDefaultModelList NOTIFY userDefaultModelListChanged)
@@ -282,7 +287,6 @@ public:
         InstalledRole,
         DefaultRole,
         OnlineRole,
-        DisableGUIRole,
         DescriptionRole,
         RequiresVersionRole,
         VersionRemovedRole,
@@ -301,6 +305,7 @@ public:
         TypeRole,
         IsCloneRole,
         IsDiscoveredRole,
+        IsEmbeddingModelRole,
         TemperatureRole,
         TopPRole,
         TopKRole,
@@ -332,7 +337,6 @@ public:
         roles[InstalledRole] = "installed";
         roles[DefaultRole] = "isDefault";
         roles[OnlineRole] = "isOnline";
-        roles[DisableGUIRole] = "disableGUI";
         roles[DescriptionRole] = "description";
         roles[RequiresVersionRole] = "requiresVersion";
         roles[VersionRemovedRole] = "versionRemoved";
@@ -351,6 +355,7 @@ public:
         roles[TypeRole] = "type";
         roles[IsCloneRole] = "isClone";
         roles[IsDiscoveredRole] = "isDiscovered";
+        roles[IsEmbeddingModelRole] = "isEmbeddingModel";
         roles[TemperatureRole] = "temperature";
         roles[TopPRole] = "topP";
         roles[MinPRole] = "minP";
@@ -373,7 +378,7 @@ public:
     QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override;
     QVariant data(const QString &id, int role) const;
     QVariant dataByFilename(const QString &filename, int role) const;
-    void updateDataByFilename(const QString &filename, const QVector<QPair<int, QVariant>> &data);
+    void updateDataByFilename(const QString &filename, QVector<QPair<int, QVariant>> data);
     void updateData(const QString &id, const QVector<QPair<int, QVariant>> &data);
 
     int count() const { return m_models.size(); }
@@ -396,6 +401,7 @@ public:
     const QList<QString> userDefaultModelList() const;
 
     EmbeddingModels *embeddingModels() const { return m_embeddingModels; }
+    EmbeddingModels *installedEmbeddingModels() const { return m_installedEmbeddingModels; }
     InstalledModels *installedModels() const { return m_installedModels; }
     DownloadableModels *downloadableModels() const { return m_downloadableModels; }
 
@@ -433,12 +439,11 @@ public:
 
 Q_SIGNALS:
     void countChanged();
-    void embeddingModelsChanged();
+    void installedEmbeddingModelsChanged();
     void installedModelsChanged();
     void downloadableModelsChanged();
     void userDefaultModelListChanged();
     void asyncModelRequestOngoingChanged();
-    void defaultEmbeddingModelIndexChanged();
     void discoverLimitChanged();
     void discoverSortDirectionChanged();
     void discoverSortChanged();
@@ -474,6 +479,7 @@ private:
     mutable QMutex m_mutex;
     QNetworkAccessManager m_networkManager;
     EmbeddingModels *m_embeddingModels;
+    EmbeddingModels *m_installedEmbeddingModels;
     InstalledModels *m_installedModels;
     DownloadableModels *m_downloadableModels;
     QList<ModelInfo*> m_models;
@@ -488,7 +494,7 @@ private:
 
 protected:
     explicit ModelList();
-    ~ModelList() {}
+    ~ModelList() { for (auto *model: m_models) { delete model; } }
     friend class MyModelList;
 };
 

gpt4all-chat/qml/AboutDialog.qml

@@ -98,4 +98,17 @@ MyDialog {
             Accessible.description: qsTr("Contains embedded link to https://home.nomic.ai")
         }
     }
+
+    MyButton {
+        id: checkForUpdatesButton
+        anchors.right: parent.right
+        anchors.bottom: parent.bottom
+        text: qsTr("Check for updates...")
+        font.pixelSize: theme.fontSizeLarge
+        Accessible.description: qsTr("Launch an external application that will check for updates to the installer")
+        onClicked: {
+            if (!LLM.checkForUpdates())
+                checkForUpdatesError.open()
+        }
+    }
 }

gpt4all-chat/qml/ChatDrawer.qml

@@ -9,24 +9,29 @@ import download
 import network
 import mysettings
 
-Drawer {
+Rectangle {
     id: chatDrawer
-    modal: false
 
     Theme {
         id: theme
     }
 
-    signal downloadClicked
-    signal aboutClicked
+    color: theme.containerBackground
 
-    background: Rectangle {
-        height: parent.height
-        color: theme.containerBackground
+    Rectangle {
+        id: borderRight
+        anchors.top: parent.top
+        anchors.bottom: parent.bottom
+        anchors.right: parent.right
+        width: 2
+        color: theme.containerForeground
     }
 
     Item {
-        anchors.fill: parent
+        anchors.top: parent.top
+        anchors.bottom: parent.bottom
+        anchors.left: parent.left
+        anchors.right: borderRight.left
         anchors.margins: 10
 
         Accessible.role: Accessible.Pane
@@ -54,7 +59,7 @@ Drawer {
             anchors.rightMargin: -10
             anchors.topMargin: 10
             anchors.top: newChat.bottom
-            anchors.bottom: checkForUpdatesButton.top
+            anchors.bottom: parent.bottom
             anchors.bottomMargin: 10
             ScrollBar.vertical.policy: ScrollBar.AlwaysOff
             clip: true
@@ -94,7 +99,7 @@ Drawer {
                         wrapMode: Text.NoWrap
                         hoverEnabled: false // Disable hover events on the TextArea
                         selectByMouse: false // Disable text selection in the TextArea
-                        font.pixelSize: theme.fontSizeLarger
+                        font.pixelSize: theme.fontSizeLarge
                         text: readOnly ? metrics.elidedText : name
                         horizontalAlignment: TextInput.AlignLeft
                         opacity: trashQuestionDisplayed ? 0.5 : 1.0
@@ -240,45 +245,5 @@ Drawer {
                 Accessible.description: qsTr("List of chats in the drawer dialog")
             }
         }
-
-        MyButton {
-            id: checkForUpdatesButton
-            anchors.left: parent.left
-            anchors.right: parent.right
-            anchors.bottom: downloadButton.top
-            anchors.bottomMargin: 10
-            text: qsTr("Updates")
-            font.pixelSize: theme.fontSizeLarge
-            Accessible.description: qsTr("Launch an external application that will check for updates to the installer")
-            onClicked: {
-                if (!LLM.checkForUpdates())
-                    checkForUpdatesError.open()
-            }
-        }
-
-        MyButton {
-            id: downloadButton
-            anchors.left: parent.left
-            anchors.right: parent.right
-            anchors.bottom: aboutButton.top
-            anchors.bottomMargin: 10
-            text: qsTr("Downloads")
-            Accessible.description: qsTr("Launch a dialog to download new models")
-            onClicked: {
-                downloadClicked()
-            }
-        }
-
-        MyButton {
-            id: aboutButton
-            anchors.left: parent.left
-            anchors.right: parent.right
-            anchors.bottom: parent.bottom
-            text: qsTr("About")
-            Accessible.description: qsTr("Launch a dialog to show the about page")
-            onClicked: {
-                aboutClicked()
-            }
-        }
     }
 }

gpt4all-chat/qml/LocalDocsSettings.qml

@@ -14,7 +14,7 @@ MySettingsTab {
         MySettings.restoreLocalDocsDefaults();
     }
 
-    property bool hasEmbeddingModel: ModelList.embeddingModels.count !== 0
+    property bool hasEmbeddingModel: ModelList.installedEmbeddingModels.count !== 0
     showAdvancedSettingsButton: hasEmbeddingModel
     showRestoreDefaultsButton: hasEmbeddingModel
 

gpt4all-chat/qml/ModelDownloaderDialog.qml

@@ -24,7 +24,7 @@ MyDialog {
         if (showEmbeddingModels) {
             ModelList.downloadableModels.expanded = true
             var targetModelIndex = ModelList.defaultEmbeddingModelIndex
-            modelListView.positionViewAtIndex(targetModelIndex, ListView.Contain)
+            modelListView.positionViewAtIndex(targetModelIndex, ListView.Beginning)
         }
     }
 

gpt4all-chat/qml/MyToolButton.qml

@@ -9,6 +9,8 @@ Button {
     padding: 10
     property color backgroundColor: theme.iconBackgroundDark
     property color backgroundColorHovered: theme.iconBackgroundHovered
+    property color toggledColor: theme.accentColor
+    property real toggledWidth: 1
     property bool toggled: false
     property alias source: image.source
     property alias fillMode: image.fillMode
@@ -27,8 +29,8 @@ Button {
             anchors.fill: parent
             color: "transparent"
             visible: myButton.toggled
-            border.color: theme.accentColor
-            border.width: 1
+            border.color: myButton.toggledColor
+            border.width: myButton.toggledWidth
             radius: 10
         }
         Image {

gpt4all-chat/qml/Theme.qml

@@ -200,6 +200,17 @@ QtObject {
         }
     }
 
+    property color viewBarBackground: {
+        switch (MySettings.chatTheme) {
+            case "LegacyDark":
+                return blue950;
+            case "Dark":
+                return darkgray300;
+            default:
+                return gray300;
+        }
+    }
+
     property color progressForeground: {
         switch (MySettings.chatTheme) {
             case "LegacyDark":
@@ -376,6 +387,39 @@ QtObject {
         }
     }
 
+    property color iconBackgroundViewBar: {
+        switch (MySettings.chatTheme) {
+            case "LegacyDark":
+                return blue200;
+            case "Dark":
+                return green400;
+            default:
+                return green700;
+        }
+    }
+
+    property color iconBackgroundViewBarToggled: {
+        switch (MySettings.chatTheme) {
+            case "LegacyDark":
+                return purple400;
+            case "Dark":
+                return accentColor;
+            default:
+                return black;
+        }
+    }
+
+    property color iconBackgroundViewBarHovered: {
+        switch (MySettings.chatTheme) {
+            case "LegacyDark":
+                return blue400;
+            case "Dark":
+                return green600;
+            default:
+                return green500;
+        }
+    }
+
     property color slugBackground: {
         switch (MySettings.chatTheme) {
             case "LegacyDark":