minor change to trigger CircleCI

The backend has a breaking change for Falcon and MPT models, so we need
to make a new release.

.circleci/continue_config.yml

@@ -287,6 +287,7 @@ jobs:
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
+            $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
             mkdir build
             cd build
             & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
@@ -348,6 +349,7 @@ jobs:
           install-yarn: true
           node-version: "18.16"
       - run: node --version
+      - run: corepack enable
       - node/install-packages:
           pkg-manager: yarn
           app-dir: gpt4all-bindings/typescript
@@ -482,8 +484,9 @@ jobs:
             cd gpt4all-backend
             mkdir build
             cd build
-            $env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
-            $env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            $Env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
+            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
             cmake -G "MinGW Makefiles" .. -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
             cmake --build . --parallel
       - run:
@@ -853,9 +856,11 @@ jobs:
           install-yarn: true
           node-version: "18.16"
       - run: node --version
+      - run: corepack enable
       - node/install-packages:
           app-dir: gpt4all-bindings/typescript
           pkg-manager: yarn
+          override-ci-command: yarn install
       - run:
           command: | 
             cd gpt4all-bindings/typescript
@@ -882,9 +887,11 @@ jobs:
           install-yarn: true
           node-version: "18.16"
       - run: node --version
+      - run: corepack enable
       - node/install-packages:
           app-dir: gpt4all-bindings/typescript
           pkg-manager: yarn
+          override-ci-command: yarn install
       - run:
           command: | 
             cd gpt4all-bindings/typescript
@@ -893,14 +900,14 @@ jobs:
           name: "Persisting all necessary things to workspace"
           command: |  
             mkdir -p gpt4all-backend/prebuilds/darwin-x64
-            mkdir -p gpt4all-backend/runtimes/darwin-x64
-            cp /tmp/gpt4all-backend/runtimes/osx-x64/*-*.* gpt4all-backend/runtimes/darwin-x64
+            mkdir -p gpt4all-backend/runtimes/darwin
+            cp /tmp/gpt4all-backend/runtimes/osx-x64/*-*.* gpt4all-backend/runtimes/darwin
             cp gpt4all-bindings/typescript/prebuilds/darwin-x64/*.node gpt4all-backend/prebuilds/darwin-x64
       - persist_to_workspace:
           root: gpt4all-backend
           paths:
             - prebuilds/darwin-x64/*.node
-            - runtimes/darwin-x64/*-*.*
+            - runtimes/darwin/*-*.*
 
   build-nodejs-windows: 
     executor:
@@ -922,6 +929,7 @@ jobs:
               nvm install 18.16.0
               nvm use 18.16.0
       - run: node --version 
+      - run: corepack enable
       - run:           
           command: |
             npm install -g yarn
@@ -955,6 +963,7 @@ jobs:
           install-yarn: true
           node-version: "18.16"
       - run: node --version
+      - run: corepack enable
       - run: 
           command: |
             cd gpt4all-bindings/typescript
@@ -969,9 +978,12 @@ jobs:
             cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so runtimes/linux-x64/native/ 
             cp /tmp/gpt4all-backend/prebuilds/linux-x64/*.node prebuilds/linux-x64/    
 
-            mkdir -p runtimes/darwin-x64/native
+            # darwin has univeral runtime libraries
+            mkdir -p runtimes/darwin/native
             mkdir -p prebuilds/darwin-x64/
-            cp /tmp/gpt4all-backend/runtimes/darwin-x64/*-*.* runtimes/darwin-x64/native/
+
+            cp /tmp/gpt4all-backend/runtimes/darwin/*-*.* runtimes/darwin/native/
+
             cp /tmp/gpt4all-backend/prebuilds/darwin-x64/*.node prebuilds/darwin-x64/    
             
             # Fallback build if user is not on above prebuilds
@@ -994,7 +1006,7 @@ jobs:
           command: |
             cd gpt4all-bindings/typescript
             npm set //registry.npmjs.org/:_authToken=$NPM_TOKEN
-            npm publish --access public --tag alpha
+            npm publish
 
 workflows:
   version: 2

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
-ignore-words-list = blong, belong, afterall, som, assistent
+ignore-words-list = blong, afterall, som, assistent, crasher
 skip = .git,*.pdf,*.svg,*.lock

.gitignore

@@ -183,4 +183,7 @@ build_*
 build-*
 
 # IntelliJ
-.idea/
+.idea/
+
+# LLM models
+*.gguf

README.md

@@ -1,11 +1,9 @@
 <h1 align="center">GPT4All</h1>
 
-<p align="center">Open-source assistant-style large language models that run locally on your CPU</p>
-
-<p align="center"><strong>New</strong>: Now with Nomic Vulkan Universal GPU support. <a href="https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan">Learn more</a>.</p>
+<p align="center">Open-source large language models that run locally on your CPU and nearly any GPU</p>
 
 <p align="center">
-<a href="https://gpt4all.io">GPT4All Website</a>
+<a href="https://gpt4all.io">GPT4All Website and Models</a>
 </p>
 
 <p align="center">
@@ -32,13 +30,24 @@ Run on an M1 macOS Device (not sped up!)
 </p>
 
 ## GPT4All: An ecosystem of open-source on-edge large language models.
-GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
+
+> [!IMPORTANT]
+> GPT4All v2.5.0 and newer only supports models in GGUF format (.gguf). Models used with a previous version of GPT4All (.bin extension) will no longer work.
+
+GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and any GPU. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
 
 Learn more in the [documentation](https://docs.gpt4all.io).
 
-The goal is simple - be the best instruction tuned assistant-style language model that any person or enterprise can freely use, distribute and build on.
+A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models.
 
-A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models. 
+### What's New ([Issue Tracker](https://github.com/orgs/nomic-ai/projects/2))
+- **October 19th, 2023**: GGUF Support Launches with Support for:
+    - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
+    - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4_0, Q6 quantizations in GGUF.
+    - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
+- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on AMD, Intel, Samsung, Qualcomm and NVIDIA GPUs.
+- **August 15th, 2023**: GPT4All API launches allowing inference of local LLMs from docker containers.
+- **July 2023**: Stable support for LocalDocs, a GPT4All Plugin that allows you to privately and locally chat with your data.
 
 
 ### Chat Client

gpt4all-api/docker-compose.yaml

@@ -7,13 +7,16 @@ services:
     restart: always #restart on error (usually code compilation from save during bad state)
     ports:
       - "4891:4891"
+    env_file:
+      - .env
     environment:
       - APP_ENVIRONMENT=dev
       - WEB_CONCURRENCY=2
       - LOGLEVEL=debug
       - PORT=4891
-      - model=ggml-mpt-7b-chat.bin
+      - model=${MODEL_BIN} # using variable from .env file
       - inference_mode=cpu
     volumes:
       - './gpt4all_api/app:/app'
+      - './gpt4all_api/models:/models' # models are mounted in the container
     command: ["/start-reload.sh"]

gpt4all-api/gpt4all_api/Dockerfile.buildkit

@@ -1,8 +1,6 @@
 # syntax=docker/dockerfile:1.0.0-experimental
 FROM tiangolo/uvicorn-gunicorn:python3.11
 
-ARG MODEL_BIN=ggml-mpt-7b-chat.bin
-
 # Put first so anytime this file changes other cached layers are invalidated.
 COPY gpt4all_api/requirements.txt /requirements.txt
 
@@ -17,7 +15,3 @@ COPY gpt4all_api/app /app
 
 RUN mkdir -p /models
 
-# Include the following line to bake a model into the image and not have to download it on API start.
-RUN wget -q --show-progress=off https://gpt4all.io/models/${MODEL_BIN} -P /models \
-  && md5sum /models/${MODEL_BIN}
-

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -1,39 +1,35 @@
 import logging
 import time
-from typing import Dict, List
-
-from api_v1.settings import settings
-from fastapi import APIRouter, Depends, Response, Security, status
+from typing import List
+from uuid import uuid4
+from fastapi import APIRouter
 from pydantic import BaseModel, Field
+from api_v1.settings import settings
+from fastapi.responses import StreamingResponse
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
 ### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
 class ChatCompletionMessage(BaseModel):
     role: str
     content: str
 
-
 class ChatCompletionRequest(BaseModel):
-    model: str = Field(..., description='The model to generate a completion from.')
-    messages: List[ChatCompletionMessage] = Field(..., description='The model to generate a completion from.')
-
+    model: str = Field(settings.model, description='The model to generate a completion from.')
+    messages: List[ChatCompletionMessage] = Field(..., description='Messages for the chat completion.')
 
 class ChatCompletionChoice(BaseModel):
     message: ChatCompletionMessage
     index: int
+    logprobs: float
     finish_reason: str
 
-
 class ChatCompletionUsage(BaseModel):
     prompt_tokens: int
     completion_tokens: int
     total_tokens: int
 
-
 class ChatCompletionResponse(BaseModel):
     id: str
     object: str = 'text_completion'
@@ -42,20 +38,38 @@ class ChatCompletionResponse(BaseModel):
     choices: List[ChatCompletionChoice]
     usage: ChatCompletionUsage
 
-
 router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
 
-
 @router.post("/completions", response_model=ChatCompletionResponse)
 async def chat_completion(request: ChatCompletionRequest):
     '''
-    Completes a GPT4All model response.
+    Completes a GPT4All model response based on the last message in the chat.
     '''
+    # Example: Echo the last message content with some modification
+    if request.messages:
+        last_message = request.messages[-1].content
+        response_content = f"Echo: {last_message}"
+    else:
+        response_content = "No messages received."
 
-    return ChatCompletionResponse(
-        id='asdf',
-        created=time.time(),
-        model=request.model,
-        choices=[{}],
-        usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
+    # Create a chat message for the response
+    response_message = ChatCompletionMessage(role="system", content=response_content)
+
+    # Create a choice object with the response message
+    response_choice = ChatCompletionChoice(
+        message=response_message,
+        index=0,
+        logprobs=-1.0,  # Placeholder value
+        finish_reason="length"  # Placeholder value
     )
+
+    # Create the response object
+    chat_response = ChatCompletionResponse(
+        id=str(uuid4()),
+        created=int(time.time()),
+        model=request.model,
+        choices=[response_choice],
+        usage=ChatCompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0),  # Placeholder values
+    )
+
+    return chat_response

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -1,40 +1,39 @@
-import logging
-from typing import Dict, List
-
-from api_v1.settings import settings
-from fastapi import APIRouter, Depends, Response, Security, status
+import requests
+from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel, Field
+from typing import List, Dict
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
+# Define the router for the engines module
+router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
 
+# Define the models for the engines module
 class ListEnginesResponse(BaseModel):
     data: List[Dict] = Field(..., description="All available models.")
 
-
 class EngineResponse(BaseModel):
     data: List[Dict] = Field(..., description="All available models.")
 
 
-router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
-
-
+# Define the routes for the engines module
 @router.get("/", response_model=ListEnginesResponse)
 async def list_engines():
-    '''
-    List all available GPT4All models from
-    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json
-    '''
-    raise NotImplementedError()
-    return ListEnginesResponse(data=[])
-
+    try:
+        response = requests.get('https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json')
+        response.raise_for_status()  # This will raise an HTTPError if the HTTP request returned an unsuccessful status code
+        engines = response.json()
+        return ListEnginesResponse(data=engines)
+    except requests.RequestException as e:
+        logger.error(f"Error fetching engine list: {e}")
+        raise HTTPException(status_code=500, detail="Error fetching engine list")
 
+# Define the routes for the engines module
 @router.get("/{engine_id}", response_model=EngineResponse)
 async def retrieve_engine(engine_id: str):
-    ''' '''
-
-    raise NotImplementedError()
-    return EngineResponse()
+    try:
+        # Implement logic to fetch a specific engine's details
+        # This is a placeholder, replace with your actual data retrieval logic
+        engine_details = {"id": engine_id, "name": "Engine Name", "description": "Engine Description"}
+        return EngineResponse(data=[engine_details])
+    except Exception as e:
+        logger.error(f"Error fetching engine details: {e}")
+        raise HTTPException(status_code=500, detail=f"Error fetching details for engine {engine_id}")

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -2,16 +2,26 @@
 Use the OpenAI python API to test gpt4all models.
 """
 from typing import List, get_args
+import os
+from dotenv import load_dotenv
 
 import openai
 
 openai.api_base = "http://localhost:4891/v1"
-
 openai.api_key = "not needed for a local LLM"
 
+# Load the .env file
+env_path = 'gpt4all-api/gpt4all_api/.env'
+load_dotenv(dotenv_path=env_path)
+
+# Fetch MODEL_ID from .env file
+model_id = os.getenv('MODEL_BIN', 'default_model_id')
+embedding = os.getenv('EMBEDDING', 'default_embedding_model_id')
+print (model_id)
+print (embedding)
 
 def test_completion():
-    model = "ggml-mpt-7b-chat.bin"
+    model = model_id
     prompt = "Who is Michael Jordan?"
     response = openai.Completion.create(
         model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
@@ -19,7 +29,7 @@ def test_completion():
     assert len(response['choices'][0]['text']) > len(prompt)
 
 def test_streaming_completion():
-    model = "ggml-mpt-7b-chat.bin"
+    model = model_id
     prompt = "Who is Michael Jordan?"
     tokens = []
     for resp in openai.Completion.create(
@@ -36,19 +46,27 @@ def test_streaming_completion():
     assert (len(tokens) > 0)
     assert (len("".join(tokens)) > len(prompt))
 
-
+# Modified test batch, problems with keyerror in response
 def test_batched_completion():
-    model = "ggml-mpt-7b-chat.bin"
+    model = model_id  # replace with your specific model ID
     prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model, prompt=[prompt] * 3, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-    assert len(response['choices']) == 3
+    responses = []
+    
+    # Loop to create completions one at a time
+    for _ in range(3):
+        response = openai.Completion.create(
+            model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
+        )
+        responses.append(response)
 
+    # Assertions to check the responses
+    for response in responses:
+        assert len(response['choices'][0]['text']) > len(prompt)
+    
+    assert len(responses) == 3
 
 def test_embedding():
-    model = "ggml-all-MiniLM-L6-v2-f16.bin"
+    model = embedding
     prompt = "Who is Michael Jordan?"
     response = openai.Embedding.create(model=model, input=prompt)
     output = response["data"][0]["embedding"]
@@ -56,4 +74,4 @@ def test_embedding():
 
     assert response["model"] == model
     assert isinstance(output, list)
-    assert all(isinstance(x, args) for x in output)
+    assert all(isinstance(x, args) for x in output)

gpt4all-api/gpt4all_api/env

@@ -0,0 +1,3 @@
+# Add your GGUF compatible model LLM here. ie: MODEL_BIN="mistral-7b-instruct-v0.1.Q4_0", rename file ".env"
+# Make sure this LLM matches the model you placed inside the models folder
+MODEL_BIN=""

gpt4all-api/gpt4all_api/models/README.md

@@ -0,0 +1 @@
+### Drop GGUF compatible models here, make sure it matches MODEL_BIN on your .env file

gpt4all-api/gpt4all_api/requirements.txt

@@ -7,6 +7,7 @@ fastapi>=0.95.0
 Jinja2>=3.0
 gpt4all>=1.0.0
 pytest
-openai
+openai==0.28.0
 black
-isort
+isort
+python-dotenv

gpt4all-api/makefile

@@ -14,7 +14,7 @@ testenv_gpu: clean_testenv test_build
 	docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up --build
 
 testenv_d: clean_testenv test_build
-	docker compose up --build -d
+	docker compose env up --build -d
 
 test:
 	docker compose exec $(APP_NAME) pytest -svv --disable-warnings -p no:cacheprovider /app/tests
@@ -28,19 +28,19 @@ clean_testenv:
 fresh_testenv: clean_testenv testenv
 
 venv:
-	if [ ! -d $(ROOT_DIR)/env ]; then $(PYTHON) -m venv $(ROOT_DIR)/env; fi
+	if [ ! -d $(ROOT_DIR)/venv ]; then $(PYTHON) -m venv $(ROOT_DIR)/venv; fi
 
 dependencies: venv
-	source $(ROOT_DIR)/env/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
+	source $(ROOT_DIR)/venv/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
 
 clean: clean_testenv
 	# Remove existing environment
-	rm -rf $(ROOT_DIR)/env;
+	rm -rf $(ROOT_DIR)/venv;
 	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
 
 
 black:
-	source $(ROOT_DIR)/env/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
+	source $(ROOT_DIR)/venv/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
 
 isort:
-	source $(ROOT_DIR)/env/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)
+	source $(ROOT_DIR)/venv/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)

gpt4all-backend/CMakeLists.txt

@@ -114,8 +114,6 @@ add_library(llmodel
     llmodel_c.h llmodel_c.cpp
     dlhandle.h
 )
-target_link_libraries(llmodel PRIVATE ggml-mainline-default)
-target_compile_definitions(llmodel PRIVATE GGML_BUILD_VARIANT="default")
 target_compile_definitions(llmodel PRIVATE LIB_FILE_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}")
 
 set_target_properties(llmodel PROPERTIES

gpt4all-backend/bert.cpp

@@ -317,7 +317,7 @@ void bert_eval(
     };
 
     struct ggml_context *ctx0 = ggml_init(params);
-    struct ggml_cgraph gf = {};
+    struct ggml_cgraph *gf = ggml_new_graph(ctx0);
 
     // Embeddings. word_embeddings + token_type_embeddings + position_embeddings
     struct ggml_tensor *token_layer = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
@@ -448,10 +448,10 @@ void bert_eval(
 
     ggml_tensor *output = inpL;
     // run the computation
-    ggml_build_forward_expand(&gf, output);
+    ggml_build_forward_expand(gf, output);
     //ggml_graph_compute_g4a()
-    ggml_graph_compute_g4a(ctx->work_buf, &gf, n_threads);
-    //ggml_graph_compute(ctx0, &gf);
+    ggml_graph_compute_g4a(ctx->work_buf, gf, n_threads);
+    //ggml_graph_compute(ctx0, gf);
 
 
     // float *dat = ggml_get_data_f32(output);
@@ -460,7 +460,7 @@ void bert_eval(
     #ifdef GGML_PERF
         // print timing information per ggml operation (for debugging purposes)
         // requires GGML_PERF to be defined
-        ggml_graph_print(&gf);
+        ggml_graph_print(gf);
     #endif
 
     if (!mem_req_mode) {
@@ -490,6 +490,11 @@ struct bert_ctx * bert_load_from_file(const char *fname)
 #endif
 
     bert_ctx * new_bert = new bert_ctx;
+#if defined(GGML_USE_KOMPUTE)
+    new_bert->buf_compute.force_cpu = true;
+    new_bert->work_buf.force_cpu = true;
+#endif
+
     bert_model & model = new_bert->model;
     bert_vocab & vocab = new_bert->vocab;
 
@@ -709,8 +714,9 @@ Bert::~Bert() {
     bert_free(d_ptr->ctx);
 }
 
-bool Bert::loadModel(const std::string &modelPath)
+bool Bert::loadModel(const std::string &modelPath, int n_ctx)
 {
+    (void)n_ctx;
     d_ptr->ctx = bert_load_from_file(modelPath.c_str());
     d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
     d_ptr->modelLoaded = d_ptr->ctx != nullptr;
@@ -723,8 +729,10 @@ bool Bert::isModelLoaded() const
     return d_ptr->modelLoaded;
 }
 
-size_t Bert::requiredMem(const std::string &/*modelPath*/)
+size_t Bert::requiredMem(const std::string &modelPath, int n_ctx)
 {
+    (void)modelPath;
+    (void)n_ctx;
     return 0;
 }
 
@@ -884,7 +892,7 @@ DLL_EXPORT bool magic_match(const char * fname) {
     if (!ctx_gguf)
         return false;
 
-    bool isValid = gguf_get_version(ctx_gguf) <= 2;
+    bool isValid = gguf_get_version(ctx_gguf) <= 3;
     isValid = isValid && get_arch_name(ctx_gguf) == "bert";
 
     gguf_free(ctx_gguf);

gpt4all-backend/bert_impl.h

@@ -18,9 +18,9 @@ public:
 
     bool supportsEmbedding() const override { return true; }
     bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
+    bool loadModel(const std::string &modelPath, int n_ctx) override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;

gpt4all-backend/dlhandle.h

@@ -53,6 +53,8 @@ public:
     }
 };
 #else
+#include <algorithm>
+#include <filesystem>
 #include <string>
 #include <exception>
 #include <stdexcept>
@@ -75,7 +77,9 @@ public:
 
     Dlhandle() : chandle(nullptr) {}
     Dlhandle(const std::string& fpath) {
-        chandle = LoadLibraryExA(fpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
+        std::string afpath = std::filesystem::absolute(fpath).string();
+        std::replace(afpath.begin(), afpath.end(), '/', '\\');
+        chandle = LoadLibraryExA(afpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
         if (!chandle) {
             throw Exception("dlopen(\""+fpath+"\"): Error");
         }

gpt4all-backend/gptj.cpp

@@ -343,7 +343,14 @@ bool gptj_eval(
     };
 
     struct ggml_context * ctx0 = ggml_init(params);
-    struct ggml_cgraph gf = {};
+    struct ggml_cgraph * gf = ggml_new_graph(ctx0);
+
+    // KQ_pos - contains the positions
+    struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    int * data = (int *) KQ_pos->data;
+    for (int i = 0; i < N; ++i) {
+        data[i] = n_past + i;
+    }
 
     struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
     memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -370,8 +377,14 @@ bool gptj_eval(
 
         // self-attention
         {
-            struct ggml_tensor * Qcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
-            struct ggml_tensor * Kcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
+            struct ggml_tensor * Qcur = ggml_rope(
+                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N),
+                KQ_pos, n_rot, 0, 0
+            );
+            struct ggml_tensor * Kcur = ggml_rope(
+                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N),
+                KQ_pos, n_rot, 0, 0
+            );
 
             // store key and value to memory
             {
@@ -382,8 +395,8 @@ bool gptj_eval(
                         (   n_ctx)*ggml_element_size(model.kv_self.v),
                         (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
 
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
+                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
+                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
             }
 
             // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
@@ -502,22 +515,22 @@ bool gptj_eval(
     // logits -> probs
     //inpL = ggml_soft_max(ctx0, inpL);
 
-    ggml_build_forward_expand(&gf, inpL);
+    ggml_build_forward_expand(gf, inpL);
 
     // run the computation
     {
         std::unique_ptr<uint8_t []> data;
-        auto plan = ggml_graph_plan(&gf, n_threads);
+        auto plan = ggml_graph_plan(gf, n_threads);
         if (plan.work_size > 0) {
             data.reset(new uint8_t[plan.work_size]);
             plan.work_data = data.get();
         }
-        ggml_graph_compute(&gf, &plan);
+        ggml_graph_compute(gf, &plan);
     }
 
     //if (n_past%100 == 0) {
-    //    ggml_graph_print   (&gf);
-    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
+    //    ggml_graph_print   (gf);
+    //    ggml_graph_dump_dot(gf, NULL, "gpt-2.dot");
     //}
 
     //embd_w.resize(n_vocab*N);
@@ -663,7 +676,8 @@ GPTJ::GPTJ()
     d_ptr->modelLoaded = false;
 }
 
-size_t GPTJ::requiredMem(const std::string &modelPath) {
+size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx) {
+    (void)n_ctx;
     gptj_model dummy_model;
     gpt_vocab dummy_vocab;
     size_t mem_req;
@@ -671,7 +685,8 @@ size_t GPTJ::requiredMem(const std::string &modelPath) {
     return mem_req;
 }
 
-bool GPTJ::loadModel(const std::string &modelPath) {
+bool GPTJ::loadModel(const std::string &modelPath, int n_ctx) {
+    (void)n_ctx;
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
 
@@ -806,7 +821,7 @@ DLL_EXPORT bool magic_match(const char * fname) {
     if (!ctx_gguf)
         return false;
 
-    bool isValid = gguf_get_version(ctx_gguf) <= 2;
+    bool isValid = gguf_get_version(ctx_gguf) <= 3;
     isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
 
     gguf_free(ctx_gguf);

gpt4all-backend/gptj_impl.h

@@ -17,9 +17,9 @@ public:
 
     bool supportsEmbedding() const override { return false; }
     bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
+    bool loadModel(const std::string &modelPath, int n_ctx) override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;

gpt4all-backend/llama.cpp.cmake

@@ -77,7 +77,6 @@ option(LLAMA_OPENBLAS               "llama: use OpenBLAS"
 #option(LLAMA_CUBLAS                 "llama: use cuBLAS"                                     OFF)
 #option(LLAMA_CLBLAST                "llama: use CLBlast"                                    OFF)
 #option(LLAMA_METAL                  "llama: use Metal"                                      OFF)
-#option(LLAMA_K_QUANTS               "llama: use k-quants"                                   ON)
 set(LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor")
 set(LLAMA_CUDA_DMMV_X "32" CACHE STRING "llama: x stride for dmmv CUDA kernels")
 set(LLAMA_CUDA_DMMV_Y "1" CACHE STRING  "llama: y block size for dmmv CUDA kernels")
@@ -228,6 +227,7 @@ if (LLAMA_KOMPUTE)
         # Compile our shaders
         compile_shader(SOURCES
           kompute/op_scale.comp
+          kompute/op_scale_8.comp
           kompute/op_add.comp
           kompute/op_addrow.comp
           kompute/op_mul.comp
@@ -249,7 +249,8 @@ if (LLAMA_KOMPUTE)
           kompute/op_getrows_q4_0.comp
           kompute/op_getrows_q4_1.comp
           kompute/op_getrows_q6_k.comp
-          kompute/op_rope.comp
+          kompute/op_rope_f16.comp
+          kompute/op_rope_f32.comp
           kompute/op_cpy_f16_f16.comp
           kompute/op_cpy_f16_f32.comp
           kompute/op_cpy_f32_f16.comp
@@ -259,6 +260,7 @@ if (LLAMA_KOMPUTE)
         # Create a custom target for our generated shaders
         add_custom_target(generated_shaders DEPENDS
           shaderop_scale.h
+          shaderop_scale_8.h
           shaderop_add.h
           shaderop_addrow.h
           shaderop_mul.h
@@ -280,7 +282,8 @@ if (LLAMA_KOMPUTE)
           shaderop_getrows_q4_0.h
           shaderop_getrows_q4_1.h
           shaderop_getrows_q6_k.h
-          shaderop_rope.h
+          shaderop_rope_f16.h
+          shaderop_rope_f32.h
           shaderop_cpy_f16_f16.h
           shaderop_cpy_f16_f32.h
           shaderop_cpy_f32_f16.h
@@ -564,33 +567,26 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
         endif()
     endif()
 
-    set(GGML_SOURCES_QUANT_K )
-    set(GGML_METAL_SOURCES )
-    if (LLAMA_K_QUANTS)
-        set(GGML_SOURCES_QUANT_K
-            ${DIRECTORY}/k_quants.h
-            ${DIRECTORY}/k_quants.c)
+    set(GGML_METAL_SOURCES)
+    if (LLAMA_METAL)
+        find_library(FOUNDATION_LIBRARY         Foundation              REQUIRED)
+        find_library(METAL_FRAMEWORK            Metal                   REQUIRED)
+        find_library(METALKIT_FRAMEWORK         MetalKit                REQUIRED)
+        find_library(METALPERFORMANCE_FRAMEWORK MetalPerformanceShaders REQUIRED)
 
-        if (LLAMA_METAL)
-            find_library(FOUNDATION_LIBRARY         Foundation              REQUIRED)
-            find_library(METAL_FRAMEWORK            Metal                   REQUIRED)
-            find_library(METALKIT_FRAMEWORK         MetalKit                REQUIRED)
-            find_library(METALPERFORMANCE_FRAMEWORK MetalPerformanceShaders REQUIRED)
+        set(GGML_METAL_SOURCES ${DIRECTORY}/ggml-metal.m ${DIRECTORY}/ggml-metal.h)
+        # get full path to the file
+        #add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
 
-            set(GGML_METAL_SOURCES ${DIRECTORY}/ggml-metal.m ${DIRECTORY}/ggml-metal.h)
-            # get full path to the file
-            #add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
+        # copy ggml-metal.metal to bin directory
+        configure_file(${DIRECTORY}/ggml-metal.metal bin/ggml-metal.metal COPYONLY)
 
-            # copy ggml-metal.metal to bin directory
-            configure_file(${DIRECTORY}/ggml-metal.metal bin/ggml-metal.metal COPYONLY)
-
-            set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS}
-                ${FOUNDATION_LIBRARY}
-                ${METAL_FRAMEWORK}
-                ${METALKIT_FRAMEWORK}
-                ${METALPERFORMANCE_FRAMEWORK}
-            )
-        endif()
+        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS}
+            ${FOUNDATION_LIBRARY}
+            ${METAL_FRAMEWORK}
+            ${METALKIT_FRAMEWORK}
+            ${METALPERFORMANCE_FRAMEWORK}
+        )
     endif()
 
     add_library(ggml${SUFFIX} OBJECT
@@ -598,16 +594,15 @@ function(include_ggml DIRECTORY SUFFIX WITH_LLAMA)
                 ${DIRECTORY}/ggml.h
                 ${DIRECTORY}/ggml-alloc.c
                 ${DIRECTORY}/ggml-alloc.h
-                ${GGML_SOURCES_QUANT_K}
+                ${DIRECTORY}/ggml-backend.c
+                ${DIRECTORY}/ggml-backend.h
+                ${DIRECTORY}/ggml-quants.h
+                ${DIRECTORY}/ggml-quants.c
                 ${GGML_SOURCES_CUDA}
                 ${GGML_METAL_SOURCES}
                 ${GGML_OPENCL_SOURCES}
                 ${GGML_SOURCES_KOMPUTE})
 
-    if (LLAMA_K_QUANTS)
-        target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_K_QUANTS)
-    endif()
-
     if (LLAMA_METAL AND GGML_METAL_SOURCES)
         target_compile_definitions(ggml${SUFFIX} PUBLIC GGML_USE_METAL GGML_METAL_NDEBUG)
     endif()

gpt4all-backend/llamamodel.cpp

@@ -71,9 +71,10 @@ static int llama_sample_top_p_top_k(
         int top_k,
         float top_p,
         float temp,
-        float repeat_penalty) {
-    auto logits = llama_get_logits(ctx);
-    auto n_vocab = llama_n_vocab(ctx);
+        float repeat_penalty,
+        int32_t pos) {
+    auto logits = llama_get_logits_ith(ctx, pos);
+    auto n_vocab = llama_n_vocab(llama_get_model(ctx));
     // Populate initial list of all candidates
     std::vector<llama_token_data> candidates;
     candidates.reserve(n_vocab);
@@ -82,21 +83,23 @@ static int llama_sample_top_p_top_k(
     }
     llama_token_data_array candidates_p = {candidates.data(), candidates.size(), false};
     // Sample repeat penalty
-    llama_sample_repetition_penalty(nullptr, &candidates_p, last_n_tokens_data, last_n_tokens_size, repeat_penalty);
+    llama_sample_repetition_penalties(nullptr, &candidates_p, last_n_tokens_data, last_n_tokens_size, repeat_penalty, 0.0f, 0.0f);
     // Temperature sampling
     llama_sample_top_k(ctx, &candidates_p, top_k, 1);
     llama_sample_tail_free(ctx, &candidates_p, 1.0f, 1);
     llama_sample_typical(ctx, &candidates_p, 1.0f, 1);
     llama_sample_top_p(ctx, &candidates_p, top_p, 1);
-    llama_sample_temperature(ctx, &candidates_p, temp);
+    llama_sample_temp(ctx, &candidates_p, temp);
     return llama_sample_token(ctx, &candidates_p);
 }
 
 struct LLamaPrivate {
     const std::string modelPath;
     bool modelLoaded;
+    llama_model *model = nullptr;
     llama_context *ctx = nullptr;
-    llama_context_params params;
+    llama_model_params model_params;
+    llama_context_params ctx_params;
     int64_t n_threads = 0;
     std::vector<LLModel::Token> end_tokens;
 };
@@ -117,7 +120,8 @@ struct llama_file_hparams {
     enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16;
 };
 
-size_t LLamaModel::requiredMem(const std::string &modelPath) {
+size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx) {
+    // TODO(cebtenzzre): update to GGUF
     auto fin = std::ifstream(modelPath, std::ios::binary);
     fin.seekg(0, std::ios_base::end);
     size_t filesize = fin.tellg();
@@ -134,45 +138,49 @@ size_t LLamaModel::requiredMem(const std::string &modelPath) {
     fin.read(reinterpret_cast<char*>(&hparams.n_layer), sizeof(hparams.n_layer));
     fin.read(reinterpret_cast<char*>(&hparams.n_rot), sizeof(hparams.n_rot));
     fin.read(reinterpret_cast<char*>(&hparams.ftype), sizeof(hparams.ftype));
-    const size_t n_ctx = 2048;
     const size_t kvcache_element_size = 2; // fp16
     const size_t est_kvcache_size = hparams.n_embd * hparams.n_layer * 2u * n_ctx * kvcache_element_size;
     return filesize + est_kvcache_size;
 }
 
-bool LLamaModel::loadModel(const std::string &modelPath)
+bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx)
 {
-    // load the model
-    d_ptr->params = llama_context_default_params();
-
     gpt_params params;
-    d_ptr->params.n_ctx      = 2048;
-    d_ptr->params.seed       = params.seed;
-    d_ptr->params.f16_kv     = params.memory_f16;
-    d_ptr->params.use_mmap   = params.use_mmap;
+
+    if (n_ctx < 8) {
+        std::cerr << "warning: minimum context size is 8, using minimum size.\n";
+        n_ctx = 8;
+    }
+
+    // -- load the model --
+
+    d_ptr->model_params = llama_model_default_params();
+
+    d_ptr->model_params.use_mmap  = params.use_mmap;
 #if defined (__APPLE__)
-    d_ptr->params.use_mlock  = true;
+    d_ptr->model_params.use_mlock = true;
 #else
-    d_ptr->params.use_mlock  = params.use_mlock;
+    d_ptr->model_params.use_mlock = params.use_mlock;
 #endif
+
 #ifdef GGML_USE_METAL
     if (llama_verbose()) {
         std::cerr << "llama.cpp: using Metal" << std::endl;
     }
     // metal always runs the whole model if n_gpu_layers is not 0, at least
     // currently
-    d_ptr->params.n_gpu_layers = 1;
+    d_ptr->model_params.n_gpu_layers = 1;
 #endif
 #ifdef GGML_USE_KOMPUTE
     if (ggml_vk_has_device()) {
         // vulkan always runs the whole model if n_gpu_layers is not 0, at least
         // currently
-        d_ptr->params.n_gpu_layers = 1;
+        d_ptr->model_params.n_gpu_layers = 1;
     }
 #endif
 
-    d_ptr->ctx = llama_init_from_file(modelPath.c_str(), d_ptr->params);
-    if (!d_ptr->ctx) {
+    d_ptr->model = llama_load_model_from_file_gpt4all(modelPath.c_str(), &d_ptr->model_params);
+    if (!d_ptr->model) {
 #ifdef GGML_USE_KOMPUTE
         // Explicitly free the device so next load it doesn't use it
         ggml_vk_free_device();
@@ -181,7 +189,39 @@ bool LLamaModel::loadModel(const std::string &modelPath)
         return false;
     }
 
-    d_ptr->end_tokens = {llama_token_eos(d_ptr->ctx)};
+    const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
+    if (n_ctx > n_ctx_train) {
+        std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
+                  << n_ctx << " specified)\n";
+    }
+
+    // -- initialize the context --
+
+    d_ptr->ctx_params = llama_context_default_params();
+
+    d_ptr->ctx_params.n_ctx  = n_ctx;
+    d_ptr->ctx_params.seed   = params.seed;
+    d_ptr->ctx_params.f16_kv = params.memory_f16;
+
+    // The new batch API provides space for n_vocab*n_tokens logits. Tell llama.cpp early
+    // that we want this many logits so the state serializes consistently.
+    d_ptr->ctx_params.logits_all = true;
+
+    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    d_ptr->ctx_params.n_threads       = d_ptr->n_threads;
+    d_ptr->ctx_params.n_threads_batch = d_ptr->n_threads;
+
+    d_ptr->ctx = llama_new_context_with_model(d_ptr->model, d_ptr->ctx_params);
+    if (!d_ptr->ctx) {
+#ifdef GGML_USE_KOMPUTE
+        // Explicitly free the device so next load it doesn't use it
+        ggml_vk_free_device();
+#endif
+        std::cerr << "LLAMA ERROR: failed to init context for model " <<  modelPath << std::endl;
+        return false;
+    }
+
+    d_ptr->end_tokens = {llama_token_eos(d_ptr->model)};
 
 #ifdef GGML_USE_KOMPUTE
     if (ggml_vk_has_device()) {
@@ -189,7 +229,6 @@ bool LLamaModel::loadModel(const std::string &modelPath)
     }
 #endif
 
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
     d_ptr->modelLoaded = true;
     fflush(stderr);
     return true;
@@ -197,6 +236,7 @@ bool LLamaModel::loadModel(const std::string &modelPath)
 
 void LLamaModel::setThreadCount(int32_t n_threads) {
     d_ptr->n_threads = n_threads;
+    llama_set_n_threads(d_ptr->ctx, n_threads, n_threads);
 }
 
 int32_t LLamaModel::threadCount() const {
@@ -208,6 +248,7 @@ LLamaModel::~LLamaModel()
     if (d_ptr->ctx) {
         llama_free(d_ptr->ctx);
     }
+    llama_free_model(d_ptr->model);
 }
 
 bool LLamaModel::isModelLoaded() const
@@ -233,16 +274,17 @@ size_t LLamaModel::restoreState(const uint8_t *src)
 
 std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str) const
 {
-    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos(d_ptr->ctx));
+    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos(d_ptr->model));
     std::vector<LLModel::Token> fres(str.size()+4);
-    auto fres_len = llama_tokenize(d_ptr->ctx, str.c_str(), str.length(), fres.data(), fres.size(), useBOS);
+    // TODO(cebtenzzre): we may want to use special=true here to process special tokens
+    auto fres_len = llama_tokenize(d_ptr->model, str.c_str(), str.length(), fres.data(), fres.size(), useBOS, false);
     fres.resize(fres_len);
     return fres;
 }
 
 std::string LLamaModel::tokenToString(Token id) const
 {
-    return llama_token_to_str(d_ptr->ctx, id);
+    return llama_token_to_piece(d_ptr->ctx, id);
 }
 
 LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
@@ -251,12 +293,32 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
     return llama_sample_top_p_top_k(d_ptr->ctx,
         promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
         n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty);
+        promptCtx.repeat_penalty, promptCtx.n_last_batch_tokens - 1);
 }
 
 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
 {
-    return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
+    llama_kv_cache_seq_rm(d_ptr->ctx, 0, ctx.n_past, -1);
+
+    llama_batch batch = llama_batch_init(tokens.size(), 0, 1);
+
+    batch.n_tokens = tokens.size();
+    ctx.n_last_batch_tokens = tokens.size();
+
+    for (int32_t i = 0; i < batch.n_tokens; i++) {
+        batch.token   [i] = tokens[i];
+        batch.pos     [i] = ctx.n_past + i;
+        batch.n_seq_id[i] = 1;
+        batch.seq_id  [i][0] = 0;
+        batch.logits  [i] = false;
+    }
+
+    // llama_decode will output logits only for the last token of the prompt
+    batch.logits[batch.n_tokens - 1] = true;
+
+    int res = llama_decode(d_ptr->ctx, batch);
+    llama_batch_free(batch);
+    return res == 0;
 }
 
 int32_t LLamaModel::contextLength() const
@@ -385,22 +447,35 @@ DLL_EXPORT const char *get_build_variant() {
 }
 
 DLL_EXPORT bool magic_match(const char * fname) {
-
     struct ggml_context * ctx_meta = NULL;
     struct gguf_init_params params = {
         /*.no_alloc = */ true,
         /*.ctx      = */ &ctx_meta,
     };
     gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
+    if (!ctx_gguf) {
+        std::cerr << __func__ << ": gguf_init_from_file failed\n";
         return false;
+    }
+
+    bool valid = true;
+
+    int gguf_ver = gguf_get_version(ctx_gguf);
+    if (valid && gguf_ver > 3) {
+        std::cerr << __func__ << ": unsupported gguf version: " << gguf_ver << "\n";
+        valid = false;
+    }
 
-    bool isValid = gguf_get_version(ctx_gguf) <= 2;
     auto arch = get_arch_name(ctx_gguf);
-    isValid = isValid && (arch == "llama" || arch == "starcoder" || arch == "falcon" || arch == "mpt");
+    if (valid && !(arch == "llama" || arch == "starcoder" || arch == "falcon" || arch == "mpt")) {
+        if (!(arch == "gptj" || arch == "bert")) { // we support these via other modules
+            std::cerr << __func__ << ": unsupported model architecture: " << arch << "\n";
+        }
+        valid = false;
+    }
 
     gguf_free(ctx_gguf);
-    return isValid;
+    return valid;
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llamamodel_impl.h

@@ -17,9 +17,9 @@ public:
 
     bool supportsEmbedding() const override { return false; }
     bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
+    bool loadModel(const std::string &modelPath, int n_ctx) override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;

gpt4all-backend/llmodel.cpp

@@ -10,6 +10,7 @@
 #include <cassert>
 #include <cstdlib>
 #include <sstream>
+#include <regex>
 #ifdef _MSC_VER
 #include <intrin.h>
 #endif
@@ -81,6 +82,13 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     static auto* libs = new std::vector<Implementation>([] () {
         std::vector<Implementation> fres;
 
+        std::string impl_name_re = "(bert|gptj|llamamodel-mainline)";
+        if (requires_avxonly()) {
+            impl_name_re += "-avxonly";
+        } else {
+            impl_name_re += "-(default|metal)";
+        }
+        std::regex re(impl_name_re);
         auto search_in_directory = [&](const std::string& paths) {
             std::stringstream ss(paths);
             std::string path;
@@ -90,7 +98,10 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
                 // Iterate over all libraries
                 for (const auto& f : std::filesystem::directory_iterator(fs_path)) {
                     const std::filesystem::path& p = f.path();
+
                     if (p.extension() != LIB_FILE_EXT) continue;
+                    if (!std::regex_search(p.stem().string(), re)) continue;
+
                     // Add to list if model implementation
                     try {
                         Dlhandle dl(p.string());
@@ -112,15 +123,22 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
 }
 
 const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
+    bool buildVariantMatched = false;
     for (const auto& i : implementationList()) {
         if (buildVariant != i.m_buildVariant) continue;
+        buildVariantMatched = true;
+
         if (!i.m_magicMatch(fname)) continue;
         return &i;
     }
+
+    if (!buildVariantMatched) {
+        std::cerr << "LLModel ERROR: Could not find any implementations for build variant: " << buildVariant << "\n";
+    }
     return nullptr;
 }
 
-LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant) {
+LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant, int n_ctx) {
     if (!has_at_least_minimal_hardware()) {
         std::cerr << "LLModel ERROR: CPU does not support AVX\n";
         return nullptr;
@@ -136,7 +154,11 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
             if(impl) {
                 LLModel* metalimpl = impl->m_construct();
                 metalimpl->m_implementation = impl;
-                size_t req_mem = metalimpl->requiredMem(modelPath);
+                /* TODO(cebtenzzre): after we fix requiredMem, we should change this to happen at
+                 * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
+                 * most (all?) places where this is called, causing underestimation of required
+                 * memory. */
+                size_t req_mem = metalimpl->requiredMem(modelPath, n_ctx);
                 float req_to_total = (float) req_mem / (float) total_mem;
                 // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
                 if (req_to_total >= 0.53) {
@@ -147,6 +169,8 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
                 }
             }
         }
+    #else
+        (void)n_ctx;
     #endif
 
     if (!impl) {
@@ -168,6 +192,27 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
     return fres;
 }
 
+LLModel *LLModel::Implementation::constructDefaultLlama() {
+    const LLModel::Implementation *impl = nullptr;
+    for (const auto &i : implementationList()) {
+        if (i.m_buildVariant == "metal" || i.m_modelType != "LLaMA") continue;
+        impl = &i;
+    }
+    if (!impl) {
+        std::cerr << "LLModel ERROR: Could not find CPU LLaMA implementation\n";
+        return nullptr;
+    }
+    auto fres = impl->m_construct();
+    fres->m_implementation = impl;
+    return fres;
+}
+
+std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices() {
+    static LLModel *llama = LLModel::Implementation::constructDefaultLlama(); // (memory leak)
+    if (llama) { return llama->availableGPUDevices(0); }
+    return {};
+}
+
 void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
     s_implementations_search_path = path;
 }

gpt4all-backend/llmodel.h

@@ -15,6 +15,15 @@ class Dlhandle;
 class LLModel {
 public:
     using Token = int32_t;
+
+    struct GPUDevice {
+        int index = 0;
+        int type = 0;
+        size_t heapSize = 0;
+        std::string name;
+        std::string vendor;
+    };
+
     class Implementation {
     public:
         Implementation(Dlhandle&&);
@@ -28,15 +37,17 @@ public:
         static bool isImplementation(const Dlhandle&);
         static const std::vector<Implementation>& implementationList();
         static const Implementation *implementation(const char *fname, const std::string& buildVariant);
-        static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
+        static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto", int n_ctx = 2048);
+        static std::vector<GPUDevice> availableGPUDevices();
         static void setImplementationsSearchPath(const std::string& path);
         static const std::string& implementationsSearchPath();
 
     private:
+        static LLModel *constructDefaultLlama();
+
         bool (*m_magicMatch)(const char *fname);
         LLModel *(*m_construct)();
 
-    private:
         std::string_view m_modelType;
         std::string_view m_buildVariant;
         Dlhandle *m_dlhandle;
@@ -54,16 +65,8 @@ public:
         int32_t n_batch = 9;
         float   repeat_penalty = 1.10f;
         int32_t repeat_last_n = 64;     // last n tokens to penalize
-        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context
-            // window
-    };
-
-    struct GPUDevice {
-        int index = 0;
-        int type = 0;
-        size_t heapSize = 0;
-        std::string name;
-        std::string vendor;
+        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context window
+        int32_t n_last_batch_tokens = 0;
     };
 
     explicit LLModel() {}
@@ -71,9 +74,9 @@ public:
 
     virtual bool supportsEmbedding() const = 0;
     virtual bool supportsCompletion() const = 0;
-    virtual bool loadModel(const std::string &modelPath) = 0;
+    virtual bool loadModel(const std::string &modelPath, int n_ctx) = 0;
     virtual bool isModelLoaded() const = 0;
-    virtual size_t requiredMem(const std::string &modelPath) = 0;
+    virtual size_t requiredMem(const std::string &modelPath, int n_ctx) = 0;
     virtual size_t stateSize() const { return 0; }
     virtual size_t saveState(uint8_t */*dest*/) const { return 0; }
     virtual size_t restoreState(const uint8_t */*src*/) { return 0; }
@@ -106,7 +109,6 @@ public:
     virtual bool initializeGPUDevice(int /*device*/) { return false; }
     virtual bool hasGPUDevice() { return false; }
     virtual bool usingGPUDevice() { return false; }
-    static std::vector<GPUDevice> availableGPUDevices();
 
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of

gpt4all-backend/llmodel_c.cpp

@@ -11,45 +11,33 @@ struct LLModelWrapper {
     ~LLModelWrapper() { delete llModel; }
 };
 
-
 thread_local static std::string last_error_message;
 
-
 llmodel_model llmodel_model_create(const char *model_path) {
-    auto fres = llmodel_model_create2(model_path, "auto", nullptr);
+    const char *error;
+    auto fres = llmodel_model_create2(model_path, "auto", &error);
     if (!fres) {
-        fprintf(stderr, "Invalid model file\n");
+        fprintf(stderr, "Unable to instantiate model: %s\n", error);
     }
     return fres;
 }
 
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error) {
+llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error) {
     auto wrapper = new LLModelWrapper;
-    int error_code = 0;
 
     try {
         wrapper->llModel = LLModel::Implementation::construct(model_path, build_variant);
+        if (!wrapper->llModel) {
+            last_error_message = "Model format not supported (no matching implementation found)";
+        }
     } catch (const std::exception& e) {
-        error_code = EINVAL;
         last_error_message = e.what();
     }
 
     if (!wrapper->llModel) {
         delete std::exchange(wrapper, nullptr);
-        // Get errno and error message if none
-        if (error_code == 0) {
-            if (errno != 0) {
-                error_code = errno;
-                last_error_message = std::strerror(error_code);
-            } else {
-                error_code = ENOTSUP;
-                last_error_message = "Model format not supported (no matching implementation found)";
-            }
-        }
-        // Set error argument
         if (error) {
-            error->message = last_error_message.c_str();
-            error->code = error_code;
+            *error = last_error_message.c_str();
         }
     }
     return reinterpret_cast<llmodel_model*>(wrapper);
@@ -59,16 +47,16 @@ void llmodel_model_destroy(llmodel_model model) {
     delete reinterpret_cast<LLModelWrapper*>(model);
 }
 
-size_t llmodel_required_mem(llmodel_model model, const char *model_path)
+size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx)
 {
     LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->requiredMem(model_path);
+    return wrapper->llModel->requiredMem(model_path, n_ctx);
 }
 
-bool llmodel_loadModel(llmodel_model model, const char *model_path)
+bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx)
 {
     LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->loadModel(model_path);
+    return wrapper->llModel->loadModel(model_path, n_ctx);
 }
 
 bool llmodel_isModelLoaded(llmodel_model model)

gpt4all-backend/llmodel_c.h

@@ -23,17 +23,6 @@ extern "C" {
  */
 typedef void *llmodel_model;
 
-/**
- * Structure containing any errors that may eventually occur
- */
-struct llmodel_error {
-    const char *message;  // Human readable error description; Thread-local; guaranteed to survive until next llmodel C API call
-    int code;             // errno; 0 if none
-};
-#ifndef __cplusplus
-typedef struct llmodel_error llmodel_error;
-#endif
-
 /**
  * llmodel_prompt_context structure for holding the prompt context.
  * NOTE: The implementation takes care of all the memory handling of the raw logits pointer and the
@@ -105,10 +94,10 @@ DEPRECATED llmodel_model llmodel_model_create(const char *model_path);
  * Recognises correct model type from file at model_path
  * @param model_path A string representing the path to the model file; will only be used to detect model type.
  * @param build_variant A string representing the implementation to use (auto, default, avxonly, ...),
- * @param error A pointer to a llmodel_error; will only be set on error.
+ * @param error A pointer to a string; will only be set on error.
  * @return A pointer to the llmodel_model instance; NULL on error.
  */
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error);
+llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error);
 
 /**
  * Destroy a llmodel instance.
@@ -121,17 +110,19 @@ void llmodel_model_destroy(llmodel_model model);
  * Estimate RAM requirement for a model file
  * @param model A pointer to the llmodel_model instance.
  * @param model_path A string representing the path to the model file.
+ * @param n_ctx Maximum size of context window
  * @return size greater than 0 if the model was parsed successfully, 0 if file could not be parsed.
  */
-size_t llmodel_required_mem(llmodel_model model, const char *model_path);
+size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx);
 
 /**
  * Load a model from a file.
  * @param model A pointer to the llmodel_model instance.
  * @param model_path A string representing the path to the model file.
+ * @param n_ctx Maximum size of context window
  * @return true if the model was loaded successfully, false otherwise.
  */
-bool llmodel_loadModel(llmodel_model model, const char *model_path);
+bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx);
 
 /**
  * Check if a model is loaded.

gpt4all-backend/llmodel_shared.cpp

@@ -4,10 +4,6 @@
 #include <iostream>
 #include <unordered_set>
 
-#ifdef GGML_USE_KOMPUTE
-#include "ggml-vulkan.h"
-#endif
-
 void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
     size_t i = 0;
     promptCtx.n_past = 0;
@@ -177,26 +173,3 @@ std::vector<float> LLModel::embedding(const std::string &/*text*/)
     }
     return std::vector<float>();
 }
-
-std::vector<LLModel::GPUDevice> LLModel::availableGPUDevices()
-{
-#if defined(GGML_USE_KOMPUTE)
-    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(0);
-
-    std::vector<LLModel::GPUDevice> devices;
-    for(const auto& vkDevice : vkDevices) {
-        LLModel::GPUDevice device;
-        device.index = vkDevice.index;
-        device.type = vkDevice.type;
-        device.heapSize = vkDevice.heapSize;
-        device.name = vkDevice.name;
-        device.vendor = vkDevice.vendor;
-
-        devices.push_back(device);
-    }
-
-    return devices;
-#else
-    return std::vector<LLModel::GPUDevice>();
-#endif
-}

gpt4all-backend/llmodel_shared.h

@@ -10,13 +10,14 @@ struct llm_buffer {
     uint8_t * addr = NULL;
     size_t size = 0;
     ggml_vk_memory memory;
+    bool force_cpu = false;
 
     llm_buffer() = default;
 
     void resize(size_t size) {
         free();
 
-        if (!ggml_vk_has_device()) {
+        if (!ggml_vk_has_device() || force_cpu) {
             this->addr = new uint8_t[size];
             this->size = size;
         } else {

gpt4all-bindings/cli/app.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """GPT4All CLI
 
 The GPT4All CLI is a self-contained script based on the `gpt4all` and `typer` packages. It offers a
@@ -53,14 +54,18 @@ def repl(
     model: Annotated[
         str,
         typer.Option("--model", "-m", help="Model to use for chatbot"),
-    ] = "ggml-gpt4all-j-v1.3-groovy",
+    ] = "mistral-7b-instruct-v0.1.Q4_0.gguf",
     n_threads: Annotated[
         int,
         typer.Option("--n-threads", "-t", help="Number of threads to use for chatbot"),
     ] = None,
+    device: Annotated[
+        str,
+        typer.Option("--device", "-d", help="Device to use for chatbot, e.g. gpu, amd, nvidia, intel. Defaults to CPU."),
+    ] = None,
 ):
     """The CLI read-eval-print loop."""
-    gpt4all_instance = GPT4All(model)
+    gpt4all_instance = GPT4All(model, device=device)
 
     # if threads are passed, set them
     if n_threads is not None:

gpt4all-bindings/csharp/Gpt4All/Bindings/LLModel.cs

@@ -188,7 +188,7 @@ public class LLModel : ILLModel
     /// <returns>true if the model was loaded successfully, false otherwise.</returns>
     public bool Load(string modelPath)
     {
-        return NativeMethods.llmodel_loadModel(_handle, modelPath);
+        return NativeMethods.llmodel_loadModel(_handle, modelPath, 2048);
     }
 
     protected void Destroy()

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeMethods.cs

@@ -70,7 +70,8 @@ internal static unsafe partial class NativeMethods
     [return: MarshalAs(UnmanagedType.I1)]
     public static extern bool llmodel_loadModel(
         [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path,
+        [NativeTypeName("int32_t")] int n_ctx);
 
     [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
 

gpt4all-bindings/csharp/Gpt4All/Model/Gpt4AllModelFactory.cs

@@ -39,7 +39,7 @@ public class Gpt4AllModelFactory : IGpt4AllModelFactory
         var handle = NativeMethods.llmodel_model_create2(modelPath, "auto", out error);
         _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
         _logger.LogInformation("Model loading started");
-        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath);
+        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath, 2048);
         _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
         if (!loadedSuccessfully)
         {

gpt4all-bindings/csharp/build_linux.sh

@@ -1,3 +1,4 @@
+#!/bin/sh
 mkdir -p runtimes
 rm -rf runtimes/linux-x64
 mkdir -p runtimes/linux-x64/native
@@ -7,4 +8,3 @@ cmake --build runtimes/linux-x64/build --parallel --config Release
 cp runtimes/linux-x64/build/libllmodel.so  runtimes/linux-x64/native/libllmodel.so
 cp runtimes/linux-x64/build/libgptj*.so  runtimes/linux-x64/native/
 cp runtimes/linux-x64/build/libllama*.so  runtimes/linux-x64/native/
-cp runtimes/linux-x64/build/libmpt*.so  runtimes/linux-x64/native/

gpt4all-bindings/golang/Makefile

@@ -139,7 +139,7 @@ $(info I CXX:      $(CXXV))
 $(info )
 
 llmodel.o:
-	mkdir buildllm
+	[ -e buildllm ] || mkdir buildllm
 	cd buildllm && cmake ../../../gpt4all-backend/ $(CMAKEFLAGS) && make
 	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel_c.cpp.o ../llmodel_c.o
 	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel.cpp.o ../llmodel.o
@@ -150,7 +150,7 @@ clean:
 	rm -rf buildllm
 	rm -rf example/main
 
-binding.o: 
+binding.o: binding.cpp binding.h
 	$(CXX) $(CXXFLAGS) binding.cpp -o binding.o -c $(LDFLAGS)
 
 libgpt4all.a: binding.o llmodel.o

gpt4all-bindings/golang/README.md

@@ -36,7 +36,7 @@ func main() {
 In order to use the bindings you will need to build `libgpt4all.a`:
 
 ```
-git clone https://github.com/nomic-ai/gpt4all
+git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
 cd gpt4all/gpt4all-bindings/golang
 make libgpt4all.a
 ```

gpt4all-bindings/golang/binding.cpp

@@ -17,14 +17,13 @@
 
 void* load_model(const char *fname, int n_threads) {
     // load the model
-    llmodel_error new_error{};
+    const char *new_error;
     auto model = llmodel_model_create2(fname, "auto", &new_error);
-    if (model == nullptr ){
-        fprintf(stderr, "%s: error '%s'\n",
-                __func__, new_error.message);
+    if (model == nullptr) {
+        fprintf(stderr, "%s: error '%s'\n", __func__, new_error);
         return nullptr;
     }
-    if (!llmodel_loadModel(model, fname)) {
+    if (!llmodel_loadModel(model, fname, 2048)) {
         llmodel_model_destroy(model);
         return nullptr;
     }

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java

@@ -1,6 +1,7 @@
 package com.hexadevlabs.gpt4all;
 
 import jnr.ffi.Pointer;
+import jnr.ffi.byref.PointerByReference;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -176,7 +177,7 @@ public  class LLModel implements AutoCloseable {
         modelName = modelPath.getFileName().toString();
         String modelPathAbs = modelPath.toAbsolutePath().toString();
 
-        LLModelLibrary.LLModelError error = new LLModelLibrary.LLModelError(jnr.ffi.Runtime.getSystemRuntime());
+        PointerByReference error = new PointerByReference();
 
         // Check if model file exists
         if(!Files.exists(modelPath)){
@@ -192,9 +193,9 @@ public  class LLModel implements AutoCloseable {
         model = library.llmodel_model_create2(modelPathAbs, "auto", error);
 
         if(model == null) {
-            throw new IllegalStateException("Could not load, gpt4all backend returned error: " + error.message);
+            throw new IllegalStateException("Could not load, gpt4all backend returned error: " + error.getValue().getString(0));
         }
-        library.llmodel_loadModel(model, modelPathAbs);
+        library.llmodel_loadModel(model, modelPathAbs, 2048);
 
         if(!library.llmodel_isModelLoaded(model)){
             throw new IllegalStateException("The model " + modelName + " could not be loaded");
@@ -631,4 +632,4 @@ public  class LLModel implements AutoCloseable {
         library.llmodel_model_destroy(model);
     }
 
-}
+}

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModelLibrary.java

@@ -1,6 +1,7 @@
 package com.hexadevlabs.gpt4all;
 
 import jnr.ffi.Pointer;
+import jnr.ffi.byref.PointerByReference;
 import jnr.ffi.Struct;
 import jnr.ffi.annotations.Delegate;
 import jnr.ffi.annotations.Encoding;
@@ -58,9 +59,9 @@ public interface LLModelLibrary {
         }
     }
 
-    Pointer llmodel_model_create2(String model_path, String build_variant, @Out LLModelError llmodel_error);
+    Pointer llmodel_model_create2(String model_path, String build_variant, PointerByReference error);
     void llmodel_model_destroy(Pointer model);
-    boolean llmodel_loadModel(Pointer model, String model_path);
+    boolean llmodel_loadModel(Pointer model, String model_path, int n_ctx);
     boolean llmodel_isModelLoaded(Pointer model);
     @u_int64_t long llmodel_get_state_size(Pointer model);
     @u_int64_t long llmodel_save_state_data(Pointer model, Pointer dest);

gpt4all-bindings/python/docs/gpt4all_chat.md

@@ -5,48 +5,46 @@ The [GPT4All Chat Client](https://gpt4all.io) lets you easily interact with any
 It is optimized to run 7-13B parameter LLMs on the CPU's of any computer running OSX/Windows/Linux.
 
 ## Running LLMs on CPU
-The GPT4All Chat UI supports models from all newer versions of `GGML`, `llama.cpp` including the `LLaMA`, `MPT`, `replit`,  `GPT-J` and `falcon` architectures
+The GPT4All Chat UI supports models from all newer versions of `llama.cpp` with `GGUF` models including the `Mistral`, `LLaMA2`, `LLaMA`, `OpenLLaMa`, `Falcon`, `MPT`, `Replit`, `Starcoder`, and `Bert` architectures
 
 GPT4All maintains an official list of recommended models located in [models2.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
 
-#### Sideloading any GGML model
+#### Sideloading any GGUF model
 If a model is compatible with the gpt4all-backend, you can sideload it into GPT4All Chat by:
 
-1. Downloading your model in GGML format. It should be a 3-8 GB file similar to the ones [here](https://huggingface.co/TheBloke/Samantha-7B-GGML/tree/main).
-2. Identifying your GPT4All model downloads folder. This is the path listed at the bottom of the downloads dialog(Three lines in top left>Downloads).
-3. Placing your downloaded model inside the GPT4All's model downloads folder.
+1. Downloading your model in GGUF format. It should be a 3-8 GB file similar to the ones [here](https://huggingface.co/TheBloke/Orca-2-7B-GGUF/tree/main).
+2. Identifying your GPT4All model downloads folder. This is the path listed at the bottom of the downloads dialog.
+3. Placing your downloaded model inside GPT4All's model downloads folder.
 4. Restarting your GPT4ALL app. Your model should appear in the model selection list.
 
 ## Plugins
 GPT4All Chat Plugins allow you to expand the capabilities of Local LLMs.
 
-### LocalDocs Beta Plugin (Chat With Your Data)
-LocalDocs is a GPT4All plugin that allows you to chat with your local files and data.
+### LocalDocs Plugin (Chat With Your Data)
+LocalDocs is a GPT4All feature that allows you to chat with your local files and data.
 It allows you to utilize powerful local LLMs to chat with private data without any data leaving your computer or server.
-When using LocalDocs, your LLM will cite the sources that most likely contributed to a given output. Note, even an LLM equipped with LocalDocs can hallucinate. If the LocalDocs plugin decides to utilize your documents to help answer a prompt, you will see references appear below the response.
+When using LocalDocs, your LLM will cite the sources that most likely contributed to a given output. Note, even an LLM equipped with LocalDocs can hallucinate. The LocalDocs plugin will utilize your documents to help answer prompts and you will see references appear below the response.
 
 <p align="center">
-  <img width="70%" src="https://github.com/nomic-ai/gpt4all/assets/13879686/f70f40b4-9684-46d8-b388-ca186f63d13e">
-</p>
-<p align="center">
-GPT4All-Snoozy with LocalDocs. Try GPT4All-Groovy for a faster experience!
+  <img width="70%" src="https://github.com/nomic-ai/gpt4all/assets/10168/fe5dd3c0-b3cc-4701-98d3-0280dfbcf26f">
 </p>
 
 #### Enabling LocalDocs
 1. Install the latest version of GPT4All Chat from [GPT4All Website](https://gpt4all.io).
 2. Go to `Settings > LocalDocs tab`.
-3. Configure a collection (folder) on your computer that contains the files your LLM should have access to. You can alter the contents of the folder/directory at anytime. As you
+3. Download the SBert model
+4. Configure a collection (folder) on your computer that contains the files your LLM should have access to. You can alter the contents of the folder/directory at anytime. As you
 add more files to your collection, your LLM will dynamically be able to access them.
-4. Spin up a chat session with any LLM (including external ones like ChatGPT but warning data will leave your machine!)
-5. At the top right, click the database icon and select which collection you want your LLM to know about during your chat session.
+5. Spin up a chat session with any LLM (including external ones like ChatGPT but warning data will leave your machine!)
+6. At the top right, click the database icon and select which collection you want your LLM to know about during your chat session.
+7. You can begin searching with your localdocs even before the collection has completed indexing, but note the search will not include those parts of the collection yet to be indexed.
 
 #### LocalDocs Capabilities
-LocalDocs allows your LLM to have context about the contents of your documentation collection. Not all prompts/question will utilize your document
-collection for context. If LocalDocs was used in your LLMs response, you will see references to the document snippets that LocalDocs used.
+LocalDocs allows your LLM to have context about the contents of your documentation collection.
 
 LocalDocs **can**:
 
-- Query your documents based upon your prompt / question. If your documents contain answers that may help answer your question/prompt LocalDocs will try to utilize snippets of your documents to provide context.
+- Query your documents based upon your prompt / question. Your documents will be searched for snippets that can be used to provide context for an answer. The most relevant snippets will be inserted into your prompts context, but it will be up to the underlying model to decide how best to use the provided context.
 
 LocalDocs **cannot**:
 
@@ -62,9 +60,6 @@ The general technique this plugin uses is called [Retrieval Augmented Generation
 
 These document chunks help your LLM respond to queries with knowledge about the contents of your data.
 The number of chunks and the size of each chunk can be configured in the LocalDocs plugin settings tab.
-For indexing speed purposes, LocalDocs uses pre-deep-learning n-gram and TF-IDF based retrieval when deciding
-what document chunks your LLM should use as context. You'll find its of comparable quality
-with embedding based retrieval approaches but magnitudes faster to ingest data.
 
 LocalDocs supports the following file types:
 ```json
@@ -82,12 +77,10 @@ LocalDocs supports the following file types:
 *My LocalDocs plugin isn't using my documents*
 
 - Make sure LocalDocs is enabled for your chat session (the DB icon on the top-right should have a border)
-- Try to modify your prompt to be more specific and use terminology that is in your document. This will increase the likelihood that LocalDocs matches document snippets for your question.
 - If your document collection is large, wait 1-2 minutes for it to finish indexing.
 
 
 #### LocalDocs Roadmap
-- Embedding based semantic search for retrieval. 
 - Customize model fine-tuned with retrieval in the loop.
 - Plugin compatibility with chat client server mode.
 

gpt4all-bindings/python/docs/gpt4all_nodejs.md

@@ -1,11 +1,14 @@
 # GPT4All Node.js API
 
+Native Node.js LLM bindings for all.
+
 ```sh
-yarn add gpt4all@alpha
+yarn add gpt4all@latest
 
-npm install gpt4all@alpha
+npm install gpt4all@latest
+
+pnpm install gpt4all@latest
 
-pnpm install gpt4all@alpha
 ```
 
 The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
@@ -15,12 +18,12 @@ The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-t
 *   Everything should work out the box.
 *   See [API Reference](#api-reference)
 
-### Chat Completion (alpha)
+### Chat Completion
 
 ```js
 import { createCompletion, loadModel } from '../src/gpt4all.js'
 
-const model = await loadModel('ggml-vicuna-7b-1.1-q4_2', { verbose: true });
+const model = await loadModel('mistral-7b-openorca.Q4_0.gguf', { verbose: true });
 
 const response = await createCompletion(model, [
     { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
@@ -29,7 +32,7 @@ const response = await createCompletion(model, [
 
 ```
 
-### Embedding (alpha)
+### Embedding
 
 ```js
 import { createEmbedding, loadModel } from '../src/gpt4all.js'
@@ -82,8 +85,6 @@ yarn
 git submodule update --init --depth 1 --recursive
 ```
 
-**AS OF NEW BACKEND** to build the backend,
-
 ```sh
 yarn build:backend
 ```
@@ -152,13 +153,16 @@ This package is in active development, and breaking changes may happen until the
 
 ##### Table of Contents
 
-*   [ModelType](#modeltype)
 *   [ModelFile](#modelfile)
     *   [gptj](#gptj)
     *   [llama](#llama)
     *   [mpt](#mpt)
     *   [replit](#replit)
 *   [type](#type)
+*   [InferenceModel](#inferencemodel)
+    *   [dispose](#dispose)
+*   [EmbeddingModel](#embeddingmodel)
+    *   [dispose](#dispose-1)
 *   [LLModel](#llmodel)
     *   [constructor](#constructor)
         *   [Parameters](#parameters)
@@ -176,12 +180,20 @@ This package is in active development, and breaking changes may happen until the
     *   [setLibraryPath](#setlibrarypath)
         *   [Parameters](#parameters-4)
     *   [getLibraryPath](#getlibrarypath)
+    *   [initGpuByString](#initgpubystring)
+        *   [Parameters](#parameters-5)
+    *   [hasGpuDevice](#hasgpudevice)
+    *   [listGpu](#listgpu)
+    *   [dispose](#dispose-2)
+*   [GpuDevice](#gpudevice)
+    *   [type](#type-2)
+*   [LoadModelOptions](#loadmodeloptions)
 *   [loadModel](#loadmodel)
-    *   [Parameters](#parameters-5)
-*   [createCompletion](#createcompletion)
     *   [Parameters](#parameters-6)
-*   [createEmbedding](#createembedding)
+*   [createCompletion](#createcompletion)
     *   [Parameters](#parameters-7)
+*   [createEmbedding](#createembedding)
+    *   [Parameters](#parameters-8)
 *   [CompletionOptions](#completionoptions)
     *   [verbose](#verbose)
     *   [systemPromptTemplate](#systemprompttemplate)
@@ -214,14 +226,14 @@ This package is in active development, and breaking changes may happen until the
     *   [repeatLastN](#repeatlastn)
     *   [contextErase](#contexterase)
 *   [createTokenStream](#createtokenstream)
-    *   [Parameters](#parameters-8)
+    *   [Parameters](#parameters-9)
 *   [DEFAULT\_DIRECTORY](#default_directory)
 *   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
 *   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
-*   [DEFAULT\_PROMT\_CONTEXT](#default_promt_context)
+*   [DEFAULT\_PROMPT\_CONTEXT](#default_prompt_context)
 *   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
 *   [downloadModel](#downloadmodel)
-    *   [Parameters](#parameters-9)
+    *   [Parameters](#parameters-10)
     *   [Examples](#examples)
 *   [DownloadModelOptions](#downloadmodeloptions)
     *   [modelPath](#modelpath)
@@ -232,16 +244,10 @@ This package is in active development, and breaking changes may happen until the
     *   [cancel](#cancel)
     *   [promise](#promise)
 
-#### ModelType
-
-Type of the model
-
-Type: (`"gptj"` | `"llama"` | `"mpt"` | `"replit"`)
-
 #### ModelFile
 
 Full list of models available
-@deprecated These model names are outdated and this type will not be maintained, please use a string literal instead
+DEPRECATED!! These model names are outdated and this type will not be maintained, please use a string literal instead
 
 ##### gptj
 
@@ -271,7 +277,27 @@ Type: `"ggml-replit-code-v1-3b.bin"`
 
 Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
 
-Type: [ModelType](#modeltype)
+Type: ModelType
+
+#### InferenceModel
+
+InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
+
+##### dispose
+
+delete and cleanup the native model
+
+Returns **void** 
+
+#### EmbeddingModel
+
+EmbeddingModel represents an LLM which can create embeddings, which are float arrays
+
+##### dispose
+
+delete and cleanup the native model
+
+Returns **void** 
 
 #### LLModel
 
@@ -294,7 +320,7 @@ Initialize a new LLModel.
 
 either 'gpt', mpt', or 'llama' or undefined
 
-Returns **([ModelType](#modeltype) | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))** 
+Returns **(ModelType | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))** 
 
 ##### name
 
@@ -376,6 +402,52 @@ Where to get the pluggable backend libraries
 
 Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 
 
+##### initGpuByString
+
+Initiate a GPU by a string identifier.
+
+###### Parameters
+
+*   `memory_required` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Should be in the range size\_t or will throw
+*   `device_name` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 'amd' | 'nvidia' | 'intel' | 'gpu' | gpu name.
+    read LoadModelOptions.device for more information
+
+Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** 
+
+##### hasGpuDevice
+
+From C documentation
+
+Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** True if a GPU device is successfully initialized, false otherwise.
+
+##### listGpu
+
+GPUs that are usable for this LLModel
+
+*   Throws **any** if hasGpuDevice returns false (i think)
+
+Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[GpuDevice](#gpudevice)>**&#x20;
+
+##### dispose
+
+delete and cleanup the native model
+
+Returns **void**&#x20;
+
+#### GpuDevice
+
+an object that contains gpu data on this machine.
+
+##### type
+
+same as VkPhysicalDeviceType
+
+Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
+
+#### LoadModelOptions
+
+Options that configure a model's behavior.
+
 #### loadModel
 
 Loads a machine learning model with the specified name. The defacto way to create a model.
@@ -384,9 +456,9 @@ By default this will download a model from the official GPT4ALL website, if a mo
 ##### Parameters
 
 *   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The name of the model to load.
-*   `options` **(LoadModelOptions | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))?** (Optional) Additional options for loading the model.
+*   `options` **([LoadModelOptions](#loadmodeloptions) | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))?** (Optional) Additional options for loading the model.
 
-Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<(InferenceModel | EmbeddingModel)>** A promise that resolves to an instance of the loaded LLModel.
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<([InferenceModel](#inferencemodel) | [EmbeddingModel](#embeddingmodel))>** A promise that resolves to an instance of the loaded LLModel.
 
 #### createCompletion
 
@@ -394,7 +466,7 @@ The nodejs equivalent to python binding's chat\_completion
 
 ##### Parameters
 
-*   `model` **InferenceModel** The language model object.
+*   `model` **[InferenceModel](#inferencemodel)** The language model object.
 *   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
 *   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
 
@@ -407,7 +479,7 @@ meow
 
 ##### Parameters
 
-*   `model` **EmbeddingModel** The language model object.
+*   `model` **[EmbeddingModel](#embeddingmodel)** The language model object.
 *   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** text to embed
 
 Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The completion result.
@@ -652,7 +724,7 @@ Default model configuration.
 
 Type: ModelConfig
 
-#### DEFAULT\_PROMT\_CONTEXT
+#### DEFAULT\_PROMPT\_CONTEXT
 
 Default prompt context.
 

gpt4all-bindings/python/gpt4all/__init__.py

@@ -1,2 +1,2 @@
-from .gpt4all import Embed4All, GPT4All  # noqa
-from .pyllmodel import LLModel  # noqa
+from .gpt4all import Embed4All as Embed4All, GPT4All as GPT4All
+from .pyllmodel import LLModel as LLModel

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -69,6 +69,7 @@ class GPT4All:
         allow_download: bool = True,
         n_threads: Optional[int] = None,
         device: Optional[str] = "cpu",
+        n_ctx: int = 2048,
         verbose: bool = False,
     ):
         """
@@ -90,15 +91,16 @@ class GPT4All:
                 Default is "cpu".
 
                 Note: If a selected GPU device does not have sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the model.
+            n_ctx: Maximum size of context window
+            verbose: If True, print debug messages.
         """
         self.model_type = model_type
         self.model = pyllmodel.LLModel()
         # Retrieve model and download if allowed
         self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download, verbose=verbose)
-        if device is not None:
-            if device != "cpu":
-                self.model.init_gpu(model_path=self.config["path"], device=device)
-        self.model.load_model(self.config["path"])
+        if device is not None and device != "cpu":
+            self.model.init_gpu(model_path=self.config["path"], device=device, n_ctx=n_ctx)
+        self.model.load_model(self.config["path"], n_ctx)
         # Set n_threads
         if n_threads is not None:
             self.model.set_thread_count(n_threads)

gpt4all-bindings/python/gpt4all/pyllmodel.py

@@ -1,4 +1,5 @@
-import atexit
+from __future__ import annotations
+
 import ctypes
 import importlib.resources
 import logging
@@ -8,20 +9,15 @@ import re
 import subprocess
 import sys
 import threading
-from contextlib import ExitStack
+from enum import Enum
 from queue import Queue
 from typing import Callable, Iterable, List
 
 logger: logging.Logger = logging.getLogger(__name__)
 
 
-file_manager = ExitStack()
-atexit.register(file_manager.close)  # clean up files on exit
-
 # TODO: provide a config file to make this more robust
-MODEL_LIB_PATH = file_manager.enter_context(importlib.resources.as_file(
-    importlib.resources.files("gpt4all") / "llmodel_DO_NOT_MODIFY" / "build",
-))
+MODEL_LIB_PATH = importlib.resources.files("gpt4all") / "llmodel_DO_NOT_MODIFY" / "build"
 
 
 def load_llmodel_library():
@@ -42,10 +38,6 @@ def load_llmodel_library():
 llmodel = load_llmodel_library()
 
 
-class LLModelError(ctypes.Structure):
-    _fields_ = [("message", ctypes.c_char_p), ("code", ctypes.c_int32)]
-
-
 class LLModelPromptContext(ctypes.Structure):
     _fields_ = [
         ("logits", ctypes.POINTER(ctypes.c_float)),
@@ -77,15 +69,15 @@ class LLModelGPUDevice(ctypes.Structure):
 llmodel.llmodel_model_create.argtypes = [ctypes.c_char_p]
 llmodel.llmodel_model_create.restype = ctypes.c_void_p
 
-llmodel.llmodel_model_create2.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.POINTER(LLModelError)]
+llmodel.llmodel_model_create2.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_char_p)]
 llmodel.llmodel_model_create2.restype = ctypes.c_void_p
 
 llmodel.llmodel_model_destroy.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_model_destroy.restype = None
 
-llmodel.llmodel_loadModel.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
+llmodel.llmodel_loadModel.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int]
 llmodel.llmodel_loadModel.restype = ctypes.c_bool
-llmodel.llmodel_required_mem.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
+llmodel.llmodel_required_mem.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int]
 llmodel.llmodel_required_mem.restype = ctypes.c_size_t
 llmodel.llmodel_isModelLoaded.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_isModelLoaded.restype = ctypes.c_bool
@@ -125,7 +117,7 @@ llmodel.llmodel_set_implementation_search_path.restype = None
 llmodel.llmodel_threadCount.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_threadCount.restype = ctypes.c_int32
 
-llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).replace("\\", r"\\").encode("utf-8"))
+llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).replace("\\", r"\\").encode())
 
 llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
 llmodel.llmodel_available_gpu_devices.restype = ctypes.POINTER(LLModelGPUDevice)
@@ -150,6 +142,20 @@ def empty_response_callback(token_id: int, response: str) -> bool:
     return True
 
 
+def _create_model(model_path: bytes) -> ctypes.c_void_p:
+    err = ctypes.c_char_p()
+    model = llmodel.llmodel_model_create2(model_path, b"auto", ctypes.byref(err))
+    if model is None:
+        s = err.value
+        raise ValueError("Unable to instantiate model: {'null' if s is None else s.decode()}")
+    return model
+
+
+# Symbol to terminate from generator
+class Sentinel(Enum):
+    TERMINATING_SYMBOL = 0
+
+
 class LLModel:
     """
     Base class and universal wrapper for GPT4All language models
@@ -176,16 +182,16 @@ class LLModel:
         if self.model is not None:
             self.llmodel_lib.llmodel_model_destroy(self.model)
 
-    def memory_needed(self, model_path: str) -> int:
-        model_path_enc = model_path.encode("utf-8")
-        self.model = llmodel.llmodel_model_create(model_path_enc)
+    def memory_needed(self, model_path: str, n_ctx: int) -> int:
+        self.model = None
+        return self._memory_needed(model_path, n_ctx)
 
-        if self.model is not None:
-            return llmodel.llmodel_required_mem(self.model, model_path_enc)
-        else:
-            raise ValueError("Unable to instantiate model")
+    def _memory_needed(self, model_path: str, n_ctx: int) -> int:
+        if self.model is None:
+            self.model = _create_model(model_path.encode())
+        return llmodel.llmodel_required_mem(self.model, model_path.encode(), n_ctx)
 
-    def list_gpu(self, model_path: str) -> list:
+    def list_gpu(self, model_path: str, n_ctx: int) -> list[LLModelGPUDevice]:
         """
         Lists available GPU devices that satisfy the model's memory requirements.
 
@@ -193,45 +199,41 @@ class LLModel:
         ----------
         model_path : str
             Path to the model.
+        n_ctx : int
+            Maximum size of context window
 
         Returns
         -------
         list
             A list of LLModelGPUDevice structures representing available GPU devices.
         """
-        if self.model is not None:
-            model_path_enc = model_path.encode("utf-8")
-            mem_required = llmodel.llmodel_required_mem(self.model, model_path_enc)
-        else:
-            mem_required = self.memory_needed(model_path)
+        mem_required = self._memory_needed(model_path, n_ctx)
+        return self._list_gpu(mem_required)
+
+    def _list_gpu(self, mem_required: int) -> list[LLModelGPUDevice]:
         num_devices = ctypes.c_int32(0)
         devices_ptr = self.llmodel_lib.llmodel_available_gpu_devices(self.model, mem_required, ctypes.byref(num_devices))
         if not devices_ptr:
             raise ValueError("Unable to retrieve available GPU devices")
-        devices = [devices_ptr[i] for i in range(num_devices.value)]
-        return devices
+        return devices_ptr[:num_devices.value]
 
-    def init_gpu(self, model_path: str, device: str):
-        if self.model is not None:
-            model_path_enc = model_path.encode("utf-8")
-            mem_required = llmodel.llmodel_required_mem(self.model, model_path_enc)
-        else:
-            mem_required = self.memory_needed(model_path)
-        device_enc = device.encode("utf-8")
-        success = self.llmodel_lib.llmodel_gpu_init_gpu_device_by_string(self.model, mem_required, device_enc)
+    def init_gpu(self, model_path: str, device: str, n_ctx: int):
+        mem_required = self._memory_needed(model_path, n_ctx)
+
+        success = self.llmodel_lib.llmodel_gpu_init_gpu_device_by_string(self.model, mem_required, device.encode())
         if not success:
             # Retrieve all GPUs without considering memory requirements.
             num_devices = ctypes.c_int32(0)
             all_devices_ptr = self.llmodel_lib.llmodel_available_gpu_devices(self.model, 0, ctypes.byref(num_devices))
             if not all_devices_ptr:
                 raise ValueError("Unable to retrieve list of all GPU devices")
-            all_gpus = [all_devices_ptr[i].name.decode('utf-8') for i in range(num_devices.value)]
+            all_gpus = [d.name.decode() for d in all_devices_ptr[:num_devices.value]]
 
             # Retrieve GPUs that meet the memory requirements using list_gpu
-            available_gpus = [device.name.decode('utf-8') for device in self.list_gpu(model_path)]
+            available_gpus = [device.name.decode() for device in self._list_gpu(mem_required)]
 
             # Identify GPUs that are unavailable due to insufficient memory or features
-            unavailable_gpus = set(all_gpus) - set(available_gpus)
+            unavailable_gpus = set(all_gpus).difference(available_gpus)
 
             # Formulate the error message
             error_msg = "Unable to initialize model on GPU: '{}'.".format(device)
@@ -239,7 +241,7 @@ class LLModel:
             error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}.".format(unavailable_gpus)
             raise ValueError(error_msg)
 
-    def load_model(self, model_path: str) -> bool:
+    def load_model(self, model_path: str, n_ctx: int) -> bool:
         """
         Load model from a file.
 
@@ -247,19 +249,16 @@ class LLModel:
         ----------
         model_path : str
             Model filepath
+        n_ctx : int
+            Maximum size of context window
 
         Returns
         -------
         True if model loaded successfully, False otherwise
         """
-        model_path_enc = model_path.encode("utf-8")
-        err = LLModelError()
-        self.model = llmodel.llmodel_model_create2(model_path_enc, b"auto", ctypes.byref(err))
+        self.model = _create_model(model_path.encode())
 
-        if self.model is None:
-            raise ValueError(f"Unable to instantiate model: code={err.code}, {err.message.decode()}")
-
-        llmodel.llmodel_loadModel(self.model, model_path_enc)
+        llmodel.llmodel_loadModel(self.model, model_path.encode(), n_ctx)
 
         filename = os.path.basename(model_path)
         self.model_name = os.path.splitext(filename)[0]
@@ -323,7 +322,7 @@ class LLModel:
             raise ValueError("Text must not be None or empty")
 
         embedding_size = ctypes.c_size_t()
-        c_text = ctypes.c_char_p(text.encode('utf-8'))
+        c_text = ctypes.c_char_p(text.encode())
         embedding_ptr = llmodel.llmodel_embedding(self.model, c_text, ctypes.byref(embedding_size))
         embedding_array = [embedding_ptr[i] for i in range(embedding_size.value)]
         llmodel.llmodel_free_embedding(embedding_ptr)
@@ -368,7 +367,7 @@ class LLModel:
             prompt,
         )
 
-        prompt_bytes = prompt.encode("utf-8")
+        prompt_bytes = prompt.encode()
         prompt_ptr = ctypes.c_char_p(prompt_bytes)
 
         self._set_context(
@@ -396,10 +395,7 @@ class LLModel:
     def prompt_model_streaming(
         self, prompt: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
     ) -> Iterable[str]:
-        # Symbol to terminate from generator
-        TERMINATING_SYMBOL = object()
-
-        output_queue: Queue = Queue()
+        output_queue: Queue[str | Sentinel] = Queue()
 
         # Put response tokens into an output queue
         def _generator_callback_wrapper(callback: ResponseCallbackType) -> ResponseCallbackType:
@@ -416,7 +412,7 @@ class LLModel:
 
         def run_llmodel_prompt(prompt: str, callback: ResponseCallbackType, **kwargs):
             self.prompt_model(prompt, callback, **kwargs)
-            output_queue.put(TERMINATING_SYMBOL)
+            output_queue.put(Sentinel.TERMINATING_SYMBOL)
 
         # Kick off llmodel_prompt in separate thread so we can return generator
         # immediately
@@ -430,7 +426,7 @@ class LLModel:
         # Generator
         while True:
             response = output_queue.get()
-            if response is TERMINATING_SYMBOL:
+            if isinstance(response, Sentinel):
                 break
             yield response
 
@@ -453,7 +449,7 @@ class LLModel:
                 else: 
                     # beginning of a byte sequence
                     if len(self.buffer) > 0:
-                        decoded.append(self.buffer.decode('utf-8', 'replace'))
+                        decoded.append(self.buffer.decode(errors='replace'))
 
                         self.buffer.clear()
 
@@ -462,7 +458,7 @@ class LLModel:
 
                 if self.buff_expecting_cont_bytes <= 0: 
                     # received the whole sequence or an out of place continuation byte
-                    decoded.append(self.buffer.decode('utf-8', 'replace'))
+                    decoded.append(self.buffer.decode(errors='replace'))
 
                     self.buffer.clear()
                     self.buff_expecting_cont_bytes = 0

gpt4all-bindings/python/gpt4all/tests/test_embed_timings.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import sys
 import time
 from io import StringIO

gpt4all-bindings/python/gpt4all/tests/test_gpt4all.py

@@ -117,7 +117,7 @@ def test_empty_embedding():
 def test_download_model(tmp_path: Path):
     import gpt4all.gpt4all
     old_default_dir = gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY
-    gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = tmp_path  # temporary pytest directory to ensure a download happens
+    gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = str(tmp_path)  # temporary pytest directory to ensure a download happens
     try:
         model = GPT4All(model_name='ggml-all-MiniLM-L6-v2-f16.bin')
         model_path = tmp_path / model.config['filename']

gpt4all-bindings/python/mkdocs.yml

@@ -14,7 +14,7 @@ nav:
       - 'GPT4All in Python':
         - 'Generation': 'gpt4all_python.md'
         - 'Embedding': 'gpt4all_python_embedding.md'
-      - 'GPT4ALL in NodeJs': 'gpt4all_typescript.md'
+      - 'GPT4ALL in NodeJs': 'gpt4all_nodejs.md'
       - 'gpt4all_cli.md'
 #    - 'Tutorials':
 #      - 'gpt4all_modal.md'

gpt4all-bindings/python/setup.py

@@ -6,7 +6,7 @@ import shutil
 package_name = "gpt4all"
 
 # Define the location of your prebuilt C library files
-SRC_CLIB_DIRECtORY = os.path.join("..", "..", "gpt4all-backend")
+SRC_CLIB_DIRECTORY = os.path.join("..", "..", "gpt4all-backend")
 SRC_CLIB_BUILD_DIRECTORY = os.path.join("..", "..", "gpt4all-backend", "build") 
 
 LIB_NAME = "llmodel"
@@ -55,13 +55,13 @@ def copy_prebuilt_C_lib(src_dir, dest_dir, dest_build_dir):
 
 # NOTE: You must provide correct path to the prebuilt llmodel C library. 
 # Specifically, the llmodel.h and C shared library are needed.
-copy_prebuilt_C_lib(SRC_CLIB_DIRECtORY,
+copy_prebuilt_C_lib(SRC_CLIB_DIRECTORY,
                     DEST_CLIB_DIRECTORY,
                     DEST_CLIB_BUILD_DIRECTORY)
 
 setup(
     name=package_name,
-    version="2.0.1",
+    version="2.1.0",
     description="Python bindings for GPT4All",
     author="Nomic and the Open Source Community",
     author_email="support@nomic.ai",

gpt4all-bindings/typescript/.gitignore

@@ -8,3 +8,4 @@ prebuilds/
 !.yarn/sdks
 !.yarn/versions
 runtimes/
+compile_flags.txt

gpt4all-bindings/typescript/.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

gpt4all-bindings/typescript/README.md

@@ -1,11 +1,14 @@
 # GPT4All Node.js API
 
+Native Node.js LLM bindings for all.
+
 ```sh
-yarn add gpt4all@alpha
+yarn add gpt4all@latest
 
-npm install gpt4all@alpha
+npm install gpt4all@latest
+
+pnpm install gpt4all@latest
 
-pnpm install gpt4all@alpha
 ```
 
 The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
@@ -20,7 +23,7 @@ The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-t
 ```js
 import { createCompletion, loadModel } from '../src/gpt4all.js'
 
-const model = await loadModel('ggml-vicuna-7b-1.1-q4_2', { verbose: true });
+const model = await loadModel('mistral-7b-openorca.Q4_0.gguf', { verbose: true });
 
 const response = await createCompletion(model, [
     { role : 'system', content: 'You are meant to be annoying and unhelpful.'  },
@@ -75,15 +78,12 @@ cd gpt4all-bindings/typescript
 ```sh
 yarn
 ```
-
 *   llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
 
 ```sh
 git submodule update --init --depth 1 --recursive
 ```
 
-**AS OF NEW BACKEND** to build the backend,
-
 ```sh
 yarn build:backend
 ```
@@ -147,587 +147,3 @@ This package is in active development, and breaking changes may happen until the
 *   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
 
 ### API Reference
-
-<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
-
-##### Table of Contents
-
-*   [ModelType](#modeltype)
-*   [ModelFile](#modelfile)
-    *   [gptj](#gptj)
-    *   [llama](#llama)
-    *   [mpt](#mpt)
-    *   [replit](#replit)
-*   [type](#type)
-*   [LLModel](#llmodel)
-    *   [constructor](#constructor)
-        *   [Parameters](#parameters)
-    *   [type](#type-1)
-    *   [name](#name)
-    *   [stateSize](#statesize)
-    *   [threadCount](#threadcount)
-    *   [setThreadCount](#setthreadcount)
-        *   [Parameters](#parameters-1)
-    *   [raw\_prompt](#raw_prompt)
-        *   [Parameters](#parameters-2)
-    *   [embed](#embed)
-        *   [Parameters](#parameters-3)
-    *   [isModelLoaded](#ismodelloaded)
-    *   [setLibraryPath](#setlibrarypath)
-        *   [Parameters](#parameters-4)
-    *   [getLibraryPath](#getlibrarypath)
-*   [loadModel](#loadmodel)
-    *   [Parameters](#parameters-5)
-*   [createCompletion](#createcompletion)
-    *   [Parameters](#parameters-6)
-*   [createEmbedding](#createembedding)
-    *   [Parameters](#parameters-7)
-*   [CompletionOptions](#completionoptions)
-    *   [verbose](#verbose)
-    *   [systemPromptTemplate](#systemprompttemplate)
-    *   [promptTemplate](#prompttemplate)
-    *   [promptHeader](#promptheader)
-    *   [promptFooter](#promptfooter)
-*   [PromptMessage](#promptmessage)
-    *   [role](#role)
-    *   [content](#content)
-*   [prompt\_tokens](#prompt_tokens)
-*   [completion\_tokens](#completion_tokens)
-*   [total\_tokens](#total_tokens)
-*   [CompletionReturn](#completionreturn)
-    *   [model](#model)
-    *   [usage](#usage)
-    *   [choices](#choices)
-*   [CompletionChoice](#completionchoice)
-    *   [message](#message)
-*   [LLModelPromptContext](#llmodelpromptcontext)
-    *   [logitsSize](#logitssize)
-    *   [tokensSize](#tokenssize)
-    *   [nPast](#npast)
-    *   [nCtx](#nctx)
-    *   [nPredict](#npredict)
-    *   [topK](#topk)
-    *   [topP](#topp)
-    *   [temp](#temp)
-    *   [nBatch](#nbatch)
-    *   [repeatPenalty](#repeatpenalty)
-    *   [repeatLastN](#repeatlastn)
-    *   [contextErase](#contexterase)
-*   [createTokenStream](#createtokenstream)
-    *   [Parameters](#parameters-8)
-*   [DEFAULT\_DIRECTORY](#default_directory)
-*   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
-*   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
-*   [DEFAULT\_PROMT\_CONTEXT](#default_promt_context)
-*   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
-*   [downloadModel](#downloadmodel)
-    *   [Parameters](#parameters-9)
-    *   [Examples](#examples)
-*   [DownloadModelOptions](#downloadmodeloptions)
-    *   [modelPath](#modelpath)
-    *   [verbose](#verbose-1)
-    *   [url](#url)
-    *   [md5sum](#md5sum)
-*   [DownloadController](#downloadcontroller)
-    *   [cancel](#cancel)
-    *   [promise](#promise)
-
-#### ModelType
-
-Type of the model
-
-Type: (`"gptj"` | `"llama"` | `"mpt"` | `"replit"`)
-
-#### ModelFile
-
-Full list of models available
-@deprecated These model names are outdated and this type will not be maintained, please use a string literal instead
-
-##### gptj
-
-List of GPT-J Models
-
-Type: (`"ggml-gpt4all-j-v1.3-groovy.bin"` | `"ggml-gpt4all-j-v1.2-jazzy.bin"` | `"ggml-gpt4all-j-v1.1-breezy.bin"` | `"ggml-gpt4all-j.bin"`)
-
-##### llama
-
-List Llama Models
-
-Type: (`"ggml-gpt4all-l13b-snoozy.bin"` | `"ggml-vicuna-7b-1.1-q4_2.bin"` | `"ggml-vicuna-13b-1.1-q4_2.bin"` | `"ggml-wizardLM-7B.q4_2.bin"` | `"ggml-stable-vicuna-13B.q4_2.bin"` | `"ggml-nous-gpt4-vicuna-13b.bin"` | `"ggml-v3-13b-hermes-q5_1.bin"`)
-
-##### mpt
-
-List of MPT Models
-
-Type: (`"ggml-mpt-7b-base.bin"` | `"ggml-mpt-7b-chat.bin"` | `"ggml-mpt-7b-instruct.bin"`)
-
-##### replit
-
-List of Replit Models
-
-Type: `"ggml-replit-code-v1-3b.bin"`
-
-#### type
-
-Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
-
-Type: [ModelType](#modeltype)
-
-#### LLModel
-
-LLModel class representing a language model.
-This is a base class that provides common functionality for different types of language models.
-
-##### constructor
-
-Initialize a new LLModel.
-
-###### Parameters
-
-*   `path` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Absolute path to the model file.
-
-<!---->
-
-*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model file does not exist.
-
-##### type
-
-either 'gpt', mpt', or 'llama' or undefined
-
-Returns **([ModelType](#modeltype) | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))**&#x20;
-
-##### name
-
-The name of the model.
-
-Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-
-##### stateSize
-
-Get the size of the internal state of the model.
-NOTE: This state data is specific to the type of model you have created.
-
-Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the size in bytes of the internal state of the model
-
-##### threadCount
-
-Get the number of threads used for model inference.
-The default is the number of physical cores your computer has.
-
-Returns **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The number of threads used for model inference.
-
-##### setThreadCount
-
-Set the number of threads used for model inference.
-
-###### Parameters
-
-*   `newNumber` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The new number of threads.
-
-Returns **void**&#x20;
-
-##### raw\_prompt
-
-Prompt the model with a given input and optional parameters.
-This is the raw output from model.
-Use the prompt function exported for a value
-
-###### Parameters
-
-*   `q` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
-*   `params` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
-*   `callback` **function (res: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): void**&#x20;
-
-Returns **void** The result of the model prompt.
-
-##### embed
-
-Embed text with the model. Keep in mind that
-not all models can embed text, (only bert can embed as of 07/16/2023 (mm/dd/yyyy))
-Use the prompt function exported for a value
-
-###### Parameters
-
-*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-*   `q`  The prompt input.
-*   `params`  Optional parameters for the prompt context.
-
-Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The result of the model prompt.
-
-##### isModelLoaded
-
-Whether the model is loaded or not.
-
-Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)**&#x20;
-
-##### setLibraryPath
-
-Where to search for the pluggable backend libraries
-
-###### Parameters
-
-*   `s` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-
-Returns **void**&#x20;
-
-##### getLibraryPath
-
-Where to get the pluggable backend libraries
-
-Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-
-#### loadModel
-
-Loads a machine learning model with the specified name. The defacto way to create a model.
-By default this will download a model from the official GPT4ALL website, if a model is not present at given path.
-
-##### Parameters
-
-*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The name of the model to load.
-*   `options` **(LoadModelOptions | [undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined))?** (Optional) Additional options for loading the model.
-
-Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<(InferenceModel | EmbeddingModel)>** A promise that resolves to an instance of the loaded LLModel.
-
-#### createCompletion
-
-The nodejs equivalent to python binding's chat\_completion
-
-##### Parameters
-
-*   `model` **InferenceModel** The language model object.
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
-*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
-
-Returns **[CompletionReturn](#completionreturn)** The completion result.
-
-#### createEmbedding
-
-The nodejs moral equivalent to python binding's Embed4All().embed()
-meow
-
-##### Parameters
-
-*   `model` **EmbeddingModel** The language model object.
-*   `text` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** text to embed
-
-Returns **[Float32Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Float32Array)** The completion result.
-
-#### CompletionOptions
-
-**Extends Partial\<LLModelPromptContext>**
-
-The options for creating the completion.
-
-##### verbose
-
-Indicates if verbose logging is enabled.
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### systemPromptTemplate
-
-Template for the system message. Will be put before the conversation with %1 being replaced by all system messages.
-Note that if this is not defined, system messages will not be included in the prompt.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### promptTemplate
-
-Template for user messages, with %1 being replaced by the message.
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### promptHeader
-
-The initial instruction for the model, on top of the prompt
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### promptFooter
-
-The last instruction for the model, appended to the end of the prompt.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### PromptMessage
-
-A message in the conversation, identical to OpenAI's chat message.
-
-##### role
-
-The role of the message.
-
-Type: (`"system"` | `"assistant"` | `"user"`)
-
-##### content
-
-The message content.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### prompt\_tokens
-
-The number of tokens used in the prompt.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### completion\_tokens
-
-The number of tokens used in the completion.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### total\_tokens
-
-The total number of tokens used.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### CompletionReturn
-
-The result of the completion, similar to OpenAI's format.
-
-##### model
-
-The model used for the completion.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### usage
-
-Token usage report.
-
-Type: {prompt\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), completion\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), total\_tokens: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)}
-
-##### choices
-
-The generated completions.
-
-Type: [Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[CompletionChoice](#completionchoice)>
-
-#### CompletionChoice
-
-A completion choice, similar to OpenAI's format.
-
-##### message
-
-Response message
-
-Type: [PromptMessage](#promptmessage)
-
-#### LLModelPromptContext
-
-Model inference arguments for generating completions.
-
-##### logitsSize
-
-The size of the raw logits vector.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### tokensSize
-
-The size of the raw tokens vector.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nPast
-
-The number of tokens in the past conversation.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nCtx
-
-The number of tokens possible in the context window.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nPredict
-
-The number of tokens to predict.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### topK
-
-The top-k logits to sample from.
-Top-K sampling selects the next token only from the top K most likely tokens predicted by the model.
-It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit
-the diversity of the output. A higher value for top-K (eg., 100) will consider more tokens and lead
-to more diverse text, while a lower value (eg., 10) will focus on the most probable tokens and generate
-more conservative text. 30 - 60 is a good range for most tasks.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### topP
-
-The nucleus sampling probability threshold.
-Top-P limits the selection of the next token to a subset of tokens with a cumulative probability
-above a threshold P. This method, also known as nucleus sampling, finds a balance between diversity
-and quality by considering both token probabilities and the number of tokens available for sampling.
-When using a higher value for top-P (eg., 0.95), the generated text becomes more diverse.
-On the other hand, a lower value (eg., 0.1) produces more focused and conservative text.
-The default value is 0.4, which is aimed to be the middle ground between focus and diversity, but
-for more creative tasks a higher top-p value will be beneficial, about 0.5-0.9 is a good range for that.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### temp
-
-The temperature to adjust the model's output distribution.
-Temperature is like a knob that adjusts how creative or focused the output becomes. Higher temperatures
-(eg., 1.2) increase randomness, resulting in more imaginative and diverse text. Lower temperatures (eg., 0.5)
-make the output more focused, predictable, and conservative. When the temperature is set to 0, the output
-becomes completely deterministic, always selecting the most probable next token and producing identical results
-each time. A safe range would be around 0.6 - 0.85, but you are free to search what value fits best for you.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### nBatch
-
-The number of predictions to generate in parallel.
-By splitting the prompt every N tokens, prompt-batch-size reduces RAM usage during processing. However,
-this can increase the processing time as a trade-off. If the N value is set too low (e.g., 10), long prompts
-with 500+ tokens will be most affected, requiring numerous processing runs to complete the prompt processing.
-To ensure optimal performance, setting the prompt-batch-size to 2048 allows processing of all tokens in a single run.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### repeatPenalty
-
-The penalty factor for repeated tokens.
-Repeat-penalty can help penalize tokens based on how frequently they occur in the text, including the input prompt.
-A token that has already appeared five times is penalized more heavily than a token that has appeared only one time.
-A value of 1 means that there is no penalty and values larger than 1 discourage repeated tokens.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### repeatLastN
-
-The number of last tokens to penalize.
-The repeat-penalty-tokens N option controls the number of tokens in the history to consider for penalizing repetition.
-A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only
-consider recent tokens.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-##### contextErase
-
-The percentage of context to erase if the context window is exceeded.
-
-Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
-
-#### createTokenStream
-
-TODO: Help wanted to implement this
-
-##### Parameters
-
-*   `llmodel` **[LLModel](#llmodel)**&#x20;
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>**&#x20;
-*   `options` **[CompletionOptions](#completionoptions)**&#x20;
-
-Returns **function (ll: [LLModel](#llmodel)): AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>**&#x20;
-
-#### DEFAULT\_DIRECTORY
-
-From python api:
-models will be stored in (homedir)/.cache/gpt4all/\`
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### DEFAULT\_LIBRARIES\_DIRECTORY
-
-From python api:
-The default path for dynamic libraries to be stored.
-You may separate paths by a semicolon to search in multiple areas.
-This searches DEFAULT\_DIRECTORY/libraries, cwd/libraries, and finally cwd.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### DEFAULT\_MODEL\_CONFIG
-
-Default model configuration.
-
-Type: ModelConfig
-
-#### DEFAULT\_PROMT\_CONTEXT
-
-Default prompt context.
-
-Type: [LLModelPromptContext](#llmodelpromptcontext)
-
-#### DEFAULT\_MODEL\_LIST\_URL
-
-Default model list url.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### downloadModel
-
-Initiates the download of a model file.
-By default this downloads without waiting. use the controller returned to alter this behavior.
-
-##### Parameters
-
-*   `modelName` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The model to be downloaded.
-*   `options` **DownloadOptions** to pass into the downloader. Default is { location: (cwd), verbose: false }.
-
-##### Examples
-
-```javascript
-const download = downloadModel('ggml-gpt4all-j-v1.3-groovy.bin')
-download.promise.then(() => console.log('Downloaded!'))
-```
-
-*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model already exists in the specified location.
-*   Throws **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** If the model cannot be found at the specified url.
-
-Returns **[DownloadController](#downloadcontroller)** object that allows controlling the download process.
-
-#### DownloadModelOptions
-
-Options for the model download process.
-
-##### modelPath
-
-location to download the model.
-Default is process.cwd(), or the current working directory
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### verbose
-
-Debug mode -- check how long it took to download in seconds
-
-Type: [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
-
-##### url
-
-Remote download url. Defaults to `https://gpt4all.io/models/gguf/<modelName>`
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-##### md5sum
-
-MD5 sum of the model file. If this is provided, the downloaded file will be checked against this sum.
-If the sums do not match, an error will be thrown and the file will be deleted.
-
-Type: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)
-
-#### DownloadController
-
-Model download controller.
-
-##### cancel
-
-Cancel the request to download if this is called.
-
-Type: function (): void
-
-##### promise
-
-A promise resolving to the downloaded models config once the download is done
-
-Type: [Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)\<ModelConfig>

gpt4all-bindings/typescript/index.cc

@@ -1,6 +1,5 @@
 #include "index.h"
 
-Napi::FunctionReference NodeModelWrapper::constructor;
 
 Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
     Napi::Function self = DefineClass(env, "LLModel", {
@@ -13,14 +12,64 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
        InstanceMethod("embed", &NodeModelWrapper::GenerateEmbedding),
        InstanceMethod("threadCount", &NodeModelWrapper::ThreadCount),
        InstanceMethod("getLibraryPath", &NodeModelWrapper::GetLibraryPath),
+       InstanceMethod("initGpuByString", &NodeModelWrapper::InitGpuByString),
+       InstanceMethod("hasGpuDevice", &NodeModelWrapper::HasGpuDevice),
+       InstanceMethod("listGpu", &NodeModelWrapper::GetGpuDevices),
+       InstanceMethod("memoryNeeded", &NodeModelWrapper::GetRequiredMemory),
+       InstanceMethod("dispose", &NodeModelWrapper::Dispose)
     });
     // Keep a static reference to the constructor
     //
-    constructor = Napi::Persistent(self);
-    constructor.SuppressDestruct();
+    Napi::FunctionReference* constructor = new Napi::FunctionReference();
+    *constructor = Napi::Persistent(self);
+    env.SetInstanceData(constructor);
     return self;
+}
+Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info) 
+{
+    auto env = info.Env();
+    return Napi::Number::New(env, static_cast<uint32_t>( llmodel_required_mem(GetInference(), full_model_path.c_str(), 2048) ));
+
+}
+  Napi::Value NodeModelWrapper::GetGpuDevices(const Napi::CallbackInfo& info) 
+  {
+    auto env = info.Env();
+    int num_devices = 0;
+    auto mem_size = llmodel_required_mem(GetInference(), full_model_path.c_str());
+    llmodel_gpu_device* all_devices = llmodel_available_gpu_devices(GetInference(), mem_size, &num_devices);
+    if(all_devices == nullptr) {
+        Napi::Error::New(
+            env, 
+            "Unable to retrieve list of all GPU devices"
+        ).ThrowAsJavaScriptException(); 
+        return env.Undefined();
+    }
+    auto js_array = Napi::Array::New(env, num_devices);
+    for(int i = 0; i < num_devices; ++i) {
+       auto gpu_device = all_devices[i];
+       /* 
+        *
+        * struct llmodel_gpu_device {
+            int index = 0;
+            int type = 0;           // same as VkPhysicalDeviceType
+            size_t heapSize = 0; 
+            const char * name;
+            const char * vendor;
+          };
+        *
+        */
+       Napi::Object js_gpu_device = Napi::Object::New(env);
+        js_gpu_device["index"] = uint32_t(gpu_device.index);
+        js_gpu_device["type"] = uint32_t(gpu_device.type);
+        js_gpu_device["heapSize"] = static_cast<uint32_t>( gpu_device.heapSize );
+        js_gpu_device["name"]= gpu_device.name;
+        js_gpu_device["vendor"] = gpu_device.vendor;
+
+        js_array[i] = js_gpu_device;
+    }
+    return js_array;
   }
- 
+
   Napi::Value NodeModelWrapper::getType(const Napi::CallbackInfo& info) 
   {
     if(type.empty()) {
@@ -29,15 +78,41 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
     return Napi::String::New(info.Env(), type);
   }
 
+  Napi::Value NodeModelWrapper::InitGpuByString(const Napi::CallbackInfo& info) 
+  {
+    auto env = info.Env();
+    size_t memory_required = static_cast<size_t>(info[0].As<Napi::Number>().Uint32Value());
+    
+    std::string gpu_device_identifier = info[1].As<Napi::String>();   
+
+    size_t converted_value;
+    if(memory_required <= std::numeric_limits<size_t>::max()) {
+        converted_value = static_cast<size_t>(memory_required);
+    } else {
+         Napi::Error::New(
+            env, 
+            "invalid number for memory size. Exceeded bounds for memory."
+        ).ThrowAsJavaScriptException(); 
+        return env.Undefined();
+    }
+    
+    auto result = llmodel_gpu_init_gpu_device_by_string(GetInference(), converted_value, gpu_device_identifier.c_str());
+    return Napi::Boolean::New(env, result);
+  }
+  Napi::Value NodeModelWrapper::HasGpuDevice(const Napi::CallbackInfo& info) 
+  {
+    return Napi::Boolean::New(info.Env(), llmodel_has_gpu_device(GetInference()));
+  }
+
   NodeModelWrapper::NodeModelWrapper(const Napi::CallbackInfo& info) : Napi::ObjectWrap<NodeModelWrapper>(info) 
   {
     auto env = info.Env();
     fs::path model_path;
 
-    std::string full_weight_path;
-    //todo
-    std::string library_path = ".";
-    std::string model_name;
+    std::string full_weight_path,
+                library_path = ".",
+                model_name, 
+                device;
     if(info[0].IsString()) {
         model_path = info[0].As<Napi::String>().Utf8Value();
         full_weight_path = model_path.string();
@@ -56,15 +131,13 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
         } else {
             library_path = ".";
         }
+        device = config_object.Get("device").As<Napi::String>();
     }
     llmodel_set_implementation_search_path(library_path.c_str());
-    llmodel_error e = {
-        .message="looks good to me",
-        .code=0,
-    };
-    inference_ = std::make_shared<llmodel_model>(llmodel_model_create2(full_weight_path.c_str(), "auto", &e));
-    if(e.code != 0) {
-       Napi::Error::New(env, e.message).ThrowAsJavaScriptException(); 
+    const char* e;
+    inference_ = llmodel_model_create2(full_weight_path.c_str(), "auto", &e);
+    if(!inference_) {
+       Napi::Error::New(env, e).ThrowAsJavaScriptException();
        return;
     }
     if(GetInference() == nullptr) {
@@ -74,18 +147,43 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
        Napi::Error::New(env, "Had an issue creating llmodel object, inference is null").ThrowAsJavaScriptException(); 
        return;
     }
+    if(device != "cpu") {
+        size_t mem = llmodel_required_mem(GetInference(), full_weight_path.c_str());
+        std::cout << "Initiating GPU\n";
 
-    auto success = llmodel_loadModel(GetInference(), full_weight_path.c_str());
+        auto success = llmodel_gpu_init_gpu_device_by_string(GetInference(), mem, device.c_str());
+        if(success) {
+            std::cout << "GPU init successfully\n";
+        } else {
+            //https://github.com/nomic-ai/gpt4all/blob/3acbef14b7c2436fe033cae9036e695d77461a16/gpt4all-bindings/python/gpt4all/pyllmodel.py#L215
+            //Haven't implemented this but it is still open to contribution
+            std::cout << "WARNING: Failed to init GPU\n";
+        }
+    }
+
+    auto success = llmodel_loadModel(GetInference(), full_weight_path.c_str(), 2048);
     if(!success) {
         Napi::Error::New(env, "Failed to load model at given path").ThrowAsJavaScriptException(); 
         return;
     }
-    name = model_name.empty() ? model_path.filename().string() : model_name;
-  };
-  //NodeModelWrapper::~NodeModelWrapper() {
-    //GetInference().reset();
-  //}
 
+    name = model_name.empty() ? model_path.filename().string() : model_name;
+    full_model_path = full_weight_path;
+  };
+
+//  NodeModelWrapper::~NodeModelWrapper() {
+//    if(GetInference() != nullptr) {
+//        std::cout << "Debug: deleting model\n";
+//        llmodel_model_destroy(inference_);
+//        std::cout << (inference_ == nullptr);
+//    }
+//  }
+//  void NodeModelWrapper::Finalize(Napi::Env env) {
+//    if(inference_ != nullptr) {
+//        std::cout << "Debug: deleting model\n";
+//
+//    } 
+//  }
   Napi::Value NodeModelWrapper::IsModelLoaded(const Napi::CallbackInfo& info) {
     return Napi::Boolean::New(info.Env(), llmodel_isModelLoaded(GetInference()));
   }
@@ -193,8 +291,9 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
     std::string copiedQuestion = question;
     PromptWorkContext pc = {
         copiedQuestion,
-        std::ref(inference_),
+        inference_,
         copiedPrompt,
+        ""
     };
     auto threadSafeContext = new TsfnContext(env, pc);
     threadSafeContext->tsfn = Napi::ThreadSafeFunction::New(
@@ -210,7 +309,9 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
     threadSafeContext->nativeThread = std::thread(threadEntry, threadSafeContext);
     return threadSafeContext->deferred_.Promise();
   }
-
+  void NodeModelWrapper::Dispose(const Napi::CallbackInfo& info) {
+    llmodel_model_destroy(inference_);
+  }
   void NodeModelWrapper::SetThreadCount(const Napi::CallbackInfo& info) {
     if(info[0].IsNumber()) {
         llmodel_setThreadCount(GetInference(), info[0].As<Napi::Number>().Int64Value());
@@ -233,7 +334,7 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
   }
 
   llmodel_model NodeModelWrapper::GetInference() {
-    return *inference_;
+    return inference_;
   }
 
 //Exports Bindings

gpt4all-bindings/typescript/index.h

@@ -6,24 +6,33 @@
 #include <atomic>
 #include <memory>
 #include <filesystem>
+#include <set>
 namespace fs = std::filesystem;
 
+
 class NodeModelWrapper: public Napi::ObjectWrap<NodeModelWrapper> {
 public:
   NodeModelWrapper(const Napi::CallbackInfo &);
-  //~NodeModelWrapper();
+  //virtual ~NodeModelWrapper();
   Napi::Value getType(const Napi::CallbackInfo& info);
   Napi::Value IsModelLoaded(const Napi::CallbackInfo& info);
   Napi::Value StateSize(const Napi::CallbackInfo& info);
+  //void Finalize(Napi::Env env) override;
   /**
    * Prompting the model. This entails spawning a new thread and adding the response tokens
    * into a thread local string variable.
    */
   Napi::Value Prompt(const Napi::CallbackInfo& info);
   void SetThreadCount(const Napi::CallbackInfo& info);
+  void Dispose(const Napi::CallbackInfo& info);
   Napi::Value getName(const Napi::CallbackInfo& info);
   Napi::Value ThreadCount(const Napi::CallbackInfo& info);
   Napi::Value GenerateEmbedding(const Napi::CallbackInfo& info);
+  Napi::Value HasGpuDevice(const Napi::CallbackInfo& info);
+  Napi::Value ListGpus(const Napi::CallbackInfo& info);
+  Napi::Value InitGpuByString(const Napi::CallbackInfo& info);
+  Napi::Value GetRequiredMemory(const Napi::CallbackInfo& info);
+  Napi::Value GetGpuDevices(const Napi::CallbackInfo& info);
   /*
    * The path that is used to search for the dynamic libraries
    */
@@ -37,10 +46,10 @@ private:
   /**
    * The underlying inference that interfaces with the C interface
    */
-  std::shared_ptr<llmodel_model> inference_;
+  llmodel_model inference_;
 
   std::string type;
   // corresponds to LLModel::name() in typescript
   std::string name;
-  static Napi::FunctionReference constructor;
+  std::string full_model_path;
 };

gpt4all-bindings/typescript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt4all",
-  "version": "2.2.0",
+  "version": "3.1.0",
   "packageManager": "yarn@3.6.1",
   "main": "src/gpt4all.js",
   "repository": "nomic-ai/gpt4all",
@@ -9,9 +9,7 @@
     "test": "jest",
     "build:backend": "node scripts/build.js",
     "build": "node-gyp-build",
-    "predocs:build": "node scripts/docs.js",
-    "docs:build": "documentation readme ./src/gpt4all.d.ts --parse-extension js d.ts --format md --section \"API Reference\" --readme-file ../python/docs/gpt4all_typescript.md",
-    "postdocs:build": "documentation readme ./src/gpt4all.d.ts --parse-extension js d.ts --format md --section \"API Reference\" --readme-file README.md"
+    "docs:build": "node scripts/docs.js && documentation readme ./src/gpt4all.d.ts --parse-extension js d.ts --format md --section \"API Reference\" --readme-file ../python/docs/gpt4all_nodejs.md"
   },
   "files": [
     "src/**/*",
@@ -47,5 +45,10 @@
   },
   "jest": {
     "verbose": true
+  },
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org/",
+    "access": "public",
+    "tag": "latest"
   }
 }

gpt4all-bindings/typescript/prompt.cc

@@ -30,7 +30,7 @@ void threadEntry(TsfnContext* context) {
     context->tsfn.BlockingCall(&context->pc,
     [](Napi::Env env, Napi::Function jsCallback, PromptWorkContext* pc) {
         llmodel_prompt(
-            *pc->inference_,
+            pc->inference_,
             pc->question.c_str(),
             &prompt_callback,
             &response_callback,

gpt4all-bindings/typescript/prompt.h

@@ -10,7 +10,7 @@
 #include <memory>
 struct PromptWorkContext {
     std::string question;
-    std::shared_ptr<llmodel_model>& inference_;
+    llmodel_model inference_;
     llmodel_prompt_context prompt_params;
     std::string res;
 

gpt4all-bindings/typescript/scripts/build_unix.sh

@@ -25,9 +25,6 @@ mkdir -p "$NATIVE_DIR" "$BUILD_DIR"
 cmake -S ../../gpt4all-backend -B "$BUILD_DIR" &&
 cmake --build "$BUILD_DIR" -j --config Release && {
   cp "$BUILD_DIR"/libbert*.$LIB_EXT   "$NATIVE_DIR"/
-  cp "$BUILD_DIR"/libfalcon*.$LIB_EXT "$NATIVE_DIR"/
-  cp "$BUILD_DIR"/libreplit*.$LIB_EXT "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libgptj*.$LIB_EXT   "$NATIVE_DIR"/
   cp "$BUILD_DIR"/libllama*.$LIB_EXT  "$NATIVE_DIR"/
-  cp "$BUILD_DIR"/libmpt*.$LIB_EXT    "$NATIVE_DIR"/
 }

gpt4all-bindings/typescript/scripts/docs.js

@@ -2,7 +2,11 @@
 
 const fs = require('fs');
 
-const newPath = '../python/docs/gpt4all_typescript.md';
-const filepath = 'README.md';
-const data = fs.readFileSync(filepath);
-fs.writeFileSync(newPath, data);
+const newPath = '../python/docs/gpt4all_nodejs.md';
+const filepath = './README.md';
+const intro = fs.readFileSync(filepath);
+
+fs.writeFileSync(
+    newPath, intro
+);
+

gpt4all-bindings/typescript/scripts/mkclangd.js

@@ -0,0 +1,43 @@
+/// makes compile_flags.txt for clangd server support with this project
+/// run this with typescript as your cwd
+//
+//for debian users make sure to install libstdc++-12-dev
+
+const nodeaddonapi=require('node-addon-api').include;
+
+const fsp = require('fs/promises');
+const { existsSync, readFileSync } = require('fs');
+const assert = require('node:assert');
+const findnodeapih = () => {
+    assert(existsSync("./build"), "Haven't built the application once yet. run node scripts/prebuild.js");
+    const dir = readFileSync("./build/config.gypi", 'utf8');
+    const nodedir_line = dir.match(/"nodedir": "([^"]+)"/);
+    assert(nodedir_line, "Found no matches")
+    assert(nodedir_line[1]);
+    console.log("node_api.h found at: ", nodedir_line[1]);
+    return nodedir_line[1]+"/include/node";
+};
+
+const knownIncludes = [
+    '-I',
+    './',
+    '-I',
+    nodeaddonapi.substring(1, nodeaddonapi.length-1),
+    '-I',
+    '../../gpt4all-backend',
+    '-I',
+    findnodeapih()
+];
+const knownFlags = [
+    "-x",
+    "c++",
+    '-std=c++17'
+];
+
+
+const output = knownFlags.join('\n')+'\n'+knownIncludes.join('\n');
+
+fsp.writeFile('./compile_flags.txt', output, 'utf8')
+    .then(() => console.log('done'))
+    .catch(() => console.err('failed'));
+

gpt4all-bindings/typescript/spec/chat.mjs

@@ -1,8 +1,8 @@
 import { LLModel, createCompletion, DEFAULT_DIRECTORY, DEFAULT_LIBRARIES_DIRECTORY, loadModel } from '../src/gpt4all.js'
 
 const model = await loadModel(
-    'orca-mini-3b-gguf2-q4_0.gguf',
-    { verbose: true }
+    'mistral-7b-openorca.Q4_0.gguf',
+    { verbose: true, device: 'gpu' }
 );
 const ll = model.llm;
 
@@ -26,7 +26,9 @@ console.log("name " + ll.name());
 console.log("type: " + ll.type());
 console.log("Default directory for models", DEFAULT_DIRECTORY);
 console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY);
-
+console.log("Has GPU", ll.hasGpuDevice());
+console.log("gpu devices", ll.listGpu())
+console.log("Required Mem in bytes", ll.memoryNeeded())
 const completion1 = await createCompletion(model, [ 
     { role : 'system', content: 'You are an advanced mathematician.'  },
     { role : 'user', content: 'What is 1 + 1?'  }, 
@@ -40,6 +42,8 @@ const completion2 = await createCompletion(model, [
 
 console.log(completion2.choices[0].message)
 
+//CALLING DISPOSE WILL INVALID THE NATIVE MODEL. USE THIS TO CLEANUP
+model.dispose()
 // At the moment, from testing this code, concurrent model prompting is not possible. 
 // Behavior: The last prompt gets answered, but the rest are cancelled
 // my experience with threading is not the best, so if anyone who is good is willing to give this a shot,
@@ -47,16 +51,16 @@ console.log(completion2.choices[0].message)
 // INFO: threading with llama.cpp is not the best maybe not even possible, so this will be left here as reference
 
 //const responses = await Promise.all([
-//    createCompletion(ll, [ 
+//    createCompletion(model, [ 
 //    { role : 'system', content: 'You are an advanced mathematician.'  },
 //    { role : 'user', content: 'What is 1 + 1?'  }, 
 //    ], { verbose: true }),
-//    createCompletion(ll, [ 
+//    createCompletion(model, [ 
 //    { role : 'system', content: 'You are an advanced mathematician.'  },
 //    { role : 'user', content: 'What is 1 + 1?'  }, 
 //    ], { verbose: true }),
 //
-//createCompletion(ll, [ 
+//createCompletion(model, [ 
 //    { role : 'system', content: 'You are an advanced mathematician.'  },
 //    { role : 'user', content: 'What is 1 + 1?'  }, 
 //], { verbose: true })

gpt4all-bindings/typescript/spec/embed.mjs

@@ -1,8 +1,6 @@
-import {  loadModel, createEmbedding } from '../src/gpt4all.js'
+import { loadModel, createEmbedding } from '../src/gpt4all.js'
 
-const embedder = await loadModel("ggml-all-MiniLM-L6-v2-f16.bin", { verbose: true })
+const embedder = await loadModel("ggml-all-MiniLM-L6-v2-f16.bin", { verbose: true, type: 'embedding'})
 
-console.log(
-    createEmbedding(embedder, "Accept your current situation")
-)
+console.log(createEmbedding(embedder, "Accept your current situation"))
 

gpt4all-bindings/typescript/src/config.js

@@ -9,7 +9,13 @@ const librarySearchPaths = [
     path.resolve(
         __dirname,
         "..",
-        `runtimes/${process.platform}-${process.arch}/native`
+        `runtimes/${process.platform}-${process.arch}/native`,
+    ),
+    //for darwin. This is hardcoded for now but it should work
+    path.resolve(
+        __dirname,
+        "..",
+        `runtimes/${process.platform}/native`,
     ),
     process.cwd(),
 ];

gpt4all-bindings/typescript/src/gpt4all.d.ts

@@ -1,13 +1,12 @@
 /// <reference types="node" />
 declare module "gpt4all";
 
-/** Type of the model */
 type ModelType = "gptj" | "llama" | "mpt" | "replit";
 
 // NOTE: "deprecated" tag in below comment breaks the doc generator https://github.com/documentationjs/documentation/issues/1596
 /**
  * Full list of models available
- * @deprecated These model names are outdated and this type will not be maintained, please use a string literal instead
+ * DEPRECATED!! These model names are outdated and this type will not be maintained, please use a string literal instead
  */
 interface ModelFile {
     /** List of GPT-J Models */
@@ -34,7 +33,6 @@ interface ModelFile {
     replit: "ggml-replit-code-v1-3b.bin";
 }
 
-//mirrors py options
 interface LLModelOptions {
     /**
      * Model architecture. This argument currently does not have any functionality and is just used as descriptive identifier for user.
@@ -51,7 +49,11 @@ interface ModelConfig {
     path: string;
     url?: string;
 }
-
+/**
+ *
+ * InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
+ *
+ */
 declare class InferenceModel {
     constructor(llm: LLModel, config: ModelConfig);
     llm: LLModel;
@@ -61,14 +63,28 @@ declare class InferenceModel {
         prompt: string,
         options?: Partial<LLModelPromptContext>
     ): Promise<string>;
+
+   /**
+     * delete and cleanup the native model
+    */
+    dispose(): void
 }
 
+/**
+ * EmbeddingModel represents an LLM which can create embeddings, which are float arrays
+ */
 declare class EmbeddingModel {
     constructor(llm: LLModel, config: ModelConfig);
     llm: LLModel;
     config: ModelConfig;
 
     embed(text: string): Float32Array;
+
+    /**
+      * delete and cleanup the native model
+     */
+    dispose(): void
+
 }
 
 /**
@@ -146,14 +162,68 @@ declare class LLModel {
      * Where to get the pluggable backend libraries
      */
     getLibraryPath(): string;
+    /**
+     * Initiate a GPU by a string identifier. 
+     * @param {number} memory_required Should be in the range size_t or will throw 
+     * @param {string} device_name  'amd' | 'nvidia' | 'intel' | 'gpu' | gpu name.
+     * read LoadModelOptions.device for more information
+     */
+    initGpuByString(memory_required: number, device_name: string): boolean
+    /**
+     * From C documentation
+     * @returns True if a GPU device is successfully initialized, false otherwise.
+     */
+    hasGpuDevice(): boolean
+    /**
+      * GPUs that are usable for this LLModel
+      * @throws if hasGpuDevice returns false (i think)
+      * @returns 
+      */
+    listGpu() : GpuDevice[]
+
+    /**
+      * delete and cleanup the native model
+     */
+    dispose(): void
+}
+/** 
+ * an object that contains gpu data on this machine.
+ */
+interface GpuDevice {
+    index: number;
+    /**
+      * same as VkPhysicalDeviceType
+     */
+    type: number;            
+    heapSize : number; 
+    name: string;
+    vendor: string;
 }
 
+/**
+  * Options that configure a model's behavior.
+  */
 interface LoadModelOptions {
     modelPath?: string;
     librariesPath?: string;
     modelConfigFile?: string;
     allowDownload?: boolean;
     verbose?: boolean;
+    /* The processing unit on which the model will run. It can be set to
+     * - "cpu": Model will run on the central processing unit.
+     * - "gpu": Model will run on the best available graphics processing unit, irrespective of its vendor.
+     * - "amd", "nvidia", "intel": Model will run on the best available GPU from the specified vendor.
+
+	   Alternatively, a specific GPU name can also be provided, and the model will run on the GPU that matches the name
+       if it's available.
+
+       Default is "cpu".
+
+	   Note: If a GPU device lacks sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All
+       instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the
+       model.
+    */ 
+    device?: string;
 }
 
 interface InferenceModelOptions extends LoadModelOptions {
@@ -184,7 +254,7 @@ declare function loadModel(
 
 declare function loadModel(
     modelName: string,
-    options?: EmbeddingOptions | InferenceOptions
+    options?: EmbeddingModelOptions | InferenceModelOptions
 ): Promise<InferenceModel | EmbeddingModel>;
 
 /**
@@ -401,7 +471,7 @@ declare const DEFAULT_MODEL_CONFIG: ModelConfig;
 /**
  * Default prompt context.
  */
-declare const DEFAULT_PROMT_CONTEXT: LLModelPromptContext;
+declare const DEFAULT_PROMPT_CONTEXT: LLModelPromptContext;
 
 /**
  * Default model list url.
@@ -502,7 +572,7 @@ export {
     DEFAULT_DIRECTORY,
     DEFAULT_LIBRARIES_DIRECTORY,
     DEFAULT_MODEL_CONFIG,
-    DEFAULT_PROMT_CONTEXT,
+    DEFAULT_PROMPT_CONTEXT,
     DEFAULT_MODEL_LIST_URL,
     downloadModel,
     retrieveModel,
@@ -510,4 +580,5 @@ export {
     DownloadController,
     RetrieveModelOptions,
     DownloadModelOptions,
+    GpuDevice
 };

gpt4all-bindings/typescript/src/gpt4all.js

@@ -18,6 +18,7 @@ const {
     DEFAULT_MODEL_LIST_URL,
 } = require("./config.js");
 const { InferenceModel, EmbeddingModel } = require("./models.js");
+const assert = require("assert");
 
 /**
  * Loads a machine learning model with the specified name. The defacto way to create a model.
@@ -34,6 +35,7 @@ async function loadModel(modelName, options = {}) {
         type: "inference",
         allowDownload: true,
         verbose: true,
+        device: 'cpu',
         ...options,
     };
 
@@ -44,30 +46,24 @@ async function loadModel(modelName, options = {}) {
         verbose: loadOptions.verbose,
     });
 
-    const libSearchPaths = loadOptions.librariesPath.split(";");
+    assert.ok(typeof loadOptions.librariesPath === 'string');
+    const existingPaths = loadOptions.librariesPath
+        .split(";")
+        .filter(existsSync)
+        .join(';');
+    console.log("Passing these paths into runtime library search:", existingPaths)
 
-    let libPath = null;
-
-    for (const searchPath of libSearchPaths) {
-        if (existsSync(searchPath)) {
-            libPath = searchPath;
-            break;
-        }
-    }
-    if (!libPath) {
-        throw Error("Could not find a valid path from " + libSearchPaths);
-    }
     const llmOptions = {
         model_name: appendBinSuffixIfMissing(modelName),
         model_path: loadOptions.modelPath,
-        library_path: libPath,
+        library_path: existingPaths,
+        device: loadOptions.device,
     };
 
     if (loadOptions.verbose) {
         console.debug("Creating LLModel with options:", llmOptions);
     }
     const llmodel = new LLModel(llmOptions);
-
     if (loadOptions.type === "embedding") {
         return new EmbeddingModel(llmodel, modelConfig);
     } else if (loadOptions.type === "inference") {

gpt4all-bindings/typescript/src/models.js

@@ -15,6 +15,10 @@ class InferenceModel {
         const result = this.llm.raw_prompt(prompt, normalizedPromptContext, () => {});
         return result;
     }
+
+    dispose() {
+        this.llm.dispose();
+    }
 }
 
 class EmbeddingModel {
@@ -29,6 +33,10 @@ class EmbeddingModel {
     embed(text) {
         return this.llm.embed(text)
     }
+
+    dispose() {
+        this.llm.dispose();
+    }
 }
 
 

gpt4all-bindings/typescript/src/util.js

@@ -43,8 +43,9 @@ async function listModels(
 }
 
 function appendBinSuffixIfMissing(name) {
-    if (!name.endsWith(".bin")) {
-        return name + ".bin";
+    const ext = path.extname(name);
+    if (![".bin", ".gguf"].includes(ext)) {
+        return name + ".gguf";
     }
     return name;
 }

gpt4all-bindings/typescript/test/gpt4all.test.js

@@ -35,6 +35,11 @@ describe("config", () => {
                 "..",
                 `runtimes/${process.platform}-${process.arch}/native`
             ),
+            path.resolve(
+                __dirname,
+                "..",
+                `runtimes/${process.platform}/native`,
+            ),
             process.cwd(),
         ];
         expect(typeof DEFAULT_LIBRARIES_DIRECTORY).toBe("string");
@@ -92,7 +97,7 @@ describe("listModels", () => {
 
 describe("appendBinSuffixIfMissing", () => {
     it("should make sure the suffix is there", () => {
-        expect(appendBinSuffixIfMissing("filename")).toBe("filename.bin");
+        expect(appendBinSuffixIfMissing("filename")).toBe("filename.gguf");
         expect(appendBinSuffixIfMissing("filename.bin")).toBe("filename.bin");
     });
 });
@@ -156,11 +161,11 @@ describe("downloadModel", () => {
     test("should successfully download a model file", async () => {
         const downloadController = downloadModel(fakeModelName);
         const modelFilePath = await downloadController.promise;
-        expect(modelFilePath).toBe(path.resolve(DEFAULT_DIRECTORY, `${fakeModelName}.bin`));
+        expect(modelFilePath).toBe(path.resolve(DEFAULT_DIRECTORY, `${fakeModelName}.gguf`));
 
         expect(global.fetch).toHaveBeenCalledTimes(1);
         expect(global.fetch).toHaveBeenCalledWith(
-            "https://gpt4all.io/models/fake-model.bin",
+            "https://gpt4all.io/models/gguf/fake-model.gguf",
             {
                 signal: "signal",
                 headers: {
@@ -189,7 +194,7 @@ describe("downloadModel", () => {
         expect(global.fetch).toHaveBeenCalledTimes(1);
         // the file should be missing
         await expect(
-            fsp.access(path.resolve(DEFAULT_DIRECTORY, `${fakeModelName}.bin`))
+            fsp.access(path.resolve(DEFAULT_DIRECTORY, `${fakeModelName}.gguf`))
         ).rejects.toThrow();
         // partial file should also be missing
         await expect(

gpt4all-bindings/typescript/test/models.json

@@ -3,8 +3,8 @@
     "order": "a",
     "md5sum": "08d6c05a21512a79a1dfeb9d2a8f262f",
     "name": "Not a real model",
-    "filename": "fake-model.bin",
+    "filename": "fake-model.gguf",
     "filesize": "4",
     "systemPrompt": " "
   }
-]
+]

gpt4all-chat/CMakeLists.txt

@@ -17,8 +17,8 @@ if(APPLE)
 endif()
 
 set(APP_VERSION_MAJOR 2)
-set(APP_VERSION_MINOR 5)
-set(APP_VERSION_PATCH 1)
+set(APP_VERSION_MINOR 6)
+set(APP_VERSION_PATCH 2)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file
@@ -75,7 +75,9 @@ qt_add_executable(chat
     chatmodel.h chatlistmodel.h chatlistmodel.cpp
     chatgpt.h chatgpt.cpp
     database.h database.cpp
+    embeddings.h embeddings.cpp
     download.h download.cpp
+    embllm.cpp embllm.h
     localdocs.h localdocs.cpp localdocsmodel.h localdocsmodel.cpp
     llm.h llm.cpp
     modellist.h modellist.cpp
@@ -90,6 +92,7 @@ qt_add_executable(chat
 qt_add_qml_module(chat
     URI gpt4all
     VERSION 1.0
+    NO_CACHEGEN
     QML_FILES
       main.qml
       qml/ChatDrawer.qml

gpt4all-chat/build_and_run.md

@@ -47,7 +47,9 @@ Under this release, select the following additional components:
 - Qt Quick 3D
 - Qt 5 Compatibility Module
 - Qt Shader Tools
-- Additional Libraries (clicking the checkbox to the left of this item enables all of them)
+- Additional Libraries:
+  - Qt HTTP Server
+  - Qt PDF
 - Qt Debug information Files
 - Qt Quick Timeline
 

gpt4all-chat/chat.cpp

@@ -10,14 +10,10 @@ Chat::Chat(QObject *parent)
     , m_id(Network::globalInstance()->generateUniqueId())
     , m_name(tr("New Chat"))
     , m_chatModel(new ChatModel(this))
-    , m_responseInProgress(false)
     , m_responseState(Chat::ResponseStopped)
     , m_creationDate(QDateTime::currentSecsSinceEpoch())
     , m_llmodel(new ChatLLM(this))
-    , m_isServer(false)
-    , m_shouldDeleteLater(false)
-    , m_isModelLoaded(false)
-    , m_shouldLoadModelWhenInstalled(false)
+    , m_collectionModel(new LocalDocsCollectionsModel(this))
 {
     connectLLM();
 }
@@ -35,6 +31,7 @@ Chat::Chat(bool isServer, QObject *parent)
     , m_shouldDeleteLater(false)
     , m_isModelLoaded(false)
     , m_shouldLoadModelWhenInstalled(false)
+    , m_collectionModel(new LocalDocsCollectionsModel(this))
 {
     connectLLM();
 }
@@ -71,6 +68,7 @@ void Chat::connectLLM()
     connect(this, &Chat::resetContextRequested, m_llmodel, &ChatLLM::resetContext, Qt::QueuedConnection);
     connect(this, &Chat::processSystemPromptRequested, m_llmodel, &ChatLLM::processSystemPrompt, Qt::QueuedConnection);
 
+    connect(this, &Chat::collectionListChanged, m_collectionModel, &LocalDocsCollectionsModel::setCollections);
     connect(ModelList::globalInstance()->installedModels(), &InstalledModels::countChanged,
         this, &Chat::handleModelInstalled, Qt::QueuedConnection);
 }
@@ -142,17 +140,9 @@ QString Chat::response() const
     return m_response;
 }
 
-QString Chat::responseState() const
+Chat::ResponseState Chat::responseState() const
 {
-    switch (m_responseState) {
-    case ResponseStopped: return QStringLiteral("response stopped");
-    case LocalDocsRetrieval: return QStringLiteral("retrieving ") + m_collections.join(", ");
-    case LocalDocsProcessing: return QStringLiteral("processing ") + m_collections.join(", ");
-    case PromptProcessing: return QStringLiteral("processing");
-    case ResponseGeneration: return QStringLiteral("generating response");
-    };
-    Q_UNREACHABLE();
-    return QString();
+    return m_responseState;
 }
 
 void Chat::handleResponseChanged(const QString &response)
@@ -403,6 +393,7 @@ bool Chat::deserialize(QDataStream &stream, int version)
     emit idChanged(m_id);
     stream >> m_name;
     stream >> m_userName;
+    m_generatedName = QLatin1String("nonempty");
     emit nameChanged();
 
     QString modelId;
@@ -439,8 +430,7 @@ bool Chat::deserialize(QDataStream &stream, int version)
     if (!m_chatModel->deserialize(stream, version))
         return false;
 
-    if (!deserializeKV || discardKV)
-        m_llmodel->setStateFromText(m_chatModel->text());
+    m_llmodel->setStateFromText(m_chatModel->text());
 
     emit chatModelChanged();
     return stream.status() == QDataStream::Ok;

gpt4all-chat/chat.h

@@ -21,12 +21,13 @@ class Chat : public QObject
     Q_PROPERTY(bool responseInProgress READ responseInProgress NOTIFY responseInProgressChanged)
     Q_PROPERTY(bool isRecalc READ isRecalc NOTIFY recalcChanged)
     Q_PROPERTY(bool isServer READ isServer NOTIFY isServerChanged)
-    Q_PROPERTY(QString responseState READ responseState NOTIFY responseStateChanged)
+    Q_PROPERTY(ResponseState responseState READ responseState NOTIFY responseStateChanged)
     Q_PROPERTY(QList<QString> collectionList READ collectionList NOTIFY collectionListChanged)
     Q_PROPERTY(QString modelLoadingError READ modelLoadingError NOTIFY modelLoadingErrorChanged)
     Q_PROPERTY(QString tokenSpeed READ tokenSpeed NOTIFY tokenSpeedChanged);
     Q_PROPERTY(QString device READ device NOTIFY deviceChanged);
     Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY fallbackReasonChanged);
+    Q_PROPERTY(LocalDocsCollectionsModel *collectionModel READ collectionModel NOTIFY collectionModelChanged)
     QML_ELEMENT
     QML_UNCREATABLE("Only creatable from c++!")
 
@@ -68,7 +69,7 @@ public:
 
     QString response() const;
     bool responseInProgress() const { return m_responseInProgress; }
-    QString responseState() const;
+    ResponseState responseState() const;
     ModelInfo modelInfo() const;
     void setModelInfo(const ModelInfo &modelInfo);
     bool isRecalc() const;
@@ -83,6 +84,7 @@ public:
     bool isServer() const { return m_isServer; }
 
     QList<QString> collectionList() const;
+    LocalDocsCollectionsModel *collectionModel() const { return m_collectionModel; }
 
     Q_INVOKABLE bool hasCollection(const QString &collection) const;
     Q_INVOKABLE void addCollection(const QString &collection);
@@ -123,6 +125,7 @@ Q_SIGNALS:
     void tokenSpeedChanged();
     void deviceChanged();
     void fallbackReasonChanged();
+    void collectionModelChanged();
 
 private Q_SLOTS:
     void handleResponseChanged(const QString &response);
@@ -152,15 +155,16 @@ private:
     QString m_response;
     QList<QString> m_collections;
     ChatModel *m_chatModel;
-    bool m_responseInProgress;
+    bool m_responseInProgress = false;
     ResponseState m_responseState;
     qint64 m_creationDate;
     ChatLLM *m_llmodel;
     QList<ResultInfo> m_databaseResults;
-    bool m_isServer;
-    bool m_shouldDeleteLater;
-    bool m_isModelLoaded;
-    bool m_shouldLoadModelWhenInstalled;
+    bool m_isServer = false;
+    bool m_shouldDeleteLater = false;
+    bool m_isModelLoaded = false;
+    bool m_shouldLoadModelWhenInstalled = false;
+    LocalDocsCollectionsModel *m_collectionModel;
 };
 
 #endif // CHAT_H

gpt4all-chat/chatgpt.cpp

@@ -20,15 +20,17 @@ ChatGPT::ChatGPT()
 {
 }
 
-size_t ChatGPT::requiredMem(const std::string &modelPath)
+size_t ChatGPT::requiredMem(const std::string &modelPath, int n_ctx)
 {
     Q_UNUSED(modelPath);
+    Q_UNUSED(n_ctx);
     return 0;
 }
 
-bool ChatGPT::loadModel(const std::string &modelPath)
+bool ChatGPT::loadModel(const std::string &modelPath, int n_ctx)
 {
     Q_UNUSED(modelPath);
+    Q_UNUSED(n_ctx);
     return true;
 }
 

gpt4all-chat/chatgpt.h

@@ -48,9 +48,9 @@ public:
 
     bool supportsEmbedding() const override { return false; }
     bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
+    bool loadModel(const std::string &modelPath, int n_ctx) override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;

gpt4all-chat/chatlistmodel.cpp

@@ -5,7 +5,7 @@
 #include <QDataStream>
 
 #define CHAT_FORMAT_MAGIC 0xF5D553CC
-#define CHAT_FORMAT_VERSION 6
+#define CHAT_FORMAT_VERSION 7
 
 class MyChatListModel: public ChatListModel { };
 Q_GLOBAL_STATIC(MyChatListModel, chatListModelInstance)
@@ -16,9 +16,6 @@ ChatListModel *ChatListModel::globalInstance()
 
 ChatListModel::ChatListModel()
     : QAbstractListModel(nullptr)
-    , m_newChat(nullptr)
-    , m_serverChat(nullptr)
-    , m_currentChat(nullptr)
 {
     addChat();
 

gpt4all-chat/chatlistmodel.h

@@ -239,9 +239,9 @@ private Q_SLOTS:
     }
 
 private:
-    Chat* m_newChat;
-    Chat* m_serverChat;
-    Chat* m_currentChat;
+    Chat* m_newChat = nullptr;
+    Chat* m_serverChat = nullptr;
+    Chat* m_currentChat = nullptr;
     QList<Chat*> m_chats;
 
 private:

gpt4all-chat/chatllm.cpp

@@ -227,7 +227,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         if (!m_isServer)
             LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
         m_llModelInfo = LLModelInfo();
-        emit modelLoadingError(QString("Previous attempt to load model resulted in crash for `%1` most likely due to insufficient memory. You should either remove this model or decrease your system RAM by closing other applications.").arg(modelInfo.filename()));
+        emit modelLoadingError(QString("Previous attempt to load model resulted in crash for `%1` most likely due to insufficient memory. You should either remove this model or decrease your system RAM usage by closing other applications.").arg(modelInfo.filename()));
     }
 
     if (fileInfo.exists()) {
@@ -248,14 +248,16 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             m_llModelInfo.model = model;
         } else {
 
+            // TODO: make configurable in UI
+            auto n_ctx = MySettings::globalInstance()->modelContextLength(modelInfo);
+            m_ctx.n_ctx = n_ctx;
+
+            std::string buildVariant = "auto";
 #if defined(Q_OS_MAC) && defined(__arm__)
             if (m_forceMetal)
-                m_llModelInfo.model = LLMImplementation::construct(filePath.toStdString(), "metal");
-            else
-                m_llModelInfo.model = LLMImplementation::construct(filePath.toStdString(), "auto");
-#else
-            m_llModelInfo.model = LLModel::Implementation::construct(filePath.toStdString(), "auto");
+                buildVariant = "metal";
 #endif
+            m_llModelInfo.model = LLModel::Implementation::construct(filePath.toStdString(), buildVariant, n_ctx);
 
             if (m_llModelInfo.model) {
                 // Update the settings that a model is being loaded and update the device list
@@ -267,7 +269,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                 if (requestedDevice == "CPU") {
                     emit reportFallbackReason(""); // fallback not applicable
                 } else {
-                    const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString());
+                    const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString(), n_ctx);
                     std::vector<LLModel::GPUDevice> availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
                     LLModel::GPUDevice *device = nullptr;
 
@@ -296,14 +298,14 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                 // Report which device we're actually using
                 emit reportDevice(actualDevice);
 
-                bool success = m_llModelInfo.model->loadModel(filePath.toStdString());
+                bool success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx);
                 if (actualDevice == "CPU") {
                     // we asked llama.cpp to use the CPU
                 } else if (!success) {
                     // llama_init_from_file returned nullptr
                     emit reportDevice("CPU");
                     emit reportFallbackReason("<br>GPU loading failed (out of VRAM?)");
-                    success = m_llModelInfo.model->loadModel(filePath.toStdString());
+                    success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx);
                 } else if (!m_llModelInfo.model->usingGPUDevice()) {
                     // ggml_vk_init was not called in llama.cpp
                     // We might have had to fallback to CPU after load if the model is not possible to accelerate
@@ -378,6 +380,32 @@ bool ChatLLM::isModelLoaded() const
     return m_llModelInfo.model && m_llModelInfo.model->isModelLoaded();
 }
 
+std::string remove_leading_whitespace(const std::string& input) {
+    auto first_non_whitespace = std::find_if(input.begin(), input.end(), [](unsigned char c) {
+        return !std::isspace(c);
+    });
+
+    if (first_non_whitespace == input.end())
+        return std::string();
+
+    return std::string(first_non_whitespace, input.end());
+}
+
+std::string trim_whitespace(const std::string& input) {
+    auto first_non_whitespace = std::find_if(input.begin(), input.end(), [](unsigned char c) {
+        return !std::isspace(c);
+    });
+
+    if (first_non_whitespace == input.end())
+        return std::string();
+
+    auto last_non_whitespace = std::find_if(input.rbegin(), input.rend(), [](unsigned char c) {
+        return !std::isspace(c);
+    }).base();
+
+    return std::string(first_non_whitespace, last_non_whitespace);
+}
+
 void ChatLLM::regenerateResponse()
 {
     // ChatGPT uses a different semantic meaning for n_past than local models. For ChatGPT, the meaning
@@ -409,29 +437,6 @@ void ChatLLM::resetContext()
     m_ctx = LLModel::PromptContext();
 }
 
-std::string remove_leading_whitespace(const std::string& input) {
-    auto first_non_whitespace = std::find_if(input.begin(), input.end(), [](unsigned char c) {
-        return !std::isspace(c);
-    });
-
-    return std::string(first_non_whitespace, input.end());
-}
-
-std::string trim_whitespace(const std::string& input) {
-    auto first_non_whitespace = std::find_if(input.begin(), input.end(), [](unsigned char c) {
-        return !std::isspace(c);
-    });
-
-    if (first_non_whitespace == input.end())
-        return std::string();
-
-    auto last_non_whitespace = std::find_if(input.rbegin(), input.rend(), [](unsigned char c) {
-        return !std::isspace(c);
-    }).base();
-
-    return std::string(first_non_whitespace, last_non_whitespace);
-}
-
 QString ChatLLM::response() const
 {
     return QString::fromStdString(remove_leading_whitespace(m_response));
@@ -476,7 +481,7 @@ bool ChatLLM::handleResponse(int32_t token, const std::string &response)
     // check for error
     if (token < 0) {
         m_response.append(response);
-        emit responseChanged(QString::fromStdString(m_response));
+        emit responseChanged(QString::fromStdString(remove_leading_whitespace(m_response)));
         return false;
     }
 
@@ -486,7 +491,7 @@ bool ChatLLM::handleResponse(int32_t token, const std::string &response)
     m_timer->inc();
     Q_ASSERT(!response.empty());
     m_response.append(response);
-    emit responseChanged(QString::fromStdString(m_response));
+    emit responseChanged(QString::fromStdString(remove_leading_whitespace(m_response)));
     return !m_stopGenerating;
 }
 
@@ -503,6 +508,11 @@ bool ChatLLM::handleRecalculate(bool isRecalc)
 }
 bool ChatLLM::prompt(const QList<QString> &collectionList, const QString &prompt)
 {
+    if (m_restoreStateFromText) {
+        Q_ASSERT(m_state.isEmpty());
+        processRestoreStateFromText();
+    }
+
     if (!m_processedSystemPrompt)
         processSystemPrompt();
     const QString promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
@@ -526,8 +536,10 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
 
     QList<ResultInfo> databaseResults;
     const int retrievalSize = MySettings::globalInstance()->localDocsRetrievalSize();
-    emit requestRetrieveFromDB(collectionList, prompt, retrievalSize, &databaseResults); // blocks
-    emit databaseResultsChanged(databaseResults);
+    if (!collectionList.isEmpty()) {
+        emit requestRetrieveFromDB(collectionList, prompt, retrievalSize, &databaseResults); // blocks
+        emit databaseResultsChanged(databaseResults);
+    }
 
     // Augment the prompt template with the results if any
     QList<QString> augmentedTemplate;
@@ -753,6 +765,8 @@ bool ChatLLM::handleRestoreStateFromTextRecalculate(bool isRecalc)
     return false;
 }
 
+// this function serialized the cached model state to disk.
+// we want to also serialize n_ctx, and read it at load time.
 bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
 {
     if (version > 1) {
@@ -780,6 +794,9 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
         stream << responseLogits;
     }
     stream << m_ctx.n_past;
+    if (version >= 6) {
+        stream << m_ctx.n_ctx;
+    }
     stream << quint64(m_ctx.logits.size());
     stream.writeRawData(reinterpret_cast<const char*>(m_ctx.logits.data()), m_ctx.logits.size() * sizeof(float));
     stream << quint64(m_ctx.tokens.size());
@@ -829,6 +846,12 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
     stream >> n_past;
     if (!discardKV) m_ctx.n_past = n_past;
 
+    if (version >= 6) {
+        uint32_t n_ctx;
+        stream >> n_ctx;
+        if (!discardKV) m_ctx.n_ctx = n_ctx;
+    }
+
     quint64 logitsSize;
     stream >> logitsSize;
     if (!discardKV) {
@@ -853,11 +876,11 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
         if (!discardKV)
             m_state = qUncompress(compressed);
     } else {
-        if (!discardKV)
+        if (!discardKV) {
             stream >> m_state;
-        else {
+        } else {
             QByteArray state;
-            stream >> m_state;
+            stream >> state;
         }
     }
 
@@ -902,32 +925,33 @@ void ChatLLM::restoreState()
         stream >> context;
         chatGPT->setContext(context);
         m_state.clear();
-        m_state.resize(0);
+        m_state.squeeze();
         return;
     }
 
-    if (m_restoreStateFromText) {
-        Q_ASSERT(m_state.isEmpty());
-        processRestoreStateFromText();
-    }
-
 #if defined(DEBUG)
     qDebug() << "restoreState" << m_llmThread.objectName() << "size:" << m_state.size();
 #endif
-    m_processedSystemPrompt = true;
 
     if (m_state.isEmpty())
         return;
 
-    m_llModelInfo.model->restoreState(static_cast<const uint8_t*>(reinterpret_cast<void*>(m_state.data())));
+    if (m_llModelInfo.model->stateSize() == m_state.size()) {
+        m_llModelInfo.model->restoreState(static_cast<const uint8_t*>(reinterpret_cast<void*>(m_state.data())));
+        m_processedSystemPrompt = true;
+    } else {
+        qWarning() << "restoring state from text because" << m_llModelInfo.model->stateSize() << "!=" << m_state.size() << "\n";
+        m_restoreStateFromText = true;
+    }
+
     m_state.clear();
-    m_state.resize(0);
+    m_state.squeeze();
 }
 
 void ChatLLM::processSystemPrompt()
 {
     Q_ASSERT(isModelLoaded());
-    if (!isModelLoaded() || m_processedSystemPrompt || m_isServer)
+    if (!isModelLoaded() || m_processedSystemPrompt || m_restoreStateFromText || m_isServer)
         return;
 
     const std::string systemPrompt = MySettings::globalInstance()->modelSystemPrompt(m_modelInfo).toStdString();
@@ -971,7 +995,7 @@ void ChatLLM::processSystemPrompt()
     fflush(stdout);
 #endif
 
-    m_processedSystemPrompt = !m_stopGenerating;
+    m_processedSystemPrompt = m_stopGenerating == false;
 }
 
 void ChatLLM::processRestoreStateFromText()

gpt4all-chat/cmake/deploy-qt-mac.cmake.in

@@ -3,20 +3,15 @@ set(COMPONENT_NAME_MAIN "@COMPONENT_NAME_MAIN@")
 set(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@")
 execute_process(COMMAND ${MACDEPLOYQT} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -verbose=2)
 file(GLOB MYGPTJLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libgptj*)
-file(GLOB MYMPTLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libmpt*)
 file(GLOB MYLLAMALIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllama*)
 file(GLOB MYBERTLLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libbert*)
 file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllmodel.*)
 file(COPY ${MYGPTJLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYMPTLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLAMALIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYBERTLLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
-file(COPY ${MYLLAMALIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLMODELLIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/favicon.icns"

gpt4all-chat/cmake/sign_dmg.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import os
 import subprocess
 import tempfile

gpt4all-chat/database.cpp

@@ -1,5 +1,7 @@
 #include "database.h"
 #include "mysettings.h"
+#include "embllm.h"
+#include "embeddings.h"
 
 #include <QTimer>
 #include <QPdfDocument>
@@ -7,18 +9,18 @@
 //#define DEBUG
 //#define DEBUG_EXAMPLE
 
-#define LOCALDOCS_VERSION 0
+#define LOCALDOCS_VERSION 1
 
 const auto INSERT_CHUNK_SQL = QLatin1String(R"(
-    insert into chunks(document_id, chunk_id, chunk_text,
-        file, title, author, subject, keywords, page, line_from, line_to,
-        embedding_id, embedding_path) values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
+    insert into chunks(document_id, chunk_text,
+        file, title, author, subject, keywords, page, line_from, line_to)
+        values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
     )");
 
 const auto INSERT_CHUNK_FTS_SQL = QLatin1String(R"(
     insert into chunks_fts(document_id, chunk_id, chunk_text,
-        file, title, author, subject, keywords, page, line_from, line_to,
-        embedding_id, embedding_path) values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
+        file, title, author, subject, keywords, page, line_from, line_to)
+        values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
     )");
 
 const auto DELETE_CHUNKS_SQL = QLatin1String(R"(
@@ -30,20 +32,33 @@ const auto DELETE_CHUNKS_FTS_SQL = QLatin1String(R"(
     )");
 
 const auto CHUNKS_SQL = QLatin1String(R"(
-    create table chunks(document_id integer, chunk_id integer, chunk_text varchar,
+    create table chunks(document_id integer, chunk_id integer primary key autoincrement, chunk_text varchar,
         file varchar, title varchar, author varchar, subject varchar, keywords varchar,
-        page integer, line_from integer, line_to integer,
-        embedding_id integer, embedding_path varchar);
+        page integer, line_from integer, line_to integer);
     )");
 
 const auto FTS_CHUNKS_SQL = QLatin1String(R"(
     create virtual table chunks_fts using fts5(document_id unindexed, chunk_id unindexed, chunk_text,
-        file, title, author, subject, keywords, page, line_from, line_to,
-        embedding_id unindexed, embedding_path unindexed, tokenize="trigram");
+        file, title, author, subject, keywords, page, line_from, line_to, tokenize="trigram");
     )");
 
-const auto SELECT_SQL = QLatin1String(R"(
-    select chunks_fts.rowid, documents.document_time,
+const auto SELECT_CHUNKS_BY_DOCUMENT_SQL = QLatin1String(R"(
+    select chunk_id from chunks WHERE document_id = ?;
+    )");
+
+const auto SELECT_CHUNKS_SQL = QLatin1String(R"(
+    select chunks.chunk_id, documents.document_time,
+        chunks.chunk_text, chunks.file, chunks.title, chunks.author, chunks.page,
+        chunks.line_from, chunks.line_to
+    from chunks
+    join documents ON chunks.document_id = documents.id
+    join folders ON documents.folder_id = folders.id
+    join collections ON folders.id = collections.folder_id
+    where chunks.chunk_id in (%1) and collections.collection_name in (%2);
+)");
+
+const auto SELECT_NGRAM_SQL = QLatin1String(R"(
+    select chunks_fts.chunk_id, documents.document_time,
         chunks_fts.chunk_text, chunks_fts.file, chunks_fts.title, chunks_fts.author, chunks_fts.page,
         chunks_fts.line_from, chunks_fts.line_to
     from chunks_fts
@@ -55,16 +70,14 @@ const auto SELECT_SQL = QLatin1String(R"(
     limit %2;
     )");
 
-bool addChunk(QSqlQuery &q, int document_id, int chunk_id, const QString &chunk_text,
+bool addChunk(QSqlQuery &q, int document_id, const QString &chunk_text,
     const QString &file, const QString &title, const QString &author, const QString &subject, const QString &keywords,
-    int page, int from, int to,
-    int embedding_id, const QString &embedding_path)
+    int page, int from, int to, int *chunk_id)
 {
     {
         if (!q.prepare(INSERT_CHUNK_SQL))
             return false;
         q.addBindValue(document_id);
-        q.addBindValue(chunk_id);
         q.addBindValue(chunk_text);
         q.addBindValue(file);
         q.addBindValue(title);
@@ -74,16 +87,19 @@ bool addChunk(QSqlQuery &q, int document_id, int chunk_id, const QString &chunk_
         q.addBindValue(page);
         q.addBindValue(from);
         q.addBindValue(to);
-        q.addBindValue(embedding_id);
-        q.addBindValue(embedding_path);
         if (!q.exec())
             return false;
     }
+    if (!q.exec("select last_insert_rowid();"))
+        return false;
+    if (!q.next())
+        return false;
+    *chunk_id = q.value(0).toInt();
     {
         if (!q.prepare(INSERT_CHUNK_FTS_SQL))
             return false;
         q.addBindValue(document_id);
-        q.addBindValue(chunk_id);
+        q.addBindValue(*chunk_id);
         q.addBindValue(chunk_text);
         q.addBindValue(file);
         q.addBindValue(title);
@@ -93,8 +109,6 @@ bool addChunk(QSqlQuery &q, int document_id, int chunk_id, const QString &chunk_
         q.addBindValue(page);
         q.addBindValue(from);
         q.addBindValue(to);
-        q.addBindValue(embedding_id);
-        q.addBindValue(embedding_path);
         if (!q.exec())
             return false;
     }
@@ -146,6 +160,18 @@ QStringList generateGrams(const QString &input, int N)
     return ngrams;
 }
 
+bool selectChunk(QSqlQuery &q, const QList<QString> &collection_names, const std::vector<qint64> &chunk_ids, int retrievalSize)
+{
+    QString chunk_ids_str = QString::number(chunk_ids[0]);
+    for (size_t i = 1; i < chunk_ids.size(); ++i)
+        chunk_ids_str += "," + QString::number(chunk_ids[i]);
+    const QString collection_names_str = collection_names.join("', '");
+    const QString formatted_query = SELECT_CHUNKS_SQL.arg(chunk_ids_str).arg("'" + collection_names_str + "'");
+    if (!q.prepare(formatted_query))
+        return false;
+    return q.exec();
+}
+
 bool selectChunk(QSqlQuery &q, const QList<QString> &collection_names, const QString &chunk_text, int retrievalSize)
 {
     static QRegularExpression spaces("\\s+");
@@ -155,7 +181,7 @@ bool selectChunk(QSqlQuery &q, const QList<QString> &collection_names, const QSt
         QList<QString> text = generateGrams(chunk_text, N);
         QString orText = text.join(" OR ");
         const QString collection_names_str = collection_names.join("', '");
-        const QString formatted_query = SELECT_SQL.arg("'" + collection_names_str + "'").arg(QString::number(retrievalSize));
+        const QString formatted_query = SELECT_NGRAM_SQL.arg("'" + collection_names_str + "'").arg(QString::number(retrievalSize));
         if (!q.prepare(formatted_query))
             return false;
         q.addBindValue(orText);
@@ -248,7 +274,8 @@ bool selectAllFromCollections(QSqlQuery &q, QList<CollectionItem> *collections)
         CollectionItem i;
         i.collection = q.value(0).toString();
         i.folder_path = q.value(1).toString();
-        i.folder_id = q.value(0).toInt();
+        i.folder_id = q.value(2).toInt();
+        i.indexing = false;
         i.installed = true;
         collections->append(i);
     }
@@ -459,6 +486,12 @@ QSqlError initDb()
         return q.lastError();
     }
 
+    CollectionItem i;
+    i.collection = collection_name;
+    i.folder_path = folder_path;
+    i.folder_id = folder_id;
+    emit addCollectionItem(i);
+
     // Add a document
     int document_time = 123456789;
     int document_id;
@@ -504,6 +537,8 @@ Database::Database(int chunkSize)
     : QObject(nullptr)
     , m_watcher(new QFileSystemWatcher(this))
     , m_chunkSize(chunkSize)
+    , m_embLLM(new EmbeddingLLM)
+    , m_embeddings(new Embeddings(this))
 {
     moveToThread(&m_dbThread);
     connect(&m_dbThread, &QThread::started, this, &Database::start);
@@ -511,22 +546,39 @@ Database::Database(int chunkSize)
     m_dbThread.start();
 }
 
-void Database::handleDocumentErrorAndScheduleNext(const QString &errorMessage,
-    int document_id, const QString &document_path, const QSqlError &error)
+Database::~Database()
 {
-    qWarning() << errorMessage << document_id << document_path << error.text();
+    m_dbThread.quit();
+    m_dbThread.wait();
+}
+
+void Database::scheduleNext(int folder_id, size_t countForFolder)
+{
+    emit updateCurrentDocsToIndex(folder_id, countForFolder);
+    if (!countForFolder) {
+        emit updateIndexing(folder_id, false);
+        emit updateInstalled(folder_id, true);
+        m_embeddings->save();
+    }
     if (!m_docsToScan.isEmpty())
         QTimer::singleShot(0, this, &Database::scanQueue);
 }
 
-void Database::chunkStream(QTextStream &stream, int document_id, const QString &file,
-    const QString &title, const QString &author, const QString &subject, const QString &keywords, int page)
+void Database::handleDocumentError(const QString &errorMessage,
+    int document_id, const QString &document_path, const QSqlError &error)
+{
+    qWarning() << errorMessage << document_id << document_path << error.text();
+}
+
+size_t Database::chunkStream(QTextStream &stream, int document_id, const QString &file,
+    const QString &title, const QString &author, const QString &subject, const QString &keywords, int page,
+    int maxChunks)
 {
-    int chunk_id = 0;
     int charCount = 0;
     int line_from = -1;
     int line_to = -1;
     QList<QString> words;
+    int chunks = 0;
 
     while (!stream.atEnd()) {
         QString word;
@@ -536,9 +588,9 @@ void Database::chunkStream(QTextStream &stream, int document_id, const QString &
         if (charCount + words.size() - 1 >= m_chunkSize || stream.atEnd()) {
             const QString chunk = words.join(" ");
             QSqlQuery q;
+            int chunk_id = 0;
             if (!addChunk(q,
                 document_id,
-                ++chunk_id,
                 chunk,
                 file,
                 title,
@@ -548,15 +600,111 @@ void Database::chunkStream(QTextStream &stream, int document_id, const QString &
                 page,
                 line_from,
                 line_to,
-                0 /*embedding_id*/,
-                QString() /*embedding_path*/
+                &chunk_id
             )) {
                 qWarning() << "ERROR: Could not insert chunk into db" << q.lastError();
             }
+
+            const std::vector<float> result = m_embLLM->generateEmbeddings(chunk);
+            if (!m_embeddings->add(result, chunk_id))
+                qWarning() << "ERROR: Cannot add point to embeddings index";
+
+            ++chunks;
+
             words.clear();
             charCount = 0;
+
+            if (maxChunks > 0 && chunks == maxChunks)
+                return stream.pos();
         }
     }
+    return stream.pos();
+}
+
+void Database::removeEmbeddingsByDocumentId(int document_id)
+{
+    QSqlQuery q;
+
+    if (!q.prepare(SELECT_CHUNKS_BY_DOCUMENT_SQL)) {
+        qWarning() << "ERROR: Cannot prepare sql for select chunks by document" << q.lastError();
+        return;
+    }
+
+    q.addBindValue(document_id);
+
+    if (!q.exec()) {
+        qWarning() << "ERROR: Cannot exec sql for select chunks by document" << q.lastError();
+        return;
+    }
+
+    while (q.next()) {
+        const int chunk_id = q.value(0).toInt();
+        m_embeddings->remove(chunk_id);
+    }
+    m_embeddings->save();
+}
+
+size_t Database::countOfDocuments(int folder_id) const
+{
+    if (!m_docsToScan.contains(folder_id))
+        return 0;
+    return m_docsToScan.value(folder_id).size();
+}
+
+size_t Database::countOfBytes(int folder_id) const
+{
+    if (!m_docsToScan.contains(folder_id))
+        return 0;
+    size_t totalBytes = 0;
+    const QQueue<DocumentInfo> &docs = m_docsToScan.value(folder_id);
+    for (const DocumentInfo &f : docs)
+        totalBytes += f.doc.size();
+    return totalBytes;
+}
+
+DocumentInfo Database::dequeueDocument()
+{
+    Q_ASSERT(!m_docsToScan.isEmpty());
+    const int firstKey = m_docsToScan.firstKey();
+    QQueue<DocumentInfo> &queue = m_docsToScan[firstKey];
+    Q_ASSERT(!queue.isEmpty());
+    DocumentInfo result = queue.dequeue();
+    if (queue.isEmpty())
+        m_docsToScan.remove(firstKey);
+    return result;
+}
+
+void Database::removeFolderFromDocumentQueue(int folder_id)
+{
+    if (!m_docsToScan.contains(folder_id))
+        return;
+    m_docsToScan.remove(folder_id);
+    emit removeFolderById(folder_id);
+    emit docsToScanChanged();
+}
+
+void Database::enqueueDocumentInternal(const DocumentInfo &info, bool prepend)
+{
+    const int key = info.folder;
+    if (!m_docsToScan.contains(key))
+        m_docsToScan[key] = QQueue<DocumentInfo>();
+    if (prepend)
+        m_docsToScan[key].prepend(info);
+    else
+        m_docsToScan[key].enqueue(info);
+}
+
+void Database::enqueueDocuments(int folder_id, const QVector<DocumentInfo> &infos)
+{
+    for (int i = 0; i < infos.size(); ++i)
+        enqueueDocumentInternal(infos[i]);
+    const size_t count = countOfDocuments(folder_id);
+    emit updateCurrentDocsToIndex(folder_id, count);
+    emit updateTotalDocsToIndex(folder_id, count);
+    const size_t bytes = countOfBytes(folder_id);
+    emit updateCurrentBytesToIndex(folder_id, bytes);
+    emit updateTotalBytesToIndex(folder_id, bytes);
+    emit docsToScanChanged();
 }
 
 void Database::scanQueue()
@@ -564,7 +712,9 @@ void Database::scanQueue()
     if (m_docsToScan.isEmpty())
         return;
 
-    DocumentInfo info = m_docsToScan.dequeue();
+    DocumentInfo info = dequeueDocument();
+    const size_t countForFolder = countOfDocuments(info.folder);
+    const int folder_id = info.folder;
 
     // Update info
     info.doc.stat();
@@ -572,99 +722,127 @@ void Database::scanQueue()
     // If the doc has since been deleted or no longer readable, then we schedule more work and return
     // leaving the cleanup for the cleanup handler
     if (!info.doc.exists() || !info.doc.isReadable()) {
-        if (!m_docsToScan.isEmpty()) QTimer::singleShot(0, this, &Database::scanQueue);
-        return;
+        return scheduleNext(folder_id, countForFolder);
     }
 
-    const int folder_id = info.folder;
     const qint64 document_time = info.doc.fileTime(QFile::FileModificationTime).toMSecsSinceEpoch();
     const QString document_path = info.doc.canonicalFilePath();
-
-#if defined(DEBUG)
-    qDebug() << "scanning document" << document_path;
-#endif
+    const bool currentlyProcessing = info.currentlyProcessing;
 
     // Check and see if we already have this document
     QSqlQuery q;
     int existing_id = -1;
     qint64 existing_time = -1;
     if (!selectDocument(q, document_path, &existing_id, &existing_time)) {
-        return handleDocumentErrorAndScheduleNext("ERROR: Cannot select document",
+        handleDocumentError("ERROR: Cannot select document",
             existing_id, document_path, q.lastError());
+        return scheduleNext(folder_id, countForFolder);
     }
 
     // If we have the document, we need to compare the last modification time and if it is newer
     // we must rescan the document, otherwise return
-    if (existing_id != -1) {
+    if (existing_id != -1 && !currentlyProcessing) {
         Q_ASSERT(existing_time != -1);
         if (document_time == existing_time) {
             // No need to rescan, but we do have to schedule next
-            if (!m_docsToScan.isEmpty()) QTimer::singleShot(0, this, &Database::scanQueue);
-            return;
+            return scheduleNext(folder_id, countForFolder);
         } else {
+            removeEmbeddingsByDocumentId(existing_id);
             if (!removeChunksByDocumentId(q, existing_id)) {
-                return handleDocumentErrorAndScheduleNext("ERROR: Cannot remove chunks of document",
+                handleDocumentError("ERROR: Cannot remove chunks of document",
                     existing_id, document_path, q.lastError());
+                return scheduleNext(folder_id, countForFolder);
             }
         }
     }
 
     // Update the document_time for an existing document, or add it for the first time now
     int document_id = existing_id;
-    if (document_id != -1) {
-        if (!updateDocument(q, document_id, document_time)) {
-            return handleDocumentErrorAndScheduleNext("ERROR: Could not update document_time",
-                document_id, document_path, q.lastError());
-        }
-    } else {
-        if (!addDocument(q, folder_id, document_time, document_path, &document_id)) {
-            return handleDocumentErrorAndScheduleNext("ERROR: Could not add document",
-                document_id, document_path, q.lastError());
+    if (!currentlyProcessing) {
+        if (document_id != -1) {
+            if (!updateDocument(q, document_id, document_time)) {
+                handleDocumentError("ERROR: Could not update document_time",
+                    document_id, document_path, q.lastError());
+                return scheduleNext(folder_id, countForFolder);
+            }
+        } else {
+            if (!addDocument(q, folder_id, document_time, document_path, &document_id)) {
+                handleDocumentError("ERROR: Could not add document",
+                    document_id, document_path, q.lastError());
+                return scheduleNext(folder_id, countForFolder);
+            }
         }
     }
 
-    QElapsedTimer timer;
-    timer.start();
-
     QSqlDatabase::database().transaction();
     Q_ASSERT(document_id != -1);
-    if (info.doc.suffix() == QLatin1String("pdf")) {
+    if (info.isPdf()) {
         QPdfDocument doc;
         if (QPdfDocument::Error::None != doc.load(info.doc.canonicalFilePath())) {
-            return handleDocumentErrorAndScheduleNext("ERROR: Could not load pdf",
+            handleDocumentError("ERROR: Could not load pdf",
                 document_id, document_path, q.lastError());
-            return;
+            return scheduleNext(folder_id, countForFolder);
         }
-        for (int i = 0; i < doc.pageCount(); ++i) {
-            const QPdfSelection selection = doc.getAllText(i);
-            QString text = selection.text();
-            QTextStream stream(&text);
-            chunkStream(stream, document_id, info.doc.fileName(),
-                doc.metaData(QPdfDocument::MetaDataField::Title).toString(),
-                doc.metaData(QPdfDocument::MetaDataField::Author).toString(),
-                doc.metaData(QPdfDocument::MetaDataField::Subject).toString(),
-                doc.metaData(QPdfDocument::MetaDataField::Keywords).toString(),
-                i + 1
-            );
+        const size_t bytes = info.doc.size();
+        const size_t bytesPerPage = std::floor(bytes / doc.pageCount());
+        const int pageIndex = info.currentPage;
+#if defined(DEBUG)
+        qDebug() << "scanning page" << pageIndex << "of" << doc.pageCount() << document_path;
+#endif
+        const QPdfSelection selection = doc.getAllText(pageIndex);
+        QString text = selection.text();
+        QTextStream stream(&text);
+        chunkStream(stream, document_id, info.doc.fileName(),
+            doc.metaData(QPdfDocument::MetaDataField::Title).toString(),
+            doc.metaData(QPdfDocument::MetaDataField::Author).toString(),
+            doc.metaData(QPdfDocument::MetaDataField::Subject).toString(),
+            doc.metaData(QPdfDocument::MetaDataField::Keywords).toString(),
+            pageIndex + 1
+        );
+        m_embeddings->save();
+        emit subtractCurrentBytesToIndex(info.folder, bytesPerPage);
+        if (info.currentPage < doc.pageCount()) {
+            info.currentPage += 1;
+            info.currentlyProcessing = true;
+            enqueueDocumentInternal(info, true /*prepend*/);
+            return scheduleNext(folder_id, countForFolder + 1);
+        } else {
+            emit subtractCurrentBytesToIndex(info.folder, bytes - (bytesPerPage * doc.pageCount()));
         }
     } else {
         QFile file(document_path);
-        if (!file.open( QIODevice::ReadOnly)) {
-            return handleDocumentErrorAndScheduleNext("ERROR: Cannot open file for scanning",
-                                                      existing_id, document_path, q.lastError());
+        if (!file.open(QIODevice::ReadOnly)) {
+            handleDocumentError("ERROR: Cannot open file for scanning",
+                                existing_id, document_path, q.lastError());
+            return scheduleNext(folder_id, countForFolder);
         }
+
+        const size_t bytes = info.doc.size();
         QTextStream stream(&file);
-        chunkStream(stream, document_id, info.doc.fileName(), QString() /*title*/, QString() /*author*/,
-            QString() /*subject*/, QString() /*keywords*/, -1 /*page*/);
+        const size_t byteIndex = info.currentPosition;
+        if (!stream.seek(byteIndex)) {
+            handleDocumentError("ERROR: Cannot seek to pos for scanning",
+                                existing_id, document_path, q.lastError());
+            return scheduleNext(folder_id, countForFolder);
+        }
+#if defined(DEBUG)
+        qDebug() << "scanning byteIndex" << byteIndex << "of" << bytes << document_path;
+#endif
+        int pos = chunkStream(stream, document_id, info.doc.fileName(), QString() /*title*/, QString() /*author*/,
+            QString() /*subject*/, QString() /*keywords*/, -1 /*page*/, 5 /*maxChunks*/);
+        m_embeddings->save();
         file.close();
+        const size_t bytesChunked = pos - byteIndex;
+        emit subtractCurrentBytesToIndex(info.folder, bytesChunked);
+        if (info.currentPosition < bytes) {
+            info.currentPosition = pos;
+            info.currentlyProcessing = true;
+            enqueueDocumentInternal(info, true /*prepend*/);
+            return scheduleNext(folder_id, countForFolder + 1);
+        }
     }
     QSqlDatabase::database().commit();
-
-#if defined(DEBUG)
-    qDebug() << "chunking" << document_path << "took" << timer.elapsed() << "ms";
-#endif
-
-    if (!m_docsToScan.isEmpty()) QTimer::singleShot(0, this, &Database::scanQueue);
+    return scheduleNext(folder_id, countForFolder);
 }
 
 void Database::scanDocuments(int folder_id, const QString &folder_path)
@@ -687,6 +865,7 @@ void Database::scanDocuments(int folder_id, const QString &folder_path)
     Q_ASSERT(dir.exists());
     Q_ASSERT(dir.isReadable());
     QDirIterator it(folder_path, QDir::Readable | QDir::Files, QDirIterator::Subdirectories);
+    QVector<DocumentInfo> infos;
     while (it.hasNext()) {
         it.next();
         QFileInfo fileInfo = it.fileInfo();
@@ -701,9 +880,13 @@ void Database::scanDocuments(int folder_id, const QString &folder_path)
         DocumentInfo info;
         info.folder = folder_id;
         info.doc = fileInfo;
-        m_docsToScan.enqueue(info);
+        infos.append(info);
+    }
+
+    if (!infos.isEmpty()) {
+        emit updateIndexing(folder_id, true);
+        enqueueDocuments(folder_id, infos);
     }
-    emit docsToScanChanged();
 }
 
 void Database::start()
@@ -717,6 +900,10 @@ void Database::start()
         if (err.type() != QSqlError::NoError)
             qWarning() << "ERROR: initializing db" << err.text();
     }
+
+    if (m_embeddings->fileExists() && !m_embeddings->load())
+        qWarning() << "ERROR: Could not load embeddings";
+
     addCurrentFolders();
 }
 
@@ -733,25 +920,12 @@ void Database::addCurrentFolders()
         return;
     }
 
+    emit collectionListUpdated(collections);
+
     for (const auto &i : collections)
         addFolder(i.collection, i.folder_path);
 }
 
-void Database::updateCollectionList()
-{
-#if defined(DEBUG)
-    qDebug() << "updateCollectionList";
-#endif
-
-    QSqlQuery q;
-    QList<CollectionItem> collections;
-    if (!selectAllFromCollections(q, &collections)) {
-        qWarning() << "ERROR: Cannot select collections" << q.lastError();
-        return;
-    }
-    emit collectionListUpdated(collections);
-}
-
 void Database::addFolder(const QString &collection, const QString &path)
 {
     QFileInfo info(path);
@@ -784,14 +958,21 @@ void Database::addFolder(const QString &collection, const QString &path)
         return;
     }
 
-    if (!folders.contains(folder_id) && !addCollection(q, collection, folder_id)) {
-        qWarning() << "ERROR: Cannot add folder to collection" << collection << path << q.lastError();
-        return;
+    if (!folders.contains(folder_id)) {
+        if (!addCollection(q, collection, folder_id)) {
+            qWarning() << "ERROR: Cannot add folder to collection" << collection << path << q.lastError();
+            return;
+        }
+
+        CollectionItem i;
+        i.collection = collection;
+        i.folder_path = path;
+        i.folder_id = folder_id;
+        emit addCollectionItem(i);
     }
 
     addFolderToWatch(path);
     scanDocuments(folder_id, path);
-    updateCollectionList();
 }
 
 void Database::removeFolder(const QString &collection, const QString &path)
@@ -840,15 +1021,8 @@ void Database::removeFolderInternal(const QString &collection, int folder_id, co
     if (collections.count() > 1)
         return;
 
-    // First remove all upcoming jobs associated with this folder by performing an opt-in filter
-    QQueue<DocumentInfo> docsToScan;
-    for (const DocumentInfo &info : m_docsToScan) {
-        if (info.folder == folder_id)
-            continue;
-        docsToScan.append(info);
-    }
-    m_docsToScan = docsToScan;
-    emit docsToScanChanged();
+    // First remove all upcoming jobs associated with this folder
+    removeFolderFromDocumentQueue(folder_id);
 
     // Get a list of all documents associated with folder
     QList<int> documentIds;
@@ -859,6 +1033,7 @@ void Database::removeFolderInternal(const QString &collection, int folder_id, co
 
     // Remove all chunks and documents associated with this folder
     for (int document_id : documentIds) {
+        removeEmbeddingsByDocumentId(document_id);
         if (!removeChunksByDocumentId(q, document_id)) {
             qWarning() << "ERROR: Cannot remove chunks of document_id" << document_id << q.lastError();
             return;
@@ -875,8 +1050,9 @@ void Database::removeFolderInternal(const QString &collection, int folder_id, co
         return;
     }
 
+    emit removeFolderById(folder_id);
+
     removeFolderFromWatch(path);
-    updateCollectionList();
 }
 
 bool Database::addFolderToWatch(const QString &path)
@@ -903,9 +1079,18 @@ void Database::retrieveFromDB(const QList<QString> &collections, const QString &
 #endif
 
     QSqlQuery q;
-    if (!selectChunk(q, collections, text, retrievalSize)) {
-        qDebug() << "ERROR: selecting chunks:" << q.lastError().text();
-        return;
+    if (m_embeddings->isLoaded()) {
+        std::vector<float> result = m_embLLM->generateEmbeddings(text);
+        std::vector<qint64> embeddings = m_embeddings->search(result, retrievalSize);
+        if (!selectChunk(q, collections, embeddings, retrievalSize)) {
+            qDebug() << "ERROR: selecting chunks:" << q.lastError().text();
+            return;
+        }
+    } else {
+        if (!selectChunk(q, collections, text, retrievalSize)) {
+            qDebug() << "ERROR: selecting chunks:" << q.lastError().text();
+            return;
+        }
     }
 
     while (q.next()) {
@@ -986,6 +1171,7 @@ void Database::cleanDB()
 
         // Remove all chunks and documents that either don't exist or have become unreadable
         QSqlQuery query;
+        removeEmbeddingsByDocumentId(document_id);
         if (!removeChunksByDocumentId(query, document_id)) {
             qWarning() << "ERROR: Cannot remove chunks of document_id" << document_id << query.lastError();
         }
@@ -994,7 +1180,6 @@ void Database::cleanDB()
             qWarning() << "ERROR: Cannot remove document_id" << document_id << query.lastError();
         }
     }
-    updateCollectionList();
 }
 
 void Database::changeChunkSize(int chunkSize)
@@ -1024,6 +1209,7 @@ void Database::changeChunkSize(int chunkSize)
         int document_id = q.value(0).toInt();
         // Remove all chunks and documents to change the chunk size
         QSqlQuery query;
+        removeEmbeddingsByDocumentId(document_id);
         if (!removeChunksByDocumentId(query, document_id)) {
             qWarning() << "ERROR: Cannot remove chunks of document_id" << document_id << query.lastError();
         }

gpt4all-chat/database.h

@@ -8,10 +8,18 @@
 #include <QThread>
 #include <QFileSystemWatcher>
 
+class Embeddings;
+class EmbeddingLLM;
 struct DocumentInfo
 {
     int folder;
     QFileInfo doc;
+    int currentPage = 0;
+    size_t currentPosition = 0;
+    bool currentlyProcessing = false;
+    bool isPdf() const {
+        return doc.suffix() == QLatin1String("pdf");
+    }
 };
 
 struct ResultInfo {
@@ -30,6 +38,11 @@ struct CollectionItem {
     QString folder_path;
     int folder_id = -1;
     bool installed = false;
+    bool indexing = false;
+    int currentDocsToIndex = 0;
+    int totalDocsToIndex = 0;
+    size_t currentBytesToIndex = 0;
+    size_t totalBytesToIndex = 0;
 };
 Q_DECLARE_METATYPE(CollectionItem)
 
@@ -38,6 +51,7 @@ class Database : public QObject
     Q_OBJECT
 public:
     Database(int chunkSize);
+    virtual ~Database();
 
 public Q_SLOTS:
     void scanQueue();
@@ -50,6 +64,16 @@ public Q_SLOTS:
 
 Q_SIGNALS:
     void docsToScanChanged();
+    void updateInstalled(int folder_id, bool b);
+    void updateIndexing(int folder_id, bool b);
+    void updateCurrentDocsToIndex(int folder_id, size_t currentDocsToIndex);
+    void updateTotalDocsToIndex(int folder_id, size_t totalDocsToIndex);
+    void subtractCurrentBytesToIndex(int folder_id, size_t subtractedBytes);
+    void updateCurrentBytesToIndex(int folder_id, size_t currentBytesToIndex);
+    void updateTotalBytesToIndex(int folder_id, size_t totalBytesToIndex);
+    void addCollectionItem(const CollectionItem &item);
+    void removeFolderById(int folder_id);
+    void removeCollectionItem(const QString &collectionName);
     void collectionListUpdated(const QList<CollectionItem> &collectionList);
 
 private Q_SLOTS:
@@ -58,21 +82,31 @@ private Q_SLOTS:
     bool addFolderToWatch(const QString &path);
     bool removeFolderFromWatch(const QString &path);
     void addCurrentFolders();
-    void updateCollectionList();
 
 private:
     void removeFolderInternal(const QString &collection, int folder_id, const QString &path);
-    void chunkStream(QTextStream &stream, int document_id, const QString &file,
-        const QString &title, const QString &author, const QString &subject, const QString &keywords, int page);
-    void handleDocumentErrorAndScheduleNext(const QString &errorMessage,
+    size_t chunkStream(QTextStream &stream, int document_id, const QString &file,
+        const QString &title, const QString &author, const QString &subject, const QString &keywords, int page,
+        int maxChunks = -1);
+    void removeEmbeddingsByDocumentId(int document_id);
+    void scheduleNext(int folder_id, size_t countForFolder);
+    void handleDocumentError(const QString &errorMessage,
         int document_id, const QString &document_path, const QSqlError &error);
+    size_t countOfDocuments(int folder_id) const;
+    size_t countOfBytes(int folder_id) const;
+    DocumentInfo dequeueDocument();
+    void removeFolderFromDocumentQueue(int folder_id);
+    void enqueueDocumentInternal(const DocumentInfo &info, bool prepend = false);
+    void enqueueDocuments(int folder_id, const QVector<DocumentInfo> &infos);
 
 private:
     int m_chunkSize;
-    QQueue<DocumentInfo> m_docsToScan;
+    QMap<int, QQueue<DocumentInfo>> m_docsToScan;
     QList<ResultInfo> m_retrieve;
     QThread m_dbThread;
     QFileSystemWatcher *m_watcher;
+    EmbeddingLLM *m_embLLM;
+    Embeddings *m_embeddings;
 };
 
 #endif // DATABASE_H

gpt4all-chat/download.cpp

@@ -108,6 +108,7 @@ void Download::downloadModel(const QString &modelFile)
         const QString error
             = QString("ERROR: Could not open temp file: %1 %2").arg(tempFile->fileName()).arg(modelFile);
         qWarning() << error;
+        clearRetry(modelFile);
         ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::DownloadErrorRole, error);
         return;
     }
@@ -140,6 +141,7 @@ void Download::downloadModel(const QString &modelFile)
     QNetworkReply *modelReply = m_networkManager.get(request);
     connect(qApp, &QCoreApplication::aboutToQuit, modelReply, &QNetworkReply::abort);
     connect(modelReply, &QNetworkReply::downloadProgress, this, &Download::handleDownloadProgress);
+    connect(modelReply, &QNetworkReply::errorOccurred, this, &Download::handleErrorOccurred);
     connect(modelReply, &QNetworkReply::finished, this, &Download::handleModelDownloadFinished);
     connect(modelReply, &QNetworkReply::readyRead, this, &Download::handleReadyRead);
     m_activeDownloads.insert(modelReply, tempFile);
@@ -254,13 +256,51 @@ void Download::parseReleaseJsonFile(const QByteArray &jsonData)
     emit releaseInfoChanged();
 }
 
+bool Download::hasRetry(const QString &filename) const
+{
+    return m_activeRetries.contains(filename);
+}
+
+bool Download::shouldRetry(const QString &filename)
+{
+    int retries = 0;
+    if (m_activeRetries.contains(filename))
+        retries = m_activeRetries.value(filename);
+
+    ++retries;
+
+    // Allow up to ten retries for now
+    if (retries < 10) {
+        m_activeRetries.insert(filename, retries);
+        return true;
+    }
+
+    return false;
+}
+
+void Download::clearRetry(const QString &filename)
+{
+    m_activeRetries.remove(filename);
+}
+
 void Download::handleErrorOccurred(QNetworkReply::NetworkError code)
 {
     QNetworkReply *modelReply = qobject_cast<QNetworkReply *>(sender());
     if (!modelReply)
         return;
 
+    // This occurs when the user explicitly cancels the download
+    if (code == QNetworkReply::OperationCanceledError)
+        return;
+
     QString modelFilename = modelReply->request().attribute(QNetworkRequest::User).toString();
+    if (shouldRetry(modelFilename)) {
+        downloadModel(modelFilename);
+        return;
+    }
+
+    clearRetry(modelFilename);
+
     const QString error
         = QString("ERROR: Network error occurred attempting to download %1 code: %2 errorString %3")
             .arg(modelFilename)
@@ -355,6 +395,7 @@ void HashAndSaveFile::hashAndSave(const QString &expectedHash, const QString &sa
     // but will only work if the destination is on the same filesystem
     if (tempFile->rename(saveFilePath)) {
         emit hashAndSaveFinished(true, QString(), tempFile, modelReply);
+        ModelList::globalInstance()->updateModelsFromDirectory();
         return;
     }
 
@@ -385,8 +426,9 @@ void HashAndSaveFile::hashAndSave(const QString &expectedHash, const QString &sa
         qWarning() << errorString;
         tempFile->close();
         emit hashAndSaveFinished(false, errorString, tempFile, modelReply);
-        return;
     }
+
+    ModelList::globalInstance()->updateModelsFromDirectory();
 }
 
 void Download::handleModelDownloadFinished()
@@ -405,11 +447,15 @@ void Download::handleModelDownloadFinished()
         qWarning() << errorString;
         modelReply->deleteLater();
         tempFile->deleteLater();
-        ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadingRole, false);
-        ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadErrorRole, errorString);
+        if (!hasRetry(modelFilename)) {
+            ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadingRole, false);
+            ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadErrorRole, errorString);
+        }
         return;
     }
 
+    clearRetry(modelFilename);
+
     // The hash and save needs the tempFile closed
     tempFile->close();
 

gpt4all-chat/download.h

@@ -78,11 +78,15 @@ Q_SIGNALS:
 private:
     void parseReleaseJsonFile(const QByteArray &jsonData);
     QString incompleteDownloadPath(const QString &modelFile);
+    bool hasRetry(const QString &filename) const;
+    bool shouldRetry(const QString &filename);
+    void clearRetry(const QString &filename);
 
     HashAndSaveFile *m_hashAndSave;
     QMap<QString, ReleaseInfo> m_releaseMap;
     QNetworkAccessManager m_networkManager;
     QMap<QNetworkReply*, QFile*> m_activeDownloads;
+    QHash<QString, int> m_activeRetries;
     QDateTime m_startTime;
 
 private:

gpt4all-chat/embeddings.cpp

@@ -0,0 +1,190 @@
+#include "embeddings.h"
+
+#include <QFile>
+#include <QFileInfo>
+#include <QDebug>
+
+#include "mysettings.h"
+#include "hnswlib/hnswlib.h"
+
+#define EMBEDDINGS_VERSION 0
+
+const int s_dim = 384;              // Dimension of the elements
+const int s_ef_construction = 200;  // Controls index search speed/build speed tradeoff
+const int s_M = 16;                 // Tightly connected with internal dimensionality of the data
+                                    // strongly affects the memory consumption
+
+Embeddings::Embeddings(QObject *parent)
+    : QObject(parent)
+    , m_space(nullptr)
+    , m_hnsw(nullptr)
+{
+    m_filePath = MySettings::globalInstance()->modelPath()
+        + QString("embeddings_v%1.dat").arg(EMBEDDINGS_VERSION);
+}
+
+Embeddings::~Embeddings()
+{
+    delete m_hnsw;
+    m_hnsw = nullptr;
+    delete m_space;
+    m_space = nullptr;
+}
+
+bool Embeddings::load()
+{
+    QFileInfo info(m_filePath);
+    if (!info.exists()) {
+        qWarning() << "ERROR: loading embeddings file does not exist" << m_filePath;
+        return false;
+    }
+
+    if (!info.isReadable()) {
+        qWarning() << "ERROR: loading embeddings file is not readable" << m_filePath;
+        return false;
+    }
+
+    if (!info.isWritable()) {
+        qWarning() << "ERROR: loading embeddings file is not writeable" << m_filePath;
+        return false;
+    }
+
+    try {
+        m_space = new hnswlib::InnerProductSpace(s_dim);
+        m_hnsw = new hnswlib::HierarchicalNSW<float>(m_space, m_filePath.toStdString(), s_M, s_ef_construction);
+    } catch (const std::exception &e) {
+        qWarning() << "ERROR: could not load hnswlib index:" << e.what();
+        return false;
+    }
+    return isLoaded();
+}
+
+bool Embeddings::load(qint64 maxElements)
+{
+    try {
+        m_space = new hnswlib::InnerProductSpace(s_dim);
+        m_hnsw = new hnswlib::HierarchicalNSW<float>(m_space, maxElements, s_M, s_ef_construction);
+    } catch (const std::exception &e) {
+        qWarning() << "ERROR: could not create hnswlib index:" << e.what();
+        return false;
+    }
+    return isLoaded();
+}
+
+bool Embeddings::save()
+{
+    if (!isLoaded())
+        return false;
+    try {
+        m_hnsw->saveIndex(m_filePath.toStdString());
+    } catch (const std::exception &e) {
+        qWarning() << "ERROR: could not save hnswlib index:" << e.what();
+        return false;
+    }
+    return true;
+}
+
+bool Embeddings::isLoaded() const
+{
+    return m_hnsw != nullptr;
+}
+
+bool Embeddings::fileExists() const
+{
+    QFileInfo info(m_filePath);
+    return info.exists();
+}
+
+bool Embeddings::resize(qint64 size)
+{
+    if (!isLoaded()) {
+        qWarning() << "ERROR: attempting to resize an embedding when the embeddings are not open!";
+        return false;
+    }
+
+    Q_ASSERT(m_hnsw);
+    try {
+        m_hnsw->resizeIndex(size);
+    } catch (const std::exception &e) {
+        qWarning() << "ERROR: could not resize hnswlib index:" << e.what();
+        return false;
+    }
+    return true;
+}
+
+bool Embeddings::add(const std::vector<float> &embedding, qint64 label)
+{
+    if (!isLoaded()) {
+        bool success = load(500);
+        if (!success) {
+            qWarning() << "ERROR: attempting to add an embedding when the embeddings are not open!";
+            return false;
+        }
+    }
+
+    Q_ASSERT(m_hnsw);
+    if (m_hnsw->cur_element_count + 1 > m_hnsw->max_elements_) {
+        if (!resize(m_hnsw->max_elements_ + 500)) {
+            return false;
+        }
+    }
+
+    try {
+        m_hnsw->addPoint(embedding.data(), label, false);
+    } catch (const std::exception &e) {
+        qWarning() << "ERROR: could not add embedding to hnswlib index:" << e.what();
+        return false;
+    }
+    return true;
+}
+
+void Embeddings::remove(qint64 label)
+{
+    if (!isLoaded()) {
+        qWarning() << "ERROR: attempting to remove an embedding when the embeddings are not open!";
+        return;
+    }
+
+    Q_ASSERT(m_hnsw);
+    try {
+        m_hnsw->markDelete(label);
+    } catch (const std::exception &e) {
+        qWarning() << "ERROR: could not add remove embedding from hnswlib index:" << e.what();
+    }
+}
+
+void Embeddings::clear()
+{
+    delete m_hnsw;
+    m_hnsw = nullptr;
+    delete m_space;
+    m_space = nullptr;
+}
+
+std::vector<qint64> Embeddings::search(const std::vector<float> &embedding, int K)
+{
+    if (!isLoaded())
+        return {};
+
+    Q_ASSERT(m_hnsw);
+    std::priority_queue<std::pair<float, hnswlib::labeltype>> result;
+    try {
+        result = m_hnsw->searchKnn(embedding.data(), K);
+    } catch (const std::exception &e) {
+        qWarning() << "ERROR: could not search hnswlib index:" << e.what();
+        return {};
+    }
+
+    std::vector<qint64> neighbors;
+    neighbors.reserve(K);
+
+    while(!result.empty()) {
+        neighbors.push_back(result.top().second);
+        result.pop();
+    }
+
+    // Reverse the neighbors, as the top of the priority queue is the farthest neighbor.
+    std::reverse(neighbors.begin(), neighbors.end());
+
+    return neighbors;
+}

gpt4all-chat/embeddings.h

@@ -0,0 +1,45 @@
+#ifndef EMBEDDINGS_H
+#define EMBEDDINGS_H
+
+#include <QObject>
+
+namespace hnswlib {
+    template <typename T>
+    class HierarchicalNSW;
+    class InnerProductSpace;
+}
+
+class Embeddings : public QObject
+{
+    Q_OBJECT
+public:
+    Embeddings(QObject *parent);
+    virtual ~Embeddings();
+
+    bool load();
+    bool load(qint64 maxElements);
+    bool save();
+    bool isLoaded() const;
+    bool fileExists() const;
+    bool resize(qint64 size);
+
+    // Adds the embedding and returns the label used
+    bool add(const std::vector<float> &embedding, qint64 label);
+
+    // Removes the embedding at label by marking it as unused
+    void remove(qint64 label);
+
+    // Clears the embeddings
+    void clear();
+
+    // Performs a nearest neighbor search of the embeddings and returns a vector of labels
+    // for the K nearest neighbors of the given embedding
+    std::vector<qint64> search(const std::vector<float> &embedding, int K);
+
+private:
+    QString m_filePath;
+    hnswlib::InnerProductSpace *m_space;
+    hnswlib::HierarchicalNSW<float> *m_hnsw;
+};
+
+#endif // EMBEDDINGS_H

gpt4all-chat/embllm.cpp

@@ -0,0 +1,64 @@
+#include "embllm.h"
+#include "modellist.h"
+
+EmbeddingLLM::EmbeddingLLM()
+    : QObject{nullptr}
+    , m_model{nullptr}
+{
+}
+
+EmbeddingLLM::~EmbeddingLLM()
+{
+    delete m_model;
+    m_model = nullptr;
+}
+
+bool EmbeddingLLM::loadModel()
+{
+    const EmbeddingModels *embeddingModels = ModelList::globalInstance()->embeddingModels();
+    if (!embeddingModels->count())
+        return false;
+
+    const ModelInfo defaultModel = embeddingModels->defaultModelInfo();
+
+    QString filePath = defaultModel.dirpath + defaultModel.filename();
+    QFileInfo fileInfo(filePath);
+    if (!fileInfo.exists()) {
+        qWarning() << "WARNING: Could not load sbert because file does not exist";
+        m_model = nullptr;
+        return false;
+    }
+
+    m_model = LLModel::Implementation::construct(filePath.toStdString());
+    bool success = m_model->loadModel(filePath.toStdString(), 2048);
+    if (!success) {
+        qWarning() << "WARNING: Could not load sbert";
+        delete m_model;
+        m_model = nullptr;
+        return false;
+    }
+
+    if (m_model->implementation().modelType() != "Bert") {
+        qWarning() << "WARNING: Model type is not sbert";
+        delete m_model;
+        m_model = nullptr;
+        return false;
+    }
+    return true;
+}
+
+bool EmbeddingLLM::hasModel() const
+{
+    return m_model;
+}
+
+std::vector<float> EmbeddingLLM::generateEmbeddings(const QString &text)
+{
+    if (!hasModel() && !loadModel()) {
+        qWarning() << "WARNING: Could not load sbert model for embeddings";
+        return std::vector<float>();
+    }
+
+    Q_ASSERT(hasModel());
+    return m_model->embedding(text.toStdString());
+}

gpt4all-chat/embllm.h

@@ -0,0 +1,27 @@
+#ifndef EMBLLM_H
+#define EMBLLM_H
+
+#include <QObject>
+#include <QThread>
+#include "../gpt4all-backend/llmodel.h"
+
+class EmbeddingLLM : public QObject
+{
+    Q_OBJECT
+public:
+    EmbeddingLLM();
+    virtual ~EmbeddingLLM();
+
+    bool hasModel() const;
+
+public Q_SLOTS:
+    std::vector<float> generateEmbeddings(const QString &text);
+
+private:
+    bool loadModel();
+
+private:
+    LLModel *m_model = nullptr;
+};
+
+#endif // EMBLLM_H

gpt4all-chat/hnswlib/bruteforce.h

@@ -0,0 +1,167 @@
+#pragma once
+#include <unordered_map>
+#include <fstream>
+#include <mutex>
+#include <algorithm>
+#include <assert.h>
+
+namespace hnswlib {
+template<typename dist_t>
+class BruteforceSearch : public AlgorithmInterface<dist_t> {
+ public:
+    char *data_;
+    size_t maxelements_;
+    size_t cur_element_count;
+    size_t size_per_element_;
+
+    size_t data_size_;
+    DISTFUNC <dist_t> fstdistfunc_;
+    void *dist_func_param_;
+    std::mutex index_lock;
+
+    std::unordered_map<labeltype, size_t > dict_external_to_internal;
+
+
+    BruteforceSearch(SpaceInterface <dist_t> *s)
+        : data_(nullptr),
+            maxelements_(0),
+            cur_element_count(0),
+            size_per_element_(0),
+            data_size_(0),
+            dist_func_param_(nullptr) {
+    }
+
+
+    BruteforceSearch(SpaceInterface<dist_t> *s, const std::string &location)
+        : data_(nullptr),
+            maxelements_(0),
+            cur_element_count(0),
+            size_per_element_(0),
+            data_size_(0),
+            dist_func_param_(nullptr) {
+        loadIndex(location, s);
+    }
+
+
+    BruteforceSearch(SpaceInterface <dist_t> *s, size_t maxElements) {
+        maxelements_ = maxElements;
+        data_size_ = s->get_data_size();
+        fstdistfunc_ = s->get_dist_func();
+        dist_func_param_ = s->get_dist_func_param();
+        size_per_element_ = data_size_ + sizeof(labeltype);
+        data_ = (char *) malloc(maxElements * size_per_element_);
+        if (data_ == nullptr)
+            throw std::runtime_error("Not enough memory: BruteforceSearch failed to allocate data");
+        cur_element_count = 0;
+    }
+
+
+    ~BruteforceSearch() {
+        free(data_);
+    }
+
+
+    void addPoint(const void *datapoint, labeltype label, bool replace_deleted = false) {
+        int idx;
+        {
+            std::unique_lock<std::mutex> lock(index_lock);
+
+            auto search = dict_external_to_internal.find(label);
+            if (search != dict_external_to_internal.end()) {
+                idx = search->second;
+            } else {
+                if (cur_element_count >= maxelements_) {
+                    throw std::runtime_error("The number of elements exceeds the specified limit\n");
+                }
+                idx = cur_element_count;
+                dict_external_to_internal[label] = idx;
+                cur_element_count++;
+            }
+        }
+        memcpy(data_ + size_per_element_ * idx + data_size_, &label, sizeof(labeltype));
+        memcpy(data_ + size_per_element_ * idx, datapoint, data_size_);
+    }
+
+
+    void removePoint(labeltype cur_external) {
+        size_t cur_c = dict_external_to_internal[cur_external];
+
+        dict_external_to_internal.erase(cur_external);
+
+        labeltype label = *((labeltype*)(data_ + size_per_element_ * (cur_element_count-1) + data_size_));
+        dict_external_to_internal[label] = cur_c;
+        memcpy(data_ + size_per_element_ * cur_c,
+                data_ + size_per_element_ * (cur_element_count-1),
+                data_size_+sizeof(labeltype));
+        cur_element_count--;
+    }
+
+
+    std::priority_queue<std::pair<dist_t, labeltype >>
+    searchKnn(const void *query_data, size_t k, BaseFilterFunctor* isIdAllowed = nullptr) const {
+        assert(k <= cur_element_count);
+        std::priority_queue<std::pair<dist_t, labeltype >> topResults;
+        if (cur_element_count == 0) return topResults;
+        for (int i = 0; i < k; i++) {
+            dist_t dist = fstdistfunc_(query_data, data_ + size_per_element_ * i, dist_func_param_);
+            labeltype label = *((labeltype*) (data_ + size_per_element_ * i + data_size_));
+            if ((!isIdAllowed) || (*isIdAllowed)(label)) {
+                topResults.push(std::pair<dist_t, labeltype>(dist, label));
+            }
+        }
+        dist_t lastdist = topResults.empty() ? std::numeric_limits<dist_t>::max() : topResults.top().first;
+        for (int i = k; i < cur_element_count; i++) {
+            dist_t dist = fstdistfunc_(query_data, data_ + size_per_element_ * i, dist_func_param_);
+            if (dist <= lastdist) {
+                labeltype label = *((labeltype *) (data_ + size_per_element_ * i + data_size_));
+                if ((!isIdAllowed) || (*isIdAllowed)(label)) {
+                    topResults.push(std::pair<dist_t, labeltype>(dist, label));
+                }
+                if (topResults.size() > k)
+                    topResults.pop();
+
+                if (!topResults.empty()) {
+                    lastdist = topResults.top().first;
+                }
+            }
+        }
+        return topResults;
+    }
+
+
+    void saveIndex(const std::string &location) {
+        std::ofstream output(location, std::ios::binary);
+        std::streampos position;
+
+        writeBinaryPOD(output, maxelements_);
+        writeBinaryPOD(output, size_per_element_);
+        writeBinaryPOD(output, cur_element_count);
+
+        output.write(data_, maxelements_ * size_per_element_);
+
+        output.close();
+    }
+
+
+    void loadIndex(const std::string &location, SpaceInterface<dist_t> *s) {
+        std::ifstream input(location, std::ios::binary);
+        std::streampos position;
+
+        readBinaryPOD(input, maxelements_);
+        readBinaryPOD(input, size_per_element_);
+        readBinaryPOD(input, cur_element_count);
+
+        data_size_ = s->get_data_size();
+        fstdistfunc_ = s->get_dist_func();
+        dist_func_param_ = s->get_dist_func_param();
+        size_per_element_ = data_size_ + sizeof(labeltype);
+        data_ = (char *) malloc(maxelements_ * size_per_element_);
+        if (data_ == nullptr)
+            throw std::runtime_error("Not enough memory: loadIndex failed to allocate data");
+
+        input.read(data_, maxelements_ * size_per_element_);
+
+        input.close();
+    }
+};
+}  // namespace hnswlib

gpt4all-chat/hnswlib/hnswlib.h

@@ -0,0 +1,199 @@
+#pragma once
+#ifndef NO_MANUAL_VECTORIZATION
+#if (defined(__SSE__) || _M_IX86_FP > 0 || defined(_M_AMD64) || defined(_M_X64))
+#define USE_SSE
+#ifdef __AVX__
+#define USE_AVX
+#ifdef __AVX512F__
+#define USE_AVX512
+#endif
+#endif
+#endif
+#endif
+
+#if defined(USE_AVX) || defined(USE_SSE)
+#ifdef _MSC_VER
+#include <intrin.h>
+#include <stdexcept>
+void cpuid(int32_t out[4], int32_t eax, int32_t ecx) {
+    __cpuidex(out, eax, ecx);
+}
+static __int64 xgetbv(unsigned int x) {
+    return _xgetbv(x);
+}
+#else
+#include <x86intrin.h>
+#include <cpuid.h>
+#include <stdint.h>
+static void cpuid(int32_t cpuInfo[4], int32_t eax, int32_t ecx) {
+    __cpuid_count(eax, ecx, cpuInfo[0], cpuInfo[1], cpuInfo[2], cpuInfo[3]);
+}
+static uint64_t xgetbv(unsigned int index) {
+    uint32_t eax, edx;
+    __asm__ __volatile__("xgetbv" : "=a"(eax), "=d"(edx) : "c"(index));
+    return ((uint64_t)edx << 32) | eax;
+}
+#endif
+
+#if defined(USE_AVX512)
+#include <immintrin.h>
+#endif
+
+#if defined(__GNUC__)
+#define PORTABLE_ALIGN32 __attribute__((aligned(32)))
+#define PORTABLE_ALIGN64 __attribute__((aligned(64)))
+#else
+#define PORTABLE_ALIGN32 __declspec(align(32))
+#define PORTABLE_ALIGN64 __declspec(align(64))
+#endif
+
+// Adapted from https://github.com/Mysticial/FeatureDetector
+#define _XCR_XFEATURE_ENABLED_MASK  0
+
+static bool AVXCapable() {
+    int cpuInfo[4];
+
+    // CPU support
+    cpuid(cpuInfo, 0, 0);
+    int nIds = cpuInfo[0];
+
+    bool HW_AVX = false;
+    if (nIds >= 0x00000001) {
+        cpuid(cpuInfo, 0x00000001, 0);
+        HW_AVX = (cpuInfo[2] & ((int)1 << 28)) != 0;
+    }
+
+    // OS support
+    cpuid(cpuInfo, 1, 0);
+
+    bool osUsesXSAVE_XRSTORE = (cpuInfo[2] & (1 << 27)) != 0;
+    bool cpuAVXSuport = (cpuInfo[2] & (1 << 28)) != 0;
+
+    bool avxSupported = false;
+    if (osUsesXSAVE_XRSTORE && cpuAVXSuport) {
+        uint64_t xcrFeatureMask = xgetbv(_XCR_XFEATURE_ENABLED_MASK);
+        avxSupported = (xcrFeatureMask & 0x6) == 0x6;
+    }
+    return HW_AVX && avxSupported;
+}
+
+static bool AVX512Capable() {
+    if (!AVXCapable()) return false;
+
+    int cpuInfo[4];
+
+    // CPU support
+    cpuid(cpuInfo, 0, 0);
+    int nIds = cpuInfo[0];
+
+    bool HW_AVX512F = false;
+    if (nIds >= 0x00000007) {  //  AVX512 Foundation
+        cpuid(cpuInfo, 0x00000007, 0);
+        HW_AVX512F = (cpuInfo[1] & ((int)1 << 16)) != 0;
+    }
+
+    // OS support
+    cpuid(cpuInfo, 1, 0);
+
+    bool osUsesXSAVE_XRSTORE = (cpuInfo[2] & (1 << 27)) != 0;
+    bool cpuAVXSuport = (cpuInfo[2] & (1 << 28)) != 0;
+
+    bool avx512Supported = false;
+    if (osUsesXSAVE_XRSTORE && cpuAVXSuport) {
+        uint64_t xcrFeatureMask = xgetbv(_XCR_XFEATURE_ENABLED_MASK);
+        avx512Supported = (xcrFeatureMask & 0xe6) == 0xe6;
+    }
+    return HW_AVX512F && avx512Supported;
+}
+#endif
+
+#include <queue>
+#include <vector>
+#include <iostream>
+#include <string.h>
+
+namespace hnswlib {
+typedef size_t labeltype;
+
+// This can be extended to store state for filtering (e.g. from a std::set)
+class BaseFilterFunctor {
+ public:
+    virtual bool operator()(hnswlib::labeltype id) { return true; }
+};
+
+template <typename T>
+class pairGreater {
+ public:
+    bool operator()(const T& p1, const T& p2) {
+        return p1.first > p2.first;
+    }
+};
+
+template<typename T>
+static void writeBinaryPOD(std::ostream &out, const T &podRef) {
+    out.write((char *) &podRef, sizeof(T));
+}
+
+template<typename T>
+static void readBinaryPOD(std::istream &in, T &podRef) {
+    in.read((char *) &podRef, sizeof(T));
+}
+
+template<typename MTYPE>
+using DISTFUNC = MTYPE(*)(const void *, const void *, const void *);
+
+template<typename MTYPE>
+class SpaceInterface {
+ public:
+    // virtual void search(void *);
+    virtual size_t get_data_size() = 0;
+
+    virtual DISTFUNC<MTYPE> get_dist_func() = 0;
+
+    virtual void *get_dist_func_param() = 0;
+
+    virtual ~SpaceInterface() {}
+};
+
+template<typename dist_t>
+class AlgorithmInterface {
+ public:
+    virtual void addPoint(const void *datapoint, labeltype label, bool replace_deleted = false) = 0;
+
+    virtual std::priority_queue<std::pair<dist_t, labeltype>>
+        searchKnn(const void*, size_t, BaseFilterFunctor* isIdAllowed = nullptr) const = 0;
+
+    // Return k nearest neighbor in the order of closer fist
+    virtual std::vector<std::pair<dist_t, labeltype>>
+        searchKnnCloserFirst(const void* query_data, size_t k, BaseFilterFunctor* isIdAllowed = nullptr) const;
+
+    virtual void saveIndex(const std::string &location) = 0;
+    virtual ~AlgorithmInterface(){
+    }
+};
+
+template<typename dist_t>
+std::vector<std::pair<dist_t, labeltype>>
+AlgorithmInterface<dist_t>::searchKnnCloserFirst(const void* query_data, size_t k,
+                                                 BaseFilterFunctor* isIdAllowed) const {
+    std::vector<std::pair<dist_t, labeltype>> result;
+
+    // here searchKnn returns the result in the order of further first
+    auto ret = searchKnn(query_data, k, isIdAllowed);
+    {
+        size_t sz = ret.size();
+        result.resize(sz);
+        while (!ret.empty()) {
+            result[--sz] = ret.top();
+            ret.pop();
+        }
+    }
+
+    return result;
+}
+}  // namespace hnswlib
+
+#include "space_l2.h"
+#include "space_ip.h"
+#include "bruteforce.h"
+#include "hnswalg.h"

gpt4all-chat/hnswlib/space_ip.h

@@ -0,0 +1,375 @@
+#pragma once
+#include "hnswlib.h"
+
+namespace hnswlib {
+
+static float
+InnerProduct(const void *pVect1, const void *pVect2, const void *qty_ptr) {
+    size_t qty = *((size_t *) qty_ptr);
+    float res = 0;
+    for (unsigned i = 0; i < qty; i++) {
+        res += ((float *) pVect1)[i] * ((float *) pVect2)[i];
+    }
+    return res;
+}
+
+static float
+InnerProductDistance(const void *pVect1, const void *pVect2, const void *qty_ptr) {
+    return 1.0f - InnerProduct(pVect1, pVect2, qty_ptr);
+}
+
+#if defined(USE_AVX)
+
+// Favor using AVX if available.
+static float
+InnerProductSIMD4ExtAVX(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    float PORTABLE_ALIGN32 TmpRes[8];
+    float *pVect1 = (float *) pVect1v;
+    float *pVect2 = (float *) pVect2v;
+    size_t qty = *((size_t *) qty_ptr);
+
+    size_t qty16 = qty / 16;
+    size_t qty4 = qty / 4;
+
+    const float *pEnd1 = pVect1 + 16 * qty16;
+    const float *pEnd2 = pVect1 + 4 * qty4;
+
+    __m256 sum256 = _mm256_set1_ps(0);
+
+    while (pVect1 < pEnd1) {
+        //_mm_prefetch((char*)(pVect2 + 16), _MM_HINT_T0);
+
+        __m256 v1 = _mm256_loadu_ps(pVect1);
+        pVect1 += 8;
+        __m256 v2 = _mm256_loadu_ps(pVect2);
+        pVect2 += 8;
+        sum256 = _mm256_add_ps(sum256, _mm256_mul_ps(v1, v2));
+
+        v1 = _mm256_loadu_ps(pVect1);
+        pVect1 += 8;
+        v2 = _mm256_loadu_ps(pVect2);
+        pVect2 += 8;
+        sum256 = _mm256_add_ps(sum256, _mm256_mul_ps(v1, v2));
+    }
+
+    __m128 v1, v2;
+    __m128 sum_prod = _mm_add_ps(_mm256_extractf128_ps(sum256, 0), _mm256_extractf128_ps(sum256, 1));
+
+    while (pVect1 < pEnd2) {
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        sum_prod = _mm_add_ps(sum_prod, _mm_mul_ps(v1, v2));
+    }
+
+    _mm_store_ps(TmpRes, sum_prod);
+    float sum = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3];
+    return sum;
+}
+
+static float
+InnerProductDistanceSIMD4ExtAVX(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    return 1.0f - InnerProductSIMD4ExtAVX(pVect1v, pVect2v, qty_ptr);
+}
+
+#endif
+
+#if defined(USE_SSE)
+
+static float
+InnerProductSIMD4ExtSSE(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    float PORTABLE_ALIGN32 TmpRes[8];
+    float *pVect1 = (float *) pVect1v;
+    float *pVect2 = (float *) pVect2v;
+    size_t qty = *((size_t *) qty_ptr);
+
+    size_t qty16 = qty / 16;
+    size_t qty4 = qty / 4;
+
+    const float *pEnd1 = pVect1 + 16 * qty16;
+    const float *pEnd2 = pVect1 + 4 * qty4;
+
+    __m128 v1, v2;
+    __m128 sum_prod = _mm_set1_ps(0);
+
+    while (pVect1 < pEnd1) {
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        sum_prod = _mm_add_ps(sum_prod, _mm_mul_ps(v1, v2));
+
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        sum_prod = _mm_add_ps(sum_prod, _mm_mul_ps(v1, v2));
+
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        sum_prod = _mm_add_ps(sum_prod, _mm_mul_ps(v1, v2));
+
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        sum_prod = _mm_add_ps(sum_prod, _mm_mul_ps(v1, v2));
+    }
+
+    while (pVect1 < pEnd2) {
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        sum_prod = _mm_add_ps(sum_prod, _mm_mul_ps(v1, v2));
+    }
+
+    _mm_store_ps(TmpRes, sum_prod);
+    float sum = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3];
+
+    return sum;
+}
+
+static float
+InnerProductDistanceSIMD4ExtSSE(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    return 1.0f - InnerProductSIMD4ExtSSE(pVect1v, pVect2v, qty_ptr);
+}
+
+#endif
+
+
+#if defined(USE_AVX512)
+
+static float
+InnerProductSIMD16ExtAVX512(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    float PORTABLE_ALIGN64 TmpRes[16];
+    float *pVect1 = (float *) pVect1v;
+    float *pVect2 = (float *) pVect2v;
+    size_t qty = *((size_t *) qty_ptr);
+
+    size_t qty16 = qty / 16;
+
+
+    const float *pEnd1 = pVect1 + 16 * qty16;
+
+    __m512 sum512 = _mm512_set1_ps(0);
+
+    while (pVect1 < pEnd1) {
+        //_mm_prefetch((char*)(pVect2 + 16), _MM_HINT_T0);
+
+        __m512 v1 = _mm512_loadu_ps(pVect1);
+        pVect1 += 16;
+        __m512 v2 = _mm512_loadu_ps(pVect2);
+        pVect2 += 16;
+        sum512 = _mm512_add_ps(sum512, _mm512_mul_ps(v1, v2));
+    }
+
+    _mm512_store_ps(TmpRes, sum512);
+    float sum = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + TmpRes[5] + TmpRes[6] + TmpRes[7] + TmpRes[8] + TmpRes[9] + TmpRes[10] + TmpRes[11] + TmpRes[12] + TmpRes[13] + TmpRes[14] + TmpRes[15];
+
+    return sum;
+}
+
+static float
+InnerProductDistanceSIMD16ExtAVX512(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    return 1.0f - InnerProductSIMD16ExtAVX512(pVect1v, pVect2v, qty_ptr);
+}
+
+#endif
+
+#if defined(USE_AVX)
+
+static float
+InnerProductSIMD16ExtAVX(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    float PORTABLE_ALIGN32 TmpRes[8];
+    float *pVect1 = (float *) pVect1v;
+    float *pVect2 = (float *) pVect2v;
+    size_t qty = *((size_t *) qty_ptr);
+
+    size_t qty16 = qty / 16;
+
+
+    const float *pEnd1 = pVect1 + 16 * qty16;
+
+    __m256 sum256 = _mm256_set1_ps(0);
+
+    while (pVect1 < pEnd1) {
+        //_mm_prefetch((char*)(pVect2 + 16), _MM_HINT_T0);
+
+        __m256 v1 = _mm256_loadu_ps(pVect1);
+        pVect1 += 8;
+        __m256 v2 = _mm256_loadu_ps(pVect2);
+        pVect2 += 8;
+        sum256 = _mm256_add_ps(sum256, _mm256_mul_ps(v1, v2));
+
+        v1 = _mm256_loadu_ps(pVect1);
+        pVect1 += 8;
+        v2 = _mm256_loadu_ps(pVect2);
+        pVect2 += 8;
+        sum256 = _mm256_add_ps(sum256, _mm256_mul_ps(v1, v2));
+    }
+
+    _mm256_store_ps(TmpRes, sum256);
+    float sum = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + TmpRes[5] + TmpRes[6] + TmpRes[7];
+
+    return sum;
+}
+
+static float
+InnerProductDistanceSIMD16ExtAVX(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    return 1.0f - InnerProductSIMD16ExtAVX(pVect1v, pVect2v, qty_ptr);
+}
+
+#endif
+
+#if defined(USE_SSE)
+
+static float
+InnerProductSIMD16ExtSSE(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    float PORTABLE_ALIGN32 TmpRes[8];
+    float *pVect1 = (float *) pVect1v;
+    float *pVect2 = (float *) pVect2v;
+    size_t qty = *((size_t *) qty_ptr);
+
+    size_t qty16 = qty / 16;
+
+    const float *pEnd1 = pVect1 + 16 * qty16;
+
+    __m128 v1, v2;
+    __m128 sum_prod = _mm_set1_ps(0);
+
+    while (pVect1 < pEnd1) {
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        sum_prod = _mm_add_ps(sum_prod, _mm_mul_ps(v1, v2));
+
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        sum_prod = _mm_add_ps(sum_prod, _mm_mul_ps(v1, v2));
+
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        sum_prod = _mm_add_ps(sum_prod, _mm_mul_ps(v1, v2));
+
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        sum_prod = _mm_add_ps(sum_prod, _mm_mul_ps(v1, v2));
+    }
+    _mm_store_ps(TmpRes, sum_prod);
+    float sum = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3];
+
+    return sum;
+}
+
+static float
+InnerProductDistanceSIMD16ExtSSE(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    return 1.0f - InnerProductSIMD16ExtSSE(pVect1v, pVect2v, qty_ptr);
+}
+
+#endif
+
+#if defined(USE_SSE) || defined(USE_AVX) || defined(USE_AVX512)
+static DISTFUNC<float> InnerProductSIMD16Ext = InnerProductSIMD16ExtSSE;
+static DISTFUNC<float> InnerProductSIMD4Ext = InnerProductSIMD4ExtSSE;
+static DISTFUNC<float> InnerProductDistanceSIMD16Ext = InnerProductDistanceSIMD16ExtSSE;
+static DISTFUNC<float> InnerProductDistanceSIMD4Ext = InnerProductDistanceSIMD4ExtSSE;
+
+static float
+InnerProductDistanceSIMD16ExtResiduals(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    size_t qty = *((size_t *) qty_ptr);
+    size_t qty16 = qty >> 4 << 4;
+    float res = InnerProductSIMD16Ext(pVect1v, pVect2v, &qty16);
+    float *pVect1 = (float *) pVect1v + qty16;
+    float *pVect2 = (float *) pVect2v + qty16;
+
+    size_t qty_left = qty - qty16;
+    float res_tail = InnerProduct(pVect1, pVect2, &qty_left);
+    return 1.0f - (res + res_tail);
+}
+
+static float
+InnerProductDistanceSIMD4ExtResiduals(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    size_t qty = *((size_t *) qty_ptr);
+    size_t qty4 = qty >> 2 << 2;
+
+    float res = InnerProductSIMD4Ext(pVect1v, pVect2v, &qty4);
+    size_t qty_left = qty - qty4;
+
+    float *pVect1 = (float *) pVect1v + qty4;
+    float *pVect2 = (float *) pVect2v + qty4;
+    float res_tail = InnerProduct(pVect1, pVect2, &qty_left);
+
+    return 1.0f - (res + res_tail);
+}
+#endif
+
+class InnerProductSpace : public SpaceInterface<float> {
+    DISTFUNC<float> fstdistfunc_;
+    size_t data_size_;
+    size_t dim_;
+
+ public:
+    InnerProductSpace(size_t dim) {
+        fstdistfunc_ = InnerProductDistance;
+#if defined(USE_AVX) || defined(USE_SSE) || defined(USE_AVX512)
+    #if defined(USE_AVX512)
+        if (AVX512Capable()) {
+            InnerProductSIMD16Ext = InnerProductSIMD16ExtAVX512;
+            InnerProductDistanceSIMD16Ext = InnerProductDistanceSIMD16ExtAVX512;
+        } else if (AVXCapable()) {
+            InnerProductSIMD16Ext = InnerProductSIMD16ExtAVX;
+            InnerProductDistanceSIMD16Ext = InnerProductDistanceSIMD16ExtAVX;
+        }
+    #elif defined(USE_AVX)
+        if (AVXCapable()) {
+            InnerProductSIMD16Ext = InnerProductSIMD16ExtAVX;
+            InnerProductDistanceSIMD16Ext = InnerProductDistanceSIMD16ExtAVX;
+        }
+    #endif
+    #if defined(USE_AVX)
+        if (AVXCapable()) {
+            InnerProductSIMD4Ext = InnerProductSIMD4ExtAVX;
+            InnerProductDistanceSIMD4Ext = InnerProductDistanceSIMD4ExtAVX;
+        }
+    #endif
+
+        if (dim % 16 == 0)
+            fstdistfunc_ = InnerProductDistanceSIMD16Ext;
+        else if (dim % 4 == 0)
+            fstdistfunc_ = InnerProductDistanceSIMD4Ext;
+        else if (dim > 16)
+            fstdistfunc_ = InnerProductDistanceSIMD16ExtResiduals;
+        else if (dim > 4)
+            fstdistfunc_ = InnerProductDistanceSIMD4ExtResiduals;
+#endif
+        dim_ = dim;
+        data_size_ = dim * sizeof(float);
+    }
+
+    size_t get_data_size() {
+        return data_size_;
+    }
+
+    DISTFUNC<float> get_dist_func() {
+        return fstdistfunc_;
+    }
+
+    void *get_dist_func_param() {
+        return &dim_;
+    }
+
+~InnerProductSpace() {}
+};
+
+}  // namespace hnswlib

gpt4all-chat/hnswlib/space_l2.h

@@ -0,0 +1,324 @@
+#pragma once
+#include "hnswlib.h"
+
+namespace hnswlib {
+
+static float
+L2Sqr(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    float *pVect1 = (float *) pVect1v;
+    float *pVect2 = (float *) pVect2v;
+    size_t qty = *((size_t *) qty_ptr);
+
+    float res = 0;
+    for (size_t i = 0; i < qty; i++) {
+        float t = *pVect1 - *pVect2;
+        pVect1++;
+        pVect2++;
+        res += t * t;
+    }
+    return (res);
+}
+
+#if defined(USE_AVX512)
+
+// Favor using AVX512 if available.
+static float
+L2SqrSIMD16ExtAVX512(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    float *pVect1 = (float *) pVect1v;
+    float *pVect2 = (float *) pVect2v;
+    size_t qty = *((size_t *) qty_ptr);
+    float PORTABLE_ALIGN64 TmpRes[16];
+    size_t qty16 = qty >> 4;
+
+    const float *pEnd1 = pVect1 + (qty16 << 4);
+
+    __m512 diff, v1, v2;
+    __m512 sum = _mm512_set1_ps(0);
+
+    while (pVect1 < pEnd1) {
+        v1 = _mm512_loadu_ps(pVect1);
+        pVect1 += 16;
+        v2 = _mm512_loadu_ps(pVect2);
+        pVect2 += 16;
+        diff = _mm512_sub_ps(v1, v2);
+        // sum = _mm512_fmadd_ps(diff, diff, sum);
+        sum = _mm512_add_ps(sum, _mm512_mul_ps(diff, diff));
+    }
+
+    _mm512_store_ps(TmpRes, sum);
+    float res = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + TmpRes[5] + TmpRes[6] +
+            TmpRes[7] + TmpRes[8] + TmpRes[9] + TmpRes[10] + TmpRes[11] + TmpRes[12] +
+            TmpRes[13] + TmpRes[14] + TmpRes[15];
+
+    return (res);
+}
+#endif
+
+#if defined(USE_AVX)
+
+// Favor using AVX if available.
+static float
+L2SqrSIMD16ExtAVX(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    float *pVect1 = (float *) pVect1v;
+    float *pVect2 = (float *) pVect2v;
+    size_t qty = *((size_t *) qty_ptr);
+    float PORTABLE_ALIGN32 TmpRes[8];
+    size_t qty16 = qty >> 4;
+
+    const float *pEnd1 = pVect1 + (qty16 << 4);
+
+    __m256 diff, v1, v2;
+    __m256 sum = _mm256_set1_ps(0);
+
+    while (pVect1 < pEnd1) {
+        v1 = _mm256_loadu_ps(pVect1);
+        pVect1 += 8;
+        v2 = _mm256_loadu_ps(pVect2);
+        pVect2 += 8;
+        diff = _mm256_sub_ps(v1, v2);
+        sum = _mm256_add_ps(sum, _mm256_mul_ps(diff, diff));
+
+        v1 = _mm256_loadu_ps(pVect1);
+        pVect1 += 8;
+        v2 = _mm256_loadu_ps(pVect2);
+        pVect2 += 8;
+        diff = _mm256_sub_ps(v1, v2);
+        sum = _mm256_add_ps(sum, _mm256_mul_ps(diff, diff));
+    }
+
+    _mm256_store_ps(TmpRes, sum);
+    return TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + TmpRes[5] + TmpRes[6] + TmpRes[7];
+}
+
+#endif
+
+#if defined(USE_SSE)
+
+static float
+L2SqrSIMD16ExtSSE(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    float *pVect1 = (float *) pVect1v;
+    float *pVect2 = (float *) pVect2v;
+    size_t qty = *((size_t *) qty_ptr);
+    float PORTABLE_ALIGN32 TmpRes[8];
+    size_t qty16 = qty >> 4;
+
+    const float *pEnd1 = pVect1 + (qty16 << 4);
+
+    __m128 diff, v1, v2;
+    __m128 sum = _mm_set1_ps(0);
+
+    while (pVect1 < pEnd1) {
+        //_mm_prefetch((char*)(pVect2 + 16), _MM_HINT_T0);
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        diff = _mm_sub_ps(v1, v2);
+        sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff));
+
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        diff = _mm_sub_ps(v1, v2);
+        sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff));
+
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        diff = _mm_sub_ps(v1, v2);
+        sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff));
+
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        diff = _mm_sub_ps(v1, v2);
+        sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff));
+    }
+
+    _mm_store_ps(TmpRes, sum);
+    return TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3];
+}
+#endif
+
+#if defined(USE_SSE) || defined(USE_AVX) || defined(USE_AVX512)
+static DISTFUNC<float> L2SqrSIMD16Ext = L2SqrSIMD16ExtSSE;
+
+static float
+L2SqrSIMD16ExtResiduals(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    size_t qty = *((size_t *) qty_ptr);
+    size_t qty16 = qty >> 4 << 4;
+    float res = L2SqrSIMD16Ext(pVect1v, pVect2v, &qty16);
+    float *pVect1 = (float *) pVect1v + qty16;
+    float *pVect2 = (float *) pVect2v + qty16;
+
+    size_t qty_left = qty - qty16;
+    float res_tail = L2Sqr(pVect1, pVect2, &qty_left);
+    return (res + res_tail);
+}
+#endif
+
+
+#if defined(USE_SSE)
+static float
+L2SqrSIMD4Ext(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    float PORTABLE_ALIGN32 TmpRes[8];
+    float *pVect1 = (float *) pVect1v;
+    float *pVect2 = (float *) pVect2v;
+    size_t qty = *((size_t *) qty_ptr);
+
+
+    size_t qty4 = qty >> 2;
+
+    const float *pEnd1 = pVect1 + (qty4 << 2);
+
+    __m128 diff, v1, v2;
+    __m128 sum = _mm_set1_ps(0);
+
+    while (pVect1 < pEnd1) {
+        v1 = _mm_loadu_ps(pVect1);
+        pVect1 += 4;
+        v2 = _mm_loadu_ps(pVect2);
+        pVect2 += 4;
+        diff = _mm_sub_ps(v1, v2);
+        sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff));
+    }
+    _mm_store_ps(TmpRes, sum);
+    return TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3];
+}
+
+static float
+L2SqrSIMD4ExtResiduals(const void *pVect1v, const void *pVect2v, const void *qty_ptr) {
+    size_t qty = *((size_t *) qty_ptr);
+    size_t qty4 = qty >> 2 << 2;
+
+    float res = L2SqrSIMD4Ext(pVect1v, pVect2v, &qty4);
+    size_t qty_left = qty - qty4;
+
+    float *pVect1 = (float *) pVect1v + qty4;
+    float *pVect2 = (float *) pVect2v + qty4;
+    float res_tail = L2Sqr(pVect1, pVect2, &qty_left);
+
+    return (res + res_tail);
+}
+#endif
+
+class L2Space : public SpaceInterface<float> {
+    DISTFUNC<float> fstdistfunc_;
+    size_t data_size_;
+    size_t dim_;
+
+ public:
+    L2Space(size_t dim) {
+        fstdistfunc_ = L2Sqr;
+#if defined(USE_SSE) || defined(USE_AVX) || defined(USE_AVX512)
+    #if defined(USE_AVX512)
+        if (AVX512Capable())
+            L2SqrSIMD16Ext = L2SqrSIMD16ExtAVX512;
+        else if (AVXCapable())
+            L2SqrSIMD16Ext = L2SqrSIMD16ExtAVX;
+    #elif defined(USE_AVX)
+        if (AVXCapable())
+            L2SqrSIMD16Ext = L2SqrSIMD16ExtAVX;
+    #endif
+
+        if (dim % 16 == 0)
+            fstdistfunc_ = L2SqrSIMD16Ext;
+        else if (dim % 4 == 0)
+            fstdistfunc_ = L2SqrSIMD4Ext;
+        else if (dim > 16)
+            fstdistfunc_ = L2SqrSIMD16ExtResiduals;
+        else if (dim > 4)
+            fstdistfunc_ = L2SqrSIMD4ExtResiduals;
+#endif
+        dim_ = dim;
+        data_size_ = dim * sizeof(float);
+    }
+
+    size_t get_data_size() {
+        return data_size_;
+    }
+
+    DISTFUNC<float> get_dist_func() {
+        return fstdistfunc_;
+    }
+
+    void *get_dist_func_param() {
+        return &dim_;
+    }
+
+    ~L2Space() {}
+};
+
+static int
+L2SqrI4x(const void *__restrict pVect1, const void *__restrict pVect2, const void *__restrict qty_ptr) {
+    size_t qty = *((size_t *) qty_ptr);
+    int res = 0;
+    unsigned char *a = (unsigned char *) pVect1;
+    unsigned char *b = (unsigned char *) pVect2;
+
+    qty = qty >> 2;
+    for (size_t i = 0; i < qty; i++) {
+        res += ((*a) - (*b)) * ((*a) - (*b));
+        a++;
+        b++;
+        res += ((*a) - (*b)) * ((*a) - (*b));
+        a++;
+        b++;
+        res += ((*a) - (*b)) * ((*a) - (*b));
+        a++;
+        b++;
+        res += ((*a) - (*b)) * ((*a) - (*b));
+        a++;
+        b++;
+    }
+    return (res);
+}
+
+static int L2SqrI(const void* __restrict pVect1, const void* __restrict pVect2, const void* __restrict qty_ptr) {
+    size_t qty = *((size_t*)qty_ptr);
+    int res = 0;
+    unsigned char* a = (unsigned char*)pVect1;
+    unsigned char* b = (unsigned char*)pVect2;
+
+    for (size_t i = 0; i < qty; i++) {
+        res += ((*a) - (*b)) * ((*a) - (*b));
+        a++;
+        b++;
+    }
+    return (res);
+}
+
+class L2SpaceI : public SpaceInterface<int> {
+    DISTFUNC<int> fstdistfunc_;
+    size_t data_size_;
+    size_t dim_;
+
+ public:
+    L2SpaceI(size_t dim) {
+        if (dim % 4 == 0) {
+            fstdistfunc_ = L2SqrI4x;
+        } else {
+            fstdistfunc_ = L2SqrI;
+        }
+        dim_ = dim;
+        data_size_ = dim * sizeof(unsigned char);
+    }
+
+    size_t get_data_size() {
+        return data_size_;
+    }
+
+    DISTFUNC<int> get_dist_func() {
+        return fstdistfunc_;
+    }
+
+    void *get_dist_func_param() {
+        return &dim_;
+    }
+
+    ~L2SpaceI() {}
+};
+}  // namespace hnswlib

gpt4all-chat/hnswlib/visited_list_pool.h

@@ -0,0 +1,78 @@
+#pragma once
+
+#include <mutex>
+#include <string.h>
+#include <deque>
+
+namespace hnswlib {
+typedef unsigned short int vl_type;
+
+class VisitedList {
+ public:
+    vl_type curV;
+    vl_type *mass;
+    unsigned int numelements;
+
+    VisitedList(int numelements1) {
+        curV = -1;
+        numelements = numelements1;
+        mass = new vl_type[numelements];
+    }
+
+    void reset() {
+        curV++;
+        if (curV == 0) {
+            memset(mass, 0, sizeof(vl_type) * numelements);
+            curV++;
+        }
+    }
+
+    ~VisitedList() { delete[] mass; }
+};
+///////////////////////////////////////////////////////////
+//
+// Class for multi-threaded pool-management of VisitedLists
+//
+/////////////////////////////////////////////////////////
+
+class VisitedListPool {
+    std::deque<VisitedList *> pool;
+    std::mutex poolguard;
+    int numelements;
+
+ public:
+    VisitedListPool(int initmaxpools, int numelements1) {
+        numelements = numelements1;
+        for (int i = 0; i < initmaxpools; i++)
+            pool.push_front(new VisitedList(numelements));
+    }
+
+    VisitedList *getFreeVisitedList() {
+        VisitedList *rez;
+        {
+            std::unique_lock <std::mutex> lock(poolguard);
+            if (pool.size() > 0) {
+                rez = pool.front();
+                pool.pop_front();
+            } else {
+                rez = new VisitedList(numelements);
+            }
+        }
+        rez->reset();
+        return rez;
+    }
+
+    void releaseVisitedList(VisitedList *vl) {
+        std::unique_lock <std::mutex> lock(poolguard);
+        pool.push_front(vl);
+    }
+
+    ~VisitedListPool() {
+        while (pool.size()) {
+            VisitedList *rez = pool.front();
+            pool.pop_front();
+            delete rez;
+        }
+    }
+};
+}  // namespace hnswlib

gpt4all-chat/llm.cpp

@@ -1,17 +1,20 @@
 #include "llm.h"
 #include "../gpt4all-backend/sysinfo.h"
-#include "../gpt4all-backend/llmodel.h"
-#include "network.h"
 
 #include <QCoreApplication>
+#include <QDesktopServices>
 #include <QDir>
 #include <QFile>
 #include <QProcess>
 #include <QResource>
 #include <QSettings>
-#include <QDesktopServices>
+#include <QUrl>
 #include <fstream>
 
+#ifndef GPT4ALL_OFFLINE_INSTALLER
+#include "network.h"
+#endif
+
 class MyLLM: public LLM { };
 Q_GLOBAL_STATIC(MyLLM, llmInstance)
 LLM *LLM::globalInstance()
@@ -23,20 +26,6 @@ LLM::LLM()
     : QObject{nullptr}
     , m_compatHardware(true)
 {
-    QString llmodelSearchPaths = QCoreApplication::applicationDirPath();
-    const QString libDir = QCoreApplication::applicationDirPath() + "/../lib/";
-    if (directoryExists(libDir))
-        llmodelSearchPaths += ";" + libDir;
-#if defined(Q_OS_MAC)
-    const QString binDir = QCoreApplication::applicationDirPath() + "/../../../";
-    if (directoryExists(binDir))
-        llmodelSearchPaths += ";" + binDir;
-    const QString frameworksDir = QCoreApplication::applicationDirPath() + "/../Frameworks/";
-    if (directoryExists(frameworksDir))
-        llmodelSearchPaths += ";" + frameworksDir;
-#endif
-    LLModel::Implementation::setImplementationsSearchPath(llmodelSearchPaths.toStdString());
-
 #if defined(__x86_64__)
     #ifndef _MSC_VER
         const bool minimal(__builtin_cpu_supports("avx"));
@@ -86,7 +75,7 @@ bool LLM::checkForUpdates() const
     #endif
 }
 
-bool LLM::directoryExists(const QString &path) const
+bool LLM::directoryExists(const QString &path)
 {
     const QUrl url(path);
     const QString localFilePath = url.isLocalFile() ? url.toLocalFile() : path;
@@ -94,7 +83,7 @@ bool LLM::directoryExists(const QString &path) const
     return info.exists() && info.isDir();
 }
 
-bool LLM::fileExists(const QString &path) const
+bool LLM::fileExists(const QString &path)
 {
     const QUrl url(path);
     const QString localFilePath = url.isLocalFile() ? url.toLocalFile() : path;

gpt4all-chat/llm.h

@@ -13,8 +13,8 @@ public:
     Q_INVOKABLE bool compatHardware() const { return m_compatHardware; }
 
     Q_INVOKABLE bool checkForUpdates() const;
-    Q_INVOKABLE bool directoryExists(const QString &path) const;
-    Q_INVOKABLE bool fileExists(const QString &path) const;
+    Q_INVOKABLE static bool directoryExists(const QString &path);
+    Q_INVOKABLE static bool fileExists(const QString &path);
     Q_INVOKABLE qint64 systemTotalRAMInGB() const;
     Q_INVOKABLE QString systemTotalRAMInGBString() const;