From a22969ad1f1c4bd3acabe05cc4bb0b3d80fdad1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Mart=C3=ADnez?= Date: Sat, 11 Nov 2023 21:39:15 +0100 Subject: [PATCH] Add sources to completions APIs and UI (#1206) --- docs/openapi.json | 38 ++++++++-- private_gpt/open_ai/openai_models.py | 28 ++++++-- private_gpt/server/chat/chat_router.py | 19 +++-- private_gpt/server/chat/chat_service.py | 72 ++++++++++--------- private_gpt/server/chunks/chunks_service.py | 41 ++++++----- .../server/completions/completions_router.py | 6 ++ private_gpt/ui/ui.py | 25 ++++++- 7 files changed, 159 insertions(+), 70 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 909b5cda..c72d7983 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "PrivateGPT", "summary": "PrivateGPT is a production-ready AI project that allows you to ask questions to your documents using the power of Large Language Models (LLMs), even in scenarios without Internet connection. 100% private, no data leaves your execution environment at any point.", - "description": "## Introduction\n\nPrivateGPT provides an **API** containing all the building blocks required to build\n**private, context-aware AI applications**. The API follows and extends OpenAI API standard, and supports\nboth normal and streaming responses.\n\nThe API is divided in two logical blocks:\n\n- High-level API, abstracting all the complexity of a RAG (Retrieval Augmented Generation) pipeline implementation:\n - Ingestion of documents: internally managing document parsing, splitting, metadata extraction,\n embedding generation and storage.\n - Chat & Completions using context from ingested documents: abstracting the retrieval of context, the prompt\n engineering and the response generation.\n- Low-level API, allowing advanced users to implement their own complex pipelines:\n - Embeddings generation: based on a piece of text.\n - Contextual chunks retrieval: given a query, returns the most relevant chunks of text from the ingested\n documents.\n\n> A working **Gradio UI client** is provided to test the API, together with a set of\n> useful tools such as bulk model download script, ingestion script, documents folder\n> watch, etc.\n\n## Quick Local Installation steps\n\nThe steps in `Installation and Settings` section are better explained and cover more\nsetup scenarios. But if you are looking for a quick setup guide, here it is:\n\n```\n# Clone the repo\ngit clone https://github.com/imartinez/privateGPT\ncd privateGPT\n\n# Install Python 3.11\npyenv install 3.11\npyenv local 3.11\n\n# Install dependencies\npoetry install --with ui,local\n\n# Download Embedding and LLM models\npoetry run python scripts/setup\n\n# (Optional) For Mac with Metal GPU, enable it. Check Installation and Settings section \nto know how to enable GPU on other platforms\nCMAKE_ARGS=\"-DLLAMA_METAL=on\" pip install --force-reinstall --no-cache-dir llama-cpp-python\n\n# Run the local server \nPGPT_PROFILES=local make run\n\n# Note: on Mac with Metal you should see a ggml_metal_add_buffer log, stating GPU is \nbeing used\n\n# Navigate to the UI and try it out! \nhttp://localhost:8001/\n```\n\n## Installation and Settings\n\n### Base requirements to run PrivateGPT\n\n* Git clone PrivateGPT repository, and navigate to it:\n\n```\n git clone https://github.com/imartinez/privateGPT\n cd privateGPT\n```\n\n* Install Python 3.11. Ideally through a python version manager like `pyenv`.\n Python 3.12\n should work too. Earlier python versions are not supported.\n * osx/linux: [pyenv](https://github.com/pyenv/pyenv)\n * windows: [pyenv-win](https://github.com/pyenv-win/pyenv-win)\n\n``` \npyenv install 3.11\npyenv local 3.11\n```\n\n* Install [Poetry](https://python-poetry.org/docs/#installing-with-the-official-installer) for dependency management:\n\n* Have a valid C++ compiler like gcc. See [Troubleshooting: C++ Compiler](#troubleshooting-c-compiler) for more details.\n\n* Install `make` for scripts:\n * osx: (Using homebrew): `brew install make`\n * windows: (Using chocolatey) `choco install make`\n\n### Install dependencies\n\nInstall the dependencies:\n\n```bash\npoetry install --with ui\n```\n\nVerify everything is working by running `make run` (or `poetry run python -m private_gpt`) and navigate to\nhttp://localhost:8001. You should see a [Gradio UI](https://gradio.app/) **configured with a mock LLM** that will\necho back the input. Later we'll see how to configure a real LLM.\n\n### Settings\n\n> Note: the default settings of PrivateGPT work out-of-the-box for a 100% local setup. Skip this section if you just\n> want to test PrivateGPT locally, and come back later to learn about more configuration options.\n\nPrivateGPT is configured through *profiles* that are defined using yaml files, and selected through env variables.\nThe full list of properties configurable can be found in `settings.yaml`\n\n#### env var `PGPT_SETTINGS_FOLDER`\n\nThe location of the settings folder. Defaults to the root of the project.\nShould contain the default `settings.yaml` and any other `settings-{profile}.yaml`.\n\n#### env var `PGPT_PROFILES`\n\nBy default, the profile definition in `settings.yaml` is loaded.\nUsing this env var you can load additional profiles; format is a comma separated list of profile names.\nThis will merge `settings-{profile}.yaml` on top of the base settings file.\n\nFor example:\n`PGPT_PROFILES=local,cuda` will load `settings-local.yaml`\nand `settings-cuda.yaml`, their contents will be merged with\nlater profiles properties overriding values of earlier ones like `settings.yaml`.\n\nDuring testing, the `test` profile will be active along with the default, therefore `settings-test.yaml`\nfile is required.\n\n#### Environment variables expansion\n\nConfiguration files can contain environment variables,\nthey will be expanded at runtime.\n\nExpansion must follow the pattern `${VARIABLE_NAME:default_value}`.\n\nFor example, the following configuration will use the value of the `PORT`\nenvironment variable or `8001` if it's not set.\nMissing variables with no default will produce an error.\n\n```yaml\nserver:\n port: ${PORT:8001}\n```\n\n### Local LLM requirements\n\nInstall extra dependencies for local execution:\n\n```bash\npoetry install --with local\n```\n\nFor PrivateGPT to run fully locally GPU acceleration is required\n(CPU execution is possible, but very slow), however,\ntypical Macbook laptops or window desktops with mid-range GPUs lack VRAM to run\neven the smallest LLMs. For that reason\n**local execution is only supported for models compatible with [llama.cpp](https://github.com/ggerganov/llama.cpp)**\n\nThese two models are known to work well:\n\n* https://huggingface.co/TheBloke/Llama-2-7B-chat-GGUF\n* https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF (recommended)\n\nTo ease the installation process, use the `setup` script that will download both\nthe embedding and the LLM model and place them in the correct location (under `models` folder):\n\n```bash\npoetry run python scripts/setup\n```\n\nIf you are ok with CPU execution, you can skip the rest of this section.\n\nAs stated before, llama.cpp is required and in\nparticular [llama-cpp-python](https://github.com/abetlen/llama-cpp-python)\nis used.\n\n> It's highly encouraged that you fully read llama-cpp and llama-cpp-python documentation relevant to your platform.\n> Running into installation issues is very likely, and you'll need to troubleshoot them yourself.\n\n#### OSX GPU support\n\nYou will need to build [llama.cpp](https://github.com/ggerganov/llama.cpp) with\nmetal support. To do that run:\n\n```bash\nCMAKE_ARGS=\"-DLLAMA_METAL=on\" pip install --force-reinstall --no-cache-dir llama-cpp-python\n```\n\n#### Windows NVIDIA GPU support\n\nWindows GPU support is done through CUDA.\nFollow the instructions on the original [llama.cpp](https://github.com/ggerganov/llama.cpp) repo to install the required\ndependencies.\n\nSome tips to get it working with an NVIDIA card and CUDA (Tested on Windows 10 with CUDA 11.5 RTX 3070):\n\n* Install latest VS2022 (and build tools) https://visualstudio.microsoft.com/vs/community/\n* Install CUDA toolkit https://developer.nvidia.com/cuda-downloads\n* Verify your installation is correct by running `nvcc --version` and `nvidia-smi`, ensure your CUDA version is up to\n date and your GPU is detected.\n* [Optional] Install CMake to troubleshoot building issues by compiling llama.cpp directly https://cmake.org/download/\n\nIf you have all required dependencies properly configured running the\nfollowing powershell command should succeed.\n\n```powershell\n$env:CMAKE_ARGS='-DLLAMA_CUBLAS=on'; poetry run pip install --force-reinstall --no-cache-dir llama-cpp-python\n```\n\nIf your installation was correct, you should see a message similar to the following next\ntime you start the server `BLAS = 1`.\n\n```\nllama_new_context_with_model: total VRAM used: 4857.93 MB (model: 4095.05 MB, context: 762.87 MB)\nAVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 0 | VSX = 0 | \n```\n\nNote that llama.cpp offloads matrix calculations to the GPU but the performance is\nstill hit heavily due to latency between CPU and GPU communication. You might need to tweak\nbatch sizes and other parameters to get the best performance for your particular system.\n\n#### Linux NVIDIA GPU support and Windows-WSL\n\nLinux GPU support is done through CUDA.\nFollow the instructions on the original [llama.cpp](https://github.com/ggerganov/llama.cpp) repo to install the required\nexternal\ndependencies.\n\nSome tips:\n\n* Make sure you have an up-to-date C++ compiler\n* Install CUDA toolkit https://developer.nvidia.com/cuda-downloads\n* Verify your installation is correct by running `nvcc --version` and `nvidia-smi`, ensure your CUDA version is up to\n date and your GPU is detected.\n\nAfter that running the following command in the repository will install llama.cpp with GPU support:\n\n`\nCMAKE_ARGS='-DLLAMA_CUBLAS=on' poetry run pip install --force-reinstall --no-cache-dir llama-cpp-python\n`\n\nIf your installation was correct, you should see a message similar to the following next\ntime you start the server `BLAS = 1`.\n\n```\nllama_new_context_with_model: total VRAM used: 4857.93 MB (model: 4095.05 MB, context: 762.87 MB)\nAVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 0 | VSX = 0 | \n```\n\n#### Known issues and Troubleshooting\n\nExecution of LLMs locally still has a lot of sharp edges, specially when running on non Linux platforms.\nYou might encounter several issues:\n\n* Performance: RAM or VRAM usage is very high, your computer might experience slowdowns or even crashes.\n* GPU Virtualization on Windows and OSX: Simply not possible with docker desktop, you have to run the server directly on\n the host.\n* Building errors: Some of PrivateGPT dependencies need to build native code, and they might fail on some platforms.\n Most likely you are missing some dev tools in your machine (updated C++ compiler, CUDA is not on PATH, etc.).\n If you encounter any of these issues, please open an issue and we'll try to help.\n\n#### Troubleshooting: C++ Compiler\n\nIf you encounter an error while building a wheel during the `pip install` process, you may need to install a C++\ncompiler on your computer.\n\n**For Windows 10/11**\n\nTo install a C++ compiler on Windows 10/11, follow these steps:\n\n1. Install Visual Studio 2022.\n2. Make sure the following components are selected:\n * Universal Windows Platform development\n * C++ CMake tools for Windows\n3. Download the MinGW installer from the [MinGW website](https://sourceforge.net/projects/mingw/).\n4. Run the installer and select the `gcc` component.\n\n** For OSX **\n\n1. Check if you have a C++ compiler installed, Xcode might have done it for you. for example running `gcc`.\n2. If not, you can install clang or gcc with homebrew `brew install gcc`\n\n#### Troubleshooting: Mac Running Intel\n\nWhen running a Mac with Intel hardware (not M1), you may run into _clang: error: the clang compiler does not support '\n-march=native'_ during pip install.\n\nIf so set your archflags during pip install. eg: _ARCHFLAGS=\"-arch x86_64\" pip3 install -r requirements.txt_\n\n## Running the Server\n\nAfter following the installation steps you should be ready to go. Here are some common run setups:\n\n### Running 100% locally\n\nMake sure you have followed the *Local LLM requirements* section before moving on.\n\nThis command will start PrivateGPT using the `settings.yaml` (default profile) together with the `settings-local.yaml`\nconfiguration files. By default, it will enable both the API and the Gradio UI. Run:\n\n```\nPGPT_PROFILES=local make run\n``` \n\nor\n\n```\nPGPT_PROFILES=local poetry run python -m private_gpt\n```\n\nWhen the server is started it will print a log *Application startup complete*.\nNavigate to http://localhost:8001 to use the Gradio UI or to http://localhost:8001/docs (API section) to try the API\nusing Swagger UI.\n\n### Local server using OpenAI as LLM\n\nIf you cannot run a local model (because you don't have a GPU, for example) or for testing purposes, you may\ndecide to run PrivateGPT using OpenAI as the LLM.\n\nIn order to do so, create a profile `settings-openai.yaml` with the following contents:\n\n```yaml\nllm:\n mode: openai\n\nopenai:\n api_key: # You could skip this configuration and use the OPENAI_API_KEY env var instead\n```\n\nAnd run PrivateGPT loading that profile you just created:\n\n```PGPT_PROFILES=openai make run```\n\nor\n\n```PGPT_PROFILES=openai poetry run python -m private_gpt```\n\n> Note this will still use the local Embeddings model, as it is ok to use it on a CPU.\n> We'll support using OpenAI embeddings in a future release.\n\nWhen the server is started it will print a log *Application startup complete*.\nNavigate to http://localhost:8001 to use the Gradio UI or to http://localhost:8001/docs (API section) to try the API.\nYou'll notice the speed and quality of response is higher, given you are using OpenAI's servers for the heavy\ncomputations.\n\n### Use AWS's Sagemaker\n\n\ud83d\udea7 Under construction \ud83d\udea7\n\n## Gradio UI user manual\n\nGradio UI is a ready to use way of testing most of PrivateGPT API functionalities.\n\n![Gradio PrivateGPT](https://lh3.googleusercontent.com/drive-viewer/AK7aPaD_Hc-A8A9ooMe-hPgm_eImgsbxAjb__8nFYj8b_WwzvL1Gy90oAnp1DfhPaN6yGiEHCOXs0r77W1bYHtPzlVwbV7fMsA=s1600)\n\n### Execution Modes\n\nIt has 3 modes of execution (you can select in the top-left):\n\n* Query Docs: uses the context from the\n ingested documents to answer the questions posted in the chat. It also takes\n into account previous chat messages as context.\n * Makes use of `/chat/completions` API with `use_context=true` and no\n `context_filter`.\n* Search in Docs: fast search that returns the 4 most related text\n chunks, together with their source document and page.\n * Makes use of `/chunks` API with no `context_filter`, `limit=4` and\n `prev_next_chunks=0`.\n* LLM Chat: simple, non-contextual chat with the LLM. The ingested documents won't\n be taken into account, only the previous messages.\n * Makes use of `/chat/completions` API with `use_context=false`.\n\n### Document Ingestion\n\nIngest documents by using the `Upload a File` button. You can check the progress of\nthe ingestion in the console logs of the server.\n\nThe list of ingested files is shown below the button.\n\nIf you want to delete the ingested documents, refer to *Reset Local documents\ndatabase* section in the documentation.\n\n### Chat\n\nNormal chat interface, self-explanatory ;)\n\nYou can check the actual prompt being passed to the LLM by looking at the logs of\nthe server. We'll add better observability in future releases.\n\n## Deployment options\n\n\ud83d\udea7 We are working on Dockerized deployment guidelines \ud83d\udea7\n\n## Observability\n\nBasic logs are enabled using LlamaIndex\nbasic logging (for example ingestion progress or LLM prompts and answers).\n\n\ud83d\udea7 We are working on improved Observability. \ud83d\udea7\n\n## Ingesting & Managing Documents\n\n\ud83d\udea7 Document Update and Delete are still WIP. \ud83d\udea7\n\nThe ingestion of documents can be done in different ways:\n\n* Using the `/ingest` API\n* Using the Gradio UI\n* Using the Bulk Local Ingestion functionality (check next section)\n\n### Bulk Local Ingestion\n\nWhen you are running PrivateGPT in a fully local setup, you can ingest a complete folder for convenience (containing\npdf, text files, etc.)\nand optionally watch changes on it with the command:\n\n```bash\nmake ingest /path/to/folder -- --watch\n```\n\nTo log the processed and failed files to an additional file, use:\n\n```bash\nmake ingest /path/to/folder -- --watch --log-file /path/to/log/file.log\n```\n\nAfter ingestion is complete, you should be able to chat with your documents\nby navigating to http://localhost:8001 and using the option `Query documents`,\nor using the completions / chat API.\n\n### Reset Local documents database\n\nWhen running in a local setup, you can remove all ingested documents by simply\ndeleting all contents of `local_data` folder (except .gitignore).\n\n## API\n\nAs explained in the introduction, the API contains high level APIs (ingestion and chat/completions) and low level APIs\n(embeddings and chunk retrieval). In this section the different specific API calls are explained.\n", + "description": "## Introduction\n\nPrivateGPT provides an **API** containing all the building blocks required to build\n**private, context-aware AI applications**. The API follows and extends OpenAI API standard, and supports\nboth normal and streaming responses.\n\nThe API is divided in two logical blocks:\n\n- High-level API, abstracting all the complexity of a RAG (Retrieval Augmented Generation) pipeline implementation:\n - Ingestion of documents: internally managing document parsing, splitting, metadata extraction,\n embedding generation and storage.\n - Chat & Completions using context from ingested documents: abstracting the retrieval of context, the prompt\n engineering and the response generation.\n- Low-level API, allowing advanced users to implement their own complex pipelines:\n - Embeddings generation: based on a piece of text.\n - Contextual chunks retrieval: given a query, returns the most relevant chunks of text from the ingested\n documents.\n\n> A working **Gradio UI client** is provided to test the API, together with a set of\n> useful tools such as bulk model download script, ingestion script, documents folder\n> watch, etc.\n\n## Quick Local Installation steps\n\nThe steps in `Installation and Settings` section are better explained and cover more\nsetup scenarios. But if you are looking for a quick setup guide, here it is:\n\n```\n# Clone the repo\ngit clone https://github.com/imartinez/privateGPT\ncd privateGPT\n\n# Install Python 3.11\npyenv install 3.11\npyenv local 3.11\n\n# Install dependencies\npoetry install --with ui,local\n\n# Download Embedding and LLM models\npoetry run python scripts/setup\n\n# (Optional) For Mac with Metal GPU, enable it. Check Installation and Settings section \nto know how to enable GPU on other platforms\nCMAKE_ARGS=\"-DLLAMA_METAL=on\" pip install --force-reinstall --no-cache-dir llama-cpp-python\n\n# Run the local server \nPGPT_PROFILES=local make run\n\n# Note: on Mac with Metal you should see a ggml_metal_add_buffer log, stating GPU is \nbeing used\n\n# Navigate to the UI and try it out! \nhttp://localhost:8001/\n```\n\n## Installation and Settings\n\n### Base requirements to run PrivateGPT\n\n* Git clone PrivateGPT repository, and navigate to it:\n\n```\n git clone https://github.com/imartinez/privateGPT\n cd privateGPT\n```\n\n* Install Python 3.11. Ideally through a python version manager like `pyenv`.\n Python 3.12\n should work too. Earlier python versions are not supported.\n * osx/linux: [pyenv](https://github.com/pyenv/pyenv)\n * windows: [pyenv-win](https://github.com/pyenv-win/pyenv-win)\n\n``` \npyenv install 3.11\npyenv local 3.11\n```\n\n* Install [Poetry](https://python-poetry.org/docs/#installing-with-the-official-installer) for dependency management:\n\n* Have a valid C++ compiler like gcc. See [Troubleshooting: C++ Compiler](#troubleshooting-c-compiler) for more details.\n\n* Install `make` for scripts:\n * osx: (Using homebrew): `brew install make`\n * windows: (Using chocolatey) `choco install make`\n\n### Install dependencies\n\nInstall the dependencies:\n\n```bash\npoetry install --with ui\n```\n\nVerify everything is working by running `make run` (or `poetry run python -m private_gpt`) and navigate to\nhttp://localhost:8001. You should see a [Gradio UI](https://gradio.app/) **configured with a mock LLM** that will\necho back the input. Later we'll see how to configure a real LLM.\n\n### Settings\n\n> Note: the default settings of PrivateGPT work out-of-the-box for a 100% local setup. Skip this section if you just\n> want to test PrivateGPT locally, and come back later to learn about more configuration options.\n\nPrivateGPT is configured through *profiles* that are defined using yaml files, and selected through env variables.\nThe full list of properties configurable can be found in `settings.yaml`\n\n#### env var `PGPT_SETTINGS_FOLDER`\n\nThe location of the settings folder. Defaults to the root of the project.\nShould contain the default `settings.yaml` and any other `settings-{profile}.yaml`.\n\n#### env var `PGPT_PROFILES`\n\nBy default, the profile definition in `settings.yaml` is loaded.\nUsing this env var you can load additional profiles; format is a comma separated list of profile names.\nThis will merge `settings-{profile}.yaml` on top of the base settings file.\n\nFor example:\n`PGPT_PROFILES=local,cuda` will load `settings-local.yaml`\nand `settings-cuda.yaml`, their contents will be merged with\nlater profiles properties overriding values of earlier ones like `settings.yaml`.\n\nDuring testing, the `test` profile will be active along with the default, therefore `settings-test.yaml`\nfile is required.\n\n#### Environment variables expansion\n\nConfiguration files can contain environment variables,\nthey will be expanded at runtime.\n\nExpansion must follow the pattern `${VARIABLE_NAME:default_value}`.\n\nFor example, the following configuration will use the value of the `PORT`\nenvironment variable or `8001` if it's not set.\nMissing variables with no default will produce an error.\n\n```yaml\nserver:\n port: ${PORT:8001}\n```\n\n### Local LLM requirements\n\nInstall extra dependencies for local execution:\n\n```bash\npoetry install --with local\n```\n\nFor PrivateGPT to run fully locally GPU acceleration is required\n(CPU execution is possible, but very slow), however,\ntypical Macbook laptops or window desktops with mid-range GPUs lack VRAM to run\neven the smallest LLMs. For that reason\n**local execution is only supported for models compatible with [llama.cpp](https://github.com/ggerganov/llama.cpp)**\n\nThese two models are known to work well:\n\n* https://huggingface.co/TheBloke/Llama-2-7B-chat-GGUF\n* https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF (recommended)\n\nTo ease the installation process, use the `setup` script that will download both\nthe embedding and the LLM model and place them in the correct location (under `models` folder):\n\n```bash\npoetry run python scripts/setup\n```\n\nIf you are ok with CPU execution, you can skip the rest of this section.\n\nAs stated before, llama.cpp is required and in\nparticular [llama-cpp-python](https://github.com/abetlen/llama-cpp-python)\nis used.\n\n> It's highly encouraged that you fully read llama-cpp and llama-cpp-python documentation relevant to your platform.\n> Running into installation issues is very likely, and you'll need to troubleshoot them yourself.\n\n#### Customizing low level parameters\n\nCurrently not all the parameters of llama-cpp and llama-cpp-python are available at PrivateGPT's `settings.yaml` file. In case you need to customize parameters such as the number of layers loaded into the GPU, you might change these at the `llm_component.py` file under the `private_gpt/components/llm/llm_component.py`. If you are getting an out of memory error, you might also try a smaller model or stick to the proposed recommended models, instead of custom tuning the parameters.\n\n#### OSX GPU support\n\nYou will need to build [llama.cpp](https://github.com/ggerganov/llama.cpp) with\nmetal support. To do that run:\n\n```bash\nCMAKE_ARGS=\"-DLLAMA_METAL=on\" pip install --force-reinstall --no-cache-dir llama-cpp-python\n```\n\n#### Windows NVIDIA GPU support\n\nWindows GPU support is done through CUDA.\nFollow the instructions on the original [llama.cpp](https://github.com/ggerganov/llama.cpp) repo to install the required\ndependencies.\n\nSome tips to get it working with an NVIDIA card and CUDA (Tested on Windows 10 with CUDA 11.5 RTX 3070):\n\n* Install latest VS2022 (and build tools) https://visualstudio.microsoft.com/vs/community/\n* Install CUDA toolkit https://developer.nvidia.com/cuda-downloads\n* Verify your installation is correct by running `nvcc --version` and `nvidia-smi`, ensure your CUDA version is up to\n date and your GPU is detected.\n* [Optional] Install CMake to troubleshoot building issues by compiling llama.cpp directly https://cmake.org/download/\n\nIf you have all required dependencies properly configured running the\nfollowing powershell command should succeed.\n\n```powershell\n$env:CMAKE_ARGS='-DLLAMA_CUBLAS=on'; poetry run pip install --force-reinstall --no-cache-dir llama-cpp-python\n```\n\nIf your installation was correct, you should see a message similar to the following next\ntime you start the server `BLAS = 1`.\n\n```\nllama_new_context_with_model: total VRAM used: 4857.93 MB (model: 4095.05 MB, context: 762.87 MB)\nAVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 0 | VSX = 0 | \n```\n\nNote that llama.cpp offloads matrix calculations to the GPU but the performance is\nstill hit heavily due to latency between CPU and GPU communication. You might need to tweak\nbatch sizes and other parameters to get the best performance for your particular system.\n\n#### Linux NVIDIA GPU support and Windows-WSL\n\nLinux GPU support is done through CUDA.\nFollow the instructions on the original [llama.cpp](https://github.com/ggerganov/llama.cpp) repo to install the required\nexternal\ndependencies.\n\nSome tips:\n\n* Make sure you have an up-to-date C++ compiler\n* Install CUDA toolkit https://developer.nvidia.com/cuda-downloads\n* Verify your installation is correct by running `nvcc --version` and `nvidia-smi`, ensure your CUDA version is up to\n date and your GPU is detected.\n\nAfter that running the following command in the repository will install llama.cpp with GPU support:\n\n`\nCMAKE_ARGS='-DLLAMA_CUBLAS=on' poetry run pip install --force-reinstall --no-cache-dir llama-cpp-python\n`\n\nIf your installation was correct, you should see a message similar to the following next\ntime you start the server `BLAS = 1`.\n\n```\nllama_new_context_with_model: total VRAM used: 4857.93 MB (model: 4095.05 MB, context: 762.87 MB)\nAVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 0 | VSX = 0 | \n```\n\n#### Known issues and Troubleshooting\n\nExecution of LLMs locally still has a lot of sharp edges, specially when running on non Linux platforms.\nYou might encounter several issues:\n\n* Performance: RAM or VRAM usage is very high, your computer might experience slowdowns or even crashes.\n* GPU Virtualization on Windows and OSX: Simply not possible with docker desktop, you have to run the server directly on\n the host.\n* Building errors: Some of PrivateGPT dependencies need to build native code, and they might fail on some platforms.\n Most likely you are missing some dev tools in your machine (updated C++ compiler, CUDA is not on PATH, etc.).\n If you encounter any of these issues, please open an issue and we'll try to help.\n\n#### Troubleshooting: C++ Compiler\n\nIf you encounter an error while building a wheel during the `pip install` process, you may need to install a C++\ncompiler on your computer.\n\n**For Windows 10/11**\n\nTo install a C++ compiler on Windows 10/11, follow these steps:\n\n1. Install Visual Studio 2022.\n2. Make sure the following components are selected:\n * Universal Windows Platform development\n * C++ CMake tools for Windows\n3. Download the MinGW installer from the [MinGW website](https://sourceforge.net/projects/mingw/).\n4. Run the installer and select the `gcc` component.\n\n** For OSX **\n\n1. Check if you have a C++ compiler installed, Xcode might have done it for you. for example running `gcc`.\n2. If not, you can install clang or gcc with homebrew `brew install gcc`\n\n#### Troubleshooting: Mac Running Intel\n\nWhen running a Mac with Intel hardware (not M1), you may run into _clang: error: the clang compiler does not support '\n-march=native'_ during pip install.\n\nIf so set your archflags during pip install. eg: _ARCHFLAGS=\"-arch x86_64\" pip3 install -r requirements.txt_\n\n## Running the Server\n\nAfter following the installation steps you should be ready to go. Here are some common run setups:\n\n### Running 100% locally\n\nMake sure you have followed the *Local LLM requirements* section before moving on.\n\nThis command will start PrivateGPT using the `settings.yaml` (default profile) together with the `settings-local.yaml`\nconfiguration files. By default, it will enable both the API and the Gradio UI. Run:\n\n```\nPGPT_PROFILES=local make run\n``` \n\nor\n\n```\nPGPT_PROFILES=local poetry run python -m private_gpt\n```\n\nWhen the server is started it will print a log *Application startup complete*.\nNavigate to http://localhost:8001 to use the Gradio UI or to http://localhost:8001/docs (API section) to try the API\nusing Swagger UI.\n\n### Local server using OpenAI as LLM\n\nIf you cannot run a local model (because you don't have a GPU, for example) or for testing purposes, you may\ndecide to run PrivateGPT using OpenAI as the LLM.\n\nIn order to do so, create a profile `settings-openai.yaml` with the following contents:\n\n```yaml\nllm:\n mode: openai\n\nopenai:\n api_key: # You could skip this configuration and use the OPENAI_API_KEY env var instead\n```\n\nAnd run PrivateGPT loading that profile you just created:\n\n```PGPT_PROFILES=openai make run```\n\nor\n\n```PGPT_PROFILES=openai poetry run python -m private_gpt```\n\n> Note this will still use the local Embeddings model, as it is ok to use it on a CPU.\n> We'll support using OpenAI embeddings in a future release.\n\nWhen the server is started it will print a log *Application startup complete*.\nNavigate to http://localhost:8001 to use the Gradio UI or to http://localhost:8001/docs (API section) to try the API.\nYou'll notice the speed and quality of response is higher, given you are using OpenAI's servers for the heavy\ncomputations.\n\n### Use AWS's Sagemaker\n\n\ud83d\udea7 Under construction \ud83d\udea7\n\n## Gradio UI user manual\n\nGradio UI is a ready to use way of testing most of PrivateGPT API functionalities.\n\n![Gradio PrivateGPT](https://lh3.googleusercontent.com/drive-viewer/AK7aPaD_Hc-A8A9ooMe-hPgm_eImgsbxAjb__8nFYj8b_WwzvL1Gy90oAnp1DfhPaN6yGiEHCOXs0r77W1bYHtPzlVwbV7fMsA=s1600)\n\n### Execution Modes\n\nIt has 3 modes of execution (you can select in the top-left):\n\n* Query Docs: uses the context from the\n ingested documents to answer the questions posted in the chat. It also takes\n into account previous chat messages as context.\n * Makes use of `/chat/completions` API with `use_context=true` and no\n `context_filter`.\n* Search in Docs: fast search that returns the 4 most related text\n chunks, together with their source document and page.\n * Makes use of `/chunks` API with no `context_filter`, `limit=4` and\n `prev_next_chunks=0`.\n* LLM Chat: simple, non-contextual chat with the LLM. The ingested documents won't\n be taken into account, only the previous messages.\n * Makes use of `/chat/completions` API with `use_context=false`.\n\n### Document Ingestion\n\nIngest documents by using the `Upload a File` button. You can check the progress of\nthe ingestion in the console logs of the server.\n\nThe list of ingested files is shown below the button.\n\nIf you want to delete the ingested documents, refer to *Reset Local documents\ndatabase* section in the documentation.\n\n### Chat\n\nNormal chat interface, self-explanatory ;)\n\nYou can check the actual prompt being passed to the LLM by looking at the logs of\nthe server. We'll add better observability in future releases.\n\n## Deployment options\n\n\ud83d\udea7 We are working on Dockerized deployment guidelines \ud83d\udea7\n\n## Observability\n\nBasic logs are enabled using LlamaIndex\nbasic logging (for example ingestion progress or LLM prompts and answers).\n\n\ud83d\udea7 We are working on improved Observability. \ud83d\udea7\n\n## Ingesting & Managing Documents\n\n\ud83d\udea7 Document Update and Delete are still WIP. \ud83d\udea7\n\nThe ingestion of documents can be done in different ways:\n\n* Using the `/ingest` API\n* Using the Gradio UI\n* Using the Bulk Local Ingestion functionality (check next section)\n\n### Bulk Local Ingestion\n\nWhen you are running PrivateGPT in a fully local setup, you can ingest a complete folder for convenience (containing\npdf, text files, etc.)\nand optionally watch changes on it with the command:\n\n```bash\nmake ingest /path/to/folder -- --watch\n```\n\nTo log the processed and failed files to an additional file, use:\n\n```bash\nmake ingest /path/to/folder -- --watch --log-file /path/to/log/file.log\n```\n\nAfter ingestion is complete, you should be able to chat with your documents\nby navigating to http://localhost:8001 and using the option `Query documents`,\nor using the completions / chat API.\n\n### Reset Local documents database\n\nWhen running in a local setup, you can remove all ingested documents by simply\ndeleting all contents of `local_data` folder (except .gitignore).\n\n## API\n\nAs explained in the introduction, the API contains high level APIs (ingestion and chat/completions) and low level APIs\n(embeddings and chunk retrieval). In this section the different specific API calls are explained.\n", "contact": { "url": "https://github.com/imartinez/privateGPT" }, @@ -23,7 +23,7 @@ "Contextual Completions" ], "summary": "Completion", - "description": "We recommend most users use our Chat completions API.\n\nGiven a prompt, the model will return one predicted completion. If `use_context`\nis set to `true`, the model will use context coming from the ingested documents\nto create the response. The documents being used can be filtered using the\n`context_filter` and passing the document IDs to be used. Ingested documents IDs\ncan be found using `/ingest/list` endpoint. If you want all ingested documents to\nbe used, remove `context_filter` altogether.\n\nWhen using `'stream': true`, the API will return data chunks following [OpenAI's\nstreaming model](https://platform.openai.com/docs/api-reference/chat/streaming):\n```\n{\"id\":\"12345\",\"object\":\"completion.chunk\",\"created\":1694268190,\n\"model\":\"private-gpt\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\n\"finish_reason\":null}]}\n```", + "description": "We recommend most users use our Chat completions API.\n\nGiven a prompt, the model will return one predicted completion. If `use_context`\nis set to `true`, the model will use context coming from the ingested documents\nto create the response. The documents being used can be filtered using the\n`context_filter` and passing the document IDs to be used. Ingested documents IDs\ncan be found using `/ingest/list` endpoint. If you want all ingested documents to\nbe used, remove `context_filter` altogether.\n\nWhen using `'include_sources': true`, the API will return the source Chunks used\nto create the response, which come from the context provided.\n\nWhen using `'stream': true`, the API will return data chunks following [OpenAI's\nstreaming model](https://platform.openai.com/docs/api-reference/chat/streaming):\n```\n{\"id\":\"12345\",\"object\":\"completion.chunk\",\"created\":1694268190,\n\"model\":\"private-gpt\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\n\"finish_reason\":null}]}\n```", "operationId": "prompt_completion_v1_completions_post", "requestBody": { "content": { @@ -65,7 +65,7 @@ "Contextual Completions" ], "summary": "Chat Completion", - "description": "Given a list of messages comprising a conversation, return a response.\n\nIf `use_context` is set to `true`, the model will use context coming\nfrom the ingested documents to create the response. The documents being used can\nbe filtered using the `context_filter` and passing the document IDs to be used.\nIngested documents IDs can be found using `/ingest/list` endpoint. If you want\nall ingested documents to be used, remove `context_filter` altogether.\n\nWhen using `'stream': true`, the API will return data chunks following [OpenAI's\nstreaming model](https://platform.openai.com/docs/api-reference/chat/streaming):\n```\n{\"id\":\"12345\",\"object\":\"completion.chunk\",\"created\":1694268190,\n\"model\":\"private-gpt\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\n\"finish_reason\":null}]}\n```", + "description": "Given a list of messages comprising a conversation, return a response.\n\nIf `use_context` is set to `true`, the model will use context coming\nfrom the ingested documents to create the response. The documents being used can\nbe filtered using the `context_filter` and passing the document IDs to be used.\nIngested documents IDs can be found using `/ingest/list` endpoint. If you want\nall ingested documents to be used, remove `context_filter` altogether.\n\nWhen using `'include_sources': true`, the API will return the source Chunks used\nto create the response, which come from the context provided.\n\nWhen using `'stream': true`, the API will return data chunks following [OpenAI's\nstreaming model](https://platform.openai.com/docs/api-reference/chat/streaming):\n```\n{\"id\":\"12345\",\"object\":\"completion.chunk\",\"created\":1694268190,\n\"model\":\"private-gpt\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\n\"finish_reason\":null}]}\n```", "operationId": "chat_completion_v1_chat_completions_post", "requestBody": { "content": { @@ -353,6 +353,11 @@ } ] }, + "include_sources": { + "type": "boolean", + "title": "Include Sources", + "default": true + }, "stream": { "type": "boolean", "title": "Stream", @@ -371,6 +376,7 @@ "c202d5e6-7b69-4869-81cc-dd574ee8ee11" ] }, + "include_sources": true, "messages": [ { "content": "How do you fry an egg?", @@ -454,9 +460,7 @@ "object", "score", "document", - "text", - "previous_texts", - "next_texts" + "text" ], "title": "Chunk" }, @@ -552,6 +556,11 @@ } ] }, + "include_sources": { + "type": "boolean", + "title": "Include Sources", + "default": true + }, "stream": { "type": "boolean", "title": "Stream", @@ -565,6 +574,7 @@ "title": "CompletionsBody", "examples": [ { + "include_sources": false, "prompt": "How do you fry an egg?", "stream": false, "use_context": false @@ -828,6 +838,20 @@ } ] }, + "sources": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Chunk" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Sources" + }, "index": { "type": "integer", "title": "Index", @@ -839,7 +863,7 @@ "finish_reason" ], "title": "OpenAIChoice", - "description": "Response from AI.\n\nEither the delta or the message will be present, but never both." + "description": "Response from AI.\n\nEither the delta or the message will be present, but never both.\nSources used will be returned in case context retrieval was enabled." }, "OpenAICompletion": { "properties": { diff --git a/private_gpt/open_ai/openai_models.py b/private_gpt/open_ai/openai_models.py index 260dde51..60ab82d3 100644 --- a/private_gpt/open_ai/openai_models.py +++ b/private_gpt/open_ai/openai_models.py @@ -5,6 +5,8 @@ from collections.abc import Iterator from llama_index.llms import ChatResponse, CompletionResponse from pydantic import BaseModel, Field +from private_gpt.server.chunks.chunks_service import Chunk + class OpenAIDelta(BaseModel): """A piece of completion that needs to be concatenated to get the full message.""" @@ -27,11 +29,13 @@ class OpenAIChoice(BaseModel): """Response from AI. Either the delta or the message will be present, but never both. + Sources used will be returned in case context retrieval was enabled. """ finish_reason: str | None = Field(examples=["stop"]) delta: OpenAIDelta | None = None message: OpenAIMessage | None = None + sources: list[Chunk] | None = None index: int = 0 @@ -49,7 +53,10 @@ class OpenAICompletion(BaseModel): @classmethod def from_text( - cls, text: str | None, finish_reason: str | None = None + cls, + text: str | None, + finish_reason: str | None = None, + sources: list[Chunk] | None = None, ) -> "OpenAICompletion": return OpenAICompletion( id=str(uuid.uuid4()), @@ -60,13 +67,18 @@ class OpenAICompletion(BaseModel): OpenAIChoice( message=OpenAIMessage(role="assistant", content=text), finish_reason=finish_reason, + sources=sources, ) ], ) @classmethod def json_from_delta( - cls, *, text: str | None, finish_reason: str | None = None + cls, + *, + text: str | None, + finish_reason: str | None = None, + sources: list[Chunk] | None = None, ) -> str: chunk = OpenAICompletion( id=str(uuid.uuid4()), @@ -77,6 +89,7 @@ class OpenAICompletion(BaseModel): OpenAIChoice( delta=OpenAIDelta(content=text), finish_reason=finish_reason, + sources=sources, ) ], ) @@ -84,20 +97,25 @@ class OpenAICompletion(BaseModel): return chunk.model_dump_json() -def to_openai_response(response: str | ChatResponse) -> OpenAICompletion: +def to_openai_response( + response: str | ChatResponse, sources: list[Chunk] | None = None +) -> OpenAICompletion: if isinstance(response, ChatResponse): return OpenAICompletion.from_text(response.delta, finish_reason="stop") else: - return OpenAICompletion.from_text(response, finish_reason="stop") + return OpenAICompletion.from_text( + response, finish_reason="stop", sources=sources + ) def to_openai_sse_stream( response_generator: Iterator[str | CompletionResponse | ChatResponse], + sources: list[Chunk] | None = None, ) -> Iterator[str]: for response in response_generator: if isinstance(response, CompletionResponse | ChatResponse): yield f"data: {OpenAICompletion.json_from_delta(text=response.delta)}\n\n" else: - yield f"data: {OpenAICompletion.json_from_delta(text=response)}\n\n" + yield f"data: {OpenAICompletion.json_from_delta(text=response, sources=sources)}\n\n" yield f"data: {OpenAICompletion.json_from_delta(text=None, finish_reason='stop')}\n\n" yield "data: [DONE]\n\n" diff --git a/private_gpt/server/chat/chat_router.py b/private_gpt/server/chat/chat_router.py index a43df438..4a0cfd44 100644 --- a/private_gpt/server/chat/chat_router.py +++ b/private_gpt/server/chat/chat_router.py @@ -20,6 +20,7 @@ class ChatBody(BaseModel): messages: list[OpenAIMessage] use_context: bool = False context_filter: ContextFilter | None = None + include_sources: bool = True stream: bool = False model_config = { @@ -34,6 +35,7 @@ class ChatBody(BaseModel): ], "stream": False, "use_context": True, + "include_sources": True, "context_filter": { "docs_ids": ["c202d5e6-7b69-4869-81cc-dd574ee8ee11"] }, @@ -58,6 +60,9 @@ def chat_completion(body: ChatBody) -> OpenAICompletion | StreamingResponse: Ingested documents IDs can be found using `/ingest/list` endpoint. If you want all ingested documents to be used, remove `context_filter` altogether. + When using `'include_sources': true`, the API will return the source Chunks used + to create the response, which come from the context provided. + When using `'stream': true`, the API will return data chunks following [OpenAI's streaming model](https://platform.openai.com/docs/api-reference/chat/streaming): ``` @@ -71,12 +76,18 @@ def chat_completion(body: ChatBody) -> OpenAICompletion | StreamingResponse: ChatMessage(content=m.content, role=MessageRole(m.role)) for m in body.messages ] if body.stream: - stream = service.stream_chat( + completion_gen = service.stream_chat( all_messages, body.use_context, body.context_filter ) return StreamingResponse( - to_openai_sse_stream(stream), media_type="text/event-stream" + to_openai_sse_stream( + completion_gen.response, + completion_gen.sources if body.include_sources else None, + ), + media_type="text/event-stream", ) else: - response = service.chat(all_messages, body.use_context, body.context_filter) - return to_openai_response(response) + completion = service.chat(all_messages, body.use_context, body.context_filter) + return to_openai_response( + completion.response, completion.sources if body.include_sources else None + ) diff --git a/private_gpt/server/chat/chat_service.py b/private_gpt/server/chat/chat_service.py index 9b83b485..cbd7f19f 100644 --- a/private_gpt/server/chat/chat_service.py +++ b/private_gpt/server/chat/chat_service.py @@ -1,13 +1,14 @@ -from collections.abc import Sequence -from typing import TYPE_CHECKING, Any - from injector import inject, singleton from llama_index import ServiceContext, StorageContext, VectorStoreIndex from llama_index.chat_engine import ContextChatEngine +from llama_index.chat_engine.types import ( + BaseChatEngine, +) from llama_index.indices.postprocessor import MetadataReplacementPostProcessor from llama_index.llm_predictor.utils import stream_chat_response_to_tokens from llama_index.llms import ChatMessage from llama_index.types import TokenGen +from pydantic import BaseModel from private_gpt.components.embedding.embedding_component import EmbeddingComponent from private_gpt.components.llm.llm_component import LLMComponent @@ -16,12 +17,17 @@ from private_gpt.components.vector_store.vector_store_component import ( VectorStoreComponent, ) from private_gpt.open_ai.extensions.context_filter import ContextFilter +from private_gpt.server.chunks.chunks_service import Chunk -if TYPE_CHECKING: - from llama_index.chat_engine.types import ( - AgentChatResponse, - StreamingAgentChatResponse, - ) + +class Completion(BaseModel): + response: str + sources: list[Chunk] | None = None + + +class CompletionGen(BaseModel): + response: TokenGen + sources: list[Chunk] | None = None @singleton @@ -51,66 +57,64 @@ class ChatService: show_progress=True, ) - def _chat_with_contex( - self, - message: str, - context_filter: ContextFilter | None = None, - chat_history: Sequence[ChatMessage] | None = None, - streaming: bool = False, - ) -> Any: + def _chat_engine( + self, context_filter: ContextFilter | None = None + ) -> BaseChatEngine: vector_index_retriever = self.vector_store_component.get_retriever( index=self.index, context_filter=context_filter ) - chat_engine = ContextChatEngine.from_defaults( + return ContextChatEngine.from_defaults( retriever=vector_index_retriever, service_context=self.service_context, node_postprocessors=[ MetadataReplacementPostProcessor(target_metadata_key="window"), ], ) - if streaming: - result = chat_engine.stream_chat(message, chat_history) - else: - result = chat_engine.chat(message, chat_history) - return result def stream_chat( self, messages: list[ChatMessage], use_context: bool = False, context_filter: ContextFilter | None = None, - ) -> TokenGen: + ) -> CompletionGen: if use_context: last_message = messages[-1].content - response: StreamingAgentChatResponse = self._chat_with_contex( + chat_engine = self._chat_engine(context_filter=context_filter) + streaming_response = chat_engine.stream_chat( message=last_message if last_message is not None else "", chat_history=messages[:-1], - context_filter=context_filter, - streaming=True, ) - response_gen = response.response_gen + sources = [ + Chunk.from_node(node) for node in streaming_response.source_nodes + ] + completion_gen = CompletionGen( + response=streaming_response.response_gen, sources=sources + ) else: stream = self.llm_service.llm.stream_chat(messages) - response_gen = stream_chat_response_to_tokens(stream) - return response_gen + completion_gen = CompletionGen( + response=stream_chat_response_to_tokens(stream) + ) + return completion_gen def chat( self, messages: list[ChatMessage], use_context: bool = False, context_filter: ContextFilter | None = None, - ) -> str: + ) -> Completion: if use_context: last_message = messages[-1].content - wrapped_response: AgentChatResponse = self._chat_with_contex( + chat_engine = self._chat_engine(context_filter=context_filter) + wrapped_response = chat_engine.chat( message=last_message if last_message is not None else "", chat_history=messages[:-1], - context_filter=context_filter, - streaming=False, ) - response = wrapped_response.response + sources = [Chunk.from_node(node) for node in wrapped_response.source_nodes] + completion = Completion(response=wrapped_response.response, sources=sources) else: chat_response = self.llm_service.llm.chat(messages) response_content = chat_response.message.content response = response_content if response_content is not None else "" - return response + completion = Completion(response=response) + return completion diff --git a/private_gpt/server/chunks/chunks_service.py b/private_gpt/server/chunks/chunks_service.py index ee011bfb..fab4bf63 100644 --- a/private_gpt/server/chunks/chunks_service.py +++ b/private_gpt/server/chunks/chunks_service.py @@ -24,17 +24,33 @@ class Chunk(BaseModel): document: IngestedDoc text: str = Field(examples=["Outbound sales increased 20%, driven by new leads."]) previous_texts: list[str] | None = Field( - examples=[["SALES REPORT 2023", "Inbound didn't show major changes."]] + default=None, + examples=[["SALES REPORT 2023", "Inbound didn't show major changes."]], ) next_texts: list[str] | None = Field( + default=None, examples=[ [ "New leads came from Google Ads campaign.", "The campaign was run by the Marketing Department", ] - ] + ], ) + @classmethod + def from_node(cls: type["Chunk"], node: NodeWithScore) -> "Chunk": + doc_id = node.node.ref_doc_id if node.node.ref_doc_id is not None else "-" + return cls( + object="context.chunk", + score=node.score or 0.0, + document=IngestedDoc( + object="ingest.document", + doc_id=doc_id, + doc_metadata=node.metadata, + ), + text=node.get_content(), + ) + @singleton class ChunksService: @@ -98,22 +114,11 @@ class ChunksService: retrieved_nodes = [] for node in nodes: - doc_id = node.node.ref_doc_id if node.node.ref_doc_id is not None else "-" - retrieved_nodes.append( - Chunk( - object="context.chunk", - score=node.score or 0.0, - document=IngestedDoc( - object="ingest.document", - doc_id=doc_id, - doc_metadata=node.metadata, - ), - text=node.get_content(), - previous_texts=self._get_sibling_nodes_text( - node, prev_next_chunks, False - ), - next_texts=self._get_sibling_nodes_text(node, prev_next_chunks), - ) + chunk = Chunk.from_node(node) + chunk.previous_texts = self._get_sibling_nodes_text( + node, prev_next_chunks, False ) + chunk.next_texts = self._get_sibling_nodes_text(node, prev_next_chunks) + retrieved_nodes.append(chunk) return retrieved_nodes diff --git a/private_gpt/server/completions/completions_router.py b/private_gpt/server/completions/completions_router.py index ce720d4a..d174ec0b 100644 --- a/private_gpt/server/completions/completions_router.py +++ b/private_gpt/server/completions/completions_router.py @@ -16,6 +16,7 @@ class CompletionsBody(BaseModel): prompt: str use_context: bool = False context_filter: ContextFilter | None = None + include_sources: bool = True stream: bool = False model_config = { @@ -25,6 +26,7 @@ class CompletionsBody(BaseModel): "prompt": "How do you fry an egg?", "stream": False, "use_context": False, + "include_sources": False, } ] } @@ -48,6 +50,9 @@ def prompt_completion(body: CompletionsBody) -> OpenAICompletion | StreamingResp can be found using `/ingest/list` endpoint. If you want all ingested documents to be used, remove `context_filter` altogether. + When using `'include_sources': true`, the API will return the source Chunks used + to create the response, which come from the context provided. + When using `'stream': true`, the API will return data chunks following [OpenAI's streaming model](https://platform.openai.com/docs/api-reference/chat/streaming): ``` @@ -61,6 +66,7 @@ def prompt_completion(body: CompletionsBody) -> OpenAICompletion | StreamingResp messages=[message], use_context=body.use_context, stream=body.stream, + include_sources=body.include_sources, context_filter=body.context_filter, ) return chat_completion(chat_body) diff --git a/private_gpt/ui/ui.py b/private_gpt/ui/ui.py index 430dbbf6..d9d96d91 100644 --- a/private_gpt/ui/ui.py +++ b/private_gpt/ui/ui.py @@ -11,7 +11,7 @@ from gradio.themes.utils.colors import slate # type: ignore from llama_index.llms import ChatMessage, ChatResponse, MessageRole from private_gpt.di import root_injector -from private_gpt.server.chat.chat_service import ChatService +from private_gpt.server.chat.chat_service import ChatService, CompletionGen from private_gpt.server.chunks.chunks_service import ChunksService from private_gpt.server.ingest.ingest_service import IngestService from private_gpt.settings.settings import settings @@ -33,8 +33,9 @@ class PrivateGptUi: self._ui_block = None def _chat(self, message: str, history: list[list[str]], mode: str, *_: Any) -> Any: - def yield_deltas(stream: Iterable[ChatResponse | str]) -> Iterable[str]: + def yield_deltas(completion_gen: CompletionGen) -> Iterable[str]: full_response: str = "" + stream = completion_gen.response for delta in stream: if isinstance(delta, str): full_response += str(delta) @@ -42,6 +43,26 @@ class PrivateGptUi: full_response += delta.delta or "" yield full_response + if completion_gen.sources: + full_response += "\n\n Sources: \n" + sources = ( + { + "file": chunk.document.doc_metadata["file_name"] + if chunk.document.doc_metadata + else "", + "page": chunk.document.doc_metadata["page_label"] + if chunk.document.doc_metadata + else "", + } + for chunk in completion_gen.sources + ) + sources_text = "\n\n\n".join( + f"{index}. {source['file']} (page {source['page']})" + for index, source in enumerate(sources, start=1) + ) + full_response += sources_text + yield full_response + def build_history() -> list[ChatMessage]: history_messages: list[ChatMessage] = list( itertools.chain(