From 8a5b2f453b23c3d805680eccb3bcb5dceaa84a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Mart=C3=ADnez?= Date: Wed, 17 May 2023 00:19:21 +0200 Subject: [PATCH 1/3] Use faster and better embeddings: sentenceTransformers --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f0f0bc6f..bc2dfe80 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # privateGPT Ask questions to your documents without an internet connection, using the power of LLMs. 100% private, no data leaves your execution environment at any point. You can ingest documents and ask questions without an internet connection! -Built with [LangChain](https://github.com/hwchase17/langchain) and [GPT4All](https://github.com/nomic-ai/gpt4all) and [LlamaCpp](https://github.com/ggerganov/llama.cpp) +Built with [LangChain](https://github.com/hwchase17/langchain), [GPT4All](https://github.com/nomic-ai/gpt4all), [LlamaCpp](https://github.com/ggerganov/llama.cpp), [Chroma](https://www.trychroma.com/) and [SentenceTransformers](https://www.sbert.net/). demo @@ -12,20 +12,19 @@ In order to set your environment up to run the code here, first install all requ pip install -r requirements.txt ``` -Then, download the 2 models and place them in a directory of your choice. +Then, download the LLM model and place it in a directory of your choice: - LLM: default to [ggml-gpt4all-j-v1.3-groovy.bin](https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin). If you prefer a different GPT4All-J compatible model, just download it and reference it in your `.env` file. -- Embedding: default to [ggml-model-q4_0.bin](https://huggingface.co/Pi3141/alpaca-native-7B-ggml/resolve/397e872bf4c83f4c642317a5bf65ce84a105786e/ggml-model-q4_0.bin). If you prefer a different compatible Embeddings model, just download it and reference it in your `.env` file. Rename `example.env` to `.env` and edit the variables appropriately. ``` MODEL_TYPE: supports LlamaCpp or GPT4All PERSIST_DIRECTORY: is the folder you want your vectorstore in -LLAMA_EMBEDDINGS_MODEL: (absolute) Path to your LlamaCpp supported embeddings model MODEL_PATH: Path to your GPT4All or LlamaCpp supported LLM -MODEL_N_CTX: Maximum token limit for both embeddings and LLM models +MODEL_N_CTX: Maximum token limit for the LLM model +EMBEDDINGS_MODEL_NAME: SentenceTransformers embeddings model name (see https://www.sbert.net/docs/pretrained_models.html) ``` -Note: because of the way `langchain` loads the `LLAMA` embeddings, you need to specify the absolute path of your embeddings model binary. This means it will not work if you use a home directory shortcut (eg. `~/` or `$HOME/`). +Note: because of the way `langchain` loads the `SentenceTransformers` embeddings, the first time you run the script it will require internet connection to download the embeddings model itself. ## Test dataset This repo uses a [state of the union transcript](https://github.com/imartinez/privateGPT/blob/main/source_documents/state_of_the_union.txt) as an example. @@ -40,11 +39,11 @@ Run the following command to ingest all the data. python ingest.py ``` -It will create a `db` folder containing the local vectorstore. Will take time, depending on the size of your documents. +It will create a `db` folder containing the local vectorstore. Will take 20-30 seconds per document, depending on the size of the document. You can ingest as many documents as you want, and all will be accumulated in the local embeddings database. If you want to start from an empty database, delete the `db` folder. -Note: during the ingest process no data leaves your local environment. You could ingest without an internet connection. +Note: during the ingest process no data leaves your local environment. You could ingest without an internet connection, except for the first time you run the ingest script, when the embeddings model is downloaded. ## Ask questions to your documents, locally! In order to ask a question, run a command like: @@ -68,7 +67,7 @@ Type `exit` to finish the script. # How does it work? Selecting the right local models and the power of `LangChain` you can run the entire pipeline locally, without any data leaving your environment, and with reasonable performance. -- `ingest.py` uses `LangChain` tools to parse the document and create embeddings locally using `LlamaCppEmbeddings`. It then stores the result in a local vector database using `Chroma` vector store. +- `ingest.py` uses `LangChain` tools to parse the document and create embeddings locally using `HuggingFaceEmbeddings` (`SentenceTransformers`). It then stores the result in a local vector database using `Chroma` vector store. - `privateGPT.py` uses a local LLM based on `GPT4All-J` or `LlamaCpp` to understand questions and create answers. The context for the answers is extracted from the local vector store using a similarity search to locate the right piece of context from the docs. - `GPT4All-J` wrapper was introduced in LangChain 0.0.162. From 23d24c88e938067191896edbcc4a31eba4955f95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Mart=C3=ADnez?= Date: Wed, 17 May 2023 00:32:41 +0200 Subject: [PATCH 2/3] Update code to use sentence-transformers through huggingfaceembeddings --- example.env | 2 +- ingest.py | 15 ++++++++------- privateGPT.py | 8 ++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/example.env b/example.env index 149eca2e..82907845 100644 --- a/example.env +++ b/example.env @@ -1,5 +1,5 @@ PERSIST_DIRECTORY=db -LLAMA_EMBEDDINGS_MODEL=models/ggml-model-q4_0.bin MODEL_TYPE=GPT4All MODEL_PATH=models/ggml-gpt4all-j-v1.3-groovy.bin +EMBEDDINGS_MODEL_NAME=all-MiniLM-L6-v2 MODEL_N_CTX=1000 \ No newline at end of file diff --git a/ingest.py b/ingest.py index 4c955862..2c703623 100644 --- a/ingest.py +++ b/ingest.py @@ -6,7 +6,7 @@ from dotenv import load_dotenv from langchain.document_loaders import TextLoader, PDFMinerLoader, CSVLoader from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.vectorstores import Chroma -from langchain.embeddings import LlamaCppEmbeddings +from langchain.embeddings import HuggingFaceEmbeddings from langchain.docstore.document import Document from constants import CHROMA_SETTINGS @@ -38,22 +38,23 @@ def main(): # Load environment variables persist_directory = os.environ.get('PERSIST_DIRECTORY') source_directory = os.environ.get('SOURCE_DIRECTORY', 'source_documents') - llama_embeddings_model = os.environ.get('LLAMA_EMBEDDINGS_MODEL') - model_n_ctx = os.environ.get('MODEL_N_CTX') + embeddings_model_name = os.environ.get('EMBEDDINGS_MODEL_NAME') # Load documents and split in chunks print(f"Loading documents from {source_directory}") + chunk_size = 500 + chunk_overlap = 50 documents = load_documents(source_directory) - text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50) + text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap) texts = text_splitter.split_documents(documents) print(f"Loaded {len(documents)} documents from {source_directory}") - print(f"Split into {len(texts)} chunks of text (max. 500 tokens each)") + print(f"Split into {len(texts)} chunks of text (max. {chunk_size} characters each)") # Create embeddings - llama = LlamaCppEmbeddings(model_path=llama_embeddings_model, n_ctx=model_n_ctx) + embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name) # Create and store locally vectorstore - db = Chroma.from_documents(texts, llama, persist_directory=persist_directory, client_settings=CHROMA_SETTINGS) + db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory, client_settings=CHROMA_SETTINGS) db.persist() db = None diff --git a/privateGPT.py b/privateGPT.py index 4c603a27..ae08bb93 100644 --- a/privateGPT.py +++ b/privateGPT.py @@ -1,6 +1,6 @@ from dotenv import load_dotenv from langchain.chains import RetrievalQA -from langchain.embeddings import LlamaCppEmbeddings +from langchain.embeddings import HuggingFaceEmbeddings from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler from langchain.vectorstores import Chroma from langchain.llms import GPT4All, LlamaCpp @@ -8,7 +8,7 @@ import os load_dotenv() -llama_embeddings_model = os.environ.get("LLAMA_EMBEDDINGS_MODEL") +embeddings_model_name = os.environ.get("EMBEDDINGS_MODEL_NAME") persist_directory = os.environ.get('PERSIST_DIRECTORY') model_type = os.environ.get('MODEL_TYPE') @@ -18,8 +18,8 @@ model_n_ctx = os.environ.get('MODEL_N_CTX') from constants import CHROMA_SETTINGS def main(): - llama = LlamaCppEmbeddings(model_path=llama_embeddings_model, n_ctx=model_n_ctx) - db = Chroma(persist_directory=persist_directory, embedding_function=llama, client_settings=CHROMA_SETTINGS) + embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name) + db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS) retriever = db.as_retriever() # Prepare the LLM callbacks = [StreamingStdOutCallbackHandler()] From bf3bddfbb67a165bc3ec2a0b947a4700641996c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Mart=C3=ADnez?= Date: Tue, 16 May 2023 20:44:30 +0200 Subject: [PATCH 3/3] More loaders, generic method - Update the README with extra formats - Add Powerpoint, requested in #138 - Add ePub requested in #138 comment - https://github.com/imartinez/privateGPT/pull/138#issuecomment-1549564535 - Update requirements --- README.md | 21 ++++++++++++++--- ingest.py | 60 +++++++++++++++++++++++++++++++++++++----------- requirements.txt | 5 ++++ 3 files changed, 70 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index bc2dfe80..ee27a902 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,22 @@ This repo uses a [state of the union transcript](https://github.com/imartinez/pr ## Instructions for ingesting your own dataset -Put any and all of your .txt, .pdf, or .csv files into the source_documents directory +Put any and all your files into the `source_documents` directory + +The supported extensions are: + + - `.csv`: CSV, + - `.docx`: Word Document, + - `.enex`: EverNote, + - `.eml`: Email, + - `.epub`: EPub, + - `.html`: HTML File, + - `.md`: Markdown, + - `.msg`: Outlook Message, + - `.odt`: Open Document Text, + - `.pdf`: Portable Document Format (PDF), + - `.pptx` : PowerPoint Document, + - `.txt`: Text file (UTF-8), Run the following command to ingest all the data. @@ -54,7 +69,7 @@ python privateGPT.py And wait for the script to require your input. -```shell +```plaintext > Enter a query: ``` @@ -87,7 +102,7 @@ To install a C++ compiler on Windows 10/11, follow these steps: * Universal Windows Platform development * C++ CMake tools for Windows 3. Download the MinGW installer from the [MinGW website](https://sourceforge.net/projects/mingw/). -4. Run the installer and select the "gcc" component. +4. Run the installer and select the `gcc` component. # Disclaimer This is a test project to validate the feasibility of a fully private solution for question answering using LLMs and Vector embeddings. It is not production ready, and it is not meant to be used in production. The models selection is not optimized for performance, but for privacy; but it is possible to use different models and vectorstores to improve performance. diff --git a/ingest.py b/ingest.py index 2c703623..d28edd50 100644 --- a/ingest.py +++ b/ingest.py @@ -3,7 +3,20 @@ import glob from typing import List from dotenv import load_dotenv -from langchain.document_loaders import TextLoader, PDFMinerLoader, CSVLoader +from langchain.document_loaders import ( + CSVLoader, + EverNoteLoader, + PDFMinerLoader, + TextLoader, + UnstructuredEmailLoader, + UnstructuredEPubLoader, + UnstructuredHTMLLoader, + UnstructuredMarkdownLoader, + UnstructuredODTLoader, + UnstructuredPowerPointLoader, + UnstructuredWordDocumentLoader, +) + from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.vectorstores import Chroma from langchain.embeddings import HuggingFaceEmbeddings @@ -14,23 +27,44 @@ from constants import CHROMA_SETTINGS load_dotenv() +# Map file extensions to document loaders and their arguments +LOADER_MAPPING = { + ".csv": (CSVLoader, {}), + # ".docx": (Docx2txtLoader, {}), + ".docx": (UnstructuredWordDocumentLoader, {}), + ".enex": (EverNoteLoader, {}), + ".eml": (UnstructuredEmailLoader, {}), + ".epub": (UnstructuredEPubLoader, {}), + ".html": (UnstructuredHTMLLoader, {}), + ".md": (UnstructuredMarkdownLoader, {}), + ".odt": (UnstructuredODTLoader, {}), + ".pdf": (PDFMinerLoader, {}), + ".pptx": (UnstructuredPowerPointLoader, {}), + ".txt": (TextLoader, {"encoding": "utf8"}), + # Add more mappings for other file extensions and loaders as needed +} + + +load_dotenv() + + def load_single_document(file_path: str) -> Document: - # Loads a single document from a file path - if file_path.endswith(".txt"): - loader = TextLoader(file_path, encoding="utf8") - elif file_path.endswith(".pdf"): - loader = PDFMinerLoader(file_path) - elif file_path.endswith(".csv"): - loader = CSVLoader(file_path) - return loader.load()[0] + ext = "." + file_path.rsplit(".", 1)[-1] + if ext in LOADER_MAPPING: + loader_class, loader_args = LOADER_MAPPING[ext] + loader = loader_class(file_path, **loader_args) + return loader.load()[0] + + raise ValueError(f"Unsupported file extension '{ext}'") def load_documents(source_dir: str) -> List[Document]: # Loads all documents from source documents directory - txt_files = glob.glob(os.path.join(source_dir, "**/*.txt"), recursive=True) - pdf_files = glob.glob(os.path.join(source_dir, "**/*.pdf"), recursive=True) - csv_files = glob.glob(os.path.join(source_dir, "**/*.csv"), recursive=True) - all_files = txt_files + pdf_files + csv_files + all_files = [] + for ext in LOADER_MAPPING: + all_files.extend( + glob.glob(os.path.join(source_dir, f"**/*{ext}"), recursive=True) + ) return [load_single_document(file_path) for file_path in all_files] diff --git a/requirements.txt b/requirements.txt index 2a48ad45..39273f81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,8 @@ llama-cpp-python==0.1.48 urllib3==1.26.6 pdfminer.six==20221105 python-dotenv==1.0.0 +unstructured==0.6.6 +extract-msg==0.41.1 +tabulate==0.9.0 +pandoc==2.3 +pandoc-binary==1.11