github/privateGPT
1
0
Fork 1
mirror of https://github.com/imartinez/privateGPT.git synced 2025-09-08 10:39:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use a different text splitter to improve results. Ingest takes an argument pointing to the doc to ingest.

Browse Source
This commit is contained in:
Iván Martínez
2023-05-05 17:32:31 +02:00
parent a05186b598
commit 92244a90b4
2 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
ingest.py
View File

@@ -1,13 +1,14 @@
from langchain.document_loaders import TextLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.text_splitter import CharacterTextSplitter
from langchain.vectorstores import Chroma
from langchain.embeddings import LlamaCppEmbeddings
from sys import argv
def main():
# Load document and split in chunks
loader = TextLoader('./source_documents/state_of_the_union.txt', encoding='utf8')
loader = TextLoader(argv[1], encoding="utf8")
documents = loader.load()
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
text_splitter = CharacterTextSplitter(chunk_size=500, chunk_overlap=50)
texts = text_splitter.split_documents(documents)
# Create embeddings
llama = LlamaCppEmbeddings(model_path="./models/ggml-model-q4_0.bin")