github/privateGPT
1
0
Fork 1
mirror of https://github.com/imartinez/privateGPT.git synced 2025-06-22 05:30:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimize load_documents function with multiprocessing

Browse Source
This commit is contained in:
jiangzhuo 2023-05-19 02:35:20 +09:00
parent ad64589c8f
commit 81b221bccb
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ingest.py
View File

@ -2,6 +2,7 @@ import os
import glob import glob
from typing import List from typing import List
from dotenv import load_dotenv from dotenv import load_dotenv
from multiprocessing import Pool
from langchain.document_loaders import ( from langchain.document_loaders import (
CSVLoader, CSVLoader,
@ -64,7 +65,9 @@ def load_documents(source_dir: str) -> List[Document]:
all_files.extend( all_files.extend(
glob.glob(os.path.join(source_dir, f"**/*{ext}"), recursive=True) glob.glob(os.path.join(source_dir, f"**/*{ext}"), recursive=True)
) )
return [load_single_document(file_path) for file_path in all_files] with Pool(processes=os.cpu_count()) as pool:
documents = pool.map(load_single_document, all_files)
return documents
def main(): def main():