diff --git a/ingest.py b/ingest.py index ee97e7c0..f609c760 100755 --- a/ingest.py +++ b/ingest.py @@ -4,6 +4,7 @@ import glob from typing import List from dotenv import load_dotenv from multiprocessing import Pool +from tqdm import tqdm from langchain.document_loaders import ( CSVLoader, @@ -80,7 +81,6 @@ def load_single_document(file_path: str) -> Document: raise ValueError(f"Unsupported file extension '{ext}'") - def load_documents(source_dir: str) -> List[Document]: # Loads all documents from source documents directory all_files = [] @@ -88,9 +88,15 @@ def load_documents(source_dir: str) -> List[Document]: all_files.extend( glob.glob(os.path.join(source_dir, f"**/*{ext}"), recursive=True) ) + with Pool(processes=os.cpu_count()) as pool: - documents = pool.map(load_single_document, all_files) - return documents + results = [] + with tqdm(total=len(all_files), desc='Loading documents', ncols=80) as pbar: + for i, doc in enumerate(pool.imap_unordered(load_single_document, all_files)): + results.append(doc) + pbar.update() + + return results def main(): diff --git a/requirements.txt b/requirements.txt index 21740bcd..204b77ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,5 @@ unstructured==0.6.6 extract-msg==0.41.1 tabulate==0.9.0 pandoc==2.3 -pypandoc==1.11 \ No newline at end of file +pypandoc==1.11 +tqdm==4.65.0 \ No newline at end of file