github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-07-08 14:05:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Let Notion document loader support utf-8 and make it default. (#10613)

Browse Source 
Use utf-8 encoding by default
This commit is contained in:
Richard Wang 2023-10-13 03:13:41 +08:00 committed by GitHub
parent 3759a34229
commit 6402c33299
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
libs/langchain/langchain/document_loaders/notion.py
View File

@ -8,16 +8,17 @@ from langchain.document_loaders.base import BaseLoader
class NotionDirectoryLoader(BaseLoader): class NotionDirectoryLoader(BaseLoader):
"""Load `Notion directory` dump.""" """Load `Notion directory` dump."""
def __init__(self, path: str): def __init__(self, path: str, *, encoding: str = "utf-8") -> None:
"""Initialize with a file path.""" """Initialize with a file path."""
self.file_path = path self.file_path = path
self.encoding = encoding
def load(self) -> List[Document]: def load(self) -> List[Document]:
"""Load documents.""" """Load documents."""
ps = list(Path(self.file_path).glob("**/*.md")) paths = list(Path(self.file_path).glob("**/*.md"))
docs = [] docs = []
for p in ps: for p in paths:
with open(p) as f: with open(p, encoding=self.encoding) as f:
text = f.read() text = f.read()
metadata = {"source": str(p)} metadata = {"source": str(p)}
docs.append(Document(page_content=text, metadata=metadata)) docs.append(Document(page_content=text, metadata=metadata))