mirror of
https://github.com/hwchase17/langchain.git
synced 2025-07-07 13:40:46 +00:00
Let Notion document loader support utf-8 and make it default. (#10613)
Use utf-8 encoding by default
This commit is contained in:
parent
3759a34229
commit
6402c33299
@ -8,16 +8,17 @@ from langchain.document_loaders.base import BaseLoader
|
||||
class NotionDirectoryLoader(BaseLoader):
|
||||
"""Load `Notion directory` dump."""
|
||||
|
||||
def __init__(self, path: str):
|
||||
def __init__(self, path: str, *, encoding: str = "utf-8") -> None:
|
||||
"""Initialize with a file path."""
|
||||
self.file_path = path
|
||||
self.encoding = encoding
|
||||
|
||||
def load(self) -> List[Document]:
|
||||
"""Load documents."""
|
||||
ps = list(Path(self.file_path).glob("**/*.md"))
|
||||
paths = list(Path(self.file_path).glob("**/*.md"))
|
||||
docs = []
|
||||
for p in ps:
|
||||
with open(p) as f:
|
||||
for p in paths:
|
||||
with open(p, encoding=self.encoding) as f:
|
||||
text = f.read()
|
||||
metadata = {"source": str(p)}
|
||||
docs.append(Document(page_content=text, metadata=metadata))
|
||||
|
Loading…
Reference in New Issue
Block a user