github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2026-01-25 23:04:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

update:format

Browse Source
This commit is contained in:
aries-ckt
2023-06-05 16:27:52 +08:00
parent be1a792d3c
commit f2f28fee42
11 changed files with 34 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pilot/source_embedding/EncodeTextLoader.py
View File

@@ -15,12 +15,12 @@ class EncodeTextLoader(BaseLoader):
def load(self) -> List[Document]:
"""Load from file path."""
with open(self.file_path, 'rb') as f:
with open(self.file_path, "rb") as f:
raw_text = f.read()
result = chardet.detect(raw_text)
if result['encoding'] is None:
text = raw_text.decode('utf-8')
if result["encoding"] is None:
text = raw_text.decode("utf-8")
else:
text = raw_text.decode(result['encoding'])
text = raw_text.decode(result["encoding"])
metadata = {"source": self.file_path}
return [Document(page_content=text, metadata=metadata)]