github/privateGPT
1
0
Fork 1
mirror of https://github.com/imartinez/privateGPT.git synced 2025-06-03 12:39:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Sanitize null bytes before ingestion (#2090)

Browse Source 
* Sanitize null bytes before ingestion

* Added comments
This commit is contained in:
Dmitri Qiu 2024-09-25 13:00:03 +03:00 committed by GitHub
parent fa3c30661d
commit 5fbb402477
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
private_gpt/components/ingest/ingest_helper.py
View File

@ -92,7 +92,13 @@ class IngestionHelper:
return string_reader.load_data([file_data.read_text()]) return string_reader.load_data([file_data.read_text()])
logger.debug("Specific reader found for extension=%s", extension) logger.debug("Specific reader found for extension=%s", extension)
return reader_cls().load_data(file_data) documents = reader_cls().load_data(file_data)
# Sanitize NUL bytes in text which can't be stored in Postgres
for i in range(len(documents)):
documents[i].text = documents[i].text.replace("\u0000", "")
return documents
@staticmethod @staticmethod
def _exclude_metadata(documents: list[Document]) -> None: def _exclude_metadata(documents: list[Document]) -> None: