mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-04-28 11:45:23 +00:00
641b1ee71a
* [devops] remove post commit ci * [misc] run pre-commit on all files * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
22 lines
594 B
Python
22 lines
594 B
Python
import os
|
|
|
|
from colossalqa.data_loader.document_loader import DocumentLoader
|
|
|
|
|
|
def test_add_document():
|
|
PATH = os.environ.get("TEST_DOCUMENT_LOADER_DATA_PATH")
|
|
files = [[PATH, "all data"]]
|
|
document_loader = DocumentLoader(files)
|
|
documents = document_loader.all_data
|
|
all_files = []
|
|
for doc in documents:
|
|
assert isinstance(doc.page_content, str) == True
|
|
if doc.metadata["source"] not in all_files:
|
|
all_files.append(doc.metadata["source"])
|
|
print(all_files)
|
|
assert len(all_files) == 6
|
|
|
|
|
|
if __name__ == "__main__":
|
|
test_add_document()
|