Add get_text_separator parameter to BSHTMLLoader (#3551)

By default get_text doesn't separate content of different HTML tag.
Adding option for specifying separator helps with document splitting.
This commit is contained in:
Maciej Bryński
2023-04-27 01:10:16 +02:00
committed by GitHub
parent 568c4f0d81
commit aa345a4bb7
2 changed files with 6 additions and 2 deletions

View File

@@ -9,15 +9,17 @@ from langchain.document_loaders.html_bs import BSHTMLLoader
def test_bs_html_loader() -> None:
"""Test unstructured loader."""
file_path = Path(__file__).parent.parent / "examples/example.html"
loader = BSHTMLLoader(str(file_path))
loader = BSHTMLLoader(str(file_path), get_text_separator="|")
docs = loader.load()
assert len(docs) == 1
metadata = docs[0].metadata
content = docs[0].page_content
assert metadata["title"] == "Chew dad's slippers"
assert metadata["source"] == str(file_path)
assert content[:2] == "\n|"
@pytest.mark.skipif(