mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-07 22:11:51 +00:00
text_splitters: Add HTMLSemanticPreservingSplitter (#25911)
**Description:** With current HTML splitters, they rely on secondary use of the `RecursiveCharacterSplitter` to further chunk the document into manageable chunks. The issue with this is it fails to maintain important structures such as tables, lists, etc within HTML. This Implementation of a HTML splitter, allows the user to define a maximum chunk size, HTML elements to preserve in full, options to preserve `<a>` href links in the output and custom handlers. The core splitting begins with headers, similar to `HTMLHeaderSplitter`. If these sections exceed the length of the `max_chunk_size` further recursive splitting is triggered. During this splitting, elements listed to preserve, will be excluded from the splitting process. This can cause chunks to be slightly larger then the max size, depending on preserved length. However, all contextual relevance of the preserved item remains intact. **Custom Handlers**: Sometimes, companies such as Atlassian have custom HTML elements, that are not parsed by default with `BeautifulSoup`. Custom handlers allows a user to provide a function to be ran whenever a specific html tag is encountered. This allows the user to preserve and gather information within custom html tags that `bs4` will potentially miss during extraction. **Dependencies:** User will need to install `bs4` in their project to utilise this class I have also added in `how_to` and unit tests, which require `bs4` to run, otherwise they will be skipped. Flowchart of process:  --------- Co-authored-by: Bagatur <baskaryan@gmail.com> Co-authored-by: Chester Curme <chester.curme@gmail.com>
This commit is contained in:
Luke
GitHub
@@ -17,7 +17,11 @@ from langchain_text_splitters import (
|
||||
)
|
||||
from langchain_text_splitters.base import split_text_on_tokens
|
||||
from langchain_text_splitters.character import CharacterTextSplitter
|
||||
from langchain_text_splitters.html import HTMLHeaderTextSplitter, HTMLSectionSplitter
|
||||
from langchain_text_splitters.html import (
|
||||
HTMLHeaderTextSplitter,
|
||||
HTMLSectionSplitter,
|
||||
HTMLSemanticPreservingSplitter,
|
||||
)
|
||||
from langchain_text_splitters.json import RecursiveJsonSplitter
|
||||
from langchain_text_splitters.markdown import (
|
||||
ExperimentalMarkdownSyntaxTextSplitter,
|
||||
@@ -2452,3 +2456,360 @@ $csvContent | ForEach-Object {
|
||||
"$csvContent | ForEach-Object {\n $_.ProcessName\n}",
|
||||
"# End of script",
|
||||
]
|
||||
|
||||
|
||||
def custom_iframe_extractor(iframe_tag: Any) -> str:
|
||||
iframe_src = iframe_tag.get("src", "")
|
||||
return f"[iframe:{iframe_src}]({iframe_src})"
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_custom_extractor() -> None:
|
||||
"""Test HTML splitting with a custom extractor."""
|
||||
html_content = """
|
||||
<h1>Section 1</h1>
|
||||
<p>This is an iframe:</p>
|
||||
<iframe src="http://example.com"></iframe>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")],
|
||||
custom_handlers={"iframe": custom_iframe_extractor},
|
||||
max_chunk_size=1000,
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(
|
||||
page_content="This is an iframe: [iframe:http://example.com](http://example.com)",
|
||||
metadata={"Header 1": "Section 1"},
|
||||
),
|
||||
]
|
||||
|
||||
assert documents == expected
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_href_links() -> None:
|
||||
"""Test HTML splitting with href links."""
|
||||
html_content = """
|
||||
<h1>Section 1</h1>
|
||||
<p>This is a link to <a href="http://example.com">example.com</a></p>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")],
|
||||
preserve_links=True,
|
||||
max_chunk_size=1000,
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(
|
||||
page_content="This is a link to [example.com](http://example.com)",
|
||||
metadata={"Header 1": "Section 1"},
|
||||
),
|
||||
]
|
||||
|
||||
assert documents == expected
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_nested_elements() -> None:
|
||||
"""Test HTML splitting with nested elements."""
|
||||
html_content = """
|
||||
<h1>Main Section</h1>
|
||||
<div>
|
||||
<p>Some text here.</p>
|
||||
<div>
|
||||
<p>Nested content.</p>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")], max_chunk_size=1000
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(
|
||||
page_content="Some text here. Nested content.",
|
||||
metadata={"Header 1": "Main Section"},
|
||||
),
|
||||
]
|
||||
|
||||
assert documents == expected
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_preserved_elements() -> None:
|
||||
"""Test HTML splitting with preserved elements like <table>, <ul> with low chunk
|
||||
size."""
|
||||
html_content = """
|
||||
<h1>Section 1</h1>
|
||||
<table>
|
||||
<tr><td>Row 1</td></tr>
|
||||
<tr><td>Row 2</td></tr>
|
||||
</table>
|
||||
<ul>
|
||||
<li>Item 1</li>
|
||||
<li>Item 2</li>
|
||||
</ul>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")],
|
||||
elements_to_preserve=["table", "ul"],
|
||||
max_chunk_size=50, # Deliberately low to test preservation
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(
|
||||
page_content="Row 1 Row 2 Item 1 Item 2",
|
||||
metadata={"Header 1": "Section 1"},
|
||||
),
|
||||
]
|
||||
|
||||
assert documents == expected # Shouldn't split the table or ul
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_no_further_splits() -> None:
|
||||
"""Test HTML splitting that requires no further splits beyond sections."""
|
||||
html_content = """
|
||||
<h1>Section 1</h1>
|
||||
<p>Some content here.</p>
|
||||
<h1>Section 2</h1>
|
||||
<p>More content here.</p>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")], max_chunk_size=1000
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(page_content="Some content here.", metadata={"Header 1": "Section 1"}),
|
||||
Document(page_content="More content here.", metadata={"Header 1": "Section 2"}),
|
||||
]
|
||||
|
||||
assert documents == expected # No further splits, just sections
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_small_chunk_size() -> None:
|
||||
"""Test HTML splitting with a very small chunk size to validate chunking."""
|
||||
html_content = """
|
||||
<h1>Section 1</h1>
|
||||
<p>This is some long text that should be split into multiple chunks due to the
|
||||
small chunk size.</p>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")], max_chunk_size=20, chunk_overlap=5
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(page_content="This is some long", metadata={"Header 1": "Section 1"}),
|
||||
Document(page_content="long text that", metadata={"Header 1": "Section 1"}),
|
||||
Document(page_content="that should be", metadata={"Header 1": "Section 1"}),
|
||||
Document(page_content="be split into", metadata={"Header 1": "Section 1"}),
|
||||
Document(page_content="into multiple", metadata={"Header 1": "Section 1"}),
|
||||
Document(page_content="chunks due to the", metadata={"Header 1": "Section 1"}),
|
||||
Document(page_content="the small chunk", metadata={"Header 1": "Section 1"}),
|
||||
Document(page_content="size.", metadata={"Header 1": "Section 1"}),
|
||||
]
|
||||
|
||||
assert documents == expected # Should split into multiple chunks
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_denylist_tags() -> None:
|
||||
"""Test HTML splitting with denylist tag filtering."""
|
||||
html_content = """
|
||||
<h1>Section 1</h1>
|
||||
<p>This paragraph should be kept.</p>
|
||||
<span>This span should be removed.</span>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")],
|
||||
denylist_tags=["span"],
|
||||
max_chunk_size=1000,
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(
|
||||
page_content="This paragraph should be kept.",
|
||||
metadata={"Header 1": "Section 1"},
|
||||
),
|
||||
]
|
||||
|
||||
assert documents == expected
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_external_metadata() -> None:
|
||||
"""Test HTML splitting with external metadata integration."""
|
||||
html_content = """
|
||||
<h1>Section 1</h1>
|
||||
<p>This is some content.</p>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")],
|
||||
external_metadata={"source": "example.com"},
|
||||
max_chunk_size=1000,
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(
|
||||
page_content="This is some content.",
|
||||
metadata={"Header 1": "Section 1", "source": "example.com"},
|
||||
),
|
||||
]
|
||||
|
||||
assert documents == expected
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_text_normalization() -> None:
|
||||
"""Test HTML splitting with text normalization."""
|
||||
html_content = """
|
||||
<h1>Section 1</h1>
|
||||
<p>This is some TEXT that should be normalized!</p>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")],
|
||||
normalize_text=True,
|
||||
max_chunk_size=1000,
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(
|
||||
page_content="this is some text that should be normalized",
|
||||
metadata={"Header 1": "Section 1"},
|
||||
),
|
||||
]
|
||||
|
||||
assert documents == expected
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_allowlist_tags() -> None:
|
||||
"""Test HTML splitting with allowlist tag filtering."""
|
||||
html_content = """
|
||||
<h1>Section 1</h1>
|
||||
<p>This paragraph should be kept.</p>
|
||||
<span>This span should be kept.</span>
|
||||
<div>This div should be removed.</div>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")],
|
||||
allowlist_tags=["p", "span"],
|
||||
max_chunk_size=1000,
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(
|
||||
page_content="This paragraph should be kept. This span should be kept.",
|
||||
metadata={"Header 1": "Section 1"},
|
||||
),
|
||||
]
|
||||
|
||||
assert documents == expected
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_mixed_preserve_and_filter() -> None:
|
||||
"""Test HTML splitting with both preserved elements and denylist tags."""
|
||||
html_content = """
|
||||
<h1>Section 1</h1>
|
||||
<table>
|
||||
<tr>
|
||||
<td>Keep this table</td>
|
||||
<td>Cell contents kept, span removed
|
||||
<span>This span should be removed.</span>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<p>This paragraph should be kept.</p>
|
||||
<span>This span should be removed.</span>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")],
|
||||
elements_to_preserve=["table"],
|
||||
denylist_tags=["span"],
|
||||
max_chunk_size=1000,
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(
|
||||
page_content="Keep this table Cell contents kept, span removed"
|
||||
" This paragraph should be kept.",
|
||||
metadata={"Header 1": "Section 1"},
|
||||
),
|
||||
]
|
||||
|
||||
assert documents == expected
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_no_headers() -> None:
|
||||
"""Test HTML splitting when there are no headers to split on."""
|
||||
html_content = """
|
||||
<p>This is content without any headers.</p>
|
||||
<p>It should still produce a valid document.</p>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[],
|
||||
max_chunk_size=1000,
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(
|
||||
page_content="This is content without any headers. It should still produce"
|
||||
" a valid document.",
|
||||
metadata={},
|
||||
),
|
||||
]
|
||||
|
||||
assert documents == expected
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_html_splitter_with_media_preservation() -> None:
|
||||
"""Test HTML splitting with media elements preserved and converted to Markdown-like
|
||||
links."""
|
||||
html_content = """
|
||||
<h1>Section 1</h1>
|
||||
<p>This is an image:</p>
|
||||
<img src="http://example.com/image.png" />
|
||||
<p>This is a video:</p>
|
||||
<video src="http://example.com/video.mp4"></video>
|
||||
<p>This is audio:</p>
|
||||
<audio src="http://example.com/audio.mp3"></audio>
|
||||
"""
|
||||
splitter = HTMLSemanticPreservingSplitter(
|
||||
headers_to_split_on=[("h1", "Header 1")],
|
||||
preserve_images=True,
|
||||
preserve_videos=True,
|
||||
preserve_audio=True,
|
||||
max_chunk_size=1000,
|
||||
)
|
||||
documents = splitter.split_text(html_content)
|
||||
|
||||
expected = [
|
||||
Document(
|
||||
page_content="This is an image: ![image:http://example.com/image.png]"
|
||||
"(http://example.com/image.png) "
|
||||
"This is a video: ![video:http://example.com/video.mp4]"
|
||||
"(http://example.com/video.mp4) "
|
||||
"This is audio: ![audio:http://example.com/audio.mp3]"
|
||||
"(http://example.com/audio.mp3)",
|
||||
metadata={"Header 1": "Section 1"},
|
||||
),
|
||||
]
|
||||
|
||||
assert documents == expected
|
||||
|
||||
Reference in New Issue
Block a user