github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-07-10 15:06:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add unit test for _merge_splits function (#3513)

Browse Source 
This commit adds a new unit test for the _merge_splits function in the
text splitter. The new test verifies that the function merges text into
chunks of the correct size and overlap, using a specified separator. The
test passes on the current implementation of the function.
This commit is contained in:
Roma 2023-04-25 14:02:59 -03:00 committed by GitHub
parent 61da2bb742
commit 2b4e9a3efa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
tests/unit_tests/test_text_splitter.py
View File

@ -68,6 +68,15 @@ def test_character_text_splitting_args() -> None:
CharacterTextSplitter(chunk_size=2, chunk_overlap=4) CharacterTextSplitter(chunk_size=2, chunk_overlap=4)
def test_merge_splits() -> None:
"""Test merging splits with a given separator."""
splitter = CharacterTextSplitter(separator=" ", chunk_size=9, chunk_overlap=2)
splits = ["foo", "bar", "baz"]
expected_output = ["foo bar", "baz"]
output = splitter._merge_splits(splits, separator=" ")
assert output == expected_output
def test_create_documents() -> None: def test_create_documents() -> None:
"""Test create documents method.""" """Test create documents method."""
texts = ["foo bar", "baz"] texts = ["foo bar", "baz"]