langchain[patch]: Add konlpy based text splitting for Korean (#16003)

- **Description:** Adds a text splitter based on
[Konlpy](https://konlpy.org/en/latest/#start) which is a Python package
for natural language processing (NLP) of the Korean language. (It is
like Spacy or NLTK for Korean)
- **Dependencies:** Konlpy would have to be installed before this
splitter is used,
  - **Twitter handle:** @untilhamza
This commit is contained in:
Hamza Kyamanywa
2024-01-20 02:44:56 +09:00
committed by GitHub
parent 9b0a531aa2
commit 39b3c6d94c
2 changed files with 131 additions and 1 deletions

View File

@@ -1427,6 +1427,37 @@ class SpacyTextSplitter(TextSplitter):
return self._merge_splits(splits, self._separator)
class KonlpyTextSplitter(TextSplitter):
"""Splitting text using Konlpy package.
It is good for splitting Korean text.
"""
def __init__(
self,
separator: str = "\n\n",
**kwargs: Any,
) -> None:
"""Initialize the Konlpy text splitter."""
super().__init__(**kwargs)
self._separator = separator
try:
from konlpy.tag import Kkma
except ImportError:
raise ImportError(
"""
Konlpy is not installed, please install it with
`pip install konlpy`
"""
)
self.kkma = Kkma()
def split_text(self, text: str) -> List[str]:
"""Split incoming text and return chunks."""
splits = self.kkma.sentences(text)
return self._merge_splits(splits, self._separator)
# For backwards compatibility
class PythonCodeTextSplitter(RecursiveCharacterTextSplitter):
"""Attempts to split the text along Python syntax."""