mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-19 17:36:00 +00:00
langchain[patch]: Add konlpy based text splitting for Korean (#16003)
- **Description:** Adds a text splitter based on [Konlpy](https://konlpy.org/en/latest/#start) which is a Python package for natural language processing (NLP) of the Korean language. (It is like Spacy or NLTK for Korean) - **Dependencies:** Konlpy would have to be installed before this splitter is used, - **Twitter handle:** @untilhamza
This commit is contained in:
@@ -1427,6 +1427,37 @@ class SpacyTextSplitter(TextSplitter):
|
||||
return self._merge_splits(splits, self._separator)
|
||||
|
||||
|
||||
class KonlpyTextSplitter(TextSplitter):
|
||||
"""Splitting text using Konlpy package.
|
||||
|
||||
It is good for splitting Korean text.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
separator: str = "\n\n",
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initialize the Konlpy text splitter."""
|
||||
super().__init__(**kwargs)
|
||||
self._separator = separator
|
||||
try:
|
||||
from konlpy.tag import Kkma
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"""
|
||||
Konlpy is not installed, please install it with
|
||||
`pip install konlpy`
|
||||
"""
|
||||
)
|
||||
self.kkma = Kkma()
|
||||
|
||||
def split_text(self, text: str) -> List[str]:
|
||||
"""Split incoming text and return chunks."""
|
||||
splits = self.kkma.sentences(text)
|
||||
return self._merge_splits(splits, self._separator)
|
||||
|
||||
|
||||
# For backwards compatibility
|
||||
class PythonCodeTextSplitter(RecursiveCharacterTextSplitter):
|
||||
"""Attempts to split the text along Python syntax."""
|
||||
|
Reference in New Issue
Block a user