fix(rag): use size chunking by default for markdown knowledge (Fixes #3030) (#3033)

Co-authored-by: honglei <honglei@users.noreply.github.com>

packages/dbgpt-core/src/dbgpt/rag/text_splitter/text_splitter.py

@@ -529,10 +529,23 @@ class MarkdownHeaderTextSplitter(TextSplitter):
                 line["content"] = f'"{subtitles}": ' + line["content"]
                 aggregated_chunks.append(line)
 
-        return [
-            Chunk(content=chunk["content"], metadata=chunk["metadata"])
-            for chunk in aggregated_chunks
-        ]
+        chunks = []
+        fallback_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self._chunk_size,
+            chunk_overlap=self._chunk_overlap,
+            length_function=self._length_function,
+        )
+        for chunk in aggregated_chunks:
+            content = chunk["content"]
+            metadata = chunk["metadata"]
+            if self._length_function(content) <= self._chunk_size:
+                chunks.append(Chunk(content=content, metadata=copy.deepcopy(metadata)))
+                continue
+            for split_content in fallback_splitter.split_text(content):
+                chunks.append(
+                    Chunk(content=split_content, metadata=copy.deepcopy(metadata))
+                )
+        return chunks
 
     def split_text(  # type: ignore
         self,

packages/dbgpt-ext/src/dbgpt_ext/rag/knowledge/tests/test_markdown.py

@@ -2,6 +2,10 @@ from unittest.mock import mock_open, patch
 
 import pytest
 
+from dbgpt.core import Document
+from dbgpt.rag.knowledge.base import ChunkStrategy
+from dbgpt_ext.rag import ChunkParameters
+
 from ..markdown import MarkdownKnowledge
 
 MOCK_MARKDOWN_DATA = """# Header 1
@@ -27,3 +31,31 @@ def test_load_from_markdown(mock_file_open):
     assert len(documents) == 1
     assert documents[0].content == MOCK_MARKDOWN_DATA
     assert documents[0].metadata["source"] == file_path
+
+
+def test_markdown_default_strategy_preserves_header_splitter():
+    assert (
+        MarkdownKnowledge.default_chunk_strategy()
+        == ChunkStrategy.CHUNK_BY_MARKDOWN_HEADER
+    )
+
+
+def test_automatic_markdown_chunking_splits_oversized_header_chunks():
+    chunk_size = 512
+    markdown_text = "# Header 1\n```yaml\n" + "a" * 1200 + "\n" + "b" * 1200 + "\n```"
+    document = Document(content=markdown_text, metadata={"source": "test.md"})
+
+    documents = MarkdownKnowledge(file_path="test.md").extract(
+        [document],
+        ChunkParameters(
+            chunk_strategy="Automatic",
+            chunk_size=chunk_size,
+            chunk_overlap=0,
+        ),
+    )
+
+    chunks = documents[0].chunks
+    assert len(chunks) > 1
+    assert max(len(chunk.content) for chunk in chunks) <= chunk_size
+    assert all(chunk.metadata["Header1"] == "Header 1" for chunk in chunks)
+    assert all(chunk.metadata["source"] == "test.md" for chunk in chunks)