From 5dc365682a6748af8fe860b3583131f54e64dbd3 Mon Sep 17 00:00:00 2001
From: Honglei <jhonglei@gmail.com>
Date: Mon, 25 May 2026 23:46:57 +0800
Subject: [PATCH] fix(rag): use size chunking by default for markdown knowledge
 (Fixes #3030) (#3033)

Co-authored-by: honglei <honglei@users.noreply.github.com>
---
 .../dbgpt/rag/text_splitter/text_splitter.py  | 21 +++++++++---
 .../rag/knowledge/tests/test_markdown.py      | 32 +++++++++++++++++++
 2 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/packages/dbgpt-core/src/dbgpt/rag/text_splitter/text_splitter.py b/packages/dbgpt-core/src/dbgpt/rag/text_splitter/text_splitter.py
index 81e2b830f..30a5ab6e6 100644
--- a/packages/dbgpt-core/src/dbgpt/rag/text_splitter/text_splitter.py
+++ b/packages/dbgpt-core/src/dbgpt/rag/text_splitter/text_splitter.py
@@ -529,10 +529,23 @@ class MarkdownHeaderTextSplitter(TextSplitter):
                 line["content"] = f'"{subtitles}": ' + line["content"]
                 aggregated_chunks.append(line)
 
-        return [
-            Chunk(content=chunk["content"], metadata=chunk["metadata"])
-            for chunk in aggregated_chunks
-        ]
+        chunks = []
+        fallback_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self._chunk_size,
+            chunk_overlap=self._chunk_overlap,
+            length_function=self._length_function,
+        )
+        for chunk in aggregated_chunks:
+            content = chunk["content"]
+            metadata = chunk["metadata"]
+            if self._length_function(content) <= self._chunk_size:
+                chunks.append(Chunk(content=content, metadata=copy.deepcopy(metadata)))
+                continue
+            for split_content in fallback_splitter.split_text(content):
+                chunks.append(
+                    Chunk(content=split_content, metadata=copy.deepcopy(metadata))
+                )
+        return chunks
 
     def split_text(  # type: ignore
         self,
diff --git a/packages/dbgpt-ext/src/dbgpt_ext/rag/knowledge/tests/test_markdown.py b/packages/dbgpt-ext/src/dbgpt_ext/rag/knowledge/tests/test_markdown.py
index 35c5c3911..11a4039fa 100644
--- a/packages/dbgpt-ext/src/dbgpt_ext/rag/knowledge/tests/test_markdown.py
+++ b/packages/dbgpt-ext/src/dbgpt_ext/rag/knowledge/tests/test_markdown.py
@@ -2,6 +2,10 @@ from unittest.mock import mock_open, patch
 
 import pytest
 
+from dbgpt.core import Document
+from dbgpt.rag.knowledge.base import ChunkStrategy
+from dbgpt_ext.rag import ChunkParameters
+
 from ..markdown import MarkdownKnowledge
 
 MOCK_MARKDOWN_DATA = """# Header 1
@@ -27,3 +31,31 @@ def test_load_from_markdown(mock_file_open):
     assert len(documents) == 1
     assert documents[0].content == MOCK_MARKDOWN_DATA
     assert documents[0].metadata["source"] == file_path
+
+
+def test_markdown_default_strategy_preserves_header_splitter():
+    assert (
+        MarkdownKnowledge.default_chunk_strategy()
+        == ChunkStrategy.CHUNK_BY_MARKDOWN_HEADER
+    )
+
+
+def test_automatic_markdown_chunking_splits_oversized_header_chunks():
+    chunk_size = 512
+    markdown_text = "# Header 1\n```yaml\n" + "a" * 1200 + "\n" + "b" * 1200 + "\n```"
+    document = Document(content=markdown_text, metadata={"source": "test.md"})
+
+    documents = MarkdownKnowledge(file_path="test.md").extract(
+        [document],
+        ChunkParameters(
+            chunk_strategy="Automatic",
+            chunk_size=chunk_size,
+            chunk_overlap=0,
+        ),
+    )
+
+    chunks = documents[0].chunks
+    assert len(chunks) > 1
+    assert max(len(chunk.content) for chunk in chunks) <= chunk_size
+    assert all(chunk.metadata["Header1"] == "Header 1" for chunk in chunks)
+    assert all(chunk.metadata["source"] == "test.md" for chunk in chunks)