feat:knowledge base support agentic search

packages/dbgpt-serve/src/dbgpt_serve/rag/domain/tests/test_domain_index_factory.py

@@ -0,0 +1,29 @@
+"""Tests for DomainKnowledgeIndexFactory."""
+
+import pytest
+
+from ..base import DomainKnowledgeIndex
+from ..factory import DomainKnowledgeIndexFactory
+from ..index import DomainGeneralIndex
+
+
+class TestDomainKnowledgeIndexFactory:
+    def test_create_normal_index(self):
+        index = DomainKnowledgeIndexFactory.create("normal")
+        assert isinstance(index, DomainGeneralIndex)
+        assert index.domain_type() == "normal"
+
+    def test_create_normal_index_case_insensitive(self):
+        index = DomainKnowledgeIndexFactory.create("Normal")
+        assert isinstance(index, DomainGeneralIndex)
+
+    def test_create_unknown_index_raises(self):
+        with pytest.raises(Exception, match="not supported"):
+            DomainKnowledgeIndexFactory.create("unknown_type")
+
+    def test_available_types_includes_normal(self):
+        types = DomainKnowledgeIndexFactory.available_types()
+        assert "normal" in types
+
+    def test_domain_general_index_is_subclass(self):
+        assert issubclass(DomainGeneralIndex, DomainKnowledgeIndex)

packages/dbgpt_ext/rag/graph_builder/__init__.py

@@ -0,0 +1,9 @@
+"""Code knowledge graph builder and query tools for git repositories.
+
+Builds a code graph from repository files using AST parsing (tree-sitter)
+and provides query tools for structural code search.
+"""
+
+from .repo_graph_builder import RepoGraphBuilder
+
+__all__ = ["RepoGraphBuilder"]

packages/dbgpt_serve/rag/domain/base.py

@@ -0,0 +1,76 @@
+"""Base class for Domain Knowledge Index.
+
+Defines the ETL (Extract-Transform-Load) pipeline interface that each data source
+must implement. Each domain type (normal, git_repo, yuque, notion, etc.) provides
+its own indexing strategy through this abstraction.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from dbgpt.core import Chunk
+from dbgpt.rag.knowledge.base import Knowledge
+from dbgpt.storage.full_text.base import FullTextStoreBase
+from dbgpt.storage.knowledge_graph.base import KnowledgeGraphBase
+from dbgpt.storage.vector_store.base import VectorStoreBase
+
+
+class DomainKnowledgeIndex(ABC):
+    """Abstract base class for domain-specific knowledge indexing.
+
+    Each data source type (local documents, git repositories, yuque, notion, etc.)
+    implements its own ETL pipeline by subclassing this and overriding the
+    extract/transform/load methods.
+
+    The factory pattern (DomainKnowledgeIndexFactory) is used to instantiate the
+    correct index based on the knowledge space's domain_type.
+    """
+
+    @abstractmethod
+    async def extract(
+        self,
+        knowledge: Knowledge,
+        chunk_parameter,
+        **kwargs,
+    ) -> list[Chunk]:
+        """Extract knowledge chunks from the data source."""
+        raise NotImplementedError
+
+    @abstractmethod
+    async def transform(
+        self,
+        chunks: list[Chunk],
+        **kwargs,
+    ) -> list[Chunk]:
+        """Transform knowledge chunks (enrichment, summarization, etc.)."""
+        raise NotImplementedError
+
+    @abstractmethod
+    async def load(
+        self,
+        chunks: list[Chunk],
+        vector_store: Optional[VectorStoreBase] = None,
+        full_text_store: Optional[FullTextStoreBase] = None,
+        kg_store: Optional[KnowledgeGraphBase] = None,
+        keywords: bool = True,
+        max_chunks_once_load: int = 10,
+        max_threads: int = 1,
+        **kwargs,
+    ) -> list[Chunk]:
+        """Load knowledge chunks into storage backends."""
+        raise NotImplementedError
+
+    async def clean(
+        self,
+        chunks: list[Chunk],
+        node_ids: Optional[list[str]],
+        with_keywords: bool = True,
+        **kwargs,
+    ):
+        """Clean up indexed chunks from storage backends."""
+        raise NotImplementedError
+
+    @classmethod
+    def domain_type(cls) -> str:
+        """Return the domain type identifier for this index."""
+        raise NotImplementedError

packages/dbgpt_serve/rag/tools/__init__.py

@@ -0,0 +1,22 @@
+"""Knowledge base search tools package.
+
+Provides structured search tools for agents:
+- kb_ls: List files and directories in a knowledge space
+- kb_glob: Search files by name pattern
+- kb_grep: Search file contents by keyword
+- kb_cat: Read file content by path
+- kb_semantic_search: Semantic search using vector retrieval
+- kb_codegraph_explore: Code knowledge graph exploration
+"""
+
+# Import tool modules to register them with the @tool decorator
+from . import kb_file_tools  # noqa: F401
+from . import semantic_search_tool  # noqa: F401
+
+# CodeGraph tools - optional, requires graph_store
+try:
+    from . import codegraph_tools  # noqa: F401
+except ImportError:
+    pass
+
+__all__ = ["kb_file_tools", "semantic_search_tool", "codegraph_tools"]

packages/dbgpt_serve/src/dbgpt_serve/rag/tools/semantic_search_tool.py

@@ -0,0 +1,64 @@
+"""Semantic search tool for knowledge spaces."""
+
+import logging
+from typing import Annotated
+from dbgpt.agent.resource.tool.base import tool
+
+logger = logging.getLogger(__name__)
+
+
+def _get_rag_service():
+    from ..service.service import Service
+    from dbgpt._private.config import Config
+    system_app = Config().SYSTEM_APP
+    if not system_app:
+        raise RuntimeError("SYSTEM_APP is not initialized yet")
+    return Service.get_instance(system_app)
+
+
+def _format_chunk_results(chunks, query, knowledge_id):
+    lines = [f"Semantic search '{query}' in {knowledge_id}:"]
+    chars = len(lines[0])
+    for i, chunk in enumerate(chunks):
+        content = chunk.content or ""
+        score = getattr(chunk, "score", None)
+        metadata = getattr(chunk, "metadata", {}) or {}
+        file_path = metadata.get("file_path", "")
+        score_str = f" (score: {score:.2f})" if score is not None else ""
+        entry = f"\n---\n### Result {i+1}{score_str} [{file_path}]\n{content}"
+        chars += len(entry)
+        if chars > 8000:
+            lines.append(f"\n... {len(chunks)-i} more results.")
+            break
+        lines.append(entry)
+    return "\n".join(lines)
+
+
+@tool(name="kb_semantic_search", description="Semantic search in knowledge base. Use only when kb_grep returns insufficient results.")
+async def kb_semantic_search(
+    knowledge_id: Annotated[str, "Knowledge space ID"],
+    query: Annotated[str, "Natural language search query"],
+    top_k: Annotated[int, "Number of results"] = 5,
+    score_threshold: Annotated[float, "Min score (0-1)"] = 0.0,
+) -> str:
+    top_k = int(top_k) if top_k else 5
+    score_threshold = float(score_threshold) if score_threshold else 0.0
+    try:
+        service = _get_rag_service()
+    except Exception as e:
+        return f"Semantic search service unavailable: {e}"
+    try:
+        from ..api.schemas import KnowledgeRetrieveRequest
+        request = KnowledgeRetrieveRequest(
+            query=query,
+            space_id=int(knowledge_id) if knowledge_id.isdigit() else knowledge_id,
+            top_k=top_k, score_threshold=score_threshold)
+        space = service.get({"id": request.space_id})
+        if space is None:
+            return f"Knowledge space {knowledge_id} not found"
+        search_res = await service.retrieve(request, space)
+    except Exception as e:
+        return f"Semantic search failed: {e}"
+    if not search_res:
+        return f"No results for '{query}' in {knowledge_id}"
+    return _format_chunk_results(search_res, query, knowledge_id)