diff --git a/packages/dbgpt-serve/src/dbgpt_serve/rag/domain/tests/__init__.py b/packages/dbgpt-serve/src/dbgpt_serve/rag/domain/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/dbgpt-serve/src/dbgpt_serve/rag/domain/tests/test_domain_index_factory.py b/packages/dbgpt-serve/src/dbgpt_serve/rag/domain/tests/test_domain_index_factory.py new file mode 100644 index 000000000..0267be8bb --- /dev/null +++ b/packages/dbgpt-serve/src/dbgpt_serve/rag/domain/tests/test_domain_index_factory.py @@ -0,0 +1,29 @@ +"""Tests for DomainKnowledgeIndexFactory.""" + +import pytest + +from ..base import DomainKnowledgeIndex +from ..factory import DomainKnowledgeIndexFactory +from ..index import DomainGeneralIndex + + +class TestDomainKnowledgeIndexFactory: + def test_create_normal_index(self): + index = DomainKnowledgeIndexFactory.create("normal") + assert isinstance(index, DomainGeneralIndex) + assert index.domain_type() == "normal" + + def test_create_normal_index_case_insensitive(self): + index = DomainKnowledgeIndexFactory.create("Normal") + assert isinstance(index, DomainGeneralIndex) + + def test_create_unknown_index_raises(self): + with pytest.raises(Exception, match="not supported"): + DomainKnowledgeIndexFactory.create("unknown_type") + + def test_available_types_includes_normal(self): + types = DomainKnowledgeIndexFactory.available_types() + assert "normal" in types + + def test_domain_general_index_is_subclass(self): + assert issubclass(DomainGeneralIndex, DomainKnowledgeIndex) diff --git a/packages/dbgpt_ext/rag/graph_builder/__init__.py b/packages/dbgpt_ext/rag/graph_builder/__init__.py new file mode 100644 index 000000000..17bd2a6d6 --- /dev/null +++ b/packages/dbgpt_ext/rag/graph_builder/__init__.py @@ -0,0 +1,9 @@ +"""Code knowledge graph builder and query tools for git repositories. + +Builds a code graph from repository files using AST parsing (tree-sitter) +and provides query tools for structural code search. +""" + +from .repo_graph_builder import RepoGraphBuilder + +__all__ = ["RepoGraphBuilder"] \ No newline at end of file diff --git a/packages/dbgpt_serve/rag/domain/base.py b/packages/dbgpt_serve/rag/domain/base.py new file mode 100644 index 000000000..5da68ffd9 --- /dev/null +++ b/packages/dbgpt_serve/rag/domain/base.py @@ -0,0 +1,76 @@ +"""Base class for Domain Knowledge Index. + +Defines the ETL (Extract-Transform-Load) pipeline interface that each data source +must implement. Each domain type (normal, git_repo, yuque, notion, etc.) provides +its own indexing strategy through this abstraction. +""" + +from abc import ABC, abstractmethod +from typing import Optional + +from dbgpt.core import Chunk +from dbgpt.rag.knowledge.base import Knowledge +from dbgpt.storage.full_text.base import FullTextStoreBase +from dbgpt.storage.knowledge_graph.base import KnowledgeGraphBase +from dbgpt.storage.vector_store.base import VectorStoreBase + + +class DomainKnowledgeIndex(ABC): + """Abstract base class for domain-specific knowledge indexing. + + Each data source type (local documents, git repositories, yuque, notion, etc.) + implements its own ETL pipeline by subclassing this and overriding the + extract/transform/load methods. + + The factory pattern (DomainKnowledgeIndexFactory) is used to instantiate the + correct index based on the knowledge space's domain_type. + """ + + @abstractmethod + async def extract( + self, + knowledge: Knowledge, + chunk_parameter, + **kwargs, + ) -> list[Chunk]: + """Extract knowledge chunks from the data source.""" + raise NotImplementedError + + @abstractmethod + async def transform( + self, + chunks: list[Chunk], + **kwargs, + ) -> list[Chunk]: + """Transform knowledge chunks (enrichment, summarization, etc.).""" + raise NotImplementedError + + @abstractmethod + async def load( + self, + chunks: list[Chunk], + vector_store: Optional[VectorStoreBase] = None, + full_text_store: Optional[FullTextStoreBase] = None, + kg_store: Optional[KnowledgeGraphBase] = None, + keywords: bool = True, + max_chunks_once_load: int = 10, + max_threads: int = 1, + **kwargs, + ) -> list[Chunk]: + """Load knowledge chunks into storage backends.""" + raise NotImplementedError + + async def clean( + self, + chunks: list[Chunk], + node_ids: Optional[list[str]], + with_keywords: bool = True, + **kwargs, + ): + """Clean up indexed chunks from storage backends.""" + raise NotImplementedError + + @classmethod + def domain_type(cls) -> str: + """Return the domain type identifier for this index.""" + raise NotImplementedError \ No newline at end of file diff --git a/packages/dbgpt_serve/rag/domain/tests/__init__.py b/packages/dbgpt_serve/rag/domain/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/dbgpt_serve/rag/tools/__init__.py b/packages/dbgpt_serve/rag/tools/__init__.py new file mode 100644 index 000000000..1e4cadf0e --- /dev/null +++ b/packages/dbgpt_serve/rag/tools/__init__.py @@ -0,0 +1,22 @@ +"""Knowledge base search tools package. + +Provides structured search tools for agents: +- kb_ls: List files and directories in a knowledge space +- kb_glob: Search files by name pattern +- kb_grep: Search file contents by keyword +- kb_cat: Read file content by path +- kb_semantic_search: Semantic search using vector retrieval +- kb_codegraph_explore: Code knowledge graph exploration +""" + +# Import tool modules to register them with the @tool decorator +from . import kb_file_tools # noqa: F401 +from . import semantic_search_tool # noqa: F401 + +# CodeGraph tools - optional, requires graph_store +try: + from . import codegraph_tools # noqa: F401 +except ImportError: + pass + +__all__ = ["kb_file_tools", "semantic_search_tool", "codegraph_tools"] \ No newline at end of file diff --git a/packages/dbgpt_serve/src/dbgpt_serve/rag/tools/semantic_search_tool.py b/packages/dbgpt_serve/src/dbgpt_serve/rag/tools/semantic_search_tool.py new file mode 100644 index 000000000..d5b9831e6 --- /dev/null +++ b/packages/dbgpt_serve/src/dbgpt_serve/rag/tools/semantic_search_tool.py @@ -0,0 +1,64 @@ +"""Semantic search tool for knowledge spaces.""" + +import logging +from typing import Annotated +from dbgpt.agent.resource.tool.base import tool + +logger = logging.getLogger(__name__) + + +def _get_rag_service(): + from ..service.service import Service + from dbgpt._private.config import Config + system_app = Config().SYSTEM_APP + if not system_app: + raise RuntimeError("SYSTEM_APP is not initialized yet") + return Service.get_instance(system_app) + + +def _format_chunk_results(chunks, query, knowledge_id): + lines = [f"Semantic search '{query}' in {knowledge_id}:"] + chars = len(lines[0]) + for i, chunk in enumerate(chunks): + content = chunk.content or "" + score = getattr(chunk, "score", None) + metadata = getattr(chunk, "metadata", {}) or {} + file_path = metadata.get("file_path", "") + score_str = f" (score: {score:.2f})" if score is not None else "" + entry = f"\n---\n### Result {i+1}{score_str} [{file_path}]\n{content}" + chars += len(entry) + if chars > 8000: + lines.append(f"\n... {len(chunks)-i} more results.") + break + lines.append(entry) + return "\n".join(lines) + + +@tool(name="kb_semantic_search", description="Semantic search in knowledge base. Use only when kb_grep returns insufficient results.") +async def kb_semantic_search( + knowledge_id: Annotated[str, "Knowledge space ID"], + query: Annotated[str, "Natural language search query"], + top_k: Annotated[int, "Number of results"] = 5, + score_threshold: Annotated[float, "Min score (0-1)"] = 0.0, +) -> str: + top_k = int(top_k) if top_k else 5 + score_threshold = float(score_threshold) if score_threshold else 0.0 + try: + service = _get_rag_service() + except Exception as e: + return f"Semantic search service unavailable: {e}" + try: + from ..api.schemas import KnowledgeRetrieveRequest + request = KnowledgeRetrieveRequest( + query=query, + space_id=int(knowledge_id) if knowledge_id.isdigit() else knowledge_id, + top_k=top_k, score_threshold=score_threshold) + space = service.get({"id": request.space_id}) + if space is None: + return f"Knowledge space {knowledge_id} not found" + search_res = await service.retrieve(request, space) + except Exception as e: + return f"Semantic search failed: {e}" + if not search_res: + return f"No results for '{query}' in {knowledge_id}" + return _format_chunk_results(search_res, query, knowledge_id) \ No newline at end of file