github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-09-12 20:53:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: evaluation compute() add query parameter

Browse Source
This commit is contained in:
aries_ckt
2024-08-08 14:45:37 +08:00
parent d51b762c6f
commit 5734f1c116
3 changed files with 14 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
dbgpt/rag/evaluation/answer.py
View File

@@ -17,7 +17,7 @@ from dbgpt.core.interface.llm import LLMClient, ModelRequest
logger = logging.getLogger(__name__)
ANSWER_RELEVANCY_EVALUATE_PROMPT_TEMPLATE = """
你是一个DBA智能答疑专家, 你的任务是根据用户的问题和已经相关的文档给问答的答案进行严格的打分.
你是一个智能答疑专家, 你的任务是根据用户的问题和已经相关的文档给问答的答案进行严格的打分.
你将会得到以下输入信息:
- 用户的问题
@@ -69,17 +69,18 @@ class LLMEvaluationMetric(EvaluationMetric):
self._model_name = model_name
self._prompt_template = prompt_template
async def compute( # type: ignore # noqa
async def compute(
self,
query: str,
prediction: str,
contexts: Optional[Sequence[str]] = None,
query: Optional[str] = None,
) -> EvaluationResult:
"""Compute the evaluation metric.
Args:
prediction(List[str]): The retrieved chunks from the retriever.
contexts(Sequence[str]): The contexts from dataset.
query:(Optional[str]) The query text.
Returns:
BaseEvaluationResult: The evaluation result.

6
dbgpt/rag/evaluation/retriever.py
View File

@@ -34,12 +34,14 @@ class RetrieverSimilarityMetric(RetrieverEvaluationMetric):
self,
prediction: List[str],
contexts: Optional[Sequence[str]] = None,
query: Optional[str] = None,
) -> BaseEvaluationResult:
"""Compute the evaluation metric.
Args:
prediction(List[str]): The retrieved chunks from the retriever.
contexts(Sequence[str]): The contexts from dataset.
query:(Optional[str]) The query text.
Returns:
BaseEvaluationResult: The evaluation result.
@@ -81,12 +83,14 @@ class RetrieverMRRMetric(RetrieverEvaluationMetric):
self,
prediction: List[str],
contexts: Optional[Sequence[str]] = None,
query: Optional[str] = None,
) -> BaseEvaluationResult:
"""Compute MRR metric.
Args:
prediction(Optional[List[str]]): The retrieved chunks from the retriever.
contexts(Optional[List[str]]): The contexts from dataset.
query:(Optional[str]) The query text.
Returns:
BaseEvaluationResult: The evaluation result.
The score is the reciprocal rank of the first relevant chunk.
@@ -119,12 +123,14 @@ class RetrieverHitRateMetric(RetrieverEvaluationMetric):
self,
prediction: List[str],
contexts: Optional[Sequence[str]] = None,
query: Optional[str] = None,
) -> BaseEvaluationResult:
"""Compute HitRate metric.
Args:
prediction(Optional[List[str]]): The retrieved chunks from the retriever.
contexts(Optional[List[str]]): The contexts from dataset.
query:(Optional[str]) The query text.
Returns:
BaseEvaluationResult: The evaluation result.
"""

4
dbgpt/serve/agent/evaluation/evaluation_metric.py
View File

@@ -28,12 +28,14 @@ class AppLinkMetric(EvaluationMetric[str, str], ABC):
self,
prediction: Optional[str] = None,
contexts: Optional[str] = None,
query: Optional[str] = None,
**kwargs: Any,
) -> BaseEvaluationResult:
"""Compute Intent metric.
Args:
prediction(Optional[str]): The retrieved chunks from the retriever.
contexts(Optional[str]): The contexts from dataset.
query:(Optional[str]) The query text.
Returns:
BaseEvaluationResult: The evaluation result.
"""
@@ -78,12 +80,14 @@ class IntentMetric(EvaluationMetric[str, str], ABC):
self,
prediction: Optional[str] = None,
contexts: Optional[str] = None,
query: Optional[str] = None,
**kwargs: Any,
) -> BaseEvaluationResult:
"""Compute Intent metric.
Args:
prediction(Optional[str]): The retrieved chunks from the retriever.
contexts(Optional[str]): The contexts from dataset.
query:(Optional[str]) The query text.
Returns:
BaseEvaluationResult: The evaluation result.
"""