mirror of
https://github.com/hwchase17/langchain.git
synced 2025-08-10 13:27:36 +00:00
Return feedback with failed response if there's an error (#9223)
In Evals
This commit is contained in:
parent
7124f2ebfa
commit
b0896210c7
@ -313,17 +313,30 @@ class StringRunEvaluatorChain(Chain, RunEvaluator):
|
||||
self, run: Run, example: Optional[Example] = None
|
||||
) -> EvaluationResult:
|
||||
"""Evaluate an example."""
|
||||
result = self({"run": run, "example": example}, include_run_info=True)
|
||||
return self._prepare_evaluator_output(result)
|
||||
try:
|
||||
result = self({"run": run, "example": example}, include_run_info=True)
|
||||
return self._prepare_evaluator_output(result)
|
||||
except Exception as e:
|
||||
return EvaluationResult(
|
||||
key=self.string_evaluator.evaluation_name,
|
||||
comment=f"Error evaluating run {run.id}: {e}",
|
||||
# TODO: Add run ID once we can declare it via callbacks
|
||||
)
|
||||
|
||||
async def aevaluate_run(
|
||||
self, run: Run, example: Optional[Example] = None
|
||||
) -> EvaluationResult:
|
||||
"""Evaluate an example."""
|
||||
result = await self.acall(
|
||||
{"run": run, "example": example}, include_run_info=True
|
||||
)
|
||||
return self._prepare_evaluator_output(result)
|
||||
try:
|
||||
result = await self.acall(
|
||||
{"run": run, "example": example}, include_run_info=True
|
||||
)
|
||||
return self._prepare_evaluator_output(result)
|
||||
except Exception as e:
|
||||
return EvaluationResult(
|
||||
key=self.string_evaluator.evaluation_name,
|
||||
comment=f"Error evaluating run {run.id}: {e}",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_run_and_data_type(
|
||||
|
@ -0,0 +1,27 @@
|
||||
"""Tests for the string run evaluator."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from langchain.evaluation import criteria
|
||||
from langchain.smith.evaluation.string_run_evaluator import (
|
||||
ChainStringRunMapper,
|
||||
StringRunEvaluatorChain,
|
||||
)
|
||||
from tests.unit_tests.llms import fake_llm
|
||||
|
||||
|
||||
def test_evaluate_run() -> None:
|
||||
run_mapper = ChainStringRunMapper()
|
||||
example_mapper = MagicMock()
|
||||
string_evaluator = criteria.CriteriaEvalChain.from_llm(fake_llm.FakeLLM())
|
||||
evaluator = StringRunEvaluatorChain(
|
||||
run_mapper=run_mapper,
|
||||
example_mapper=example_mapper,
|
||||
name="test_evaluator",
|
||||
string_evaluator=string_evaluator,
|
||||
)
|
||||
run = MagicMock()
|
||||
example = MagicMock()
|
||||
res = evaluator.evaluate_run(run, example)
|
||||
assert res.comment.startswith("Error evaluating run ")
|
||||
assert res.key == string_evaluator.evaluation_name
|
Loading…
Reference in New Issue
Block a user