github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-06-23 15:19:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Always add run ID (#12046)

Browse Source 
in eval callback handler.

Useful if you're using a custom run evaluator and don't want to thread
things through.
This commit is contained in:
William FH 2023-10-20 04:38:07 +09:00 committed by GitHub
parent 44d7763580
commit 25418b9b4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 22 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
libs/langchain/langchain/callbacks/tracers/evaluation.py
View File

@ -116,8 +116,28 @@ class EvaluatorCallbackHandler(BaseTracer):
eval_result = self.client.evaluate_run(run, evaluator) eval_result = self.client.evaluate_run(run, evaluator)
with manager.tracing_v2_enabled( with manager.tracing_v2_enabled(
project_name=self.project_name, tags=["eval"], client=self.client project_name=self.project_name, tags=["eval"], client=self.client
): ) as cb:
eval_result = self.client.evaluate_run(run, evaluator) reference_example = (
self.client.read_example(run.reference_example_id)
if run.reference_example_id
else None
)
evaluation_result = evaluator.evaluate_run(
run,
example=reference_example,
)
run_id = cb.latest_run.id if cb.latest_run is not None else None
self.client.create_feedback(
run.id,
evaluation_result.key,
score=evaluation_result.score,
value=evaluation_result.value,
comment=evaluation_result.comment,
correction=evaluation_result.correction,
source_info=evaluation_result.evaluator_info,
source_run_id=evaluation_result.source_run_id or run_id,
feedback_source_type=langsmith.schemas.FeedbackSourceType.MODEL,
)
except Exception as e: except Exception as e:
logger.error( logger.error(
f"Error evaluating run {run.id} with " f"Error evaluating run {run.id} with "