From a28e888b36ba51ce15d0eaf707917dcb5983b575 Mon Sep 17 00:00:00 2001 From: Buckler89 Date: Wed, 30 Aug 2023 15:35:23 +0200 Subject: [PATCH] fix call _get_keys for custom_evaluator (#9763) In the function _load_run_evaluators the function _get_keys was not called if only custom_evaluators parameter is used - Description: In the function _load_run_evaluators the function _get_keys was not called if only custom_evaluators parameter is used, - Issue: no issue created for this yet, - Dependencies: None, - Tag maintainer: @vowelparrot, - Twitter handle: Buckler89 --------- Co-authored-by: ddroghini --- .../langchain/langchain/smith/evaluation/runner_utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libs/langchain/langchain/smith/evaluation/runner_utils.py b/libs/langchain/langchain/smith/evaluation/runner_utils.py index 438bc791400..f20f5cea4ba 100644 --- a/libs/langchain/langchain/smith/evaluation/runner_utils.py +++ b/libs/langchain/langchain/smith/evaluation/runner_utils.py @@ -566,8 +566,13 @@ def _load_run_evaluators( eval_llm = config.eval_llm or ChatOpenAI(model="gpt-4", temperature=0.0) run_evaluators = [] input_key, prediction_key, reference_key = None, None, None - if config.evaluators or any( - [isinstance(e, EvaluatorType) for e in config.evaluators] + if ( + config.evaluators + or any([isinstance(e, EvaluatorType) for e in config.evaluators]) + or ( + config.custom_evaluators + and any([isinstance(e, StringEvaluator) for e in config.custom_evaluators]) + ) ): input_key, prediction_key, reference_key = _get_keys( config, run_inputs, run_outputs, example_outputs