From 57dd4daa9ae272664dbcd30070961e1ae892f6c9 Mon Sep 17 00:00:00 2001 From: William FH <13333726+hinthornw@users.noreply.github.com> Date: Thu, 10 Aug 2023 17:07:02 -0700 Subject: [PATCH] Add string example mapper (#9086) Now that we accept any runnable or arbitrary function to evaluate, we don't always look up the input keys. If an evaluator requires references, we should try to infer if there's one key present. We only have delayed validation here but it's better than nothing --- .../langchain/smith/evaluation/string_run_evaluator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/langchain/langchain/smith/evaluation/string_run_evaluator.py b/libs/langchain/langchain/smith/evaluation/string_run_evaluator.py index 4016482c191..0319433808c 100644 --- a/libs/langchain/langchain/smith/evaluation/string_run_evaluator.py +++ b/libs/langchain/langchain/smith/evaluation/string_run_evaluator.py @@ -375,7 +375,11 @@ class StringRunEvaluatorChain(Chain, RunEvaluator): ) # Configure how example rows are fed as a reference string to the evaluator - if reference_key is not None or data_type in (DataType.llm, DataType.chat): + if ( + reference_key is not None + or data_type in (DataType.llm, DataType.chat) + or evaluator.requires_reference + ): example_mapper = StringExampleMapper(reference_key=reference_key) elif evaluator.requires_reference: raise ValueError(