github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-06-24 07:35:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Throw error if evaluation key not present (#6874)

Browse Source
This commit is contained in:
Zander Chase 2023-06-29 10:30:39 -07:00 committed by GitHub
parent be164b20d8
commit f5663603cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
langchain/evaluation/run_evaluators/implementations.py
View File

@ -48,17 +48,14 @@ class StringRunEvaluatorInputMapper(RunEvaluatorInputMapper, BaseModel):
"""Maps the Run and Optional[Example] to a dictionary""" """Maps the Run and Optional[Example] to a dictionary"""
if run.outputs is None and self.prediction_map: if run.outputs is None and self.prediction_map:
raise ValueError(f"Run {run.id} has no outputs.") raise ValueError(f"Run {run.id} has no outputs.")
if self.answer_map and (not example or not example.outputs):
raise ValueError("This evaluator requires references, but none were given.")
outputs = run.outputs or {} outputs = run.outputs or {}
data = {value: outputs.get(key) for key, value in self.prediction_map.items()} data = {value: outputs[key] for key, value in self.prediction_map.items()}
data.update( data.update({value: run.inputs[key] for key, value in self.input_map.items()})
{value: run.inputs.get(key) for key, value in self.input_map.items()}
)
if self.answer_map and example and example.outputs: if self.answer_map and example and example.outputs:
data.update( data.update(
{ {value: example.outputs[key] for key, value in self.answer_map.items()}
value: example.outputs.get(key)
for key, value in self.answer_map.items()
}
) )
return data return data
@ -77,7 +74,7 @@ class ChoicesOutputParser(RunEvaluatorOutputParser):
"""Parse the last line of the text and return an evaluation result.""" """Parse the last line of the text and return an evaluation result."""
lines = text.strip().split() lines = text.strip().split()
value = lines[-1].strip() value = lines[-1].strip()
score = self.choices_map.get(value, 0) if self.choices_map else None score = self.choices_map.get(value) if self.choices_map else None
comment = " ".join(lines[:-1]) if len(lines) > 1 else None comment = " ".join(lines[:-1]) if len(lines) > 1 else None
return EvaluationResult( return EvaluationResult(
key=self.evaluation_name, key=self.evaluation_name,
@ -144,9 +141,9 @@ class CriteriaOutputParser(RunEvaluatorOutputParser):
parsed_output_ = parsed_output parsed_output_ = parsed_output
return EvaluationResult( return EvaluationResult(
key=self.evaluation_name, key=self.evaluation_name,
score=parsed_output_.get("score"), score=parsed_output_["score"],
value=parsed_output_.get("value"), value=parsed_output_["value"],
comment=parsed_output_.get("reasoning"), comment=parsed_output_["reasoning"],
) )