github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-07-18 10:43:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Accept no 'reasoning' response in qa evaluator (#7107)

Browse Source 
Re add since #6995 inadvertently undid #7031
This commit is contained in:
William FH 2023-07-03 14:47:17 -07:00 committed by GitHub
parent fd3f8efec7
commit 3f9744c9f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
langchain/evaluation/qa/eval_chain.py
View File

@ -19,14 +19,20 @@ def _parse_string_eval_output(text: str) -> dict:
Returns: Returns:
Any: The parsed output. Any: The parsed output.
""" """
reasoning, verdict = text.strip().rsplit("\n", maxsplit=1) splits = text.strip().rsplit("\n", maxsplit=1)
if len(splits) == 1:
verdict = splits[0]
reasoning = None
else:
reasoning, verdict = splits
reasoning = reasoning.strip()
score = ( score = (
1 1
if verdict.upper() == "CORRECT" if verdict.upper() == "CORRECT"
else (0 if verdict.upper() == "INCORRECT" else None) else (0 if verdict.upper() == "INCORRECT" else None)
) )
return { return {
"reasoning": reasoning.strip(), "reasoning": reasoning,
"value": verdict, "value": verdict,
"score": score, "score": score,
} }