Update error message (#9970)

in evals
This commit is contained in:
William FH 2023-08-30 17:42:55 -07:00 committed by GitHub
parent b82ad19ed2
commit 5341b04d68
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 12 additions and 6 deletions

View File

@ -108,7 +108,6 @@ def load_evaluator(
>>> from langchain.evaluation import load_evaluator, EvaluatorType
>>> evaluator = load_evaluator(EvaluatorType.QA)
"""
llm = llm or ChatOpenAI(model="gpt-4", temperature=0)
if evaluator not in _EVALUATOR_MAP:
raise ValueError(
f"Unknown evaluator type: {evaluator}"
@ -116,6 +115,16 @@ def load_evaluator(
)
evaluator_cls = _EVALUATOR_MAP[evaluator]
if issubclass(evaluator_cls, LLMEvalChain):
try:
llm = llm or ChatOpenAI(model="gpt-4", temperature=0)
except Exception as e:
raise ValueError(
f"Evaluation with the {evaluator_cls} requires a "
"language model to function."
" Failed to create the default 'gpt-4' model."
" Please manually provide an evaluation LLM"
" or check your openai credentials."
) from e
return evaluator_cls.from_llm(llm=llm, **kwargs)
else:
return evaluator_cls(**kwargs)
@ -154,7 +163,6 @@ def load_evaluators(
>>> evaluators = [EvaluatorType.QA, EvaluatorType.CRITERIA]
>>> loaded_evaluators = load_evaluators(evaluators, criteria="helpfulness")
"""
llm = llm or ChatOpenAI(model="gpt-4", temperature=0)
loaded = []
for evaluator in evaluators:
_kwargs = config.get(evaluator, {}) if config else {}

View File

@ -35,7 +35,6 @@ from langchain.callbacks.tracers.base import BaseTracer
from langchain.callbacks.tracers.evaluation import EvaluatorCallbackHandler
from langchain.callbacks.tracers.langchain import LangChainTracer
from langchain.chains.base import Chain
from langchain.chat_models.openai import ChatOpenAI
from langchain.evaluation.loading import load_evaluator
from langchain.evaluation.schema import EvaluatorType, StringEvaluator
from langchain.schema import ChatResult, LLMResult
@ -493,7 +492,7 @@ def _determine_reference_key(
def _construct_run_evaluator(
eval_config: Union[EvaluatorType, str, EvalConfig],
eval_llm: BaseLanguageModel,
eval_llm: Optional[BaseLanguageModel],
run_type: str,
data_type: DataType,
example_outputs: Optional[List[str]],
@ -563,7 +562,6 @@ def _load_run_evaluators(
Returns:
A list of run evaluators.
"""
eval_llm = config.eval_llm or ChatOpenAI(model="gpt-4", temperature=0.0)
run_evaluators = []
input_key, prediction_key, reference_key = None, None, None
if (
@ -580,7 +578,7 @@ def _load_run_evaluators(
for eval_config in config.evaluators:
run_evaluator = _construct_run_evaluator(
eval_config,
eval_llm,
config.eval_llm,
run_type,
data_type,
example_outputs,