Compare commits

...

1 Commits

Author SHA1 Message Date
William Fu-Hinthorn
2f679723cb Rm not implemented async in eval methods 2023-08-08 16:23:31 -07:00

View File

@@ -1,5 +1,6 @@
"""Interfaces to be implemented by general evaluators."""
from __future__ import annotations
import asyncio
import logging
from abc import ABC, abstractmethod
@@ -168,9 +169,13 @@ class StringEvaluator(_EvalArgsMixin, ABC):
- value: the string value of the evaluation, if applicable.
- reasoning: the reasoning for the evaluation, if applicable.
""" # noqa: E501
raise NotImplementedError(
f"{self.__class__.__name__} hasn't implemented an async "
"aevaluate_strings method."
return await asyncio.get_running_loop().run_in_executor(
None,
self._evaluate_strings,
prediction=prediction,
reference=reference,
input=input,
**kwargs,
)
def evaluate_strings(
@@ -265,9 +270,14 @@ class PairwiseStringEvaluator(_EvalArgsMixin, ABC):
Returns:
dict: A dictionary containing the preference, scores, and/or other information.
""" # noqa: E501
raise NotImplementedError(
f"{self.__class__.__name__} hasn't implemented an async "
"aevaluate_string_pairs method."
return await asyncio.get_running_loop().run_in_executor(
None,
self._evaluate_string_pairs,
prediction=prediction,
prediction_b=prediction_b,
reference=reference,
input=input,
**kwargs,
)
def evaluate_string_pairs(
@@ -381,9 +391,14 @@ class AgentTrajectoryEvaluator(_EvalArgsMixin, ABC):
Returns:
dict: The evaluation result.
"""
raise NotImplementedError(
f"{self.__class__.__name__} hasn't implemented an async "
"aevaluate_agent_trajectory method."
raise asyncio.get_running_loop().run_in_executor(
None,
self._evaluate_agent_trajectory,
prediction=prediction,
agent_trajectory=agent_trajectory,
input=input,
reference=reference,
**kwargs,
)
def evaluate_agent_trajectory(