github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-06-05 22:53:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Evals] Add Project Tags (#13982)

Browse Source 
Add them to project extra
This commit is contained in:
William FH 2023-11-28 11:38:59 -08:00 committed by GitHub
parent 9e017ff6ba
commit e5256bcb69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
libs/langchain/langchain/smith/evaluation/runner_utils.py
View File

@ -906,14 +906,18 @@ def _prepare_eval_run(
llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY, llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY,
project_name: str, project_name: str,
project_metadata: Optional[Dict[str, Any]] = None, project_metadata: Optional[Dict[str, Any]] = None,
tags: Optional[List[str]] = None,
) -> Tuple[MCF, str, Dataset, List[Example]]: ) -> Tuple[MCF, str, Dataset, List[Example]]:
wrapped_model = _wrap_in_chain_factory(llm_or_chain_factory, dataset_name) wrapped_model = _wrap_in_chain_factory(llm_or_chain_factory, dataset_name)
dataset = client.read_dataset(dataset_name=dataset_name) dataset = client.read_dataset(dataset_name=dataset_name)
try: try:
project_extra: dict = {"metadata": project_metadata} if project_metadata else {}
if tags:
project_extra["tags"] = tags
project = client.create_project( project = client.create_project(
project_name, project_name,
reference_dataset_id=dataset.id, reference_dataset_id=dataset.id,
project_extra={"metadata": project_metadata} if project_metadata else {}, project_extra=project_extra,
) )
except (HTTPError, ValueError, LangSmithError) as e: except (HTTPError, ValueError, LangSmithError) as e:
if "already exists " not in str(e): if "already exists " not in str(e):
@ -959,6 +963,7 @@ def _prepare_run_on_dataset(
llm_or_chain_factory, llm_or_chain_factory,
project_name, project_name,
project_metadata=project_metadata, project_metadata=project_metadata,
tags=tags,
) )
wrapped_model = _wrap_in_chain_factory(llm_or_chain_factory) wrapped_model = _wrap_in_chain_factory(llm_or_chain_factory)
run_evaluators = _setup_evaluation( run_evaluators = _setup_evaluation(
@ -1026,7 +1031,7 @@ def _collect_test_results(
"execution_time": all_execution_time.get(str(example.id)), "execution_time": all_execution_time.get(str(example.id)),
} }
if isinstance(output, EvalError): if isinstance(output, EvalError):
results[str(example.id)]["Error"] = output.error results[str(example.id)]["Error"] = output.Error
else: else:
results[str(example.id)]["output"] = output results[str(example.id)]["output"] = output
if example.outputs: if example.outputs: