github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-07-08 14:05:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Comet ml updates 17 04 2023 (#3074)

Browse Source 
I made a couple of improvements to the Comet tracker:

* The Comet project name is configurable in various ways (code,
environment variable or file), having a default value in code meant that
users couldn't set the project name in an environment variable or in a
file.
* I added error catching when the `flush_tracker` is called in order to
avoid crashing the whole process. Instead we are gonna display a warning
or error log message (`extra={"show_traceback": True}` is an internal
convention to force the display of the traceback when using our own
logger).

I decided to add the error catching after seeing the following error in
the third example of the notebook:
```
COMET ERROR: Failed to export agent or LLM to Comet
Traceback (most recent call last):
  File "/home/lothiraldan/project/cometml/langchain/langchain/callbacks/comet_ml_callback.py", line 484, in _log_model
    langchain_asset.save(langchain_asset_path)
  File "/home/lothiraldan/project/cometml/langchain/langchain/agents/agent.py", line 591, in save
    raise ValueError(
ValueError: Saving not supported for agent executors. If you are trying to save the agent, please use the `.save_agent(...)`

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/lothiraldan/project/cometml/langchain/langchain/callbacks/comet_ml_callback.py", line 449, in flush_tracker
    self._log_model(langchain_asset)
  File "/home/lothiraldan/project/cometml/langchain/langchain/callbacks/comet_ml_callback.py", line 488, in _log_model
    langchain_asset.save_agent(langchain_asset_path)
  File "/home/lothiraldan/project/cometml/langchain/langchain/agents/agent.py", line 599, in save_agent
    return self.agent.save(file_path)
  File "/home/lothiraldan/project/cometml/langchain/langchain/agents/agent.py", line 145, in save
    agent_dict = self.dict()
  File "/home/lothiraldan/project/cometml/langchain/langchain/agents/agent.py", line 119, in dict
    _dict = super().dict()
  File "pydantic/main.py", line 449, in pydantic.main.BaseModel.dict
  File "pydantic/main.py", line 868, in _iter
  File "pydantic/main.py", line 743, in pydantic.main.BaseModel._get_value
  File "/home/lothiraldan/project/cometml/langchain/langchain/schema.py", line 381, in dict
    output_parser_dict["_type"] = self._type
  File "/home/lothiraldan/project/cometml/langchain/langchain/schema.py", line 376, in _type
    raise NotImplementedError
NotImplementedError
```

I still need to investigate and try to fix it, it looks related to
saving an agent to a file.
This commit is contained in:
Boris Feld 2023-04-18 16:32:29 +02:00 committed by GitHub
parent fe68051d34
commit 14e4d30659
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 47 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
langchain/callbacks/comet_ml_callback.py
View File

@ -96,7 +96,7 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
self, self,
task_type: Optional[str] = "inference", task_type: Optional[str] = "inference",
workspace: Optional[str] = None, workspace: Optional[str] = None,
project_name: Optional[str] = "comet-langchain-demo", project_name: Optional[str] = None,
tags: Optional[Sequence] = None, tags: Optional[Sequence] = None,
name: Optional[str] = None, name: Optional[str] = None,
visualizations: Optional[List[str]] = None, visualizations: Optional[List[str]] = None,
@ -106,7 +106,7 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
) -> None: ) -> None:
"""Initialize callback handler.""" """Initialize callback handler."""
comet_ml = import_comet_ml() self.comet_ml = import_comet_ml()
super().__init__() super().__init__()
self.task_type = task_type self.task_type = task_type
@ -133,7 +133,7 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
"https://github.com/comet-ml/issue_tracking/issues with the tag " "https://github.com/comet-ml/issue_tracking/issues with the tag "
"`langchain`." "`langchain`."
) )
comet_ml.LOGGER.warning(warning) self.comet_ml.LOGGER.warning(warning)
self.callback_columns: list = [] self.callback_columns: list = []
self.action_records: list = [] self.action_records: list = []
@ -242,8 +242,6 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
resp.update(flatten_dict(serialized)) resp.update(flatten_dict(serialized))
resp.update(self.get_custom_callback_meta()) resp.update(self.get_custom_callback_meta())
comet_ml = import_comet_ml()
for chain_input_key, chain_input_val in inputs.items(): for chain_input_key, chain_input_val in inputs.items():
if isinstance(chain_input_val, str): if isinstance(chain_input_val, str):
input_resp = deepcopy(resp) input_resp = deepcopy(resp)
@ -253,7 +251,7 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
self.action_records.append(input_resp) self.action_records.append(input_resp)
else: else:
comet_ml.LOGGER.warning( self.comet_ml.LOGGER.warning(
f"Unexpected data format provided! " f"Unexpected data format provided! "
f"Input Value for {chain_input_key} will not be logged" f"Input Value for {chain_input_key} will not be logged"
) )
@ -268,8 +266,6 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
resp.update({"action": "on_chain_end"}) resp.update({"action": "on_chain_end"})
resp.update(self.get_custom_callback_meta()) resp.update(self.get_custom_callback_meta())
comet_ml = import_comet_ml()
for chain_output_key, chain_output_val in outputs.items(): for chain_output_key, chain_output_val in outputs.items():
if isinstance(chain_output_val, str): if isinstance(chain_output_val, str):
output_resp = deepcopy(resp) output_resp = deepcopy(resp)
@ -278,7 +274,7 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
output_resp.update({chain_output_key: chain_output_val}) output_resp.update({chain_output_key: chain_output_val})
self.action_records.append(output_resp) self.action_records.append(output_resp)
else: else:
comet_ml.LOGGER.warning( self.comet_ml.LOGGER.warning(
f"Unexpected data format provided! " f"Unexpected data format provided! "
f"Output Value for {chain_output_key} will not be logged" f"Output Value for {chain_output_key} will not be logged"
) )
@ -449,7 +445,14 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
self._log_session(langchain_asset) self._log_session(langchain_asset)
if langchain_asset: if langchain_asset:
self._log_model(langchain_asset) try:
self._log_model(langchain_asset)
except Exception:
self.comet_ml.LOGGER.error(
"Failed to export agent or LLM to Comet",
exc_info=True,
extra={"show_traceback": True},
)
if finish: if finish:
self.experiment.end() self.experiment.end()
@ -470,8 +473,6 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
self.experiment.log_text(prompt, metadata=metadata, step=step) self.experiment.log_text(prompt, metadata=metadata, step=step)
def _log_model(self, langchain_asset: Any) -> None: def _log_model(self, langchain_asset: Any) -> None:
comet_ml = import_comet_ml()
model_parameters = self._get_llm_parameters(langchain_asset) model_parameters = self._get_llm_parameters(langchain_asset)
self.experiment.log_parameters(model_parameters, prefix="model") self.experiment.log_parameters(model_parameters, prefix="model")
@ -487,24 +488,45 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
langchain_asset.save_agent(langchain_asset_path) langchain_asset.save_agent(langchain_asset_path)
self.experiment.log_model(model_name, str(langchain_asset_path)) self.experiment.log_model(model_name, str(langchain_asset_path))
else: else:
comet_ml.LOGGER.warning( self.comet_ml.LOGGER.error(
f"{e}" f"{e}"
" Could not save Langchain Asset " " Could not save Langchain Asset "
f"for {langchain_asset.__class__.__name__}" f"for {langchain_asset.__class__.__name__}"
) )
def _log_session(self, langchain_asset: Optional[Any] = None) -> None: def _log_session(self, langchain_asset: Optional[Any] = None) -> None:
llm_session_df = self._create_session_analysis_dataframe(langchain_asset) try:
# Log the cleaned dataframe as a table llm_session_df = self._create_session_analysis_dataframe(langchain_asset)
self.experiment.log_table("langchain-llm-session.csv", llm_session_df) # Log the cleaned dataframe as a table
self.experiment.log_table("langchain-llm-session.csv", llm_session_df)
except Exception:
self.comet_ml.LOGGER.warning(
"Failed to log session data to Comet",
exc_info=True,
extra={"show_traceback": True},
)
metadata = {"langchain_version": str(langchain.__version__)} try:
# Log the langchain low-level records as a JSON file directly metadata = {"langchain_version": str(langchain.__version__)}
self.experiment.log_asset_data( # Log the langchain low-level records as a JSON file directly
self.action_records, "langchain-action_records.json", metadata=metadata self.experiment.log_asset_data(
) self.action_records, "langchain-action_records.json", metadata=metadata
)
except Exception:
self.comet_ml.LOGGER.warning(
"Failed to log session data to Comet",
exc_info=True,
extra={"show_traceback": True},
)
self._log_visualizations(llm_session_df) try:
self._log_visualizations(llm_session_df)
except Exception:
self.comet_ml.LOGGER.warning(
"Failed to log visualizations to Comet",
exc_info=True,
extra={"show_traceback": True},
)
def _log_text_metrics(self, metrics: Sequence[dict], step: int) -> None: def _log_text_metrics(self, metrics: Sequence[dict], step: int) -> None:
if not metrics: if not metrics:
@ -519,7 +541,6 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
return return
spacy = import_spacy() spacy = import_spacy()
comet_ml = import_comet_ml()
prompts = session_df["prompts"].tolist() prompts = session_df["prompts"].tolist()
outputs = session_df["text"].tolist() outputs = session_df["text"].tolist()
@ -544,7 +565,9 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
step=idx, step=idx,
) )
except Exception as e: except Exception as e:
comet_ml.LOGGER.warning(e) self.comet_ml.LOGGER.warning(
e, exc_info=True, extra={"show_traceback": True}
)
return return