Merge branch 'master' into bagatur/add_tool_call_metadata

libs/core/langchain_core/language_models/chat_models.py

@@ -708,9 +708,7 @@ class BaseChatModel(BaseLanguageModel[AIMessage], ABC):
             await run_manager.on_llm_error(err, response=LLMResult(generations=[]))
             raise err
 
-        await run_manager.on_llm_end(
-            LLMResult(generations=[[generation]]),
-        )
+        await run_manager.on_llm_end(LLMResult(generations=[[generation]]))
 
     # --- Custom methods ---
 

libs/core/langchain_core/tracers/core.py

@@ -15,6 +15,7 @@ from typing import (
 
 from langchain_core.exceptions import TracerException
 from langchain_core.load import dumpd
+from langchain_core.messages import AIMessage
 from langchain_core.outputs import (
     ChatGeneration,
     ChatGenerationChunk,
@@ -274,13 +275,18 @@ class _TracerCore(ABC):
             llm_run.outputs = cast("dict[str, Any]", llm_run.outputs)
         if not llm_run.extra.get("__omit_auto_outputs", False):
             llm_run.outputs.update(response.model_dump())
+        if "ls_tools_called" not in llm_run.metadata:
+            llm_run.metadata["ls_tools_called"] = []
         for i, generations in enumerate(response.generations):
             for j, generation in enumerate(generations):
                 output_generation = llm_run.outputs["generations"][i][j]
                 if "message" in output_generation:
-                    output_generation["message"] = dumpd(
-                        cast("ChatGeneration", generation).message
-                    )
+                    msg = cast("ChatGeneration", generation).message
+                    output_generation["message"] = dumpd(msg)
+                    if i == 0 and j == 0 and isinstance(msg, AIMessage):
+                        llm_run.metadata["ls_tools_called"].extend(
+                            [tc["name"] for tc in msg.tool_calls]
+                        )
         llm_run.end_time = datetime.now(timezone.utc)
         llm_run.events.append({"name": "end", "time": llm_run.end_time})
 

libs/core/tests/unit_tests/runnables/test_runnable.py

@@ -2144,7 +2144,11 @@ async def test_prompt_with_llm(
                 "value": {
                     "end_time": None,
                     "final_output": None,
-                    "metadata": {"ls_model_type": "llm", "ls_provider": "fakelist"},
+                    "metadata": {
+                        "ls_model_type": "llm",
+                        "ls_provider": "fakelist",
+                        "ls_tools_called": [],
+                    },
                     "name": "FakeListLLM",
                     "start_time": "2023-01-01T00:00:00.000+00:00",
                     "streamed_output": [],
@@ -2358,6 +2362,7 @@ async def test_prompt_with_llm_parser(
                     "metadata": {
                         "ls_model_type": "llm",
                         "ls_provider": "fakestreaminglist",
+                        "ls_tools_called": [],
                     },
                     "name": "FakeStreamingListLLM",
                     "start_time": "2023-01-01T00:00:00.000+00:00",

libs/core/tests/unit_tests/tracers/test_async_base_tracer.py

@@ -65,7 +65,7 @@ async def test_tracer_llm_run() -> None:
             {"name": "start", "time": datetime.now(timezone.utc)},
             {"name": "end", "time": datetime.now(timezone.utc)},
         ],
-        extra={},
+        extra={"metadata": {"ls_tools_called": []}},
         serialized=SERIALIZED,
         inputs={"prompts": []},
         outputs=LLMResult(generations=[[]]),  # type: ignore[arg-type]
@@ -98,7 +98,7 @@ async def test_tracer_chat_model_run() -> None:
             {"name": "start", "time": datetime.now(timezone.utc)},
             {"name": "end", "time": datetime.now(timezone.utc)},
         ],
-        extra={},
+        extra={"metadata": {"ls_tools_called": []}},
         serialized=SERIALIZED_CHAT,
         inputs={"prompts": ["Human: "]},
         outputs=LLMResult(generations=[[]]),  # type: ignore[arg-type]
@@ -134,7 +134,7 @@ async def test_tracer_multiple_llm_runs() -> None:
             {"name": "start", "time": datetime.now(timezone.utc)},
             {"name": "end", "time": datetime.now(timezone.utc)},
         ],
-        extra={},
+        extra={"metadata": {"ls_tools_called": []}},
         serialized=SERIALIZED,
         inputs={"prompts": []},
         outputs=LLMResult(generations=[[]]),  # type: ignore[arg-type]
@@ -291,7 +291,7 @@ async def test_tracer_nested_run() -> None:
                             {"name": "start", "time": datetime.now(timezone.utc)},
                             {"name": "end", "time": datetime.now(timezone.utc)},
                         ],
-                        extra={},
+                        extra={"metadata": {"ls_tools_called": []}},
                         serialized=SERIALIZED,
                         inputs={"prompts": []},
                         outputs=LLMResult(generations=[[]]),  # type: ignore[arg-type]
@@ -311,7 +311,7 @@ async def test_tracer_nested_run() -> None:
                     {"name": "start", "time": datetime.now(timezone.utc)},
                     {"name": "end", "time": datetime.now(timezone.utc)},
                 ],
-                extra={},
+                extra={"metadata": {"ls_tools_called": []}},
                 serialized=SERIALIZED,
                 inputs={"prompts": []},
                 outputs=LLMResult(generations=[[]]),  # type: ignore[arg-type]
@@ -526,7 +526,7 @@ async def test_tracer_nested_runs_on_error() -> None:
                     {"name": "start", "time": datetime.now(timezone.utc)},
                     {"name": "end", "time": datetime.now(timezone.utc)},
                 ],
-                extra={},
+                extra={"metadata": {"ls_tools_called": []}},
                 serialized=SERIALIZED,
                 error=None,
                 inputs={"prompts": []},
@@ -544,7 +544,7 @@ async def test_tracer_nested_runs_on_error() -> None:
                     {"name": "start", "time": datetime.now(timezone.utc)},
                     {"name": "end", "time": datetime.now(timezone.utc)},
                 ],
-                extra={},
+                extra={"metadata": {"ls_tools_called": []}},
                 serialized=SERIALIZED,
                 error=None,
                 inputs={"prompts": []},

libs/core/tests/unit_tests/tracers/test_base_tracer.py

@@ -70,7 +70,7 @@ def test_tracer_llm_run() -> None:
             {"name": "start", "time": datetime.now(timezone.utc)},
             {"name": "end", "time": datetime.now(timezone.utc)},
         ],
-        extra={},
+        extra={"metadata": {"ls_tools_called": []}},
         serialized=SERIALIZED,
         inputs={"prompts": []},
         outputs=LLMResult(generations=[[]]),  # type: ignore[arg-type]
@@ -103,7 +103,7 @@ def test_tracer_chat_model_run() -> None:
             {"name": "start", "time": datetime.now(timezone.utc)},
             {"name": "end", "time": datetime.now(timezone.utc)},
         ],
-        extra={},
+        extra={"metadata": {"ls_tools_called": []}},
         serialized=SERIALIZED_CHAT,
         inputs={"prompts": ["Human: "]},
         outputs=LLMResult(generations=[[]]),  # type: ignore[arg-type]
@@ -139,7 +139,7 @@ def test_tracer_multiple_llm_runs() -> None:
             {"name": "start", "time": datetime.now(timezone.utc)},
             {"name": "end", "time": datetime.now(timezone.utc)},
         ],
-        extra={},
+        extra={"metadata": {"ls_tools_called": []}},
         serialized=SERIALIZED,
         inputs={"prompts": []},
         outputs=LLMResult(generations=[[]]),  # type: ignore[arg-type]
@@ -294,7 +294,7 @@ def test_tracer_nested_run() -> None:
                             {"name": "start", "time": datetime.now(timezone.utc)},
                             {"name": "end", "time": datetime.now(timezone.utc)},
                         ],
-                        extra={},
+                        extra={"metadata": {"ls_tools_called": []}},
                         serialized=SERIALIZED,
                         inputs={"prompts": []},
                         outputs=LLMResult(generations=[[]]),  # type: ignore[arg-type]
@@ -314,7 +314,7 @@ def test_tracer_nested_run() -> None:
                     {"name": "start", "time": datetime.now(timezone.utc)},
                     {"name": "end", "time": datetime.now(timezone.utc)},
                 ],
-                extra={},
+                extra={"metadata": {"ls_tools_called": []}},
                 serialized=SERIALIZED,
                 inputs={"prompts": []},
                 outputs=LLMResult(generations=[[]]),  # type: ignore[arg-type]
@@ -527,7 +527,7 @@ def test_tracer_nested_runs_on_error() -> None:
                     {"name": "start", "time": datetime.now(timezone.utc)},
                     {"name": "end", "time": datetime.now(timezone.utc)},
                 ],
-                extra={},
+                extra={"metadata": {"ls_tools_called": []}},
                 serialized=SERIALIZED,
                 error=None,
                 inputs={"prompts": []},
@@ -545,7 +545,7 @@ def test_tracer_nested_runs_on_error() -> None:
                     {"name": "start", "time": datetime.now(timezone.utc)},
                     {"name": "end", "time": datetime.now(timezone.utc)},
                 ],
-                extra={},
+                extra={"metadata": {"ls_tools_called": []}},
                 serialized=SERIALIZED,
                 error=None,
                 inputs={"prompts": []},