diff --git a/docs/docs/how_to/example_selectors_langsmith.ipynb b/docs/docs/how_to/example_selectors_langsmith.ipynb index 7ca5d136876..d8e7a905465 100644 --- a/docs/docs/how_to/example_selectors_langsmith.ipynb +++ b/docs/docs/how_to/example_selectors_langsmith.ipynb @@ -18,7 +18,7 @@ "\n", "\n", "\n", "\n", "\n", @@ -33,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "85445e0e", "metadata": {}, "outputs": [ @@ -72,7 +72,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -qU langsmith>=0.1.101 langchain langchain-openai langchain-benchmarks" + "%pip install -qU langsmith>=0.1.100 langchain langchain-openai langchain-benchmarks" ] }, { @@ -84,7 +84,7 @@ "\n", "We'll clone the [Multiverse math few shot example dataset](https://blog.langchain.dev/few-shot-prompting-to-improve-tool-calling-performance/).\n", "\n", - "This enables searching over the dataset, and will make sure that anytime we update/add examples they are also indexed." + "This enables searching over the dataset and will make sure that anytime we update/add examples they are also indexed." ] }, { @@ -94,20 +94,19 @@ "metadata": {}, "outputs": [], "source": [ - "from langsmith import AsyncClient as AsyncLangSmith\n", "from langsmith import Client as LangSmith\n", "\n", "ls_client = LangSmith()\n", - "async_ls_client = AsyncLangSmith()\n", "\n", - "dataset_name = \"multiverse-math-examples-for-few-shot\"\n", + "dataset_name = \"multiverse-math-few-shot-examples-v2\"\n", "dataset_public_url = (\n", - " \"https://smith.langchain.com/public/0df59e49-d226-4ef2-9ecd-8c0fc9cd0288/d\"\n", + " \"https://smith.langchain.com/public/620596ee-570b-4d2b-8c8f-f828adbe5242/d\"\n", ")\n", "\n", "ls_client.clone_public_dataset(dataset_public_url)\n", "\n", "dataset_id = ls_client.read_dataset(dataset_name=dataset_name).id\n", + "\n", "ls_client.index_dataset(dataset_id=dataset_id)" ] }, @@ -116,12 +115,12 @@ "id": "5767d171", "metadata": {}, "source": [ - "Indexing can take a few seconds. Once the dataset is indexed, we can search for similar examples like so:" + "Indexing can take a few seconds. Once the dataset is indexed, we can search for similar examples. Note that the input to the `similar_examples` method must have the same schema as the examples inputs. In this case our example inputs are a dictionary with a \"question\" key:" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 12, "id": "5013a56f", "metadata": {}, "outputs": [ @@ -131,14 +130,14 @@ "3" ] }, - "execution_count": 29, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "examples = ls_client.similar_examples(\n", - " {\"input\": \"whats the negation of the negation of the negation of 3\"},\n", + " {\"question\": \"whats the negation of the negation of the negation of 3\"},\n", " limit=3,\n", " dataset_id=dataset_id,\n", ")\n", @@ -147,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 13, "id": "a142db06", "metadata": {}, "outputs": [ @@ -157,13 +156,13 @@ "'evaluate the negation of -100'" ] }, - "execution_count": 34, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "examples[0].inputs[\"input\"]" + "examples[0].inputs[\"question\"]" ] }, { @@ -171,28 +170,51 @@ "id": "d2627125", "metadata": {}, "source": [ - "For this dataset the outputs are an entire chat history:" + "For this dataset, the outputs are the conversation that followed the question in OpenAI message format:" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 14, "id": "af5b9191", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "9" + "[{'role': 'assistant',\n", + " 'content': None,\n", + " 'tool_calls': [{'id': 'toolu_01HTpq4cYNUac6F7omUc2Wz3',\n", + " 'type': 'function',\n", + " 'function': {'name': 'negate', 'arguments': '{\"a\": -100}'}}]},\n", + " {'role': 'tool',\n", + " 'content': '-100.0',\n", + " 'tool_call_id': 'toolu_01HTpq4cYNUac6F7omUc2Wz3'},\n", + " {'role': 'assistant', 'content': 'So the answer is 100.'},\n", + " {'role': 'user',\n", + " 'content': '100 is incorrect. Please refer to the output of your tool call.'},\n", + " {'role': 'assistant',\n", + " 'content': [{'text': \"You're right, my previous answer was incorrect. Let me re-evaluate using the tool output:\",\n", + " 'type': 'text'}],\n", + " 'tool_calls': [{'id': 'toolu_01XsJQboYghGDygQpPjJkeRq',\n", + " 'type': 'function',\n", + " 'function': {'name': 'negate', 'arguments': '{\"a\": -100}'}}]},\n", + " {'role': 'tool',\n", + " 'content': '-100.0',\n", + " 'tool_call_id': 'toolu_01XsJQboYghGDygQpPjJkeRq'},\n", + " {'role': 'assistant', 'content': 'The answer is -100.0'},\n", + " {'role': 'user',\n", + " 'content': 'You have the correct numerical answer but are returning additional text. Please only respond with the numerical answer.'},\n", + " {'role': 'assistant', 'content': '-100.0'}]" ] }, - "execution_count": 33, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "len(examples[1].outputs[\"output\"])" + "examples[0].outputs[\"conversation\"]" ] }, { @@ -205,7 +227,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 20, "id": "12cba1e1", "metadata": {}, "outputs": [], @@ -223,8 +245,10 @@ " sin,\n", " subtract,\n", ")\n", - "from langchain_core.messages import HumanMessage, SystemMessage, convert_to_messages\n", "from langchain_core.runnables import RunnableLambda\n", + "from langsmith import AsyncClient as AsyncLangSmith\n", + "\n", + "async_ls_client = AsyncLangSmith()\n", "\n", "\n", "def similar_examples(input_: dict) -> dict:\n", @@ -241,20 +265,24 @@ "\n", "def construct_prompt(input_: dict) -> list:\n", " instructions = \"\"\"You are great at using mathematical tools.\"\"\"\n", - " messages = []\n", + " examples = []\n", " for ex in input_[\"examples\"]:\n", - " # For this dataset, a multi-turn conversation is stored as output.\n", - " messages.extend(convert_to_messages(ex.outputs[\"output\"]))\n", - " examples = [msg for msg in messages if not isinstance(msg, SystemMessage)]\n", - " for ex in examples:\n", - " ex.name = (\n", - " \"example_user\" if isinstance(ex, HumanMessage) else \"example_assistant\"\n", - " )\n", - " return [SystemMessage(instructions), *examples, HumanMessage(input_[\"input\"])]\n", + " examples.append({\"role\": \"user\", \"content\": ex.inputs[\"question\"]})\n", + " for msg in ex.outputs[\"conversation\"]:\n", + " if msg[\"role\"] == \"assistant\":\n", + " msg[\"name\"] = \"example_assistant\"\n", + " if msg[\"role\"] == \"user\":\n", + " msg[\"name\"] = \"example_user\"\n", + " examples.append(msg)\n", + " return [\n", + " {\"role\": \"system\", \"content\": instructions},\n", + " *examples,\n", + " {\"role\": \"user\", \"content\": input_[\"question\"]},\n", + " ]\n", "\n", "\n", "tools = [add, cos, divide, log, multiply, negate, pi, power, sin, subtract]\n", - "llm = init_chat_model(\"gpt-4o\")\n", + "llm = init_chat_model(\"gpt-4o-2024-08-06\")\n", "llm_with_tools = llm.bind_tools(tools)\n", "\n", "example_selector = RunnableLambda(func=similar_examples, afunc=asimilar_examples)\n", @@ -264,7 +292,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 21, "id": "c423b367", "metadata": {}, "outputs": [ @@ -273,17 +301,17 @@ "text/plain": [ "[{'name': 'negate',\n", " 'args': {'a': 3},\n", - " 'id': 'call_ehmx3Z4Cj6HFpI8FV4pYZ5Oo',\n", + " 'id': 'call_uMSdoTl6ehfHh5a6JQUb2NoZ',\n", " 'type': 'tool_call'}]" ] }, - "execution_count": 52, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ai_msg = await chain.ainvoke({\"input\": \"whats the negation of the negation of 3\"})\n", + "ai_msg = await chain.ainvoke({\"question\": \"whats the negation of the negation of 3\"})\n", "ai_msg.tool_calls" ] }, @@ -292,7 +320,7 @@ "id": "94489b4a", "metadata": {}, "source": [ - "Looking at the LangSmith trace, we can see that relevant examples were pulled in in the `similar_examples` step and passed as messages to ChatOpenAI: https://smith.langchain.com/public/05af2ce8-1a45-4f3a-8d54-6524ff919279/r." + "Looking at the LangSmith trace, we can see that relevant examples were pulled in in the `similar_examples` step and passed as messages to ChatOpenAI: https://smith.langchain.com/public/9585e30f-765a-4ed9-b964-2211420cd2f8/r." ] } ], diff --git a/libs/core/langchain_core/messages/utils.py b/libs/core/langchain_core/messages/utils.py index 18225c53228..c7d4d58a149 100644 --- a/libs/core/langchain_core/messages/utils.py +++ b/libs/core/langchain_core/messages/utils.py @@ -10,6 +10,7 @@ Some examples of what you can do with these functions include: from __future__ import annotations import inspect +import json from functools import partial from typing import ( TYPE_CHECKING, @@ -213,7 +214,23 @@ def _create_message_from_message_type( if id is not None: kwargs["id"] = id if tool_calls is not None: - kwargs["tool_calls"] = tool_calls + kwargs["tool_calls"] = [] + for tool_call in tool_calls: + # Convert OpenAI-format tool call to LangChain format. + if "function" in tool_call: + args = tool_call["function"]["arguments"] + if isinstance(args, str): + args = json.loads(args, strict=False) + kwargs["tool_calls"].append( + { + "name": tool_call["function"]["name"], + "args": args, + "id": tool_call["id"], + "type": "tool_call", + } + ) + else: + kwargs["tool_calls"].append(tool_call) if message_type in ("human", "user"): message: BaseMessage = HumanMessage(content=content, **kwargs) elif message_type in ("ai", "assistant"): @@ -271,7 +288,8 @@ def _convert_to_message(message: MessageLikeRepresentation) -> BaseMessage: msg_type = msg_kwargs.pop("role") except KeyError: msg_type = msg_kwargs.pop("type") - msg_content = msg_kwargs.pop("content") + # None msg content is not allowed + msg_content = msg_kwargs.pop("content") or "" except KeyError: raise ValueError( f"Message dict must contain 'role' and 'content' keys, got {message}" diff --git a/libs/core/langchain_core/output_parsers/openai_tools.py b/libs/core/langchain_core/output_parsers/openai_tools.py index 11bb5518cbe..a20707eb14b 100644 --- a/libs/core/langchain_core/output_parsers/openai_tools.py +++ b/libs/core/langchain_core/output_parsers/openai_tools.py @@ -5,12 +5,8 @@ from typing import Any, Dict, List, Optional from langchain_core.exceptions import OutputParserException from langchain_core.messages import AIMessage, InvalidToolCall -from langchain_core.messages.tool import ( - invalid_tool_call, -) -from langchain_core.messages.tool import ( - tool_call as create_tool_call, -) +from langchain_core.messages.tool import invalid_tool_call +from langchain_core.messages.tool import tool_call as create_tool_call from langchain_core.output_parsers.transform import BaseCumulativeTransformOutputParser from langchain_core.outputs import ChatGeneration, Generation from langchain_core.pydantic_v1 import ValidationError