diff --git a/docs/docs/how_to/example_selectors_langsmith.ipynb b/docs/docs/how_to/example_selectors_langsmith.ipynb new file mode 100644 index 00000000000..7ca5d136876 --- /dev/null +++ b/docs/docs/how_to/example_selectors_langsmith.ipynb @@ -0,0 +1,320 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4f7e423b", + "metadata": {}, + "source": [ + "# How to select examples from a LangSmith dataset\n", + "\n", + "import Prerequisites from \"@theme/Prerequisites\";\n", + "import Compatibility from \"@theme/Compatibility\";\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "LangSmith datasets have built-in support for similarity search, making them a great tool for building and querying few-shot examples.\n", + "\n", + "In this guide we'll see how to use an indexed LangSmith dataset as a few-shot example selector.\n", + "\n", + "## Setup\n", + "\n", + "Before getting started make sure you've [created a LangSmith account](https://smith.langchain.com/) and set your credentials:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "85445e0e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set LangSmith API key:\n", + "\n", + "········\n" + ] + } + ], + "source": [ + "import getpass\n", + "import os\n", + "\n", + "if not os.environ.get(\"LANGSMITH_API_KEY\"):\n", + " os.environ[\"LANGSMITH_API_KEY\"] = getpass.getpass(\"Set LangSmith API key:\\n\\n\")\n", + "\n", + "os.environ[\"LANGSMITH_TRACING\"] = \"true\"" + ] + }, + { + "cell_type": "markdown", + "id": "ca899e29", + "metadata": {}, + "source": [ + "We'll need to install the `langsmith` SDK. In this example we'll also make use of `langchain`, `langchain-openai`, and `langchain-benchmarks`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4fa7810", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -qU langsmith>=0.1.101 langchain langchain-openai langchain-benchmarks" + ] + }, + { + "cell_type": "markdown", + "id": "fc716e12", + "metadata": {}, + "source": [ + "Now we'll clone a public dataset and turn on indexing for the dataset. We can also turn on indexing via the [LangSmith UI](https://docs.smith.langchain.com/how_to_guides/datasets/index_datasets_for_dynamic_few_shot_example_selection).\n", + "\n", + "We'll clone the [Multiverse math few shot example dataset](https://blog.langchain.dev/few-shot-prompting-to-improve-tool-calling-performance/).\n", + "\n", + "This enables searching over the dataset, and will make sure that anytime we update/add examples they are also indexed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf53d280", + "metadata": {}, + "outputs": [], + "source": [ + "from langsmith import AsyncClient as AsyncLangSmith\n", + "from langsmith import Client as LangSmith\n", + "\n", + "ls_client = LangSmith()\n", + "async_ls_client = AsyncLangSmith()\n", + "\n", + "dataset_name = \"multiverse-math-examples-for-few-shot\"\n", + "dataset_public_url = (\n", + " \"https://smith.langchain.com/public/0df59e49-d226-4ef2-9ecd-8c0fc9cd0288/d\"\n", + ")\n", + "\n", + "ls_client.clone_public_dataset(dataset_public_url)\n", + "\n", + "dataset_id = ls_client.read_dataset(dataset_name=dataset_name).id\n", + "ls_client.index_dataset(dataset_id=dataset_id)" + ] + }, + { + "cell_type": "markdown", + "id": "5767d171", + "metadata": {}, + "source": [ + "Indexing can take a few seconds. Once the dataset is indexed, we can search for similar examples like so:" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "5013a56f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "examples = ls_client.similar_examples(\n", + " {\"input\": \"whats the negation of the negation of the negation of 3\"},\n", + " limit=3,\n", + " dataset_id=dataset_id,\n", + ")\n", + "len(examples)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "a142db06", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'evaluate the negation of -100'" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "examples[0].inputs[\"input\"]" + ] + }, + { + "cell_type": "markdown", + "id": "d2627125", + "metadata": {}, + "source": [ + "For this dataset the outputs are an entire chat history:" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "af5b9191", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "9" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(examples[1].outputs[\"output\"])" + ] + }, + { + "cell_type": "markdown", + "id": "e852c8ef", + "metadata": {}, + "source": [ + "The search returns the examples whose inputs are most similar to the query input. We can use this for few-shot prompting a model like so:" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "12cba1e1", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.chat_models import init_chat_model\n", + "from langchain_benchmarks.tool_usage.tasks.multiverse_math import (\n", + " add,\n", + " cos,\n", + " divide,\n", + " log,\n", + " multiply,\n", + " negate,\n", + " pi,\n", + " power,\n", + " sin,\n", + " subtract,\n", + ")\n", + "from langchain_core.messages import HumanMessage, SystemMessage, convert_to_messages\n", + "from langchain_core.runnables import RunnableLambda\n", + "\n", + "\n", + "def similar_examples(input_: dict) -> dict:\n", + " examples = ls_client.similar_examples(input_, limit=5, dataset_id=dataset_id)\n", + " return {**input_, \"examples\": examples}\n", + "\n", + "\n", + "async def asimilar_examples(input_: dict) -> dict:\n", + " examples = await async_ls_client.similar_examples(\n", + " input_, limit=5, dataset_id=dataset_id\n", + " )\n", + " return {**input_, \"examples\": examples}\n", + "\n", + "\n", + "def construct_prompt(input_: dict) -> list:\n", + " instructions = \"\"\"You are great at using mathematical tools.\"\"\"\n", + " messages = []\n", + " for ex in input_[\"examples\"]:\n", + " # For this dataset, a multi-turn conversation is stored as output.\n", + " messages.extend(convert_to_messages(ex.outputs[\"output\"]))\n", + " examples = [msg for msg in messages if not isinstance(msg, SystemMessage)]\n", + " for ex in examples:\n", + " ex.name = (\n", + " \"example_user\" if isinstance(ex, HumanMessage) else \"example_assistant\"\n", + " )\n", + " return [SystemMessage(instructions), *examples, HumanMessage(input_[\"input\"])]\n", + "\n", + "\n", + "tools = [add, cos, divide, log, multiply, negate, pi, power, sin, subtract]\n", + "llm = init_chat_model(\"gpt-4o\")\n", + "llm_with_tools = llm.bind_tools(tools)\n", + "\n", + "example_selector = RunnableLambda(func=similar_examples, afunc=asimilar_examples)\n", + "\n", + "chain = example_selector | construct_prompt | llm_with_tools" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "id": "c423b367", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'negate',\n", + " 'args': {'a': 3},\n", + " 'id': 'call_ehmx3Z4Cj6HFpI8FV4pYZ5Oo',\n", + " 'type': 'tool_call'}]" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ai_msg = await chain.ainvoke({\"input\": \"whats the negation of the negation of 3\"})\n", + "ai_msg.tool_calls" + ] + }, + { + "cell_type": "markdown", + "id": "94489b4a", + "metadata": {}, + "source": [ + "Looking at the LangSmith trace, we can see that relevant examples were pulled in in the `similar_examples` step and passed as messages to ChatOpenAI: https://smith.langchain.com/public/05af2ce8-1a45-4f3a-8d54-6524ff919279/r." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "poetry-venv-311", + "language": "python", + "name": "poetry-venv-311" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/docs/how_to/index.mdx b/docs/docs/how_to/index.mdx index 05f90a7d24d..b7ab308dff4 100644 --- a/docs/docs/how_to/index.mdx +++ b/docs/docs/how_to/index.mdx @@ -69,6 +69,7 @@ These are the core building blocks you can use when building applications. - [How to: select examples by semantic similarity](/docs/how_to/example_selectors_similarity) - [How to: select examples by semantic ngram overlap](/docs/how_to/example_selectors_ngram) - [How to: select examples by maximal marginal relevance](/docs/how_to/example_selectors_mmr) +- [How to: select examples from LangSmith few-shot datasets](/docs/how_to/example_selectors_langsmith/) ### Chat models