diff --git a/docs/docs/langsmith/img/log_traces.png b/docs/docs/langsmith/img/log_traces.png index 2ded1d8bca0..7eca9cd5b27 100644 Binary files a/docs/docs/langsmith/img/log_traces.png and b/docs/docs/langsmith/img/log_traces.png differ diff --git a/docs/docs/langsmith/img/test_results.png b/docs/docs/langsmith/img/test_results.png index b842bbc659f..326ee795433 100644 Binary files a/docs/docs/langsmith/img/test_results.png and b/docs/docs/langsmith/img/test_results.png differ diff --git a/docs/docs/langsmith/walkthrough.ipynb b/docs/docs/langsmith/walkthrough.ipynb index 24f4f9a1334..0e9b6c5004f 100644 --- a/docs/docs/langsmith/walkthrough.ipynb +++ b/docs/docs/langsmith/walkthrough.ipynb @@ -129,15 +129,17 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "id": "a0fbfbba-3c82-4298-a312-9cec016d9d2e", "metadata": {}, "outputs": [], "source": [ "from langchain import hub\n", "from langchain.agents import AgentExecutor\n", - "from langchain.agents.format_scratchpad import format_to_openai_function_messages\n", - "from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser\n", + "from langchain.agents.format_scratchpad.openai_tools import (\n", + " format_to_openai_tool_messages,\n", + ")\n", + "from langchain.agents.output_parsers.openai_tools import OpenAIToolsAgentOutputParser\n", "from langchain_community.tools import DuckDuckGoSearchResults\n", "from langchain_openai import ChatOpenAI\n", "\n", @@ -155,18 +157,18 @@ " ), # General internet search using DuckDuckGo\n", "]\n", "\n", - "llm_with_tools = llm.bind_functions(tools)\n", + "llm_with_tools = llm.bind_tools(tools)\n", "\n", "runnable_agent = (\n", " {\n", " \"input\": lambda x: x[\"input\"],\n", - " \"agent_scratchpad\": lambda x: format_to_openai_function_messages(\n", + " \"agent_scratchpad\": lambda x: format_to_openai_tool_messages(\n", " x[\"intermediate_steps\"]\n", " ),\n", " }\n", " | prompt\n", " | llm_with_tools\n", - " | OpenAIFunctionsAgentOutputParser()\n", + " | OpenAIToolsAgentOutputParser()\n", ")\n", "\n", "agent_executor = AgentExecutor(\n", @@ -184,7 +186,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "id": "19537902-b95c-4390-80a4-f6c9a937081e", "metadata": { "tags": [] @@ -204,7 +206,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 5, "id": "9a6a764c-5d7a-4de7-a916-3ecc987d5bb6", "metadata": {}, "outputs": [ @@ -217,7 +219,7 @@ " 'output': 'I\\'m sorry, but I couldn\\'t find any information about \"LangSmith\". It could be a company, a product, or a person. Can you provide more context or details about what you are referring to?'}]" ] }, - "execution_count": 8, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -269,7 +271,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 6, "id": "43fd40b2-3f02-4e51-9343-705aafe90a36", "metadata": {}, "outputs": [], @@ -285,7 +287,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 7, "id": "17580c4b-bd04-4dde-9d21-9d4edd25b00d", "metadata": { "tags": [] @@ -322,7 +324,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 8, "id": "f42d8ecc-d46a-448b-a89c-04b0f6907f75", "metadata": { "tags": [] @@ -331,8 +333,6 @@ "source": [ "from langchain import hub\n", "from langchain.agents import AgentExecutor, AgentType, initialize_agent, load_tools\n", - "from langchain.agents.format_scratchpad import format_to_openai_function_messages\n", - "from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser\n", "from langchain_openai import ChatOpenAI\n", "\n", "\n", @@ -343,13 +343,13 @@ " runnable_agent = (\n", " {\n", " \"input\": lambda x: x[\"input\"],\n", - " \"agent_scratchpad\": lambda x: format_to_openai_function_messages(\n", + " \"agent_scratchpad\": lambda x: format_to_openai_tool_messages(\n", " x[\"intermediate_steps\"]\n", " ),\n", " }\n", " | prompt\n", " | llm_with_tools\n", - " | OpenAIFunctionsAgentOutputParser()\n", + " | OpenAIToolsAgentOutputParser()\n", " )\n", " return AgentExecutor(agent=runnable_agent, tools=tools, handle_parsing_errors=True)" ] @@ -371,16 +371,15 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "331c3c53-949d-405e-8ba5-38bab1ce413b", "metadata": {}, "outputs": [], "source": [ - "from langsmith.evaluation import EvaluationResult, run_evaluator\n", + "from langsmith.evaluation import EvaluationResult\n", "from langsmith.schemas import Example, Run\n", "\n", "\n", - "@run_evaluator\n", "def check_not_idk(run: Run, example: Example):\n", " \"\"\"Illustration of a custom evaluator.\"\"\"\n", " agent_response = run.outputs[\"output\"]\n", @@ -396,6 +395,34 @@ " )" ] }, + { + "cell_type": "markdown", + "id": "1cc51d0a-4982-4ff9-89c1-b294d5cce8f6", + "metadata": {}, + "source": [ + "#### Aggregate Evaluators\n", + "\n", + "Some metrics are only performed in aggregate or full \"test session\" level. These could be simple \n", + "classification metrics like Precision, Recall, or AUC, or other custom metrics.\n", + "\n", + "You can define any custom metric on an aggregate level using a callable that accepts a list of Runs (system traces) and list of Examples (dataset records)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cada62c4-c237-4f85-aa33-789cbcd1e8c1", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import List\n", + "\n", + "\n", + "def max_pred_length(runs: List[Run], examples: List[Example]):\n", + " predictions = [len(run.outputs[\"output\"]) for run in runs]\n", + " return EvaluationResult(key=\"max_pred_length\", score=max(predictions))" + ] + }, { "cell_type": "markdown", "id": "ad9c4791-570b-4adf-a23f-d025ff383254", @@ -412,7 +439,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "a25dc281", "metadata": { "tags": [] @@ -425,6 +452,7 @@ "evaluation_config = RunEvalConfig(\n", " # Evaluators can either be an evaluator type (e.g., \"qa\", \"criteria\", \"embedding_distance\", etc.) or a configuration for that evaluator\n", " evaluators=[\n", + " check_not_idk,\n", " # Measures whether a QA response is \"Correct\", based on a reference answer\n", " # You can also select via the raw string \"qa\"\n", " EvaluatorType.QA,\n", @@ -448,9 +476,7 @@ " normalize_by=10,\n", " ),\n", " ],\n", - " # You can add custom StringEvaluator or RunEvaluator objects here as well, which will automatically be\n", - " # applied to each prediction. Check out the docs for examples.\n", - " custom_evaluators=[check_not_idk],\n", + " aggregate_evaluators=[max_pred_length],\n", ")" ] }, @@ -473,7 +499,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "af8c8469-d70d-46d9-8fcd-517a1ccc7c4b", "metadata": {}, "outputs": [], @@ -486,265 +512,12 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "id": "3733269b-8085-4644-9d5d-baedcff13a2f", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "View the evaluation results for project 'runnable-agent-test-5d466cbc-97e1' at:\n", - "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/14d8a382-3c0f-48e7-b212-33489ee8a13e/compare?selectedSessions=62f0a0c0-73bf-420c-a907-2c6b2f4625c4\n", - "\n", - "View all tests for Dataset agent-qa-e2d24144 at:\n", - "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/14d8a382-3c0f-48e7-b212-33489ee8a13e\n", - "[> ] 0/5" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Server error caused failure to patch https://api.smith.langchain.com/runs/e9d26fe6-bf4a-4f88-81c5-f5d0f70977f0 in LangSmith API. HTTPError('500 Server Error: Internal Server Error for url: https://api.smith.langchain.com/runs/e9d26fe6-bf4a-4f88-81c5-f5d0f70977f0', '{\"detail\":\"Internal server error\"}')\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[------------------------------------------------->] 5/5" - ] - }, - { - "data": { - "text/html": [ - "

Experiment Results:

" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
feedback.correctnessfeedback.embedding_cosine_distancefeedback.helpfulnessfeedback.score_string:accuracyfeedback.not_uncertainerrorexecution_timerun_id
count5.0000005.0000005.0000005.0000005.005.0000005
uniqueNaNNaNNaNNaNNaN0NaN5
topNaNNaNNaNNaNNaNNaNNaNe9440e33-e2c6-4bec-a190-2fa41947e9c5
freqNaNNaNNaNNaNNaNNaNNaN1
mean0.8000000.1102680.8000000.6400001.0NaN6.338926NaN
std0.4472140.0606800.4472140.3286340.0NaN2.525439NaN
min0.0000000.0494420.0000000.1000001.0NaN3.858924NaN
25%1.0000000.0641861.0000000.7000001.0NaN5.250572NaN
50%1.0000000.0922561.0000000.7000001.0NaN5.817807NaN
75%1.0000000.1530031.0000000.7000001.0NaN6.199289NaN
max1.0000000.1924531.0000001.0000001.0NaN10.568036NaN
\n", - "
" - ], - "text/plain": [ - " feedback.correctness feedback.embedding_cosine_distance \\\n", - "count 5.000000 5.000000 \n", - "unique NaN NaN \n", - "top NaN NaN \n", - "freq NaN NaN \n", - "mean 0.800000 0.110268 \n", - "std 0.447214 0.060680 \n", - "min 0.000000 0.049442 \n", - "25% 1.000000 0.064186 \n", - "50% 1.000000 0.092256 \n", - "75% 1.000000 0.153003 \n", - "max 1.000000 0.192453 \n", - "\n", - " feedback.helpfulness feedback.score_string:accuracy \\\n", - "count 5.000000 5.000000 \n", - "unique NaN NaN \n", - "top NaN NaN \n", - "freq NaN NaN \n", - "mean 0.800000 0.640000 \n", - "std 0.447214 0.328634 \n", - "min 0.000000 0.100000 \n", - "25% 1.000000 0.700000 \n", - "50% 1.000000 0.700000 \n", - "75% 1.000000 0.700000 \n", - "max 1.000000 1.000000 \n", - "\n", - " feedback.not_uncertain error execution_time \\\n", - "count 5.0 0 5.000000 \n", - "unique NaN 0 NaN \n", - "top NaN NaN NaN \n", - "freq NaN NaN NaN \n", - "mean 1.0 NaN 6.338926 \n", - "std 0.0 NaN 2.525439 \n", - "min 1.0 NaN 3.858924 \n", - "25% 1.0 NaN 5.250572 \n", - "50% 1.0 NaN 5.817807 \n", - "75% 1.0 NaN 6.199289 \n", - "max 1.0 NaN 10.568036 \n", - "\n", - " run_id \n", - "count 5 \n", - "unique 5 \n", - "top e9440e33-e2c6-4bec-a190-2fa41947e9c5 \n", - "freq 1 \n", - "mean NaN \n", - "std NaN \n", - "min NaN \n", - "25% NaN \n", - "50% NaN \n", - "75% NaN \n", - "max NaN " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import functools\n", "\n", @@ -758,7 +531,7 @@ " evaluation=evaluation_config,\n", " verbose=True,\n", " client=client,\n", - " project_name=f\"runnable-agent-test-5d466cbc-{unique_id}\",\n", + " project_name=f\"tools-agent-test-5d466cbc-{unique_id}\",\n", " # Project metadata communicates the experiment parameters,\n", " # Useful for reviewing the test results\n", " project_metadata={\n", @@ -790,209 +563,10 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "id": "9da60638-5be8-4b5f-a721-2c6627aeaf0c", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
inputs.inputoutputs.inputoutputs.outputreference.outputfeedback.correctnessfeedback.embedding_cosine_distancefeedback.helpfulnessfeedback.score_string:accuracyfeedback.not_uncertainerrorexecution_timerun_id
63e7ff81-b3f6-40aa-81c8-3600c504d54fWhen did langchain first announce the hub?When did langchain first announce the hub?LangChain first announced the LangChain Hub on...September 5, 202310.15300311.01None5.250572e9440e33-e2c6-4bec-a190-2fa41947e9c5
48c2b719-93a3-47cb-baa6-ee93ecb1ba30What is the langsmith cookbook?What is the langsmith cookbook?The LangSmith Cookbook is a collection of reci...The langsmith cookbook is a github repository ...10.04944210.71None10.568036ad1d3050-dffb-45a2-ab9d-bbc88a702682
96ed16c3-cbc6-4ebb-a335-ec70e64db109When was Llama-v2 released?When was Llama-v2 released?Llama-v2 was released in 2023.July 18, 202310.19245310.71None6.199289646518d4-b4be-4b20-b5cb-9b86e0a8fe86
7342bb9c-9733-47bd-b88e-cd4ab4f4f82dWhat's LangSmith?What's LangSmith?LangSmith is a platform that helps developers ...LangSmith is a unified platform for debugging,...10.06418610.71None5.81780707d157f1-a18c-4173-bda0-1b3ac8d8dcf2
aee03fdf-c63e-4f63-9075-a90be3922c7fWhat is LangChain?What is LangChain?LangChain is a decentralized blockchain platfo...LangChain is an open-source framework for buil...00.09225600.11None3.858924bf1bc257-2ab6-464e-b6d1-99f25b042383
\n", - "
" - ], - "text/plain": [ - " inputs.input \\\n", - "63e7ff81-b3f6-40aa-81c8-3600c504d54f When did langchain first announce the hub? \n", - "48c2b719-93a3-47cb-baa6-ee93ecb1ba30 What is the langsmith cookbook? \n", - "96ed16c3-cbc6-4ebb-a335-ec70e64db109 When was Llama-v2 released? \n", - "7342bb9c-9733-47bd-b88e-cd4ab4f4f82d What's LangSmith? \n", - "aee03fdf-c63e-4f63-9075-a90be3922c7f What is LangChain? \n", - "\n", - " outputs.input \\\n", - "63e7ff81-b3f6-40aa-81c8-3600c504d54f When did langchain first announce the hub? \n", - "48c2b719-93a3-47cb-baa6-ee93ecb1ba30 What is the langsmith cookbook? \n", - "96ed16c3-cbc6-4ebb-a335-ec70e64db109 When was Llama-v2 released? \n", - "7342bb9c-9733-47bd-b88e-cd4ab4f4f82d What's LangSmith? \n", - "aee03fdf-c63e-4f63-9075-a90be3922c7f What is LangChain? \n", - "\n", - " outputs.output \\\n", - "63e7ff81-b3f6-40aa-81c8-3600c504d54f LangChain first announced the LangChain Hub on... \n", - "48c2b719-93a3-47cb-baa6-ee93ecb1ba30 The LangSmith Cookbook is a collection of reci... \n", - "96ed16c3-cbc6-4ebb-a335-ec70e64db109 Llama-v2 was released in 2023. \n", - "7342bb9c-9733-47bd-b88e-cd4ab4f4f82d LangSmith is a platform that helps developers ... \n", - "aee03fdf-c63e-4f63-9075-a90be3922c7f LangChain is a decentralized blockchain platfo... \n", - "\n", - " reference.output \\\n", - "63e7ff81-b3f6-40aa-81c8-3600c504d54f September 5, 2023 \n", - "48c2b719-93a3-47cb-baa6-ee93ecb1ba30 The langsmith cookbook is a github repository ... \n", - "96ed16c3-cbc6-4ebb-a335-ec70e64db109 July 18, 2023 \n", - "7342bb9c-9733-47bd-b88e-cd4ab4f4f82d LangSmith is a unified platform for debugging,... \n", - "aee03fdf-c63e-4f63-9075-a90be3922c7f LangChain is an open-source framework for buil... \n", - "\n", - " feedback.correctness \\\n", - "63e7ff81-b3f6-40aa-81c8-3600c504d54f 1 \n", - "48c2b719-93a3-47cb-baa6-ee93ecb1ba30 1 \n", - "96ed16c3-cbc6-4ebb-a335-ec70e64db109 1 \n", - "7342bb9c-9733-47bd-b88e-cd4ab4f4f82d 1 \n", - "aee03fdf-c63e-4f63-9075-a90be3922c7f 0 \n", - "\n", - " feedback.embedding_cosine_distance \\\n", - "63e7ff81-b3f6-40aa-81c8-3600c504d54f 0.153003 \n", - "48c2b719-93a3-47cb-baa6-ee93ecb1ba30 0.049442 \n", - "96ed16c3-cbc6-4ebb-a335-ec70e64db109 0.192453 \n", - "7342bb9c-9733-47bd-b88e-cd4ab4f4f82d 0.064186 \n", - "aee03fdf-c63e-4f63-9075-a90be3922c7f 0.092256 \n", - "\n", - " feedback.helpfulness \\\n", - "63e7ff81-b3f6-40aa-81c8-3600c504d54f 1 \n", - "48c2b719-93a3-47cb-baa6-ee93ecb1ba30 1 \n", - "96ed16c3-cbc6-4ebb-a335-ec70e64db109 1 \n", - "7342bb9c-9733-47bd-b88e-cd4ab4f4f82d 1 \n", - "aee03fdf-c63e-4f63-9075-a90be3922c7f 0 \n", - "\n", - " feedback.score_string:accuracy \\\n", - "63e7ff81-b3f6-40aa-81c8-3600c504d54f 1.0 \n", - "48c2b719-93a3-47cb-baa6-ee93ecb1ba30 0.7 \n", - "96ed16c3-cbc6-4ebb-a335-ec70e64db109 0.7 \n", - "7342bb9c-9733-47bd-b88e-cd4ab4f4f82d 0.7 \n", - "aee03fdf-c63e-4f63-9075-a90be3922c7f 0.1 \n", - "\n", - " feedback.not_uncertain error \\\n", - "63e7ff81-b3f6-40aa-81c8-3600c504d54f 1 None \n", - "48c2b719-93a3-47cb-baa6-ee93ecb1ba30 1 None \n", - "96ed16c3-cbc6-4ebb-a335-ec70e64db109 1 None \n", - "7342bb9c-9733-47bd-b88e-cd4ab4f4f82d 1 None \n", - "aee03fdf-c63e-4f63-9075-a90be3922c7f 1 None \n", - "\n", - " execution_time \\\n", - "63e7ff81-b3f6-40aa-81c8-3600c504d54f 5.250572 \n", - "48c2b719-93a3-47cb-baa6-ee93ecb1ba30 10.568036 \n", - "96ed16c3-cbc6-4ebb-a335-ec70e64db109 6.199289 \n", - "7342bb9c-9733-47bd-b88e-cd4ab4f4f82d 5.817807 \n", - "aee03fdf-c63e-4f63-9075-a90be3922c7f 3.858924 \n", - "\n", - " run_id \n", - "63e7ff81-b3f6-40aa-81c8-3600c504d54f e9440e33-e2c6-4bec-a190-2fa41947e9c5 \n", - "48c2b719-93a3-47cb-baa6-ee93ecb1ba30 ad1d3050-dffb-45a2-ab9d-bbc88a702682 \n", - "96ed16c3-cbc6-4ebb-a335-ec70e64db109 646518d4-b4be-4b20-b5cb-9b86e0a8fe86 \n", - "7342bb9c-9733-47bd-b88e-cd4ab4f4f82d 07d157f1-a18c-4173-bda0-1b3ac8d8dcf2 \n", - "aee03fdf-c63e-4f63-9075-a90be3922c7f bf1bc257-2ab6-464e-b6d1-99f25b042383 " - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "chain_results.to_dataframe()" ] @@ -1009,249 +583,10 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "id": "5eeb023f-ded2-4d0f-b910-2a57d9675853", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "View the evaluation results for project 'runnable-agent-test-39f3bbd0-97e1' at:\n", - "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/14d8a382-3c0f-48e7-b212-33489ee8a13e/compare?selectedSessions=7753a05e-8235-4bc2-a227-d0622c1a36a4\n", - "\n", - "View all tests for Dataset agent-qa-e2d24144 at:\n", - "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/14d8a382-3c0f-48e7-b212-33489ee8a13e\n", - "[------------------------------------------------->] 5/5" - ] - }, - { - "data": { - "text/html": [ - "

Experiment Results:

" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
feedback.correctnessfeedback.embedding_cosine_distancefeedback.helpfulnessfeedback.score_string:accuracyfeedback.not_uncertainerrorexecution_timerun_id
count5.0000005.0000005.05.0000005.005.0000005
uniqueNaNNaNNaNNaNNaN0NaN5
topNaNNaNNaNNaNNaNNaNNaNc7128691-de05-41f4-8ffa-ddd12a640ba9
freqNaNNaNNaNNaNNaNNaNNaN1
mean0.8000000.1192821.00.8200001.0NaN7.655640NaN
std0.4472140.0801450.00.2167950.0NaN2.665656NaN
min0.0000000.0433681.00.5000001.0NaN5.409900NaN
25%1.0000000.0533111.00.7000001.0NaN5.633545NaN
50%1.0000000.1078261.00.9000001.0NaN6.451075NaN
75%1.0000000.1530031.01.0000001.0NaN9.194739NaN
max1.0000000.2389031.01.0000001.0NaN11.588943NaN
\n", - "
" - ], - "text/plain": [ - " feedback.correctness feedback.embedding_cosine_distance \\\n", - "count 5.000000 5.000000 \n", - "unique NaN NaN \n", - "top NaN NaN \n", - "freq NaN NaN \n", - "mean 0.800000 0.119282 \n", - "std 0.447214 0.080145 \n", - "min 0.000000 0.043368 \n", - "25% 1.000000 0.053311 \n", - "50% 1.000000 0.107826 \n", - "75% 1.000000 0.153003 \n", - "max 1.000000 0.238903 \n", - "\n", - " feedback.helpfulness feedback.score_string:accuracy \\\n", - "count 5.0 5.000000 \n", - "unique NaN NaN \n", - "top NaN NaN \n", - "freq NaN NaN \n", - "mean 1.0 0.820000 \n", - "std 0.0 0.216795 \n", - "min 1.0 0.500000 \n", - "25% 1.0 0.700000 \n", - "50% 1.0 0.900000 \n", - "75% 1.0 1.000000 \n", - "max 1.0 1.000000 \n", - "\n", - " feedback.not_uncertain error execution_time \\\n", - "count 5.0 0 5.000000 \n", - "unique NaN 0 NaN \n", - "top NaN NaN NaN \n", - "freq NaN NaN NaN \n", - "mean 1.0 NaN 7.655640 \n", - "std 0.0 NaN 2.665656 \n", - "min 1.0 NaN 5.409900 \n", - "25% 1.0 NaN 5.633545 \n", - "50% 1.0 NaN 6.451075 \n", - "75% 1.0 NaN 9.194739 \n", - "max 1.0 NaN 11.588943 \n", - "\n", - " run_id \n", - "count 5 \n", - "unique 5 \n", - "top c7128691-de05-41f4-8ffa-ddd12a640ba9 \n", - "freq 1 \n", - "mean NaN \n", - "std NaN \n", - "min NaN \n", - "25% NaN \n", - "50% NaN \n", - "75% NaN \n", - "max NaN " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "candidate_prompt = hub.pull(\"wfh/langsmith-agent-prompt:39f3bbd0\")\n", "\n", @@ -1263,7 +598,7 @@ " evaluation=evaluation_config,\n", " verbose=True,\n", " client=client,\n", - " project_name=f\"runnable-agent-test-39f3bbd0-{unique_id}\",\n", + " project_name=f\"tools-agent-test-39f3bbd0-{unique_id}\",\n", " project_metadata={\n", " \"env\": \"testing-notebook\",\n", " \"model\": \"gpt-3.5-turbo\",\n", @@ -1286,7 +621,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "id": "33bfefde-d1bb-4f50-9f7a-fd572ee76820", "metadata": { "tags": [] @@ -1298,29 +633,27 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, + "id": "f45de493-8f2a-43b9-a979-6fe694948532", + "metadata": {}, + "outputs": [], + "source": [ + "# The resulting tests are stored in a project. You can programmatically\n", + "# access important metadata from the test, such as the dataset version it was run on\n", + "# or your application's revision ID.\n", + "client.read_project(project_name=chain_results[\"project_name\"]).metadata" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "6595c888-1f5c-4ae3-9390-0a559f5575d1", "metadata": { "tags": [] }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'correctness': {'n': 5, 'avg': 0.8},\n", - " 'embedding_cosine_distance': {'n': 5, 'avg': 0.11926},\n", - " 'helpfulness': {'n': 5, 'avg': 1.0},\n", - " 'not_uncertain': {'n': 5, 'avg': 1.0},\n", - " 'score_string:accuracy': {'n': 5, 'avg': 0.82}}" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "# After some time, these will be populated.\n", + "# After some time, the test metrics will be populated as well.\n", "client.read_project(project_name=chain_results[\"project_name\"]).feedback_stats" ] }, @@ -1343,9 +676,9 @@ ], "metadata": { "kernelspec": { - "display_name": "poetry-venv", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "poetry-venv" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1357,7 +690,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.1" + "version": "3.11.2" } }, "nbformat": 4, diff --git a/libs/langchain/langchain/smith/evaluation/config.py b/libs/langchain/langchain/smith/evaluation/config.py index 69dc1c9efa1..202a94fddea 100644 --- a/libs/langchain/langchain/smith/evaluation/config.py +++ b/libs/langchain/langchain/smith/evaluation/config.py @@ -1,12 +1,14 @@ """Configuration for run evaluators.""" -from typing import Any, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Sequence, Union from langchain_core.embeddings import Embeddings from langchain_core.language_models import BaseLanguageModel from langchain_core.prompts import BasePromptTemplate from langchain_core.pydantic_v1 import BaseModel, Field from langsmith import RunEvaluator +from langsmith.evaluation.evaluator import EvaluationResult, EvaluationResults +from langsmith.schemas import Example, Run from langchain.evaluation.criteria.eval_chain import CRITERIA_TYPE from langchain.evaluation.embedding_distance.base import ( @@ -17,6 +19,14 @@ from langchain.evaluation.string_distance.base import ( StringDistance as StringDistanceEnum, ) +RUN_EVALUATOR_LIKE = Callable[ + [Run, Optional[Example]], Union[EvaluationResult, EvaluationResults, dict] +] +BATCH_EVALUATOR_LIKE = Callable[ + [Sequence[Run], Optional[Sequence[Example]]], + Union[EvaluationResult, EvaluationResults, dict], +] + class EvalConfig(BaseModel): """Configuration for a given run evaluator. @@ -76,12 +86,16 @@ class SingleKeyEvalConfig(EvalConfig): return kwargs +CUSTOM_EVALUATOR_TYPE = Union[RUN_EVALUATOR_LIKE, RunEvaluator, StringEvaluator] +SINGLE_EVAL_CONFIG_TYPE = Union[EvaluatorType, str, EvalConfig] + + class RunEvalConfig(BaseModel): """Configuration for a run evaluation. Parameters ---------- - evaluators : List[Union[EvaluatorType, EvalConfig]] + evaluators : List[Union[EvaluatorType, EvalConfig, RunEvaluator, Callable]] Configurations for which evaluators to apply to the dataset run. Each can be the string of an :class:`EvaluatorType `, such as EvaluatorType.QA, the evaluator type string ("qa"), or a configuration for a @@ -107,9 +121,12 @@ class RunEvalConfig(BaseModel): The language model to pass to any evaluators that use a language model. """ # noqa: E501 - evaluators: List[Union[EvaluatorType, str, EvalConfig]] = Field( - default_factory=list - ) + evaluators: List[ + Union[ + SINGLE_EVAL_CONFIG_TYPE, + CUSTOM_EVALUATOR_TYPE, + ] + ] = Field(default_factory=list) """Configurations for which evaluators to apply to the dataset run. Each can be the string of an :class:`EvaluatorType `, such @@ -117,8 +134,15 @@ class RunEvalConfig(BaseModel): given evaluator (e.g., :class:`RunEvalConfig.QA `).""" # noqa: E501 - custom_evaluators: Optional[List[Union[RunEvaluator, StringEvaluator]]] = None + custom_evaluators: Optional[List[CUSTOM_EVALUATOR_TYPE]] = None """Custom evaluators to apply to the dataset run.""" + batch_evaluators: Optional[List[BATCH_EVALUATOR_LIKE]] = None + """Evaluators that run on an aggregate/batch level. + + These generate 1 or more metrics that are assigned to the full test run. + As a result, they are not associated with individual traces. + """ + reference_key: Optional[str] = None """The key in the dataset run to use as the reference string. If not provided, we will attempt to infer automatically.""" diff --git a/libs/langchain/langchain/smith/evaluation/runner_utils.py b/libs/langchain/langchain/smith/evaluation/runner_utils.py index 2c1a044c2a8..2797adaf405 100644 --- a/libs/langchain/langchain/smith/evaluation/runner_utils.py +++ b/libs/langchain/langchain/smith/evaluation/runner_utils.py @@ -2,6 +2,7 @@ from __future__ import annotations +import concurrent.futures import dataclasses import functools import inspect @@ -35,9 +36,15 @@ from langchain_core.tracers.evaluation import ( from langchain_core.tracers.langchain import LangChainTracer from langsmith.client import Client from langsmith.env import get_git_info, get_langchain_env_var_metadata -from langsmith.evaluation import EvaluationResult, RunEvaluator +from langsmith.evaluation import ( + EvaluationResult, + RunEvaluator, +) +from langsmith.evaluation import ( + run_evaluator as run_evaluator_dec, +) from langsmith.run_helpers import as_runnable, is_traceable_function -from langsmith.schemas import Dataset, DataType, Example, TracerSession +from langsmith.schemas import Dataset, DataType, Example, Run, TracerSession from langsmith.utils import LangSmithError from requests import HTTPError from typing_extensions import TypedDict @@ -513,7 +520,10 @@ def _determine_reference_key( def _construct_run_evaluator( - eval_config: Union[EvaluatorType, str, smith_eval_config.EvalConfig], + eval_config: Union[ + smith_eval_config.SINGLE_EVAL_CONFIG_TYPE, + smith_eval_config.CUSTOM_EVALUATOR_TYPE, + ], eval_llm: Optional[BaseLanguageModel], run_type: str, data_type: DataType, @@ -522,12 +532,14 @@ def _construct_run_evaluator( input_key: Optional[str], prediction_key: Optional[str], ) -> RunEvaluator: + if isinstance(eval_config, RunEvaluator): + return eval_config if isinstance(eval_config, (EvaluatorType, str)): if not isinstance(eval_config, EvaluatorType): eval_config = EvaluatorType(eval_config) evaluator_ = load_evaluator(eval_config, llm=eval_llm) eval_type_tag = eval_config.value - else: + elif isinstance(eval_config, smith_eval_config.EvalConfig): kwargs = {"llm": eval_llm, **eval_config.get_kwargs()} evaluator_ = load_evaluator(eval_config.evaluator_type, **kwargs) eval_type_tag = eval_config.evaluator_type.value @@ -536,6 +548,11 @@ def _construct_run_evaluator( input_key = eval_config.input_key or input_key prediction_key = eval_config.prediction_key or prediction_key reference_key = eval_config.reference_key or reference_key + elif callable(eval_config): + # Assume we can decorate + return run_evaluator_dec(eval_config) + else: + raise ValueError(f"Unknown evaluator type: {type(eval_config)}") if isinstance(evaluator_, StringEvaluator): if evaluator_.requires_reference and reference_key is None: @@ -600,13 +617,9 @@ def _load_run_evaluators( """ run_evaluators = [] input_key, prediction_key, reference_key = None, None, None - if ( - config.evaluators - or any([isinstance(e, EvaluatorType) for e in config.evaluators]) - or ( - config.custom_evaluators - and any([isinstance(e, StringEvaluator) for e in config.custom_evaluators]) - ) + if config.evaluators or ( + config.custom_evaluators + and any([isinstance(e, StringEvaluator) for e in config.custom_evaluators]) ): input_key, prediction_key, reference_key = _get_keys( config, run_inputs, run_outputs, example_outputs @@ -638,6 +651,8 @@ def _load_run_evaluators( reference_key=reference_key, ) ) + elif callable(custom_evaluator): + run_evaluators.append(run_evaluator_dec(custom_evaluator)) else: raise ValueError( f"Unsupported custom evaluator: {custom_evaluator}." @@ -1032,6 +1047,7 @@ class _DatasetRunContainer: wrapped_model: MCF examples: List[Example] configs: List[RunnableConfig] + batch_evaluators: Optional[List[smith_eval_config.BATCH_EVALUATOR_LIKE]] = None def _merge_test_outputs( self, @@ -1055,8 +1071,34 @@ class _DatasetRunContainer: results[str(example.id)]["reference"] = example.outputs return results - def _collect_metrics(self) -> Dict[str, _RowResult]: + def _run_batch_evaluators(self, runs: Dict[str, Run]) -> List[dict]: + evaluators = self.batch_evaluators + if not evaluators: + return [] + runs_list = [runs[str(example.id)] for example in self.examples] + aggregate_feedback = [] + with concurrent.futures.ThreadPoolExecutor() as executor: + for evaluator in evaluators: + try: + result = evaluator(runs_list, self.examples) + if isinstance(result, EvaluationResult): + result = result.dict() + aggregate_feedback.append(cast(dict, result)) + executor.submit( + self.client.create_feedback, + **result, + run_id=None, + project_id=self.project.id, + ) + except Exception as e: + logger.error( + f"Error running batch evaluator {repr(evaluator)}: {e}" + ) + return aggregate_feedback + + def _collect_metrics(self) -> Tuple[Dict[str, _RowResult], Dict[str, Run]]: all_eval_results: dict = {} + all_runs: dict = {} for c in self.configs: for callback in cast(list, c["callbacks"]): if isinstance(callback, EvaluatorCallbackHandler): @@ -1077,20 +1119,28 @@ class _DatasetRunContainer: { "execution_time": execution_time, "run_id": run_id, + "run": run, } ) - return cast(Dict[str, _RowResult], all_eval_results) + all_runs[str(callback.example_id)] = run + return cast(Dict[str, _RowResult], all_eval_results), all_runs def _collect_test_results( self, batch_results: List[Union[dict, str, LLMResult, ChatResult]], ) -> TestResult: + logger.info("Waiting for evaluators to complete.") wait_for_all_evaluators() - all_eval_results = self._collect_metrics() + all_eval_results, all_runs = self._collect_metrics() + aggregate_feedback = None + if self.batch_evaluators: + logger.info("Running session evaluators.") + aggregate_feedback = self._run_batch_evaluators(all_runs) results = self._merge_test_outputs(batch_results, all_eval_results) return TestResult( project_name=self.project.name, results=results, + aggregate_metrics=aggregate_feedback, ) def finish(self, batch_results: list, verbose: bool = False) -> TestResult: @@ -1140,6 +1190,9 @@ class _DatasetRunContainer: tags = tags or [] for k, v in (project.metadata.get("git") or {}).items(): tags.append(f"git:{k}={v}") + run_metadata = {"dataset_version": project.metadata["dataset_version"]} + if revision_id: + run_metadata["revision_id"] = revision_id wrapped_model = _wrap_in_chain_factory(llm_or_chain_factory) run_evaluators = _setup_evaluation( wrapped_model, examples, evaluation, dataset.data_type or DataType.kv @@ -1164,7 +1217,7 @@ class _DatasetRunContainer: ], tags=tags, max_concurrency=concurrency_level, - metadata={"revision_id": revision_id} if revision_id else {}, + metadata=run_metadata, ) for example in examples ] @@ -1174,6 +1227,7 @@ class _DatasetRunContainer: wrapped_model=wrapped_model, examples=examples, configs=configs, + batch_evaluators=evaluation.batch_evaluators if evaluation else None, ) diff --git a/libs/langchain/poetry.lock b/libs/langchain/poetry.lock index b1bcf76ef93..55565f298fb 100644 --- a/libs/langchain/poetry.lock +++ b/libs/langchain/poetry.lock @@ -3038,6 +3038,7 @@ files = [ {file = "jq-1.6.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:227b178b22a7f91ae88525810441791b1ca1fc71c86f03190911793be15cec3d"}, {file = "jq-1.6.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:780eb6383fbae12afa819ef676fc93e1548ae4b076c004a393af26a04b460742"}, {file = "jq-1.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:08ded6467f4ef89fec35b2bf310f210f8cd13fbd9d80e521500889edf8d22441"}, + {file = "jq-1.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49e44ed677713f4115bd5bf2dbae23baa4cd503be350e12a1c1f506b0687848f"}, {file = "jq-1.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:984f33862af285ad3e41e23179ac4795f1701822473e1a26bf87ff023e5a89ea"}, {file = "jq-1.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f42264fafc6166efb5611b5d4cb01058887d050a6c19334f6a3f8a13bb369df5"}, {file = "jq-1.6.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a67154f150aaf76cc1294032ed588436eb002097dd4fd1e283824bf753a05080"}, @@ -3121,7 +3122,6 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, - {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, ] [[package]] @@ -3525,16 +3525,17 @@ url = "../text-splitters" [[package]] name = "langsmith" -version = "0.1.1" +version = "0.1.14" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langsmith-0.1.1-py3-none-any.whl", hash = "sha256:10ff2b977a41e3f6351d1a4239d9bd57af0547aa909e839d2791e16cc197a6f9"}, - {file = "langsmith-0.1.1.tar.gz", hash = "sha256:09df0c2ca9085105f97a4e4f281b083e312c99d162f3fe2b2d5eefd5c3692e60"}, + {file = "langsmith-0.1.14-py3-none-any.whl", hash = "sha256:ecb243057d2a43c2da0524fe395585bc3421bb5d24f1cdd53eb06fbe63e43a69"}, + {file = "langsmith-0.1.14.tar.gz", hash = "sha256:b95f267d25681f4c9862bb68236fba8a57a60ec7921ecfdaa125936807e51bde"}, ] [package.dependencies] +orjson = ">=3.9.14,<4.0.0" pydantic = ">=1,<3" requests = ">=2,<3" @@ -3751,16 +3752,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -4723,61 +4714,61 @@ requests = ">=2,<3" [[package]] name = "orjson" -version = "3.9.10" +version = "3.9.15" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" -optional = true +optional = false python-versions = ">=3.8" files = [ - {file = "orjson-3.9.10-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:c18a4da2f50050a03d1da5317388ef84a16013302a5281d6f64e4a3f406aabc4"}, - {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5148bab4d71f58948c7c39d12b14a9005b6ab35a0bdf317a8ade9a9e4d9d0bd5"}, - {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4cf7837c3b11a2dfb589f8530b3cff2bd0307ace4c301e8997e95c7468c1378e"}, - {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c62b6fa2961a1dcc51ebe88771be5319a93fd89bd247c9ddf732bc250507bc2b"}, - {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:deeb3922a7a804755bbe6b5be9b312e746137a03600f488290318936c1a2d4dc"}, - {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1234dc92d011d3554d929b6cf058ac4a24d188d97be5e04355f1b9223e98bbe9"}, - {file = "orjson-3.9.10-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:06ad5543217e0e46fd7ab7ea45d506c76f878b87b1b4e369006bdb01acc05a83"}, - {file = "orjson-3.9.10-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4fd72fab7bddce46c6826994ce1e7de145ae1e9e106ebb8eb9ce1393ca01444d"}, - {file = "orjson-3.9.10-cp310-none-win32.whl", hash = "sha256:b5b7d4a44cc0e6ff98da5d56cde794385bdd212a86563ac321ca64d7f80c80d1"}, - {file = "orjson-3.9.10-cp310-none-win_amd64.whl", hash = "sha256:61804231099214e2f84998316f3238c4c2c4aaec302df12b21a64d72e2a135c7"}, - {file = "orjson-3.9.10-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cff7570d492bcf4b64cc862a6e2fb77edd5e5748ad715f487628f102815165e9"}, - {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed8bc367f725dfc5cabeed1ae079d00369900231fbb5a5280cf0736c30e2adf7"}, - {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c812312847867b6335cfb264772f2a7e85b3b502d3a6b0586aa35e1858528ab1"}, - {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9edd2856611e5050004f4722922b7b1cd6268da34102667bd49d2a2b18bafb81"}, - {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:674eb520f02422546c40401f4efaf8207b5e29e420c17051cddf6c02783ff5ca"}, - {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d0dc4310da8b5f6415949bd5ef937e60aeb0eb6b16f95041b5e43e6200821fb"}, - {file = "orjson-3.9.10-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e99c625b8c95d7741fe057585176b1b8783d46ed4b8932cf98ee145c4facf499"}, - {file = "orjson-3.9.10-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ec6f18f96b47299c11203edfbdc34e1b69085070d9a3d1f302810cc23ad36bf3"}, - {file = "orjson-3.9.10-cp311-none-win32.whl", hash = "sha256:ce0a29c28dfb8eccd0f16219360530bc3cfdf6bf70ca384dacd36e6c650ef8e8"}, - {file = "orjson-3.9.10-cp311-none-win_amd64.whl", hash = "sha256:cf80b550092cc480a0cbd0750e8189247ff45457e5a023305f7ef1bcec811616"}, - {file = "orjson-3.9.10-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:602a8001bdf60e1a7d544be29c82560a7b49319a0b31d62586548835bbe2c862"}, - {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f295efcd47b6124b01255d1491f9e46f17ef40d3d7eabf7364099e463fb45f0f"}, - {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:92af0d00091e744587221e79f68d617b432425a7e59328ca4c496f774a356071"}, - {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5a02360e73e7208a872bf65a7554c9f15df5fe063dc047f79738998b0506a14"}, - {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:858379cbb08d84fe7583231077d9a36a1a20eb72f8c9076a45df8b083724ad1d"}, - {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666c6fdcaac1f13eb982b649e1c311c08d7097cbda24f32612dae43648d8db8d"}, - {file = "orjson-3.9.10-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3fb205ab52a2e30354640780ce4587157a9563a68c9beaf52153e1cea9aa0921"}, - {file = "orjson-3.9.10-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7ec960b1b942ee3c69323b8721df2a3ce28ff40e7ca47873ae35bfafeb4555ca"}, - {file = "orjson-3.9.10-cp312-none-win_amd64.whl", hash = "sha256:3e892621434392199efb54e69edfff9f699f6cc36dd9553c5bf796058b14b20d"}, - {file = "orjson-3.9.10-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:8b9ba0ccd5a7f4219e67fbbe25e6b4a46ceef783c42af7dbc1da548eb28b6531"}, - {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e2ecd1d349e62e3960695214f40939bbfdcaeaaa62ccc638f8e651cf0970e5f"}, - {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f433be3b3f4c66016d5a20e5b4444ef833a1f802ced13a2d852c637f69729c1"}, - {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4689270c35d4bb3102e103ac43c3f0b76b169760aff8bcf2d401a3e0e58cdb7f"}, - {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4bd176f528a8151a6efc5359b853ba3cc0e82d4cd1fab9c1300c5d957dc8f48c"}, - {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a2ce5ea4f71681623f04e2b7dadede3c7435dfb5e5e2d1d0ec25b35530e277b"}, - {file = "orjson-3.9.10-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:49f8ad582da6e8d2cf663c4ba5bf9f83cc052570a3a767487fec6af839b0e777"}, - {file = "orjson-3.9.10-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2a11b4b1a8415f105d989876a19b173f6cdc89ca13855ccc67c18efbd7cbd1f8"}, - {file = "orjson-3.9.10-cp38-none-win32.whl", hash = "sha256:a353bf1f565ed27ba71a419b2cd3db9d6151da426b61b289b6ba1422a702e643"}, - {file = "orjson-3.9.10-cp38-none-win_amd64.whl", hash = "sha256:e28a50b5be854e18d54f75ef1bb13e1abf4bc650ab9d635e4258c58e71eb6ad5"}, - {file = "orjson-3.9.10-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ee5926746232f627a3be1cc175b2cfad24d0170d520361f4ce3fa2fd83f09e1d"}, - {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a73160e823151f33cdc05fe2cea557c5ef12fdf276ce29bb4f1c571c8368a60"}, - {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c338ed69ad0b8f8f8920c13f529889fe0771abbb46550013e3c3d01e5174deef"}, - {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5869e8e130e99687d9e4be835116c4ebd83ca92e52e55810962446d841aba8de"}, - {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2c1e559d96a7f94a4f581e2a32d6d610df5840881a8cba8f25e446f4d792df3"}, - {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a3a3a72c9811b56adf8bcc829b010163bb2fc308877e50e9910c9357e78521"}, - {file = "orjson-3.9.10-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7f8fb7f5ecf4f6355683ac6881fd64b5bb2b8a60e3ccde6ff799e48791d8f864"}, - {file = "orjson-3.9.10-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c943b35ecdf7123b2d81d225397efddf0bce2e81db2f3ae633ead38e85cd5ade"}, - {file = "orjson-3.9.10-cp39-none-win32.whl", hash = "sha256:fb0b361d73f6b8eeceba47cd37070b5e6c9de5beaeaa63a1cb35c7e1a73ef088"}, - {file = "orjson-3.9.10-cp39-none-win_amd64.whl", hash = "sha256:b90f340cb6397ec7a854157fac03f0c82b744abdd1c0941a024c3c29d1340aff"}, - {file = "orjson-3.9.10.tar.gz", hash = "sha256:9ebbdbd6a046c304b1845e96fbcc5559cd296b4dfd3ad2509e33c4d9ce07d6a1"}, + {file = "orjson-3.9.15-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:d61f7ce4727a9fa7680cd6f3986b0e2c732639f46a5e0156e550e35258aa313a"}, + {file = "orjson-3.9.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4feeb41882e8aa17634b589533baafdceb387e01e117b1ec65534ec724023d04"}, + {file = "orjson-3.9.15-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fbbeb3c9b2edb5fd044b2a070f127a0ac456ffd079cb82746fc84af01ef021a4"}, + {file = "orjson-3.9.15-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b66bcc5670e8a6b78f0313bcb74774c8291f6f8aeef10fe70e910b8040f3ab75"}, + {file = "orjson-3.9.15-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2973474811db7b35c30248d1129c64fd2bdf40d57d84beed2a9a379a6f57d0ab"}, + {file = "orjson-3.9.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fe41b6f72f52d3da4db524c8653e46243c8c92df826ab5ffaece2dba9cccd58"}, + {file = "orjson-3.9.15-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4228aace81781cc9d05a3ec3a6d2673a1ad0d8725b4e915f1089803e9efd2b99"}, + {file = "orjson-3.9.15-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6f7b65bfaf69493c73423ce9db66cfe9138b2f9ef62897486417a8fcb0a92bfe"}, + {file = "orjson-3.9.15-cp310-none-win32.whl", hash = "sha256:2d99e3c4c13a7b0fb3792cc04c2829c9db07838fb6973e578b85c1745e7d0ce7"}, + {file = "orjson-3.9.15-cp310-none-win_amd64.whl", hash = "sha256:b725da33e6e58e4a5d27958568484aa766e825e93aa20c26c91168be58e08cbb"}, + {file = "orjson-3.9.15-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:c8e8fe01e435005d4421f183038fc70ca85d2c1e490f51fb972db92af6e047c2"}, + {file = "orjson-3.9.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87f1097acb569dde17f246faa268759a71a2cb8c96dd392cd25c668b104cad2f"}, + {file = "orjson-3.9.15-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff0f9913d82e1d1fadbd976424c316fbc4d9c525c81d047bbdd16bd27dd98cfc"}, + {file = "orjson-3.9.15-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8055ec598605b0077e29652ccfe9372247474375e0e3f5775c91d9434e12d6b1"}, + {file = "orjson-3.9.15-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d6768a327ea1ba44c9114dba5fdda4a214bdb70129065cd0807eb5f010bfcbb5"}, + {file = "orjson-3.9.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12365576039b1a5a47df01aadb353b68223da413e2e7f98c02403061aad34bde"}, + {file = "orjson-3.9.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:71c6b009d431b3839d7c14c3af86788b3cfac41e969e3e1c22f8a6ea13139404"}, + {file = "orjson-3.9.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e18668f1bd39e69b7fed19fa7cd1cd110a121ec25439328b5c89934e6d30d357"}, + {file = "orjson-3.9.15-cp311-none-win32.whl", hash = "sha256:62482873e0289cf7313461009bf62ac8b2e54bc6f00c6fabcde785709231a5d7"}, + {file = "orjson-3.9.15-cp311-none-win_amd64.whl", hash = "sha256:b3d336ed75d17c7b1af233a6561cf421dee41d9204aa3cfcc6c9c65cd5bb69a8"}, + {file = "orjson-3.9.15-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:82425dd5c7bd3adfe4e94c78e27e2fa02971750c2b7ffba648b0f5d5cc016a73"}, + {file = "orjson-3.9.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c51378d4a8255b2e7c1e5cc430644f0939539deddfa77f6fac7b56a9784160a"}, + {file = "orjson-3.9.15-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6ae4e06be04dc00618247c4ae3f7c3e561d5bc19ab6941427f6d3722a0875ef7"}, + {file = "orjson-3.9.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bcef128f970bb63ecf9a65f7beafd9b55e3aaf0efc271a4154050fc15cdb386e"}, + {file = "orjson-3.9.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b72758f3ffc36ca566ba98a8e7f4f373b6c17c646ff8ad9b21ad10c29186f00d"}, + {file = "orjson-3.9.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c57bc7b946cf2efa67ac55766e41764b66d40cbd9489041e637c1304400494"}, + {file = "orjson-3.9.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:946c3a1ef25338e78107fba746f299f926db408d34553b4754e90a7de1d44068"}, + {file = "orjson-3.9.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2f256d03957075fcb5923410058982aea85455d035607486ccb847f095442bda"}, + {file = "orjson-3.9.15-cp312-none-win_amd64.whl", hash = "sha256:5bb399e1b49db120653a31463b4a7b27cf2fbfe60469546baf681d1b39f4edf2"}, + {file = "orjson-3.9.15-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b17f0f14a9c0ba55ff6279a922d1932e24b13fc218a3e968ecdbf791b3682b25"}, + {file = "orjson-3.9.15-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f6cbd8e6e446fb7e4ed5bac4661a29e43f38aeecbf60c4b900b825a353276a1"}, + {file = "orjson-3.9.15-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:76bc6356d07c1d9f4b782813094d0caf1703b729d876ab6a676f3aaa9a47e37c"}, + {file = "orjson-3.9.15-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fdfa97090e2d6f73dced247a2f2d8004ac6449df6568f30e7fa1a045767c69a6"}, + {file = "orjson-3.9.15-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7413070a3e927e4207d00bd65f42d1b780fb0d32d7b1d951f6dc6ade318e1b5a"}, + {file = "orjson-3.9.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9cf1596680ac1f01839dba32d496136bdd5d8ffb858c280fa82bbfeb173bdd40"}, + {file = "orjson-3.9.15-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:809d653c155e2cc4fd39ad69c08fdff7f4016c355ae4b88905219d3579e31eb7"}, + {file = "orjson-3.9.15-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:920fa5a0c5175ab14b9c78f6f820b75804fb4984423ee4c4f1e6d748f8b22bc1"}, + {file = "orjson-3.9.15-cp38-none-win32.whl", hash = "sha256:2b5c0f532905e60cf22a511120e3719b85d9c25d0e1c2a8abb20c4dede3b05a5"}, + {file = "orjson-3.9.15-cp38-none-win_amd64.whl", hash = "sha256:67384f588f7f8daf040114337d34a5188346e3fae6c38b6a19a2fe8c663a2f9b"}, + {file = "orjson-3.9.15-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:6fc2fe4647927070df3d93f561d7e588a38865ea0040027662e3e541d592811e"}, + {file = "orjson-3.9.15-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34cbcd216e7af5270f2ffa63a963346845eb71e174ea530867b7443892d77180"}, + {file = "orjson-3.9.15-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f541587f5c558abd93cb0de491ce99a9ef8d1ae29dd6ab4dbb5a13281ae04cbd"}, + {file = "orjson-3.9.15-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92255879280ef9c3c0bcb327c5a1b8ed694c290d61a6a532458264f887f052cb"}, + {file = "orjson-3.9.15-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:05a1f57fb601c426635fcae9ddbe90dfc1ed42245eb4c75e4960440cac667262"}, + {file = "orjson-3.9.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ede0bde16cc6e9b96633df1631fbcd66491d1063667f260a4f2386a098393790"}, + {file = "orjson-3.9.15-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e88b97ef13910e5f87bcbc4dd7979a7de9ba8702b54d3204ac587e83639c0c2b"}, + {file = "orjson-3.9.15-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:57d5d8cf9c27f7ef6bc56a5925c7fbc76b61288ab674eb352c26ac780caa5b10"}, + {file = "orjson-3.9.15-cp39-none-win32.whl", hash = "sha256:001f4eb0ecd8e9ebd295722d0cbedf0748680fb9998d3993abaed2f40587257a"}, + {file = "orjson-3.9.15-cp39-none-win_amd64.whl", hash = "sha256:ea0b183a5fe6b2b45f3b854b0d19c4e932d6f5934ae1f723b07cf9560edd4ec7"}, + {file = "orjson-3.9.15.tar.gz", hash = "sha256:95cae920959d772f30ab36d3b25f83bb0f3be671e986c72ce22f8fa700dae061"}, ] [[package]] @@ -5287,8 +5278,6 @@ files = [ {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, - {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, - {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, @@ -5331,7 +5320,6 @@ files = [ {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, @@ -5340,8 +5328,6 @@ files = [ {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, @@ -5814,7 +5800,6 @@ files = [ {file = "pymongo-4.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6422b6763b016f2ef2beedded0e546d6aa6ba87910f9244d86e0ac7690f75c96"}, {file = "pymongo-4.5.0-cp312-cp312-win32.whl", hash = "sha256:77cfff95c1fafd09e940b3fdcb7b65f11442662fad611d0e69b4dd5d17a81c60"}, {file = "pymongo-4.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:e57d859b972c75ee44ea2ef4758f12821243e99de814030f69a3decb2aa86807"}, - {file = "pymongo-4.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8443f3a8ab2d929efa761c6ebce39a6c1dca1c9ac186ebf11b62c8fe1aef53f4"}, {file = "pymongo-4.5.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:2b0176f9233a5927084c79ff80b51bd70bfd57e4f3d564f50f80238e797f0c8a"}, {file = "pymongo-4.5.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:89b3f2da57a27913d15d2a07d58482f33d0a5b28abd20b8e643ab4d625e36257"}, {file = "pymongo-4.5.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:5caee7bd08c3d36ec54617832b44985bd70c4cbd77c5b313de6f7fce0bb34f93"}, @@ -6331,7 +6316,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -6339,16 +6323,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -6365,7 +6341,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -6373,7 +6348,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -9158,4 +9132,4 @@ text-helpers = ["chardet"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "86f58f5567562ea0058941a6066f9c687f1bc39117df7a0308b9006f190e8197" +content-hash = "095a661dc5f767d2a3c92541d66c2d3070e86fb455b98f313e6c9a40b699b4ef" diff --git a/libs/langchain/pyproject.toml b/libs/langchain/pyproject.toml index f372bd5a742..6bca8997d72 100644 --- a/libs/langchain/pyproject.toml +++ b/libs/langchain/pyproject.toml @@ -15,7 +15,7 @@ python = ">=3.8.1,<4.0" langchain-core = ">=0.1.28,<0.2" langchain-text-splitters = ">=0.0.1,<0.1" langchain-community = ">=0.0.25,<0.1" -langsmith = "^0.1.0" +langsmith = "^0.1.14" pydantic = ">=1,<3" SQLAlchemy = ">=1.4,<3" requests = "^2"