From 404abf139a24a7e519bd98f18f2298e206ae0624 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Tue, 23 Jan 2024 00:17:16 +0100 Subject: [PATCH] community: Add CometLLM tracing context var (#15765) I also added LANGCHAIN_COMET_TRACING to enable the CometLLM tracing integration similar to other tracing integrations. This is easier for end-users to enable it rather than importing the callback and pass it manually. (This is the same content as https://github.com/langchain-ai/langchain/pull/14650 but rebased and squashed as something seems to confuse Github Action). --- .../callbacks/comet_tracing.ipynb | 138 ++++++++++++++++++ .../langchain_community/callbacks/manager.py | 7 + 2 files changed, 145 insertions(+) create mode 100644 docs/docs/integrations/callbacks/comet_tracing.ipynb diff --git a/docs/docs/integrations/callbacks/comet_tracing.ipynb b/docs/docs/integrations/callbacks/comet_tracing.ipynb new file mode 100644 index 00000000000..aff2be56102 --- /dev/null +++ b/docs/docs/integrations/callbacks/comet_tracing.ipynb @@ -0,0 +1,138 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "5371a9bb", + "metadata": {}, + "source": [ + "# Comet Tracing\n", + "\n", + "There are two ways to trace your LangChains executions with Comet:\n", + "\n", + "1. Setting the `LANGCHAIN_COMET_TRACING` environment variable to \"true\". This is the recommended way.\n", + "2. Import the `CometTracer` manually and pass it explicitely." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17c04cc6-c93d-4b6c-a033-e897577f4ed1", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-18T12:47:46.580776Z", + "start_time": "2023-05-18T12:47:46.577833Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import comet_llm\n", + "\n", + "os.environ[\"LANGCHAIN_COMET_TRACING\"] = \"true\"\n", + "\n", + "# Connect to Comet if no API Key is set\n", + "comet_llm.init()\n", + "\n", + "# comet documentation to configure comet using env variables\n", + "# https://www.comet.com/docs/v2/api-and-sdk/llm-sdk/configuration/\n", + "# here we are configuring the comet project\n", + "os.environ[\"COMET_PROJECT_NAME\"] = \"comet-example-langchain-tracing\"\n", + "\n", + "from langchain.agents import AgentType, initialize_agent, load_tools\n", + "from langchain.llms import OpenAI" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1b62cd48", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-18T12:47:47.445229Z", + "start_time": "2023-05-18T12:47:47.436424Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Agent run with tracing. Ensure that OPENAI_API_KEY is set appropriately to run this example.\n", + "\n", + "llm = OpenAI(temperature=0)\n", + "tools = load_tools([\"llm-math\"], llm=llm)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfa16b79-aa4b-4d41-a067-70d1f593f667", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-18T12:48:01.816137Z", + "start_time": "2023-05-18T12:47:49.109574Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "agent = initialize_agent(\n", + " tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True\n", + ")\n", + "\n", + "agent.run(\"What is 2 raised to .123243 power?\") # this should be traced\n", + "# An url for the chain like the following should print in your console:\n", + "# https://www.comet.com//\n", + "# The url can be used to view the LLM chain in Comet." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e212e7d", + "metadata": {}, + "outputs": [], + "source": [ + "# Now, we unset the environment variable and use a context manager.\n", + "if \"LANGCHAIN_COMET_TRACING\" in os.environ:\n", + " del os.environ[\"LANGCHAIN_COMET_TRACING\"]\n", + "\n", + "from langchain.callbacks.tracers.comet import CometTracer\n", + "\n", + "tracer = CometTracer()\n", + "\n", + "# Recreate the LLM, tools and agent and passing the callback to each of them\n", + "llm = OpenAI(temperature=0)\n", + "tools = load_tools([\"llm-math\"], llm=llm)\n", + "agent = initialize_agent(\n", + " tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True\n", + ")\n", + "\n", + "agent.run(\n", + " \"What is 2 raised to .123243 power?\", callbacks=[tracer]\n", + ") # this should be traced" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/libs/community/langchain_community/callbacks/manager.py b/libs/community/langchain_community/callbacks/manager.py index 196afed3d0f..ec03a823453 100644 --- a/libs/community/langchain_community/callbacks/manager.py +++ b/libs/community/langchain_community/callbacks/manager.py @@ -11,6 +11,7 @@ from typing import ( from langchain_core.tracers.context import register_configure_hook from langchain_community.callbacks.openai_info import OpenAICallbackHandler +from langchain_community.callbacks.tracers.comet import CometTracer from langchain_community.callbacks.tracers.wandb import WandbTracer logger = logging.getLogger(__name__) @@ -21,11 +22,17 @@ openai_callback_var: ContextVar[Optional[OpenAICallbackHandler]] = ContextVar( wandb_tracing_callback_var: ContextVar[Optional[WandbTracer]] = ContextVar( # noqa: E501 "tracing_wandb_callback", default=None ) +comet_tracing_callback_var: ContextVar[Optional[CometTracer]] = ContextVar( # noqa: E501 + "tracing_comet_callback", default=None +) register_configure_hook(openai_callback_var, True) register_configure_hook( wandb_tracing_callback_var, True, WandbTracer, "LANGCHAIN_WANDB_TRACING" ) +register_configure_hook( + comet_tracing_callback_var, True, CometTracer, "LANGCHAIN_COMET_TRACING" +) @contextmanager