From 6950b44bfccb3170523d846462d80e2c64c97076 Mon Sep 17 00:00:00 2001 From: William FH <13333726+hinthornw@users.noreply.github.com> Date: Tue, 3 Oct 2023 06:20:58 -0700 Subject: [PATCH] Consolidate run collector. Add link helper (#11269) Instead of: ``` client = Client() with collect_runs() as cb: chain.invoke() run = cb.traced_runs[0] client.get_run_url(run) ``` it's ``` with tracing_v2_enabled() as cb: chain.invoke() cb.get_run_url() ``` --- libs/langchain/langchain/callbacks/manager.py | 14 ++++++-- .../langchain/callbacks/tracers/langchain.py | 33 +++++++++++++++++-- libs/langchain/poetry.lock | 9 +++-- libs/langchain/pyproject.toml | 2 +- 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/libs/langchain/langchain/callbacks/manager.py b/libs/langchain/langchain/callbacks/manager.py index 19f041005aa..ebfce11e31c 100644 --- a/libs/langchain/langchain/callbacks/manager.py +++ b/libs/langchain/langchain/callbacks/manager.py @@ -41,7 +41,9 @@ from langchain.callbacks.base import ( from langchain.callbacks.openai_info import OpenAICallbackHandler from langchain.callbacks.stdout import StdOutCallbackHandler from langchain.callbacks.tracers import run_collector -from langchain.callbacks.tracers.langchain import LangChainTracer +from langchain.callbacks.tracers.langchain import ( + LangChainTracer, +) from langchain.callbacks.tracers.langchain_v1 import LangChainTracerV1, TracerSessionV1 from langchain.callbacks.tracers.stdout import ConsoleCallbackHandler from langchain.callbacks.tracers.wandb import WandbTracer @@ -161,7 +163,7 @@ def tracing_v2_enabled( example_id: Optional[Union[str, UUID]] = None, tags: Optional[List[str]] = None, client: Optional[LangSmithClient] = None, -) -> Generator[None, None, None]: +) -> Generator[LangChainTracer, None, None]: """Instruct LangChain to log all runs in context to LangSmith. Args: @@ -178,6 +180,12 @@ def tracing_v2_enabled( Example: >>> with tracing_v2_enabled(): ... # LangChain code will automatically be traced + + You can use this to fetch the LangSmith run URL: + + >>> with tracing_v2_enabled() as cb: + ... chain.invoke("foo") + ... run_url = cb.get_run_url() """ if isinstance(example_id, str): example_id = UUID(example_id) @@ -188,7 +196,7 @@ def tracing_v2_enabled( client=client, ) tracing_v2_callback_var.set(cb) - yield + yield cb tracing_v2_callback_var.set(None) diff --git a/libs/langchain/langchain/callbacks/tracers/langchain.py b/libs/langchain/langchain/callbacks/tracers/langchain.py index fd91fa38424..4091eeae692 100644 --- a/libs/langchain/langchain/callbacks/tracers/langchain.py +++ b/libs/langchain/langchain/callbacks/tracers/langchain.py @@ -10,9 +10,16 @@ from typing import Any, Callable, Dict, List, Optional, Union from uuid import UUID from langsmith import Client +from langsmith.utils import LangSmithError +from tenacity import ( + Retrying, + retry_if_exception_type, + stop_after_attempt, + wait_exponential_jitter, +) from langchain.callbacks.tracers.base import BaseTracer -from langchain.callbacks.tracers.schemas import Run, TracerSession +from langchain.callbacks.tracers.schemas import Run from langchain.env import get_runtime_environment from langchain.load.dump import dumpd from langchain.schema.messages import BaseMessage @@ -71,7 +78,6 @@ class LangChainTracer(BaseTracer): ) -> None: """Initialize the LangChain tracer.""" super().__init__(**kwargs) - self.session: Optional[TracerSession] = None self.example_id = ( UUID(example_id) if isinstance(example_id, str) else example_id ) @@ -82,6 +88,7 @@ class LangChainTracer(BaseTracer): self._futures: weakref.WeakSet[Future] = weakref.WeakSet() self.tags = tags or [] self.executor = _get_executor() if use_threading else None + self.latest_run: Optional[Run] = None global _TRACERS _TRACERS.add(self) @@ -121,7 +128,27 @@ class LangChainTracer(BaseTracer): self._on_chat_model_start(chat_model_run) def _persist_run(self, run: Run) -> None: - """The Langchain Tracer uses Post/Patch rather than persist.""" + run_ = run.copy() + run_.reference_example_id = self.example_id + self.latest_run = run_ + + def get_run_url(self) -> str: + """Get the LangSmith root run URL""" + if not self.latest_run: + raise ValueError("No traced run found.") + # If this is the first run in a project, the project may not yet be created. + # This method is only really useful for debugging flows, so we will assume + # there is some tolerace for latency. + for attempt in Retrying( + stop=stop_after_attempt(5), + wait=wait_exponential_jitter(), + retry=retry_if_exception_type(LangSmithError), + ): + with attempt: + return self.client.get_run_url( + run=self.latest_run, project_name=self.project_name + ) + raise ValueError("Failed to get run URL.") def _get_tags(self, run: Run) -> List[str]: """Get combined tags for a run.""" diff --git a/libs/langchain/poetry.lock b/libs/langchain/poetry.lock index c46e0258669..e3955d35213 100644 --- a/libs/langchain/poetry.lock +++ b/libs/langchain/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "absl-py" @@ -3671,7 +3671,6 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, - {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, ] [[package]] @@ -5843,7 +5842,7 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] python-dateutil = ">=2.8.2" @@ -8803,7 +8802,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} +greenlet = {version = "!=0.4.17", markers = "platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\""} typing-extensions = ">=4.2.0" [package.extras] @@ -10616,4 +10615,4 @@ text-helpers = ["chardet"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "5dbafef78ce5d6eb643885038ee5f95c95684c4097cad3d00e8bacb7e668e950" +content-hash = "cd1e7089bf95e9eb885c392790b2afa85168b3af8ee8227727f8a95a4b60419b" diff --git a/libs/langchain/pyproject.toml b/libs/langchain/pyproject.toml index 25e22945172..f3d9f67cb72 100644 --- a/libs/langchain/pyproject.toml +++ b/libs/langchain/pyproject.toml @@ -113,7 +113,7 @@ cassio = {version = "^0.1.0", optional = true} rdflib = {version = "^6.3.2", optional = true} sympy = {version = "^1.12", optional = true} rapidfuzz = {version = "^3.1.1", optional = true} -langsmith = "~0.0.38" +langsmith = "~0.0.40" rank-bm25 = {version = "^0.2.2", optional = true} amadeus = {version = ">=8.1.0", optional = true} geopandas = {version = "^0.13.1", optional = true}