thread more

libs/langchain/Untitled.ipynb

@@ -0,0 +1,166 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "e2629b03-534a-439f-9895-7e551817be7d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "0c35e2fb-c3b9-4ff6-ba42-8919cc789545",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.llms import OpenAI\n",
+    "from langchain.chat_models import ChatOpenAI\n",
+    "\n",
+    "llm = OpenAI()\n",
+    "chat_model = ChatOpenAI()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "5530b714-1e5e-45d5-bab7-96a90b99e71c",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit error.\n",
+      "Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit error.\n",
+      "Retrying langchain.llms.openai.acompletion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit error.\n",
+      "Error in on_retry: asyncio.run() cannot be called from a running event loop\n",
+      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tenacity/__init__.py:338: RuntimeWarning: coroutine 'AsyncCallbackManagerForLLMRun.on_retry' was never awaited\n",
+      "  self.before_sleep(retry_state)\n",
+      "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n",
+      "Retrying langchain.llms.openai.acompletion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit error.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'NR responder\\n\\nLLMNR (Link-Local Multicast Name Resolution) is a protocol that allows devices to resolve the names of other devices on the local network. An LLMNR responder is a piece of software that responds to queries from other devices using the LLMNR protocol. LLMNR responders are typically installed as part of the operating system on Windows, macOS, and Linux systems. Common open-source LLMNR responders include LMNRD and Avahi.'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "llm.predict(\"Sync LLM\")\n",
+    "await llm.apredict(\"Async LLM\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "ff512507-c6be-4235-9327-a7fa12d7c3a5",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit error.\n",
+      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit error.\n",
+      "Retrying langchain.chat_models.openai.acompletion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit error.\n",
+      "Retrying langchain.chat_models.openai.acompletion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit error.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'Hello! How can I assist you today?'"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chat_model.predict(\"Sync Chat\")\n",
+    "await chat_model.apredict(\"Async Chat\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "a60786bc-3b6f-4432-9fa0-0b405e912ff5",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langsmith import Client\n",
+    "from itertools import islice\n",
+    "\n",
+    "client = Client()\n",
+    "runs = list(islice(client.list_runs(), 4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "801b8282-8a40-4c43-a511-6c078edd7c49",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[{'name': 'start', 'time': '2023-07-21T20:51:36.214448'}, {'name': 'end', 'time': '2023-07-21T20:51:44.967533'}]\n",
+      "[{'name': 'start', 'time': '2023-07-21T20:51:26.928913'}, {'name': 'retry', 'time': '2023-07-21T20:51:26.930591', 'kwargs': {'slept': 4.0, 'attempt': 1, 'outcome': 'failed', 'exception': 'Rate limit error', 'exception_cls': 'RateLimitError', 'outcome_timestamp': 584768.725407458}}, {'name': 'retry', 'time': '2023-07-21T20:51:30.938017', 'kwargs': {'slept': 8.0, 'attempt': 2, 'outcome': 'failed', 'exception': 'Rate limit error', 'exception_cls': 'RateLimitError', 'outcome_timestamp': 584772.73134625}}, {'name': 'end', 'time': '2023-07-21T20:51:36.203614'}]\n",
+      "[{'name': 'start', 'time': '2023-07-21T20:50:40.005962'}, {'name': 'end', 'time': '2023-07-21T20:50:48.876328'}]\n",
+      "[{'name': 'start', 'time': '2023-07-21T20:50:30.969464'}, {'name': 'retry', 'time': '2023-07-21T20:50:30.971400', 'kwargs': {'slept': 4.0, 'attempt': 1, 'outcome': 'failed', 'exception': 'Rate limit error', 'exception_cls': 'RateLimitError', 'outcome_timestamp': 584712.766095791}}, {'name': 'retry', 'time': '2023-07-21T20:50:34.990150', 'kwargs': {'slept': 8.0, 'attempt': 2, 'outcome': 'failed', 'exception': 'Rate limit error', 'exception_cls': 'RateLimitError', 'outcome_timestamp': 584716.784364416}}, {'name': 'end', 'time': '2023-07-21T20:50:39.997683'}]\n"
+     ]
+    }
+   ],
+   "source": [
+    "for run in runs:\n",
+    "    print(run.events)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

libs/langchain/langchain/agents/agent_iterator.py

@@ -95,7 +95,7 @@ class AgentExecutorIterator(BaseAgentExecutorIterator):
 
     @inputs.setter
     def inputs(self, inputs: Any) -> None:
-        self._inputs = self.agent_executor.prep_inputs(inputs)
+        self._inputs = self.agent_executor.prep_inputs(inputs, callbacks=None)
 
     @property
     def callbacks(self) -> Callbacks:

libs/langchain/langchain/chains/base.py

@@ -217,7 +217,6 @@ class Chain(Serializable, ABC):
             A dict of named outputs. Should contain all outputs specified in
                 `Chain.output_keys`.
         """
-        inputs = self.prep_inputs(inputs)
         callback_manager = CallbackManager.configure(
             callbacks,
             self.callbacks,
@@ -227,6 +226,7 @@ class Chain(Serializable, ABC):
             metadata,
             self.metadata,
         )
+        inputs = self.prep_inputs(inputs, callbacks=callback_manager)
         new_arg_supported = inspect.signature(self._call).parameters.get("run_manager")
         run_manager = callback_manager.on_chain_start(
             dumpd(self),
@@ -284,7 +284,6 @@ class Chain(Serializable, ABC):
             A dict of named outputs. Should contain all outputs specified in
                 `Chain.output_keys`.
         """
-        inputs = self.prep_inputs(inputs)
         callback_manager = AsyncCallbackManager.configure(
             callbacks,
             self.callbacks,
@@ -294,6 +293,7 @@ class Chain(Serializable, ABC):
             metadata,
             self.metadata,
         )
+        inputs = self.prep_inputs(inputs, callbacks=callback_manager)
         new_arg_supported = inspect.signature(self._acall).parameters.get("run_manager")
         run_manager = await callback_manager.on_chain_start(
             dumpd(self),
@@ -342,7 +342,9 @@ class Chain(Serializable, ABC):
         else:
             return {**inputs, **outputs}
 
-    def prep_inputs(self, inputs: Union[Dict[str, Any], Any]) -> Dict[str, str]:
+    def prep_inputs(
+        self, inputs: Union[Dict[str, Any], Any], *, callbacks: Callbacks
+    ) -> Dict[str, str]:
         """Validate and prepare chain inputs, including adding inputs from memory.
 
         Args:
@@ -369,7 +371,9 @@ class Chain(Serializable, ABC):
                 )
             inputs = {list(_input_keys)[0]: inputs}
         if self.memory is not None:
-            external_context = self.memory.load_memory_variables(inputs)
+            external_context = self.memory.load_memory_variables(
+                inputs, callbacks=callbacks
+            )
             inputs = dict(inputs, **external_context)
         self._validate_inputs(inputs)
         return inputs

libs/langchain/langchain/evaluation/agents/trajectory_eval_chain.py

@@ -262,11 +262,13 @@ The following is the expected answer. Use this to measure correctness:
             return ["score", "reasoning"]
         return ["score"]
 
-    def prep_inputs(self, inputs: Union[Dict[str, Any], Any]) -> Dict[str, str]:
+    def prep_inputs(
+        self, inputs: Union[Dict[str, Any], Any], *, callbacks: Callbacks
+    ) -> Dict[str, str]:
         """Validate and prep inputs."""
         if "reference" not in inputs:
             inputs["reference"] = self._format_reference(inputs.get("reference"))
-        return super().prep_inputs(inputs)
+        return super().prep_inputs(inputs, callbacks=callbacks)
 
     def _call(
         self,

libs/langchain/langchain/memory/combined.py

@@ -63,7 +63,7 @@ class CombinedMemory(BaseMemory):
 
         # Collect vars from all sub-memories
         for memory in self.memories:
-            data = memory.load_memory_variables(inputs)
+            data = memory.load_memory_variables(inputs, callbacks=callbacks)
             memory_data = {
                 **memory_data,
                 **data,

libs/langchain/langchain/memory/readonly.py

@@ -18,7 +18,7 @@ class ReadOnlySharedMemory(BaseMemory):
         self, inputs: Dict[str, Any], callbacks: Callbacks = None
     ) -> Dict[str, str]:
         """Load memory variables from memory."""
-        return self.memory.load_memory_variables(inputs)
+        return self.memory.load_memory_variables(inputs, callbacks=callbacks)
 
     def save_context(self, inputs: Dict[str, Any], outputs: Dict[str, str]) -> None:
         """Nothing should be saved or changed"""

libs/langchain/langchain/retrievers/self_query/base.py

@@ -94,7 +94,9 @@ class SelfQueryRetriever(BaseRetriever, BaseModel):
         Returns:
             List of relevant documents
         """
-        inputs = self.llm_chain.prep_inputs({"query": query})
+        inputs = self.llm_chain.prep_inputs(
+            {"query": query}, callbacks=run_manager.get_child()
+        )
         structured_query = cast(
             StructuredQuery,
             self.llm_chain.predict_and_parse(

libs/langchain/langchain/schema/memory.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List
 
-from langchain.callbacks.manager import Callbacks
+import langchain
 from langchain.load.serializable import Serializable
 from langchain.schema.messages import AIMessage, BaseMessage, HumanMessage
 
@@ -50,7 +50,9 @@ class BaseMemory(Serializable, ABC):
 
     @abstractmethod
     def load_memory_variables(
-        self, inputs: Dict[str, Any], callbacks: Callbacks = None
+        self,
+        inputs: Dict[str, Any],
+        callbacks: "langchain.callbacks.manager.Callbacks" = None,
     ) -> Dict[str, Any]:
         """Return key-value pairs given the text input to the chain."""