From df8702fead4a26263f8a3f0a6fc36db1a8c37c7b Mon Sep 17 00:00:00 2001 From: Aratako <127325395+Aratako@users.noreply.github.com> Date: Mon, 20 Mar 2023 03:01:03 +0900 Subject: [PATCH 01/11] Small fix: Remove unused variable `summary_message_role` (#1789) After the changes in #1783, `summary_message_role` is no longer used in `ConversationSummaryBufferMemory`, so this PR removes it. --- langchain/memory/summary_buffer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/langchain/memory/summary_buffer.py b/langchain/memory/summary_buffer.py index ae29f38a0fb..0e5b4734ee1 100644 --- a/langchain/memory/summary_buffer.py +++ b/langchain/memory/summary_buffer.py @@ -12,7 +12,6 @@ class ConversationSummaryBufferMemory(BaseChatMemory, SummarizerMixin, BaseModel max_token_limit: int = 2000 moving_summary_buffer: str = "" - summary_message_role: str = "system" memory_key: str = "history" @property From 8e5c4ac8672b98b3c53884843a4fe57a0b79dcae Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Sun, 19 Mar 2023 11:01:16 -0700 Subject: [PATCH 02/11] bump version to 0.0.116 (#1788) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 421ef7b6d92..cd76e5e07e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langchain" -version = "0.0.115" +version = "0.0.116" description = "Building applications with LLMs through composability" authors = [] license = "MIT" From 04acda55ec86949c6e096577635549121a7f38b6 Mon Sep 17 00:00:00 2001 From: Ankush Gola <9536492+agola11@users.noreply.github.com> Date: Sun, 19 Mar 2023 20:12:33 -0700 Subject: [PATCH 03/11] Don't use dynamic api endpoint for Zapier NLA (#1803) From Robert "Right now the dynamic/ route for specifically the above endpoints is acting on all providers a user has set up, not just the provider for the supplied API key." --- langchain/utilities/zapier.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/langchain/utilities/zapier.py b/langchain/utilities/zapier.py index 0ffb04f99cc..64a21514da5 100644 --- a/langchain/utilities/zapier.py +++ b/langchain/utilities/zapier.py @@ -38,7 +38,6 @@ class ZapierNLAWrapper(BaseModel): zapier_nla_api_key: str zapier_nla_api_base: str = "https://nla.zapier.com/api/v1/" - zapier_nla_api_dynamic_base: str = "https://nla.zapier.com/api/v1/dynamic/" class Config: """Configuration for this pydantic object.""" @@ -101,7 +100,7 @@ class ZapierNLAWrapper(BaseModel): https://nla.zapier.com/api/v1/dynamic/docs) """ session = self._get_session() - response = session.get(self.zapier_nla_api_dynamic_base + "exposed/") + response = session.get(self.zapier_nla_api_base + "exposed/") response.raise_for_status() return response.json()["results"] @@ -148,8 +147,8 @@ class ZapierNLAWrapper(BaseModel): data = self.preview(*args, **kwargs) return json.dumps(data) - def list_as_str(self, *args, **kwargs) -> str: # type: ignore[no-untyped-def] + def list_as_str(self) -> str: # type: ignore[no-untyped-def] """Same as list, but returns a stringified version of the JSON for insertting back into an LLM.""" - actions = self.list(*args, **kwargs) + actions = self.list() return json.dumps(actions) From b6ba989f2fe06b02077adcd9194595cc3d12246c Mon Sep 17 00:00:00 2001 From: Daniel Chalef <131175+danielchalef@users.noreply.github.com> Date: Sun, 19 Mar 2023 21:19:42 -0600 Subject: [PATCH 04/11] Add request timeout to ChatOpenAI (#1798) Add request_timeout field to ChatOpenAI. Defaults to 60s. --------- Co-authored-by: Daniel Chalef --- langchain/chat_models/openai.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/langchain/chat_models/openai.py b/langchain/chat_models/openai.py index 62736b53ef1..bea56fbf5f6 100644 --- a/langchain/chat_models/openai.py +++ b/langchain/chat_models/openai.py @@ -123,6 +123,8 @@ class ChatOpenAI(BaseChatModel, BaseModel): model_kwargs: Dict[str, Any] = Field(default_factory=dict) """Holds any model parameters valid for `create` call not explicitly specified.""" openai_api_key: Optional[str] = None + request_timeout: int = 60 + """Timeout in seconds for the OpenAPI request.""" max_retries: int = 6 """Maximum number of retries to make when generating.""" streaming: bool = False @@ -185,6 +187,7 @@ class ChatOpenAI(BaseChatModel, BaseModel): """Get the default parameters for calling OpenAI API.""" return { "model": self.model_name, + "request_timeout": self.request_timeout, "max_tokens": self.max_tokens, "stream": self.streaming, "n": self.n, From b1c4480d7cdf0b86d204641fc126188929a46439 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Mon, 20 Mar 2023 07:50:49 -0700 Subject: [PATCH 05/11] fix typing (#1807) --- langchain/chains/api/base.py | 4 ++-- langchain/chains/constitutional_ai/base.py | 4 ++-- langchain/chains/llm_bash/base.py | 4 ++-- langchain/chains/pal/base.py | 10 ++++++---- langchain/chains/qa_with_sources/base.py | 6 +++--- langchain/chains/qa_with_sources/loading.py | 22 +++++++++++---------- langchain/chains/sql_database/base.py | 6 +++--- langchain/chains/summarize/__init__.py | 20 ++++++++++--------- langchain/chains/vector_db_qa/base.py | 9 ++++++--- 9 files changed, 47 insertions(+), 38 deletions(-) diff --git a/langchain/chains/api/base.py b/langchain/chains/api/base.py index 5036a72c26b..5cbded4e0b1 100644 --- a/langchain/chains/api/base.py +++ b/langchain/chains/api/base.py @@ -8,9 +8,9 @@ from pydantic import BaseModel, Field, root_validator from langchain.chains.api.prompt import API_RESPONSE_PROMPT, API_URL_PROMPT from langchain.chains.base import Chain from langchain.chains.llm import LLMChain -from langchain.llms.base import BaseLLM from langchain.prompts import BasePromptTemplate from langchain.requests import RequestsWrapper +from langchain.schema import BaseLanguageModel class APIChain(Chain, BaseModel): @@ -84,7 +84,7 @@ class APIChain(Chain, BaseModel): @classmethod def from_llm_and_api_docs( cls, - llm: BaseLLM, + llm: BaseLanguageModel, api_docs: str, headers: Optional[dict] = None, api_url_prompt: BasePromptTemplate = API_URL_PROMPT, diff --git a/langchain/chains/constitutional_ai/base.py b/langchain/chains/constitutional_ai/base.py index b68e22de112..b78aa3a0ab2 100644 --- a/langchain/chains/constitutional_ai/base.py +++ b/langchain/chains/constitutional_ai/base.py @@ -5,8 +5,8 @@ from langchain.chains.base import Chain from langchain.chains.constitutional_ai.models import ConstitutionalPrinciple from langchain.chains.constitutional_ai.prompts import CRITIQUE_PROMPT, REVISION_PROMPT from langchain.chains.llm import LLMChain -from langchain.llms.base import BaseLLM from langchain.prompts.base import BasePromptTemplate +from langchain.schema import BaseLanguageModel class ConstitutionalChain(Chain): @@ -45,7 +45,7 @@ class ConstitutionalChain(Chain): @classmethod def from_llm( cls, - llm: BaseLLM, + llm: BaseLanguageModel, chain: LLMChain, critique_prompt: BasePromptTemplate = CRITIQUE_PROMPT, revision_prompt: BasePromptTemplate = REVISION_PROMPT, diff --git a/langchain/chains/llm_bash/base.py b/langchain/chains/llm_bash/base.py index 5a0d88eb560..994df302188 100644 --- a/langchain/chains/llm_bash/base.py +++ b/langchain/chains/llm_bash/base.py @@ -6,8 +6,8 @@ from pydantic import BaseModel, Extra from langchain.chains.base import Chain from langchain.chains.llm import LLMChain from langchain.chains.llm_bash.prompt import PROMPT -from langchain.llms.base import BaseLLM from langchain.prompts.base import BasePromptTemplate +from langchain.schema import BaseLanguageModel from langchain.utilities.bash import BashProcess @@ -21,7 +21,7 @@ class LLMBashChain(Chain, BaseModel): llm_bash = LLMBashChain(llm=OpenAI()) """ - llm: BaseLLM + llm: BaseLanguageModel """LLM wrapper to use.""" input_key: str = "question" #: :meta private: output_key: str = "answer" #: :meta private: diff --git a/langchain/chains/pal/base.py b/langchain/chains/pal/base.py index 5f574aaa9bb..443dd137de4 100644 --- a/langchain/chains/pal/base.py +++ b/langchain/chains/pal/base.py @@ -12,15 +12,15 @@ from langchain.chains.base import Chain from langchain.chains.llm import LLMChain from langchain.chains.pal.colored_object_prompt import COLORED_OBJECT_PROMPT from langchain.chains.pal.math_prompt import MATH_PROMPT -from langchain.llms.base import BaseLLM from langchain.prompts.base import BasePromptTemplate from langchain.python import PythonREPL +from langchain.schema import BaseLanguageModel class PALChain(Chain, BaseModel): """Implements Program-Aided Language Models.""" - llm: BaseLLM + llm: BaseLanguageModel prompt: BasePromptTemplate stop: str = "\n\n" get_answer_expr: str = "print(solution())" @@ -68,7 +68,7 @@ class PALChain(Chain, BaseModel): return output @classmethod - def from_math_prompt(cls, llm: BaseLLM, **kwargs: Any) -> PALChain: + def from_math_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PALChain: """Load PAL from math prompt.""" return cls( llm=llm, @@ -79,7 +79,9 @@ class PALChain(Chain, BaseModel): ) @classmethod - def from_colored_object_prompt(cls, llm: BaseLLM, **kwargs: Any) -> PALChain: + def from_colored_object_prompt( + cls, llm: BaseLanguageModel, **kwargs: Any + ) -> PALChain: """Load PAL from colored object prompt.""" return cls( llm=llm, diff --git a/langchain/chains/qa_with_sources/base.py b/langchain/chains/qa_with_sources/base.py index 3cc15147fde..628a7b36035 100644 --- a/langchain/chains/qa_with_sources/base.py +++ b/langchain/chains/qa_with_sources/base.py @@ -19,8 +19,8 @@ from langchain.chains.qa_with_sources.map_reduce_prompt import ( QUESTION_PROMPT, ) from langchain.docstore.document import Document -from langchain.llms.base import BaseLLM from langchain.prompts.base import BasePromptTemplate +from langchain.schema import BaseLanguageModel class BaseQAWithSourcesChain(Chain, BaseModel, ABC): @@ -38,7 +38,7 @@ class BaseQAWithSourcesChain(Chain, BaseModel, ABC): @classmethod def from_llm( cls, - llm: BaseLLM, + llm: BaseLanguageModel, document_prompt: BasePromptTemplate = EXAMPLE_PROMPT, question_prompt: BasePromptTemplate = QUESTION_PROMPT, combine_prompt: BasePromptTemplate = COMBINE_PROMPT, @@ -65,7 +65,7 @@ class BaseQAWithSourcesChain(Chain, BaseModel, ABC): @classmethod def from_chain_type( cls, - llm: BaseLLM, + llm: BaseLanguageModel, chain_type: str = "stuff", chain_type_kwargs: Optional[dict] = None, **kwargs: Any, diff --git a/langchain/chains/qa_with_sources/loading.py b/langchain/chains/qa_with_sources/loading.py index 4af17329798..c1d923aefb4 100644 --- a/langchain/chains/qa_with_sources/loading.py +++ b/langchain/chains/qa_with_sources/loading.py @@ -13,19 +13,21 @@ from langchain.chains.qa_with_sources import ( stuff_prompt, ) from langchain.chains.question_answering import map_rerank_prompt -from langchain.llms.base import BaseLLM from langchain.prompts.base import BasePromptTemplate +from langchain.schema import BaseLanguageModel class LoadingCallable(Protocol): """Interface for loading the combine documents chain.""" - def __call__(self, llm: BaseLLM, **kwargs: Any) -> BaseCombineDocumentsChain: + def __call__( + self, llm: BaseLanguageModel, **kwargs: Any + ) -> BaseCombineDocumentsChain: """Callable to load the combine documents chain.""" def _load_map_rerank_chain( - llm: BaseLLM, + llm: BaseLanguageModel, prompt: BasePromptTemplate = map_rerank_prompt.PROMPT, verbose: bool = False, document_variable_name: str = "context", @@ -44,7 +46,7 @@ def _load_map_rerank_chain( def _load_stuff_chain( - llm: BaseLLM, + llm: BaseLanguageModel, prompt: BasePromptTemplate = stuff_prompt.PROMPT, document_prompt: BasePromptTemplate = stuff_prompt.EXAMPLE_PROMPT, document_variable_name: str = "summaries", @@ -62,15 +64,15 @@ def _load_stuff_chain( def _load_map_reduce_chain( - llm: BaseLLM, + llm: BaseLanguageModel, question_prompt: BasePromptTemplate = map_reduce_prompt.QUESTION_PROMPT, combine_prompt: BasePromptTemplate = map_reduce_prompt.COMBINE_PROMPT, document_prompt: BasePromptTemplate = map_reduce_prompt.EXAMPLE_PROMPT, combine_document_variable_name: str = "summaries", map_reduce_document_variable_name: str = "context", collapse_prompt: Optional[BasePromptTemplate] = None, - reduce_llm: Optional[BaseLLM] = None, - collapse_llm: Optional[BaseLLM] = None, + reduce_llm: Optional[BaseLanguageModel] = None, + collapse_llm: Optional[BaseLanguageModel] = None, verbose: Optional[bool] = None, **kwargs: Any, ) -> MapReduceDocumentsChain: @@ -112,13 +114,13 @@ def _load_map_reduce_chain( def _load_refine_chain( - llm: BaseLLM, + llm: BaseLanguageModel, question_prompt: BasePromptTemplate = refine_prompts.DEFAULT_TEXT_QA_PROMPT, refine_prompt: BasePromptTemplate = refine_prompts.DEFAULT_REFINE_PROMPT, document_prompt: BasePromptTemplate = refine_prompts.EXAMPLE_PROMPT, document_variable_name: str = "context_str", initial_response_name: str = "existing_answer", - refine_llm: Optional[BaseLLM] = None, + refine_llm: Optional[BaseLanguageModel] = None, verbose: Optional[bool] = None, **kwargs: Any, ) -> RefineDocumentsChain: @@ -137,7 +139,7 @@ def _load_refine_chain( def load_qa_with_sources_chain( - llm: BaseLLM, + llm: BaseLanguageModel, chain_type: str = "stuff", verbose: Optional[bool] = None, **kwargs: Any, diff --git a/langchain/chains/sql_database/base.py b/langchain/chains/sql_database/base.py index 37a3bd171dd..6f95901456f 100644 --- a/langchain/chains/sql_database/base.py +++ b/langchain/chains/sql_database/base.py @@ -8,8 +8,8 @@ from pydantic import BaseModel, Extra, Field from langchain.chains.base import Chain from langchain.chains.llm import LLMChain from langchain.chains.sql_database.prompt import DECIDER_PROMPT, PROMPT -from langchain.llms.base import BaseLLM from langchain.prompts.base import BasePromptTemplate +from langchain.schema import BaseLanguageModel from langchain.sql_database import SQLDatabase @@ -24,7 +24,7 @@ class SQLDatabaseChain(Chain, BaseModel): db_chain = SQLDatabaseChain(llm=OpenAI(), database=db) """ - llm: BaseLLM + llm: BaseLanguageModel """LLM wrapper to use.""" database: SQLDatabase = Field(exclude=True) """SQL Database to connect to.""" @@ -122,7 +122,7 @@ class SQLDatabaseSequentialChain(Chain, BaseModel): @classmethod def from_llm( cls, - llm: BaseLLM, + llm: BaseLanguageModel, database: SQLDatabase, query_prompt: BasePromptTemplate = PROMPT, decider_prompt: BasePromptTemplate = DECIDER_PROMPT, diff --git a/langchain/chains/summarize/__init__.py b/langchain/chains/summarize/__init__.py index c6446f6880b..c31fda479f9 100644 --- a/langchain/chains/summarize/__init__.py +++ b/langchain/chains/summarize/__init__.py @@ -7,19 +7,21 @@ from langchain.chains.combine_documents.refine import RefineDocumentsChain from langchain.chains.combine_documents.stuff import StuffDocumentsChain from langchain.chains.llm import LLMChain from langchain.chains.summarize import map_reduce_prompt, refine_prompts, stuff_prompt -from langchain.llms.base import BaseLLM from langchain.prompts.base import BasePromptTemplate +from langchain.schema import BaseLanguageModel class LoadingCallable(Protocol): """Interface for loading the combine documents chain.""" - def __call__(self, llm: BaseLLM, **kwargs: Any) -> BaseCombineDocumentsChain: + def __call__( + self, llm: BaseLanguageModel, **kwargs: Any + ) -> BaseCombineDocumentsChain: """Callable to load the combine documents chain.""" def _load_stuff_chain( - llm: BaseLLM, + llm: BaseLanguageModel, prompt: BasePromptTemplate = stuff_prompt.PROMPT, document_variable_name: str = "text", verbose: Optional[bool] = None, @@ -36,14 +38,14 @@ def _load_stuff_chain( def _load_map_reduce_chain( - llm: BaseLLM, + llm: BaseLanguageModel, map_prompt: BasePromptTemplate = map_reduce_prompt.PROMPT, combine_prompt: BasePromptTemplate = map_reduce_prompt.PROMPT, combine_document_variable_name: str = "text", map_reduce_document_variable_name: str = "text", collapse_prompt: Optional[BasePromptTemplate] = None, - reduce_llm: Optional[BaseLLM] = None, - collapse_llm: Optional[BaseLLM] = None, + reduce_llm: Optional[BaseLanguageModel] = None, + collapse_llm: Optional[BaseLanguageModel] = None, verbose: Optional[bool] = None, **kwargs: Any, ) -> MapReduceDocumentsChain: @@ -84,12 +86,12 @@ def _load_map_reduce_chain( def _load_refine_chain( - llm: BaseLLM, + llm: BaseLanguageModel, question_prompt: BasePromptTemplate = refine_prompts.PROMPT, refine_prompt: BasePromptTemplate = refine_prompts.REFINE_PROMPT, document_variable_name: str = "text", initial_response_name: str = "existing_answer", - refine_llm: Optional[BaseLLM] = None, + refine_llm: Optional[BaseLanguageModel] = None, verbose: Optional[bool] = None, **kwargs: Any, ) -> RefineDocumentsChain: @@ -107,7 +109,7 @@ def _load_refine_chain( def load_summarize_chain( - llm: BaseLLM, + llm: BaseLanguageModel, chain_type: str = "stuff", verbose: Optional[bool] = None, **kwargs: Any, diff --git a/langchain/chains/vector_db_qa/base.py b/langchain/chains/vector_db_qa/base.py index 3da04659161..16182b78241 100644 --- a/langchain/chains/vector_db_qa/base.py +++ b/langchain/chains/vector_db_qa/base.py @@ -11,8 +11,8 @@ from langchain.chains.combine_documents.stuff import StuffDocumentsChain from langchain.chains.llm import LLMChain from langchain.chains.question_answering import load_qa_chain from langchain.chains.question_answering.stuff_prompt import PROMPT_SELECTOR -from langchain.llms.base import BaseLLM from langchain.prompts import PromptTemplate +from langchain.schema import BaseLanguageModel from langchain.vectorstores.base import VectorStore @@ -103,7 +103,10 @@ class VectorDBQA(Chain, BaseModel): @classmethod def from_llm( - cls, llm: BaseLLM, prompt: Optional[PromptTemplate] = None, **kwargs: Any + cls, + llm: BaseLanguageModel, + prompt: Optional[PromptTemplate] = None, + **kwargs: Any, ) -> VectorDBQA: """Initialize from LLM.""" _prompt = prompt or PROMPT_SELECTOR.get_prompt(llm) @@ -122,7 +125,7 @@ class VectorDBQA(Chain, BaseModel): @classmethod def from_chain_type( cls, - llm: BaseLLM, + llm: BaseLanguageModel, chain_type: str = "stuff", chain_type_kwargs: Optional[dict] = None, **kwargs: Any, From f6d24d5740f2b19654b3001dde13b100c04a95e0 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Mon, 20 Mar 2023 07:51:18 -0700 Subject: [PATCH 06/11] fix bug with openai token count (#1806) --- langchain/chat_models/openai.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/langchain/chat_models/openai.py b/langchain/chat_models/openai.py index bea56fbf5f6..24d19d7a048 100644 --- a/langchain/chat_models/openai.py +++ b/langchain/chat_models/openai.py @@ -229,7 +229,8 @@ class ChatOpenAI(BaseChatModel, BaseModel): overall_token_usage: dict = {} for output in llm_outputs: if output is None: - raise ValueError("Should always be something for OpenAI.") + # Happens in streaming + continue token_usage = output["token_usage"] for k, v in token_usage.items(): if k in overall_token_usage: From 5aa8ece21186039d91a8ebe6346d06c7ef8ae3e1 Mon Sep 17 00:00:00 2001 From: Paul <57178183+CodeJunkie01@users.noreply.github.com> Date: Mon, 20 Mar 2023 15:51:35 +0100 Subject: [PATCH 07/11] Corrected small typo in error message. (#1791) --- langchain/docstore/wikipedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain/docstore/wikipedia.py b/langchain/docstore/wikipedia.py index 5575b8c78d2..8882fb23b68 100644 --- a/langchain/docstore/wikipedia.py +++ b/langchain/docstore/wikipedia.py @@ -17,7 +17,7 @@ class Wikipedia(Docstore): except ImportError: raise ValueError( "Could not import wikipedia python package. " - "Please it install it with `pip install wikipedia`." + "Please install it with `pip install wikipedia`." ) def search(self, search: str) -> Union[str, Document]: From 76c7b1f677eefdd884e91f4db11c997c73b7b665 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Mon, 20 Mar 2023 07:52:27 -0700 Subject: [PATCH 08/11] Harrison/wandb (#1764) Co-authored-by: Anish Shah <93145909+ash0ts@users.noreply.github.com> --- .gitignore | 5 +- docs/ecosystem/wandb_tracking.ipynb | 625 ++++++++++++++++++++ langchain/callbacks/__init__.py | 21 +- langchain/callbacks/wandb_callback.py | 819 ++++++++++++++++++++++++++ 4 files changed, 1468 insertions(+), 2 deletions(-) create mode 100644 docs/ecosystem/wandb_tracking.ipynb create mode 100644 langchain/callbacks/wandb_callback.py diff --git a/.gitignore b/.gitignore index 5bdf0d1b533..354e08c2d86 100644 --- a/.gitignore +++ b/.gitignore @@ -136,5 +136,8 @@ dmypy.json # macOS display setting files .DS_Store +# Wandb directory +wandb/ + # asdf tool versions -.tool-versions \ No newline at end of file +.tool-versions diff --git a/docs/ecosystem/wandb_tracking.ipynb b/docs/ecosystem/wandb_tracking.ipynb new file mode 100644 index 00000000000..a923ad668b4 --- /dev/null +++ b/docs/ecosystem/wandb_tracking.ipynb @@ -0,0 +1,625 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Weights & Biases\n", + "\n", + "This notebook goes over how to track your LangChain experiments into one centralized Weights and Biases dashboard. To learn more about prompt engineering and the callback please refer to this Report which explains both alongside the resultant dashboards you can expect to see.\n", + "\n", + "Run in Colab: https://colab.research.google.com/drive/1DXH4beT4HFaRKy_Vm4PoxhXVDRf7Ym8L?usp=sharing\n", + "\n", + "View Report: https://wandb.ai/a-sh0ts/langchain_callback_demo/reports/Prompt-Engineering-LLMs-with-LangChain-and-W-B--VmlldzozNjk1NTUw#👋-how-to-build-a-callback-in-langchain-for-better-prompt-engineering" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install wandb\n", + "!pip install pandas\n", + "!pip install textstat\n", + "!pip install spacy\n", + "!python -m spacy download en_core_web_sm" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "T1bSmKd6V2If" + }, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"WANDB_API_KEY\"] = \"3310fceb9c83df474d00e0a2aeb54e04238cf6f7\"\n", + "# os.environ[\"OPENAI_API_KEY\"] = \"\"\n", + "# os.environ[\"SERPAPI_API_KEY\"] = \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "8WAGnTWpUUnD" + }, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from langchain.callbacks import WandbCallbackHandler, StdOutCallbackHandler\n", + "from langchain.callbacks.base import CallbackManager\n", + "from langchain.llms import OpenAI" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "Callback Handler that logs to Weights and Biases.\n", + "\n", + "Parameters:\n", + " job_type (str): The type of job.\n", + " project (str): The project to log to.\n", + " entity (str): The entity to log to.\n", + " tags (list): The tags to log.\n", + " group (str): The group to log to.\n", + " name (str): The name of the run.\n", + " notes (str): The notes to log.\n", + " visualize (bool): Whether to visualize the run.\n", + " complexity_metrics (bool): Whether to log complexity metrics.\n", + " stream_logs (bool): Whether to stream callback actions to W&B\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cxBFfZR8d9FC" + }, + "source": [ + "```\n", + "Default values for WandbCallbackHandler(...)\n", + "\n", + "visualize: bool = False,\n", + "complexity_metrics: bool = False,\n", + "stream_logs: bool = False,\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "NOTE: For beta workflows we have made the default analysis based on textstat and the visualizations based on spacy" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "KAz8weWuUeXF" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mharrison-chase\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n" + ] + }, + { + "data": { + "text/html": [ + "Tracking run with wandb version 0.14.0" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run data is saved locally in /Users/harrisonchase/workplace/langchain/docs/ecosystem/wandb/run-20230318_150408-e47j1914" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Syncing run llm to Weights & Biases (docs)
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View project at https://wandb.ai/harrison-chase/langchain_callback_demo" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run at https://wandb.ai/harrison-chase/langchain_callback_demo/runs/e47j1914" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The wandb callback is currently in beta and is subject to change based on updates to `langchain`. Please report any issues to https://github.com/wandb/wandb/issues with the tag `langchain`.\n" + ] + } + ], + "source": [ + "\"\"\"Main function.\n", + "\n", + "This function is used to try the callback handler.\n", + "Scenarios:\n", + "1. OpenAI LLM\n", + "2. Chain with multiple SubChains on multiple generations\n", + "3. Agent with Tools\n", + "\"\"\"\n", + "session_group = datetime.now().strftime(\"%m.%d.%Y_%H.%M.%S\")\n", + "wandb_callback = WandbCallbackHandler(\n", + " job_type=\"inference\",\n", + " project=\"langchain_callback_demo\",\n", + " group=f\"minimal_{session_group}\",\n", + " name=\"llm\",\n", + " tags=[\"test\"],\n", + ")\n", + "manager = CallbackManager([StdOutCallbackHandler(), wandb_callback])\n", + "llm = OpenAI(temperature=0, callback_manager=manager, verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Q-65jwrDeK6w" + }, + "source": [ + "\n", + "\n", + "```\n", + "# Defaults for WandbCallbackHandler.flush_tracker(...)\n", + "\n", + "reset: bool = True,\n", + "finish: bool = False,\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `flush_tracker` function is used to log LangChain sessions to Weights & Biases. It takes in the LangChain module or agent, and logs at minimum the prompts and generations alongside the serialized form of the LangChain module to the specified Weights & Biases project. By default we reset the session as opposed to concluding the session outright." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "o_VmneyIUyx8" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Waiting for W&B process to finish... (success)." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run llm at: https://wandb.ai/harrison-chase/langchain_callback_demo/runs/e47j1914
Synced 5 W&B file(s), 2 media file(s), 5 artifact file(s) and 0 other file(s)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Find logs at: ./wandb/run-20230318_150408-e47j1914/logs" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0d7b4307ccdb450ea631497174fca2d1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.016745895149999985, max=1.0…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Tracking run with wandb version 0.14.0" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run data is saved locally in /Users/harrisonchase/workplace/langchain/docs/ecosystem/wandb/run-20230318_150534-jyxma7hu" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Syncing run simple_sequential to Weights & Biases (docs)
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View project at https://wandb.ai/harrison-chase/langchain_callback_demo" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run at https://wandb.ai/harrison-chase/langchain_callback_demo/runs/jyxma7hu" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# SCENARIO 1 - LLM\n", + "llm_result = llm.generate([\"Tell me a joke\", \"Tell me a poem\"] * 3)\n", + "wandb_callback.flush_tracker(llm, name=\"simple_sequential\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "trxslyb1U28Y" + }, + "outputs": [], + "source": [ + "from langchain.prompts import PromptTemplate\n", + "from langchain.chains import LLMChain" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "uauQk10SUzF6" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Waiting for W&B process to finish... (success)." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run simple_sequential at: https://wandb.ai/harrison-chase/langchain_callback_demo/runs/jyxma7hu
Synced 4 W&B file(s), 2 media file(s), 6 artifact file(s) and 0 other file(s)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Find logs at: ./wandb/run-20230318_150534-jyxma7hu/logs" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "dbdbf28fb8ed40a3a60218d2e6d1a987", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.016736786816666675, max=1.0…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Tracking run with wandb version 0.14.0" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run data is saved locally in /Users/harrisonchase/workplace/langchain/docs/ecosystem/wandb/run-20230318_150550-wzy59zjq" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Syncing run agent to Weights & Biases (docs)
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View project at https://wandb.ai/harrison-chase/langchain_callback_demo" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run at https://wandb.ai/harrison-chase/langchain_callback_demo/runs/wzy59zjq" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# SCENARIO 2 - Chain\n", + "template = \"\"\"You are a playwright. Given the title of play, it is your job to write a synopsis for that title.\n", + "Title: {title}\n", + "Playwright: This is a synopsis for the above play:\"\"\"\n", + "prompt_template = PromptTemplate(input_variables=[\"title\"], template=template)\n", + "synopsis_chain = LLMChain(llm=llm, prompt=prompt_template, callback_manager=manager)\n", + "\n", + "test_prompts = [\n", + " {\n", + " \"title\": \"documentary about good video games that push the boundary of game design\"\n", + " },\n", + " {\"title\": \"cocaine bear vs heroin wolf\"},\n", + " {\"title\": \"the best in class mlops tooling\"},\n", + "]\n", + "synopsis_chain.apply(test_prompts)\n", + "wandb_callback.flush_tracker(synopsis_chain, name=\"agent\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "_jN73xcPVEpI" + }, + "outputs": [], + "source": [ + "from langchain.agents import initialize_agent, load_tools" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "Gpq4rk6VT9cu" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n", + "\u001b[32;1m\u001b[1;3m I need to find out who Leo DiCaprio's girlfriend is and then calculate her age raised to the 0.43 power.\n", + "Action: Search\n", + "Action Input: \"Leo DiCaprio girlfriend\"\u001b[0m\n", + "Observation: \u001b[36;1m\u001b[1;3mDiCaprio had a steady girlfriend in Camila Morrone. He had been with the model turned actress for nearly five years, as they were first said to be dating at the end of 2017. And the now 26-year-old Morrone is no stranger to Hollywood.\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m I need to calculate her age raised to the 0.43 power.\n", + "Action: Calculator\n", + "Action Input: 26^0.43\u001b[0m\n", + "Observation: \u001b[33;1m\u001b[1;3mAnswer: 4.059182145592686\n", + "\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m I now know the final answer.\n", + "Final Answer: Leo DiCaprio's girlfriend is Camila Morrone and her current age raised to the 0.43 power is 4.059182145592686.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + }, + { + "data": { + "text/html": [ + "Waiting for W&B process to finish... (success)." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + " View run agent at: https://wandb.ai/harrison-chase/langchain_callback_demo/runs/wzy59zjq
Synced 5 W&B file(s), 2 media file(s), 7 artifact file(s) and 0 other file(s)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Find logs at: ./wandb/run-20230318_150550-wzy59zjq/logs" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# SCENARIO 3 - Agent with Tools\n", + "tools = load_tools([\"serpapi\", \"llm-math\"], llm=llm, callback_manager=manager)\n", + "agent = initialize_agent(\n", + " tools,\n", + " llm,\n", + " agent=\"zero-shot-react-description\",\n", + " callback_manager=manager,\n", + " verbose=True,\n", + ")\n", + "agent.run(\n", + " \"Who is Leo DiCaprio's girlfriend? What is her current age raised to the 0.43 power?\"\n", + ")\n", + "wandb_callback.flush_tracker(agent, reset=False, finish=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/langchain/callbacks/__init__.py b/langchain/callbacks/__init__.py index 2bc79d6aacf..8796409475f 100644 --- a/langchain/callbacks/__init__.py +++ b/langchain/callbacks/__init__.py @@ -3,11 +3,16 @@ import os from contextlib import contextmanager from typing import Generator, Optional -from langchain.callbacks.base import BaseCallbackHandler, BaseCallbackManager +from langchain.callbacks.base import ( + BaseCallbackHandler, + BaseCallbackManager, + CallbackManager, +) from langchain.callbacks.openai_info import OpenAICallbackHandler from langchain.callbacks.shared import SharedCallbackManager from langchain.callbacks.stdout import StdOutCallbackHandler from langchain.callbacks.tracers import SharedLangChainTracer +from langchain.callbacks.wandb_callback import WandbCallbackHandler def get_callback_manager() -> BaseCallbackManager: @@ -58,3 +63,17 @@ def get_openai_callback() -> Generator[OpenAICallbackHandler, None, None]: manager.add_handler(handler) yield handler manager.remove_handler(handler) + + +__all__ = [ + "CallbackManager", + "OpenAICallbackHandler", + "SharedCallbackManager", + "StdOutCallbackHandler", + "WandbCallbackHandler", + "get_openai_callback", + "set_tracing_callback_manager", + "set_default_callback_manager", + "set_handler", + "get_callback_manager", +] diff --git a/langchain/callbacks/wandb_callback.py b/langchain/callbacks/wandb_callback.py new file mode 100644 index 00000000000..991075146b6 --- /dev/null +++ b/langchain/callbacks/wandb_callback.py @@ -0,0 +1,819 @@ +import hashlib +import json +import tempfile +from copy import deepcopy +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union + +from langchain.callbacks.base import BaseCallbackHandler +from langchain.schema import AgentAction, AgentFinish, LLMResult + + +def import_wandb() -> Any: + try: + import wandb # noqa: F401 + except ImportError: + raise ImportError( + "To use the wandb callback manager you need to have the `wandb` python " + "package installed. Please install it with `pip install wandb`" + ) + return wandb + + +def import_spacy() -> Any: + try: + import spacy # noqa: F401 + except ImportError: + raise ImportError( + "To use the wandb callback manager you need to have the `spacy` python " + "package installed. Please install it with `pip install spacy`" + ) + return spacy + + +def import_pandas() -> Any: + try: + import pandas # noqa: F401 + except ImportError: + raise ImportError( + "To use the wandb callback manager you need to have the `pandas` python " + "package installed. Please install it with `pip install pandas`" + ) + return pandas + + +def import_textstat() -> Any: + try: + import textstat # noqa: F401 + except ImportError: + raise ImportError( + "To use the wandb callback manager you need to have the `textstat` python " + "package installed. Please install it with `pip install textstat`" + ) + return textstat + + +def _flatten_dict( + nested_dict: Dict[str, Any], parent_key: str = "", sep: str = "_" +) -> Iterable[Tuple[str, Any]]: + """ + Generator that yields flattened items from a nested dictionary for a flat dict. + + Parameters: + nested_dict (dict): The nested dictionary to flatten. + parent_key (str): The prefix to prepend to the keys of the flattened dict. + sep (str): The separator to use between the parent key and the key of the + flattened dictionary. + + Yields: + (str, any): A key-value pair from the flattened dictionary. + """ + for key, value in nested_dict.items(): + new_key = parent_key + sep + key if parent_key else key + if isinstance(value, dict): + yield from _flatten_dict(value, new_key, sep) + else: + yield new_key, value + + +def flatten_dict( + nested_dict: Dict[str, Any], parent_key: str = "", sep: str = "_" +) -> Dict[str, Any]: + """Flattens a nested dictionary into a flat dictionary. + + Parameters: + nested_dict (dict): The nested dictionary to flatten. + parent_key (str): The prefix to prepend to the keys of the flattened dict. + sep (str): The separator to use between the parent key and the key of the + flattened dictionary. + + Returns: + (dict): A flat dictionary. + + """ + flat_dict = {k: v for k, v in _flatten_dict(nested_dict, parent_key, sep)} + return flat_dict + + +def hash_string(s: str) -> str: + """Hash a string using sha1. + + Parameters: + s (str): The string to hash. + + Returns: + (str): The hashed string. + """ + return hashlib.sha1(s.encode("utf-8")).hexdigest() + + +def load_json_to_dict(json_path: Union[str, Path]) -> dict: + """Load json file to a dictionary. + + Parameters: + json_path (str): The path to the json file. + + Returns: + (dict): The dictionary representation of the json file. + """ + with open(json_path, "r") as f: + data = json.load(f) + return data + + +def analyze_text( + text: str, + complexity_metrics: bool = True, + visualize: bool = True, + nlp: Any = None, + output_dir: Optional[Union[str, Path]] = None, +) -> dict: + """Analyze text using textstat and spacy. + + Parameters: + text (str): The text to analyze. + complexity_metrics (bool): Whether to compute complexity metrics. + visualize (bool): Whether to visualize the text. + nlp (spacy.lang): The spacy language model to use for visualization. + output_dir (str): The directory to save the visualization files to. + + Returns: + (dict): A dictionary containing the complexity metrics and visualization + files serialized in a wandb.Html element. + """ + resp = {} + textstat = import_textstat() + wandb = import_wandb() + spacy = import_spacy() + if complexity_metrics: + text_complexity_metrics = { + "flesch_reading_ease": textstat.flesch_reading_ease(text), + "flesch_kincaid_grade": textstat.flesch_kincaid_grade(text), + "smog_index": textstat.smog_index(text), + "coleman_liau_index": textstat.coleman_liau_index(text), + "automated_readability_index": textstat.automated_readability_index(text), + "dale_chall_readability_score": textstat.dale_chall_readability_score(text), + "difficult_words": textstat.difficult_words(text), + "linsear_write_formula": textstat.linsear_write_formula(text), + "gunning_fog": textstat.gunning_fog(text), + "text_standard": textstat.text_standard(text), + "fernandez_huerta": textstat.fernandez_huerta(text), + "szigriszt_pazos": textstat.szigriszt_pazos(text), + "gutierrez_polini": textstat.gutierrez_polini(text), + "crawford": textstat.crawford(text), + "gulpease_index": textstat.gulpease_index(text), + "osman": textstat.osman(text), + } + resp.update(text_complexity_metrics) + + if visualize and nlp and output_dir is not None: + doc = nlp(text) + + dep_out = spacy.displacy.render( # type: ignore + doc, style="dep", jupyter=False, page=True + ) + dep_output_path = Path(output_dir, hash_string(f"dep-{text}") + ".html") + dep_output_path.open("w", encoding="utf-8").write(dep_out) + + ent_out = spacy.displacy.render( # type: ignore + doc, style="ent", jupyter=False, page=True + ) + ent_output_path = Path(output_dir, hash_string(f"ent-{text}") + ".html") + ent_output_path.open("w", encoding="utf-8").write(ent_out) + + text_visualizations = { + "dependency_tree": wandb.Html(str(dep_output_path)), + "entities": wandb.Html(str(ent_output_path)), + } + resp.update(text_visualizations) + + return resp + + +def construct_html_from_prompt_and_generation(prompt: str, generation: str) -> Any: + """Construct an html element from a prompt and a generation. + + Parameters: + prompt (str): The prompt. + generation (str): The generation. + + Returns: + (wandb.Html): The html element.""" + wandb = import_wandb() + formatted_prompt = prompt.replace("\n", "
") + formatted_generation = generation.replace("\n", "
") + + return wandb.Html( + f""" +

{formatted_prompt}:

+
+

+ {formatted_generation} +

+
+ """, + inject=False, + ) + + +class BaseMetadataCallbackHandler: + """This class handles the metadata and associated function states for callbacks. + + Attributes: + step (int): The current step. + starts (int): The number of times the start method has been called. + ends (int): The number of times the end method has been called. + errors (int): The number of times the error method has been called. + text_ctr (int): The number of times the text method has been called. + ignore_llm_ (bool): Whether to ignore llm callbacks. + ignore_chain_ (bool): Whether to ignore chain callbacks. + ignore_agent_ (bool): Whether to ignore agent callbacks. + always_verbose_ (bool): Whether to always be verbose. + chain_starts (int): The number of times the chain start method has been called. + chain_ends (int): The number of times the chain end method has been called. + llm_starts (int): The number of times the llm start method has been called. + llm_ends (int): The number of times the llm end method has been called. + llm_streams (int): The number of times the text method has been called. + tool_starts (int): The number of times the tool start method has been called. + tool_ends (int): The number of times the tool end method has been called. + agent_ends (int): The number of times the agent end method has been called. + on_llm_start_records (list): A list of records of the on_llm_start method. + on_llm_token_records (list): A list of records of the on_llm_token method. + on_llm_end_records (list): A list of records of the on_llm_end method. + on_chain_start_records (list): A list of records of the on_chain_start method. + on_chain_end_records (list): A list of records of the on_chain_end method. + on_tool_start_records (list): A list of records of the on_tool_start method. + on_tool_end_records (list): A list of records of the on_tool_end method. + on_agent_finish_records (list): A list of records of the on_agent_end method. + """ + + def __init__(self) -> None: + self.step = 0 + + self.starts = 0 + self.ends = 0 + self.errors = 0 + self.text_ctr = 0 + + self.ignore_llm_ = False + self.ignore_chain_ = False + self.ignore_agent_ = False + self.always_verbose_ = False + + self.chain_starts = 0 + self.chain_ends = 0 + + self.llm_starts = 0 + self.llm_ends = 0 + self.llm_streams = 0 + + self.tool_starts = 0 + self.tool_ends = 0 + + self.agent_ends = 0 + + self.on_llm_start_records: list = [] + self.on_llm_token_records: list = [] + self.on_llm_end_records: list = [] + + self.on_chain_start_records: list = [] + self.on_chain_end_records: list = [] + + self.on_tool_start_records: list = [] + self.on_tool_end_records: list = [] + + self.on_text_records: list = [] + self.on_agent_finish_records: list = [] + self.on_agent_action_records: list = [] + + @property + def always_verbose(self) -> bool: + """Whether to call verbose callbacks even if verbose is False.""" + return self.always_verbose_ + + @property + def ignore_llm(self) -> bool: + """Whether to ignore LLM callbacks.""" + return self.ignore_llm_ + + @property + def ignore_chain(self) -> bool: + """Whether to ignore chain callbacks.""" + return self.ignore_chain_ + + @property + def ignore_agent(self) -> bool: + """Whether to ignore agent callbacks.""" + return self.ignore_agent_ + + def get_custom_callback_meta(self) -> Dict[str, Any]: + return { + "step": self.step, + "starts": self.starts, + "ends": self.ends, + "errors": self.errors, + "text_ctr": self.text_ctr, + "chain_starts": self.chain_starts, + "chain_ends": self.chain_ends, + "llm_starts": self.llm_starts, + "llm_ends": self.llm_ends, + "llm_streams": self.llm_streams, + "tool_starts": self.tool_starts, + "tool_ends": self.tool_ends, + "agent_ends": self.agent_ends, + } + + def reset_callback_meta(self) -> None: + """Reset the callback metadata.""" + self.step = 0 + + self.starts = 0 + self.ends = 0 + self.errors = 0 + self.text_ctr = 0 + + self.ignore_llm_ = False + self.ignore_chain_ = False + self.ignore_agent_ = False + self.always_verbose_ = False + + self.chain_starts = 0 + self.chain_ends = 0 + + self.llm_starts = 0 + self.llm_ends = 0 + self.llm_streams = 0 + + self.tool_starts = 0 + self.tool_ends = 0 + + self.agent_ends = 0 + + self.on_llm_start_records = [] + self.on_llm_token_records = [] + self.on_llm_end_records = [] + + self.on_chain_start_records = [] + self.on_chain_end_records = [] + + self.on_tool_start_records = [] + self.on_tool_end_records = [] + + self.on_text_records = [] + self.on_agent_finish_records = [] + self.on_agent_action_records = [] + return None + + +class WandbCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): + """Callback Handler that logs to Weights and Biases. + + Parameters: + job_type (str): The type of job. + project (str): The project to log to. + entity (str): The entity to log to. + tags (list): The tags to log. + group (str): The group to log to. + name (str): The name of the run. + notes (str): The notes to log. + visualize (bool): Whether to visualize the run. + complexity_metrics (bool): Whether to log complexity metrics. + stream_logs (bool): Whether to stream callback actions to W&B + + This handler will utilize the associated callback method called and formats + the input of each callback function with metadata regarding the state of LLM run, + and adds the response to the list of records for both the {method}_records and + action. It then logs the response using the run.log() method to Weights and Biases. + """ + + def __init__( + self, + job_type: Optional[str] = None, + project: Optional[str] = "langchain_callback_demo", + entity: Optional[str] = None, + tags: Optional[Sequence] = None, + group: Optional[str] = None, + name: Optional[str] = None, + notes: Optional[str] = None, + visualize: bool = False, + complexity_metrics: bool = False, + stream_logs: bool = False, + ) -> None: + """Initialize callback handler.""" + + wandb = import_wandb() + import_pandas() + import_textstat() + spacy = import_spacy() + super().__init__() + + self.job_type = job_type + self.project = project + self.entity = entity + self.tags = tags + self.group = group + self.name = name + self.notes = notes + self.visualize = visualize + self.complexity_metrics = complexity_metrics + self.stream_logs = stream_logs + + self.temp_dir = tempfile.TemporaryDirectory() + self.run: wandb.sdk.wandb_run.Run = wandb.init( # type: ignore + job_type=self.job_type, + project=self.project, + entity=self.entity, + tags=self.tags, + group=self.group, + name=self.name, + notes=self.notes, + ) + warning = ( + "The wandb callback is currently in beta and is subject to change " + "based on updates to `langchain`. Please report any issues to " + "https://github.com/wandb/wandb/issues with the tag `langchain`." + ) + wandb.termwarn( + warning, + repeat=False, + ) + self.callback_columns: list = [] + self.action_records: list = [] + self.complexity_metrics = complexity_metrics + self.visualize = visualize + self.nlp = spacy.load("en_core_web_sm") + + def _init_resp(self) -> Dict: + return {k: None for k in self.callback_columns} + + def on_llm_start( + self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any + ) -> None: + """Run when LLM starts.""" + self.step += 1 + self.llm_starts += 1 + self.starts += 1 + + resp = self._init_resp() + resp.update({"action": "on_llm_start"}) + resp.update(flatten_dict(serialized)) + resp.update(self.get_custom_callback_meta()) + + for prompt in prompts: + prompt_resp = deepcopy(resp) + prompt_resp["prompts"] = prompt + self.on_llm_start_records.append(prompt_resp) + self.action_records.append(prompt_resp) + if self.stream_logs: + self.run.log(prompt_resp) + + def on_llm_new_token(self, token: str, **kwargs: Any) -> None: + """Run when LLM generates a new token.""" + self.step += 1 + self.llm_streams += 1 + + resp = self._init_resp() + resp.update({"action": "on_llm_new_token", "token": token}) + resp.update(self.get_custom_callback_meta()) + + self.on_llm_token_records.append(resp) + self.action_records.append(resp) + if self.stream_logs: + self.run.log(resp) + + def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None: + """Run when LLM ends running.""" + self.step += 1 + self.llm_ends += 1 + self.ends += 1 + + resp = self._init_resp() + resp.update({"action": "on_llm_end"}) + resp.update(flatten_dict(response.llm_output or {})) + resp.update(self.get_custom_callback_meta()) + + for generations in response.generations: + for generation in generations: + generation_resp = deepcopy(resp) + generation_resp.update(flatten_dict(generation.dict())) + generation_resp.update( + analyze_text( + generation.text, + complexity_metrics=self.complexity_metrics, + visualize=self.visualize, + nlp=self.nlp, + output_dir=self.temp_dir.name, + ) + ) + self.on_llm_end_records.append(generation_resp) + self.action_records.append(generation_resp) + if self.stream_logs: + self.run.log(generation_resp) + + def on_llm_error( + self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any + ) -> None: + """Run when LLM errors.""" + self.step += 1 + self.errors += 1 + + def on_chain_start( + self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any + ) -> None: + """Run when chain starts running.""" + self.step += 1 + self.chain_starts += 1 + self.starts += 1 + + resp = self._init_resp() + resp.update({"action": "on_chain_start"}) + resp.update(flatten_dict(serialized)) + resp.update(self.get_custom_callback_meta()) + + chain_input = inputs["input"] + + if isinstance(chain_input, str): + input_resp = deepcopy(resp) + input_resp["input"] = chain_input + self.on_chain_start_records.append(input_resp) + self.action_records.append(input_resp) + if self.stream_logs: + self.run.log(input_resp) + elif isinstance(chain_input, list): + for inp in chain_input: + input_resp = deepcopy(resp) + input_resp.update(inp) + self.on_chain_start_records.append(input_resp) + self.action_records.append(input_resp) + if self.stream_logs: + self.run.log(input_resp) + else: + raise ValueError("Unexpected data format provided!") + + def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None: + """Run when chain ends running.""" + self.step += 1 + self.chain_ends += 1 + self.ends += 1 + + resp = self._init_resp() + resp.update({"action": "on_chain_end", "outputs": outputs["output"]}) + resp.update(self.get_custom_callback_meta()) + + self.on_chain_end_records.append(resp) + self.action_records.append(resp) + if self.stream_logs: + self.run.log(resp) + + def on_chain_error( + self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any + ) -> None: + """Run when chain errors.""" + self.step += 1 + self.errors += 1 + + def on_tool_start( + self, serialized: Dict[str, Any], input_str: str, **kwargs: Any + ) -> None: + """Run when tool starts running.""" + self.step += 1 + self.tool_starts += 1 + self.starts += 1 + + resp = self._init_resp() + resp.update({"action": "on_tool_start", "input_str": input_str}) + resp.update(flatten_dict(serialized)) + resp.update(self.get_custom_callback_meta()) + + self.on_tool_start_records.append(resp) + self.action_records.append(resp) + if self.stream_logs: + self.run.log(resp) + + def on_tool_end(self, output: str, **kwargs: Any) -> None: + """Run when tool ends running.""" + self.step += 1 + self.tool_ends += 1 + self.ends += 1 + + resp = self._init_resp() + resp.update({"action": "on_tool_end", "output": output}) + resp.update(self.get_custom_callback_meta()) + + self.on_tool_end_records.append(resp) + self.action_records.append(resp) + if self.stream_logs: + self.run.log(resp) + + def on_tool_error( + self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any + ) -> None: + """Run when tool errors.""" + self.step += 1 + self.errors += 1 + + def on_text(self, text: str, **kwargs: Any) -> None: + """ + Run when agent is ending. + """ + self.step += 1 + self.text_ctr += 1 + + resp = self._init_resp() + resp.update({"action": "on_text", "text": text}) + resp.update(self.get_custom_callback_meta()) + + self.on_text_records.append(resp) + self.action_records.append(resp) + if self.stream_logs: + self.run.log(resp) + + def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> None: + """Run when agent ends running.""" + self.step += 1 + self.agent_ends += 1 + self.ends += 1 + + resp = self._init_resp() + resp.update( + { + "action": "on_agent_finish", + "output": finish.return_values["output"], + "log": finish.log, + } + ) + resp.update(self.get_custom_callback_meta()) + + self.on_agent_finish_records.append(resp) + self.action_records.append(resp) + if self.stream_logs: + self.run.log(resp) + + def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any: + """Run on agent action.""" + self.step += 1 + self.tool_starts += 1 + self.starts += 1 + + resp = self._init_resp() + resp.update( + { + "action": "on_agent_action", + "tool": action.tool, + "tool_input": action.tool_input, + "log": action.log, + } + ) + resp.update(self.get_custom_callback_meta()) + self.on_agent_action_records.append(resp) + self.action_records.append(resp) + if self.stream_logs: + self.run.log(resp) + + def _create_session_analysis_df(self) -> Any: + """Create a dataframe with all the information from the session.""" + pd = import_pandas() + on_llm_start_records_df = pd.DataFrame(self.on_llm_start_records) + on_llm_end_records_df = pd.DataFrame(self.on_llm_end_records) + + llm_input_prompts_df = ( + on_llm_start_records_df[["step", "prompts", "name"]] + .dropna(axis=1) + .rename({"step": "prompt_step"}, axis=1) + ) + complexity_metrics_columns = [] + visualizations_columns = [] + + if self.complexity_metrics: + complexity_metrics_columns = [ + "flesch_reading_ease", + "flesch_kincaid_grade", + "smog_index", + "coleman_liau_index", + "automated_readability_index", + "dale_chall_readability_score", + "difficult_words", + "linsear_write_formula", + "gunning_fog", + "text_standard", + "fernandez_huerta", + "szigriszt_pazos", + "gutierrez_polini", + "crawford", + "gulpease_index", + "osman", + ] + + if self.visualize: + visualizations_columns = ["dependency_tree", "entities"] + + llm_outputs_df = ( + on_llm_end_records_df[ + [ + "step", + "text", + "token_usage_total_tokens", + "token_usage_prompt_tokens", + "token_usage_completion_tokens", + ] + + complexity_metrics_columns + + visualizations_columns + ] + .dropna(axis=1) + .rename({"step": "output_step", "text": "output"}, axis=1) + ) + session_analysis_df = pd.concat([llm_input_prompts_df, llm_outputs_df], axis=1) + session_analysis_df["chat_html"] = session_analysis_df[ + ["prompts", "output"] + ].apply( + lambda row: construct_html_from_prompt_and_generation( + row["prompts"], row["output"] + ), + axis=1, + ) + return session_analysis_df + + def flush_tracker( + self, + langchain_asset: Any = None, + reset: bool = True, + finish: bool = False, + job_type: Optional[str] = None, + project: Optional[str] = None, + entity: Optional[str] = None, + tags: Optional[Sequence] = None, + group: Optional[str] = None, + name: Optional[str] = None, + notes: Optional[str] = None, + visualize: Optional[bool] = None, + complexity_metrics: Optional[bool] = None, + ) -> None: + """Flush the tracker and reset the session. + + Args: + langchain_asset: The langchain asset to save. + reset: Whether to reset the session. + finish: Whether to finish the run. + job_type: The job type. + project: The project. + entity: The entity. + tags: The tags. + group: The group. + name: The name. + notes: The notes. + visualize: Whether to visualize. + complexity_metrics: Whether to compute complexity metrics. + + Returns: + None + """ + pd = import_pandas() + wandb = import_wandb() + action_records_table = wandb.Table(dataframe=pd.DataFrame(self.action_records)) + session_analysis_table = wandb.Table( + dataframe=self._create_session_analysis_df() + ) + self.run.log( + { + "action_records": action_records_table, + "session_analysis": session_analysis_table, + } + ) + + if langchain_asset: + langchain_asset_path = Path(self.temp_dir.name, "model.json") + model_artifact = wandb.Artifact(name="model", type="model") + model_artifact.add(action_records_table, name="action_records") + model_artifact.add(session_analysis_table, name="session_analysis") + try: + langchain_asset.save(langchain_asset_path) + model_artifact.add_file(str(langchain_asset_path)) + model_artifact.metadata = load_json_to_dict(langchain_asset_path) + except ValueError: + langchain_asset.save_agent(langchain_asset_path) + model_artifact.add_file(str(langchain_asset_path)) + model_artifact.metadata = load_json_to_dict(langchain_asset_path) + except NotImplementedError as e: + print("Could not save model.") + print(repr(e)) + pass + self.run.log_artifact(model_artifact) + + if finish or reset: + self.run.finish() + self.temp_dir.cleanup() + self.reset_callback_meta() + if reset: + self.__init__( # type: ignore + job_type=job_type if job_type else self.job_type, + project=project if project else self.project, + entity=entity if entity else self.entity, + tags=tags if tags else self.tags, + group=group if group else self.group, + name=name if name else self.name, + notes=notes if notes else self.notes, + visualize=visualize if visualize else self.visualize, + complexity_metrics=complexity_metrics + if complexity_metrics + else self.complexity_metrics, + ) From 7b6ff7fe0034bd7132f049946a3d087842ec844d Mon Sep 17 00:00:00 2001 From: Bryan Helmig Date: Mon, 20 Mar 2023 09:52:41 -0500 Subject: [PATCH 09/11] Follow up to #1803 to remove dynamic docs route. (#1818) The base docs are going to be more stable and familiar for folks. Dynamic route is currently in flux. --- docs/modules/utils/examples/zapier.ipynb | 4 ++-- langchain/tools/zapier/tool.py | 6 +++--- langchain/utilities/zapier.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/modules/utils/examples/zapier.ipynb b/docs/modules/utils/examples/zapier.ipynb index ea4bed063ea..9a0e95eafb4 100644 --- a/docs/modules/utils/examples/zapier.ipynb +++ b/docs/modules/utils/examples/zapier.ipynb @@ -7,7 +7,7 @@ "source": [ "## Zapier Natural Language Actions API\n", "\\\n", - "Full docs here: https://nla.zapier.com/api/v1/dynamic/docs\n", + "Full docs here: https://nla.zapier.com/api/v1/docs\n", "\n", "**Zapier Natural Language Actions** gives you access to the 5k+ apps, 20k+ actions on Zapier's platform through a natural language API interface.\n", "\n", @@ -21,7 +21,7 @@ "\n", "2. User-facing (Oauth): for production scenarios where you are deploying an end-user facing application and LangChain needs access to end-user's exposed actions and connected accounts on Zapier.com\n", "\n", - "This quick start will focus on the server-side use case for brevity. Review [full docs](https://nla.zapier.com/api/v1/dynamic/docs) or reach out to nla@zapier.com for user-facing oauth developer support.\n", + "This quick start will focus on the server-side use case for brevity. Review [full docs](https://nla.zapier.com/api/v1/docs) or reach out to nla@zapier.com for user-facing oauth developer support.\n", "\n", "This example goes over how to use the Zapier integration with a `SimpleSequentialChain`, then an `Agent`.\n", "In code, below:" diff --git a/langchain/tools/zapier/tool.py b/langchain/tools/zapier/tool.py index a3c3be322a6..275c90020cf 100644 --- a/langchain/tools/zapier/tool.py +++ b/langchain/tools/zapier/tool.py @@ -1,6 +1,6 @@ """## Zapier Natural Language Actions API \ -Full docs here: https://nla.zapier.com/api/v1/dynamic/docs +Full docs here: https://nla.zapier.com/api/v1/docs **Zapier Natural Language Actions** gives you access to the 5k+ apps, 20k+ actions on Zapier's platform through a natural language API interface. @@ -24,7 +24,7 @@ NLA offers both API Key and OAuth for signing NLA API requests. connected accounts on Zapier.com This quick start will focus on the server-side use case for brevity. -Review [full docs](https://nla.zapier.com/api/v1/dynamic/docs) or reach out to +Review [full docs](https://nla.zapier.com/api/v1/docs) or reach out to nla@zapier.com for user-facing oauth developer support. Typically you'd use SequentialChain, here's a basic example: @@ -92,7 +92,7 @@ class ZapierNLARunAction(BaseTool): (eg. "get the latest email from Mike Knoop" for "Gmail: find email" action) params: a dict, optional. Any params provided will *override* AI guesses from `instructions` (see "understanding the AI guessing flow" here: - https://nla.zapier.com/api/v1/dynamic/docs) + https://nla.zapier.com/api/v1/docs) """ api_wrapper: ZapierNLAWrapper = Field(default_factory=ZapierNLAWrapper) diff --git a/langchain/utilities/zapier.py b/langchain/utilities/zapier.py index 64a21514da5..d2d3ed74e3a 100644 --- a/langchain/utilities/zapier.py +++ b/langchain/utilities/zapier.py @@ -1,6 +1,6 @@ """Util that can interact with Zapier NLA. -Full docs here: https://nla.zapier.com/api/v1/dynamic/docs +Full docs here: https://nla.zapier.com/api/v1/docs Note: this wrapper currently only implemented the `api_key` auth method for testing and server-side production use cases (using the developer's connected accounts on @@ -24,7 +24,7 @@ from langchain.utils import get_from_dict_or_env class ZapierNLAWrapper(BaseModel): """Wrapper for Zapier NLA. - Full docs here: https://nla.zapier.com/api/v1/dynamic/docs + Full docs here: https://nla.zapier.com/api/v1/docs Note: this wrapper currently only implemented the `api_key` auth method for testingand server-side production use cases (using the developer's connected @@ -97,7 +97,7 @@ class ZapierNLAWrapper(BaseModel): `params` will always contain an `instructions` key, the only required param. All others optional and if provided will override any AI guesses (see "understanding the AI guessing flow" here: - https://nla.zapier.com/api/v1/dynamic/docs) + https://nla.zapier.com/api/v1/docs) """ session = self._get_session() response = session.get(self.zapier_nla_api_base + "exposed/") From d5b4393bb2f3bba4b1f8d3cc1b6e416c7ad27b7a Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Mon, 20 Mar 2023 07:53:26 -0700 Subject: [PATCH 10/11] Harrison/llm math (#1808) Co-authored-by: Vadym Barda --- docs/use_cases/evaluation/llm_math.ipynb | 306 +++++++++++++++++++++++ langchain/chains/llm_math/prompt.py | 17 +- 2 files changed, 310 insertions(+), 13 deletions(-) create mode 100644 docs/use_cases/evaluation/llm_math.ipynb diff --git a/docs/use_cases/evaluation/llm_math.ipynb b/docs/use_cases/evaluation/llm_math.ipynb new file mode 100644 index 00000000000..a2a623d0e56 --- /dev/null +++ b/docs/use_cases/evaluation/llm_math.ipynb @@ -0,0 +1,306 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a4734146", + "metadata": {}, + "source": [ + "# LLM Math\n", + "\n", + "Evaluating chains that know how to do math." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "fdd7afae", + "metadata": {}, + "outputs": [], + "source": [ + "# Comment this out if you are NOT using tracing\n", + "import os\n", + "os.environ[\"LANGCHAIN_HANDLER\"] = \"langchain\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ce05ffea", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d028a511cede4de2b845b9a9954d6bea", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading readme: 0%| | 0.00/21.0 [00:00 Date: Mon, 20 Mar 2023 08:04:04 -0700 Subject: [PATCH 11/11] release 0.0.117 (#1819) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cd76e5e07e3..560d5ba448a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langchain" -version = "0.0.116" +version = "0.0.117" description = "Building applications with LLMs through composability" authors = [] license = "MIT"