bump 245 (#8359)

## Description
This commit introduces the `DropboxLoader` class, a new document loader
that allows loading files from Dropbox into the application. The loader
relies on a Dropbox app, which requires creating an app on Dropbox,
obtaining the necessary scope permissions, and generating an access
token. Additionally, the dropbox Python package is required.

The `DropboxLoader` class is designed to be used as a document loader
for processing various file types, including text files, PDFs, and
Dropbox Paper files.

## Dependencies
`pip install dropbox` and `pip install unstructured` for PDF reading.

## Tag maintainer
@rlancemartin, @eyurtsev (from Data Loaders). I'd appreciate some
feedback here 🙏 .

## Social Networks
https://github.com/rubenbarragan
https://www.linkedin.com/in/rgbarragan/
https://twitter.com/RubenBarraganP

---------

Co-authored-by: Ruben Barragan <rbarragan@Rubens-MacBook-Air.local>

.devcontainer/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 services:
   langchain:
     build:
-      dockerfile: dev.Dockerfile
+      dockerfile: libs/langchain/dev.Dockerfile
       context: ..
     volumes:
    # Update this to wherever you want VS Code to mount the folder of your project

.github/CONTRIBUTING.md

@@ -69,6 +69,14 @@ This project uses [Poetry](https://python-poetry.org/) as a dependency manager.
 3. Tell Poetry to use the virtualenv python environment (`poetry config virtualenvs.prefer-active-python true`)
 4. Continue with the following steps.
 
+There are two separate projects in this repository:
+- `langchain`: core langchain code, abstractions, and use cases
+- `langchain.experimental`: more experimental code
+
+Each of these has their OWN development environment. 
+In order to run any of the commands below, please move into their respective directories.
+For example, to contribute to `langchain` run `cd libs/langchain` before getting started with the below.
+
 To install requirements:
 
 ```bash
@@ -248,6 +256,9 @@ When you run `poetry install`, the `langchain` package is installed as editable
 
 ## Documentation
 
+While the code is split between `langchain` and `langchain.experimental`, the documentation is one holistic thing.
+This covers how to get started contributing to documentation.
+
 ### Contribute Documentation
 
 The docs directory contains Documentation and API Reference.

.github/PULL_REQUEST_TEMPLATE.md

@@ -7,6 +7,8 @@ Replace this comment with:
   - Tag maintainer: for a quicker response, tag the relevant maintainer (see below),
   - Twitter handle: we announce bigger features on Twitter. If your PR gets announced and you'd like a mention, we'll gladly shout you out!
 
+Please make sure you're PR is passing linting and testing before submitting. Run `make format`, `make lint` and `make test` to check this locally.
+
 If you're adding a new integration, please include:
   1. a test for the integration, preferably unit tests that do not rely on network access,
   2. an example notebook showing its use.

.github/actions/poetry_setup/action.yml

@@ -52,11 +52,13 @@ runs:
 
     - name: Check Poetry File
       shell: bash
+      working-directory: ${{ inputs.working-directory }}
       run: |
         poetry check
 
     - name: Check lock file
       shell: bash
+      working-directory: ${{ inputs.working-directory }}
       run: |
         poetry lock --check
 

.github/workflows/_lint.yml

@@ -1,15 +1,21 @@
 name: lint
 
 on:
-  push:
-    branches: [master]
-  pull_request:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
 
 env:
   POETRY_VERSION: "1.4.2"
 
 jobs:
   build:
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -31,6 +37,10 @@ jobs:
       - name: Install dependencies
         run: |
           poetry install
+      - name: Install langchain editable
+        if: ${{ inputs.working-directory != 'langchain' }}
+        run: |
+          pip install -e ../langchain
       - name: Analysing the code with our lint
         run: |
           make lint

.github/workflows/_release.yml

@@ -1,13 +1,12 @@
 name: release
 
 on:
-  pull_request:
-    types:
-      - closed
-    branches:
-      - master
-    paths:
-      - 'pyproject.toml'
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
 
 env:
   POETRY_VERSION: "1.4.2"
@@ -18,6 +17,9 @@ jobs:
         ${{ github.event.pull_request.merged == true }}
         && ${{ contains(github.event.pull_request.labels.*.name, 'release') }}
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
     steps:
       - uses: actions/checkout@v3
       - name: Install poetry

.github/workflows/_test.yml

@@ -1,16 +1,25 @@
 name: test
 
 on:
-  push:
-    branches: [master]
-  pull_request:
-  workflow_dispatch:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+      test_type:
+        type: string
+        description: "Test types to run"
+        default: '["core", "extended"]'
 
 env:
   POETRY_VERSION: "1.4.2"
 
 jobs:
   build:
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -19,9 +28,7 @@ jobs:
           - "3.9"
           - "3.10"
           - "3.11"
-        test_type:
-          - "core"
-          - "extended"
+        test_type: ${{ fromJSON(inputs.test_type) }}
     name: Python ${{ matrix.python-version }} ${{ matrix.test_type }}
     steps:
       - uses: actions/checkout@v3
@@ -29,6 +36,7 @@ jobs:
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
+          working-directory: ${{ inputs.working-directory }}
           poetry-version: "1.4.2"
           cache-key: ${{ matrix.test_type }}
           install-command: |
@@ -39,6 +47,10 @@ jobs:
                 echo "Running extended tests, installing dependencies with poetry..."
                 poetry install -E extended_testing
               fi
+      - name: Install langchain editable
+        if: ${{ inputs.working-directory != 'langchain' }}
+        run: |
+          pip install -e ../langchain
       - name: Run ${{matrix.test_type}} tests
         run: |
           if [ "${{ matrix.test_type }}" == "core" ]; then

.github/workflows/codespell.yml

@@ -20,3 +20,5 @@ jobs:
         uses: actions/checkout@v3
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
+        with:
+          skip: guide_imports.json

.github/workflows/langchain_ci.yml

@@ -0,0 +1,27 @@
+---
+name: libs/langchain CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    paths:
+      - '.github/workflows/_lint.yml'
+      - '.github/workflows/_test.yml'
+      - '.github/workflows/langchain_ci.yml'
+      - 'libs/langchain/**'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  lint:
+    uses:
+      ./.github/workflows/_lint.yml
+    with:
+      working-directory: libs/langchain
+    secrets: inherit
+  test:
+    uses:
+      ./.github/workflows/_test.yml
+    with:
+      working-directory: libs/langchain
+    secrets: inherit

.github/workflows/langchain_experimental_ci.yml

@@ -0,0 +1,29 @@
+---
+name: libs/langchain-experimental CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    paths:
+      - '.github/workflows/_lint.yml'
+      - '.github/workflows/_test.yml'
+      - '.github/workflows/langchain_experimental_ci.yml'
+      - 'libs/langchain/**'
+      - 'libs/experimental/**'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  lint:
+    uses:
+      ./.github/workflows/_lint.yml
+    with:
+      working-directory: libs/experimental
+    secrets: inherit
+  test:
+    uses:
+      ./.github/workflows/_test.yml
+    with:
+      working-directory: libs/experimental
+      test_type: '["core"]'
+    secrets: inherit

.github/workflows/langchain_experimental_release.yml

@@ -0,0 +1,20 @@
+---
+name: libs/langchain-experimental Release
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - master
+    paths:
+      - 'libs/experimental/pyproject.toml'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  release:
+    uses:
+      ./.github/workflows/_release.yml
+    with:
+      working-directory: libs/experimental
+    secrets: inherit

.github/workflows/langchain_release.yml

@@ -0,0 +1,20 @@
+---
+name: libs/langchain Release
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - master
+    paths:
+      - 'libs/langchain/pyproject.toml'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  release:
+    uses:
+      ./.github/workflows/_release.yml
+    with:
+      working-directory: libs/langchain
+    secrets: inherit

.readthedocs.yaml

@@ -24,6 +24,6 @@ sphinx:
 # Optionally declare the Python requirements required to build your docs
 python:
    install:
-   - requirements: docs/requirements.txt
+   - requirements: docs/api_reference/requirements.txt
    - method: pip
      path: .

MIGRATE.md

@@ -0,0 +1,57 @@
+# Migrating to `langchain_experimental`
+
+We are moving any experimental components of LangChain, or components with vulnerability issues, into `langchain_experimental`.
+This guide covers how to migrate.
+
+## Installation
+
+Previously:
+
+`pip install -U langchain`
+
+Now (only if you want to access things in experimental):
+
+`pip install -U langchain langchain_experimental`
+
+## Things in `langchain.experimental`
+
+Previously:
+
+`from langchain.experimental import ...`
+
+Now:
+
+`from langchain_experimental import ...`
+
+## PALChain
+
+Previously:
+
+`from langchain.chains import PALChain`
+
+Now:
+
+`from langchain_experimental.pal_chain import PALChain`
+
+## SQLDatabaseChain
+
+Previously:
+
+`from langchain.chains import SQLDatabaseChain`
+
+Now:
+
+`from langchain_experimental.sql import SQLDatabaseChain`
+
+## `load_prompt` for Python files
+
+Note: this only applies if you want to load Python files as prompts.
+If you want to load json/yaml files, no change is needed.
+
+Previously:
+
+`from langchain.prompts import load_prompt`
+
+Now:
+
+`from langchain_experimental.prompts import load_prompt`

Makefile

@@ -1,18 +1,8 @@
-.PHONY: all clean docs_build docs_clean docs_linkcheck api_docs_build api_docs_clean api_docs_linkcheck format lint test tests test_watch integration_tests docker_tests help extended_tests
+.PHONY: all clean docs_build docs_clean docs_linkcheck api_docs_build api_docs_clean api_docs_linkcheck
 
 # Default target executed when no arguments are given to make.
 all: help
 
-######################
-# TESTING AND COVERAGE
-######################
-
-# Run unit tests and generate a coverage report.
-coverage:
-	poetry run pytest --cov \
-		--cov-config=.coveragerc \
-		--cov-report xml \
-		--cov-report term-missing:skip-covered
 
 ######################
 # DOCUMENTATION
@@ -41,46 +31,6 @@ api_docs_clean:
 api_docs_linkcheck:
 	poetry run linkchecker docs/api_reference/_build/html/index.html
 
-# Define a variable for the test file path.
-TEST_FILE ?= tests/unit_tests/
-
-test:
-	poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
-
-tests: 
-	poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
-
-extended_tests:
-	poetry run pytest --disable-socket --allow-unix-socket --only-extended tests/unit_tests
-
-test_watch:
-	poetry run ptw --now . -- tests/unit_tests
-
-integration_tests:
-	poetry run pytest tests/integration_tests
-
-docker_tests:
-	docker build -t my-langchain-image:test .
-	docker run --rm my-langchain-image:test
-
-######################
-# LINTING AND FORMATTING
-######################
-
-# Define a variable for Python and notebook files.
-PYTHON_FILES=.
-lint format: PYTHON_FILES=.
-lint_diff format_diff: PYTHON_FILES=$(shell git diff --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
-
-lint lint_diff:
-	poetry run mypy $(PYTHON_FILES)
-	poetry run black $(PYTHON_FILES) --check
-	poetry run ruff .
-
-format format_diff:
-	poetry run black $(PYTHON_FILES)
-	poetry run ruff --select I --fix $(PYTHON_FILES)
-
 spell_check:
 	poetry run codespell --toml pyproject.toml
 
@@ -97,12 +47,3 @@ help:
 	@echo 'docs_build                   - build the documentation'
 	@echo 'docs_clean                   - clean the documentation build artifacts'
 	@echo 'docs_linkcheck               - run linkchecker on the documentation'
-	@echo 'format                       - run code formatters'
-	@echo 'lint                         - run linters'
-	@echo 'test                         - run unit tests'
-	@echo 'tests                        - run unit tests'
-	@echo 'test TEST_FILE=<test_file>   - run all tests in file'
-	@echo 'extended_tests               - run only extended unit tests'
-	@echo 'test_watch                   - run unit tests in watch mode'
-	@echo 'integration_tests            - run integration tests'
-	@echo 'docker_tests                 - run unit tests in docker'

README.md

@@ -3,8 +3,8 @@
 ⚡ Building applications with LLMs through composability ⚡
 
 [![Release Notes](https://img.shields.io/github/release/hwchase17/langchain)](https://github.com/hwchase17/langchain/releases)
-[![lint](https://github.com/hwchase17/langchain/actions/workflows/lint.yml/badge.svg)](https://github.com/hwchase17/langchain/actions/workflows/lint.yml)
-[![test](https://github.com/hwchase17/langchain/actions/workflows/test.yml/badge.svg)](https://github.com/hwchase17/langchain/actions/workflows/test.yml)
+[![CI](https://github.com/hwchase17/langchain/actions/workflows/langchain_ci.yml/badge.svg)](https://github.com/hwchase17/langchain/actions/workflows/langchain_ci.yml)
+[![Experimental CI](https://github.com/hwchase17/langchain/actions/workflows/langchain_experimental_ci.yml/badge.svg)](https://github.com/hwchase17/langchain/actions/workflows/langchain_experimental_ci.yml)
 [![Downloads](https://static.pepy.tech/badge/langchain/month)](https://pepy.tech/project/langchain)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/langchainai.svg?style=social&label=Follow%20%40LangChainAI)](https://twitter.com/langchainai)
@@ -19,7 +19,15 @@
 Looking for the JS/TS version? Check out [LangChain.js](https://github.com/hwchase17/langchainjs).
 
 **Production Support:** As you move your LangChains into production, we'd love to offer more comprehensive support.
-Please fill out [this form](https://forms.gle/57d8AmXBYp8PP8tZA) and we'll set up a dedicated support Slack channel.
+Please fill out [this form](https://6w1pwbss0py.typeform.com/to/rrbrdTH2) and we'll set up a dedicated support Slack channel.
+
+## 🚨Breaking Changes for select chains (SQLDatabase) on 7/28
+
+In an effort to make `langchain` leaner and safer, we are moving select chains to `langchain_experimental`.
+This migration has already started, but we are remaining backwards compatible until 7/28.
+On that date, we will remove functionality from `langchain`.
+Read more about the motivation and the progress [here](https://github.com/hwchase17/langchain/discussions/8043).
+Read how to migrate your code [here](MIGRATE.md).
 
 ## Quick Install
 

docs/api_reference/conf.py

@@ -7,19 +7,66 @@
 
 # -- Path setup --------------------------------------------------------------
 
+import json
+import os
+import sys
+from pathlib import Path
+
+import toml
+from docutils import nodes
+from sphinx.util.docutils import SphinxDirective
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-import os
-import sys
-
-import toml
 
+_DIR = Path(__file__).parent.absolute()
 sys.path.insert(0, os.path.abspath("."))
+sys.path.insert(0, os.path.abspath("../../libs/langchain"))
 
-with open("../../pyproject.toml") as f:
+with (_DIR.parents[1] / "libs" / "langchain" / "pyproject.toml").open("r") as f:
     data = toml.load(f)
+with (_DIR / "guide_imports.json").open("r") as f:
+    imported_classes = json.load(f)
+
+
+class ExampleLinksDirective(SphinxDirective):
+    """Directive to generate a list of links to examples.
+
+    We have a script that extracts links to API reference docs
+    from our notebook examples. This directive uses that information
+    to backlink to the examples from the API reference docs."""
+
+    has_content = False
+    required_arguments = 1
+
+    def run(self):
+        """Run the directive.
+
+        Called any time :example_links:`ClassName` is used
+        in the template *.rst files."""
+        class_or_func_name = self.arguments[0]
+        links = imported_classes.get(class_or_func_name, {})
+        list_node = nodes.bullet_list()
+        for doc_name, link in links.items():
+            item_node = nodes.list_item()
+            para_node = nodes.paragraph()
+            link_node = nodes.reference()
+            link_node["refuri"] = link
+            link_node.append(nodes.Text(doc_name))
+            para_node.append(link_node)
+            item_node.append(para_node)
+            list_node.append(item_node)
+        if list_node.children:
+            title_node = nodes.title()
+            title_node.append(nodes.Text(f"Examples using {class_or_func_name}"))
+            return [title_node, list_node]
+        return [list_node]
+
+
+def setup(app):
+    app.add_directive("example_links", ExampleLinksDirective)
+
 
 # -- Project information -----------------------------------------------------
 

docs/api_reference/create_api_rst.py

@@ -4,7 +4,7 @@ import re
 from pathlib import Path
 
 ROOT_DIR = Path(__file__).parents[2].absolute()
-PKG_DIR = ROOT_DIR / "langchain"
+PKG_DIR = ROOT_DIR / "libs" / "langchain" / "langchain"
 WRITE_FILE = Path(__file__).parent / "api_reference.rst"
 
 
@@ -40,11 +40,7 @@ API Reference
         functions = _members["functions"]
         if not (classes or functions):
             continue
-
-        module_title = module.replace("_", " ").title()
-        if module_title == "Llms":
-            module_title = "LLMs"
-        section = f":mod:`langchain.{module}`: {module_title}"
+        section = f":mod:`langchain.{module}`"
         full_doc += f"""\
 {section}
 {'=' * (len(section) + 1)}
@@ -78,6 +74,7 @@ Functions
 
 .. autosummary::
     :toctree: {module}
+    :template: function.rst
 
     {fstring}
 

docs/api_reference/requirements.txt

@@ -1,3 +1,4 @@
+-e libs/langchain
 autodoc_pydantic==1.8.0
 myst_parser
 nbsphinx==0.8.9
@@ -9,6 +10,4 @@ sphinx-panels
 toml
 myst_nb
 sphinx_copybutton
-pydata-sphinx-theme==0.13.1
-nbdoc
-urllib3<2
+pydata-sphinx-theme==0.13.1

docs/api_reference/templates/class.rst

@@ -26,3 +26,5 @@
    {%- endfor %}
    {% endif %}
    {% endblock %}
+
+.. example_links:: {{ objname }}

docs/api_reference/templates/function.rst

@@ -0,0 +1,8 @@
+:mod:`{{module}}`.{{objname}}
+{{ underline }}==============
+
+.. currentmodule:: {{ module }}
+
+.. autofunction:: {{ objname }}
+
+.. example_links:: {{ objname }}

docs/api_reference/themes/scikit-learn-modern/static/css/theme.css

@@ -745,6 +745,11 @@ span.descname {
   background-color: transparent;
   padding: 0;
   font-family: monospace;
+  font-size: 1.2rem;
+}
+
+em.property {
+  font-weight: normal;
 }
 
 span.descclassname {

docs/docs_skeleton/docs/ecosystem/integrations/index.mdx

@@ -1,10 +0,0 @@
----
-sidebar_position: 0
----
-# Integrations
-
-Visit the [Integrations Hub](https://integrations.langchain.com) to further explore, upvote and request integrations across key LangChain components.
-
-import DocCardList from "@theme/DocCardList";
-
-<DocCardList />

docs/docs_skeleton/docs/get_started/introduction.mdx

@@ -51,7 +51,7 @@ Walkthroughs and best-practices for common end-to-end use cases, like:
 Learn best practices for developing with LangChain.
 
 ### [Ecosystem](/docs/ecosystem/)
-LangChain is part of a rich ecosystem of tools that integrate with our framework and build on top of it. Check out our growing list of [integrations](/docs/ecosystem/integrations/) and [dependent repos](/docs/ecosystem/dependents.html).
+LangChain is part of a rich ecosystem of tools that integrate with our framework and build on top of it. Check out our growing list of [integrations](/docs/integrations/) and [dependent repos](/docs/ecosystem/dependents).
 
 ### [Additional resources](/docs/additional_resources/)
 Our community is full of prolific developers, creative builders, and fantastic teachers. Check out [YouTube tutorials](/docs/additional_resources/youtube.html) for great tutorials from folks in the community, and [Gallery](https://github.com/kyrolabs/awesome-langchain) for a list of awesome LangChain projects, compiled by the folks at [KyroLabs](https://kyrolabs.com).

docs/docs_skeleton/docs/get_started/quickstart.mdx

@@ -22,28 +22,74 @@ import OpenAISetup from "@snippets/get_started/quickstart/openai_setup.mdx"
 
 ## Building an application
 
-Now we can start building our language model application. LangChain provides many modules that can be used to build language model applications. Modules can be used as stand-alones in simple applications and they can be combined for more complex use cases.
+Now we can start building our language model application. LangChain provides many modules that can be used to build language model applications.
+Modules can be used as stand-alones in simple applications and they can be combined for more complex use cases.
+
+The core building block of LangChain applications is the LLMChain.
+This combines three things:
+- LLM: The language model is the core reasoning engine here. In order to work with LangChain, you need to understand the different types of language models and how to work with them.
+- Prompt Templates: This provides instructions to the language model. This controls what the language model outputs, so understanding how to construct prompts and different prompting strategies is crucial.
+- Output Parsers: These translate the raw response from the LLM to a more workable format, making it easy to use the output downstream.
+
+In this getting started guide we will cover those three components by themselves, and then cover the LLMChain which combines all of them.
+Understanding these concepts will set you up well for being able to use and customize LangChain applications.
+Most LangChain applications allow you to configure the LLM and/or the prompt used, so knowing how to take advantage of this will be a big enabler.
 
 ## LLMs
-#### Get predictions from a language model
 
-The basic building block of LangChain is the LLM, which takes in text and generates more text.
+There are two types of language models, which in LangChain are called:
 
-As an example, suppose we're building an application that generates a company name based on a company description. In order to do this, we need to initialize an OpenAI model wrapper. In this case, since we want the outputs to be MORE random, we'll initialize our model with a HIGH temperature.
+- LLMs: this is a language model which takes a string as input and returns a string
+- ChatModels: this is a language model which takes a list of messages as input and returns a message
 
-import LLM from "@snippets/get_started/quickstart/llm.mdx"
+The input/output for LLMs is simple and easy to understand - a string.
+But what about ChatModels? The input there is a list of `ChatMessage`s, and the output is a single `ChatMessage`.
+A `ChatMessage` has two required components:
 
-<LLM/>
+- `content`: This is the content of the message.
+- `role`: This is the role of the entity from which the `ChatMessage` is coming from.
 
-## Chat models
+LangChain provides several objects to easily distinguish between different roles:
 
-Chat models are a variation on language models. While chat models use language models under the hood, the interface they expose is a bit different: rather than expose a "text in, text out" API, they expose an interface where "chat messages" are the inputs and outputs.
+- `HumanMessage`: A `ChatMessage` coming from a human/user.
+- `AIMessage`: A `ChatMessage` coming from an AI/assistant.
+- `SystemMessage`: A `ChatMessage` coming from the system.
+- `FunctionMessage`: A `ChatMessage` coming from a function call.
 
-You can get chat completions by passing one or more messages to the chat model. The response will be a message. The types of messages currently supported in LangChain are `AIMessage`, `HumanMessage`, `SystemMessage`, and `ChatMessage` -- `ChatMessage` takes in an arbitrary role parameter. Most of the time, you'll just be dealing with `HumanMessage`, `AIMessage`, and `SystemMessage`.
+If none of those roles sound right, there is also a `ChatMessage` class where you can specify the role manually.
+For more information on how to use these different messages most effectively, see our prompting guide.
 
-import ChatModel from "@snippets/get_started/quickstart/chat_model.mdx"
+LangChain exposes a standard interface for both, but it's useful to understand this difference in order to construct prompts for a given language model.
+The standard interface that LangChain exposes has two methods:
+- `predict`: Takes in a string, returns a string
+- `predict_messages`: Takes in a list of messages, returns a message.
+
+Let's see how to work with these different types of models and these different types of inputs.
+First, let's import an LLM and a ChatModel.
+
+import ImportLLMs from "@snippets/get_started/quickstart/import_llms.mdx"
+
+<ImportLLMs/>
+
+The `OpenAI` and `ChatOpenAI` objects are basically just configuration objects.
+You can initialize them with parameters like `temperature` and others, and pass them around.
+
+Next, let's use the `predict` method to run over a string input.
+
+import InputString from "@snippets/get_started/quickstart/input_string.mdx"
+
+<InputString/>
+
+Finally, let's use the `predict_messages` method to run over a list of messages.
+
+import InputMessages from "@snippets/get_started/quickstart/input_messages.mdx"
+
+<InputMessages/>
+
+For both these methods, you can also pass in parameters as key word arguments.
+For example, you could pass in `temperature=0` to adjust the temperature that is used from what the object was configured with.
+Whatever values are passed in during run time will always override what the object was configured with.
 
-<ChatModel/>
 
 ## Prompt templates
 
@@ -51,108 +97,66 @@ Most LLM applications do not pass user input directly into an LLM. Usually they
 
 In the previous example, the text we passed to the model contained instructions to generate a company name. For our application, it'd be great if the user only had to provide the description of a company/product, without having to worry about giving the model instructions.
 
+PromptTemplates help with exactly this!
+They bundle up all the logic for going from user input into a fully formatted prompt.
+This can start off very simple - for example, a prompt to produce the above string would just be:
+
 import PromptTemplateLLM from "@snippets/get_started/quickstart/prompt_templates_llms.mdx"
 import PromptTemplateChatModel from "@snippets/get_started/quickstart/prompt_templates_chat_models.mdx"
 
-<Tabs>
-    <TabItem value="llms" label="LLMs" default>
-
-With PromptTemplates this is easy! In this case our template would be very simple:
-
 <PromptTemplateLLM/>
-</TabItem>
-<TabItem value="chat_models" label="Chat models">
 
-Similar to LLMs, you can make use of templating by using a `MessagePromptTemplate`. You can build a `ChatPromptTemplate` from one or more `MessagePromptTemplate`s. You can use `ChatPromptTemplate`'s `format_messages` method to generate the formatted messages.
+However, the advantages of using these over raw string formatting are several.
+You can "partial" out variables - eg you can format only some of the variables at a time.
+You can compose them together, easily combining different templates into a single prompt.
+For explanations of these functionalities, see the [section on prompts](/docs/modules/model_io/prompts) for more detail.
 
-Because this is generating a list of messages, it is slightly more complex than the normal prompt template which is generating only a string. Please see the detailed guides on prompts to understand more options available to you here.
+PromptTemplates can also be used to produce a list of messages.
+In this case, the prompt not only contains information about the content, but also each message (its role, its position in the list, etc)
+Here, what happens most often is a ChatPromptTemplate is a list of ChatMessageTemplates.
+Each ChatMessageTemplate contains instructions for how to format that ChatMessage - its role, and then also its content.
+Let's take a look at this below:
 
 <PromptTemplateChatModel/>
-    </TabItem>
-</Tabs>
 
-## Chains
+ChatPromptTemplates can also include other things besides ChatMessageTemplates - see the [section on prompts](/docs/modules/model_io/prompts) for more detail.
 
-Now that we've got a model and a prompt template, we'll want to combine the two. Chains give us a way to link (or chain) together multiple primitives, like models, prompts, and other chains.
+## Output Parsers
 
-import ChainLLM from "@snippets/get_started/quickstart/chains_llms.mdx"
-import ChainChatModel from "@snippets/get_started/quickstart/chains_chat_models.mdx"
+OutputParsers convert the raw output of an LLM into a format that can be used downstream.
+There are few main type of OutputParsers, including:
 
-<Tabs>
-<TabItem value="llms" label="LLMs" default>
+- Convert text from LLM -> structured information (eg JSON)
+- Convert a ChatMessage into just a string
+- Convert the extra information returned from a call besides the message (like OpenAI function invocation) into a string.
 
-The simplest and most common type of chain is an LLMChain, which passes an input first to a PromptTemplate and then to an LLM. We can construct an LLM chain from our existing model and prompt template.
+For full information on this, see the [section on output parsers](/docs/modules/model_io/output_parsers)
 
-<ChainLLM/>
+In this getting started guide, we will write our own output parser - one that converts a comma separated list into a list.
 
-There we go, our first chain! Understanding how this simple chain works will set you up well for working with more complex chains.
+import OutputParser from "@snippets/get_started/quickstart/output_parser.mdx"
 
-</TabItem>
-<TabItem value="chat_models" label="Chat models">
+<OutputParser/>
 
-The `LLMChain` can be used with chat models as well:
+## LLMChain
 
-<ChainChatModel/>
-</TabItem>
-</Tabs>
+We can now combine all these into one chain.
+This chain will take input variables, pass those to a prompt template to create a prompt, pass the prompt to an LLM, and then pass the output through an (optional) output parser.
+This is a convenient way to bundle up a modular piece of logic.
+Let's see it in action!
 
-## Agents
+import LLMChain from "@snippets/get_started/quickstart/llm_chain.mdx"
 
-import AgentLLM from "@snippets/get_started/quickstart/agents_llms.mdx"
-import AgentChatModel from "@snippets/get_started/quickstart/agents_chat_models.mdx"
+<LLMChain/>
 
-Our first chain ran a pre-determined sequence of steps. To handle complex workflows, we need to be able to dynamically choose actions based on inputs.
+## Next Steps
 
-Agents do just this: they use a language model to determine which actions to take and in what order. Agents are given access to tools, and they repeatedly choose a tool, run the tool, and observe the output until they come up with a final answer.
+This is it!
+We've now gone over how to create the core building block of LangChain applications - the LLMChains.
+There is a lot more nuance in all these components (LLMs, prompts, output parsers) and a lot more different components to learn about as well.
+To continue on your journey:
 
-To load an agent, you need to choose a(n):
-- LLM/Chat model: The language model powering the agent.
-- Tool(s): A function that performs a specific duty. This can be things like: Google Search, Database lookup, Python REPL, other chains. For a list of predefined tools and their specifications, see the [Tools documentation](/docs/modules/agents/tools/).
-- Agent name: A string that references a supported agent class. An agent class is largely parameterized by the prompt the language model uses to determine which action to take. Because this notebook focuses on the simplest, highest level API, this only covers using the standard supported agents. If you want to implement a custom agent, see [here](/docs/modules/agents/how_to/custom_agent.html). For a list of supported agents and their specifications, see [here](/docs/modules/agents/agent_types/).
-
-For this example, we'll be using SerpAPI to query a search engine.
-
-You'll need to install the SerpAPI Python package:
-
-```bash
-pip install google-search-results
-```
-
-And set the `SERPAPI_API_KEY` environment variable.
-
-<Tabs>
-<TabItem value="llms" label="LLMs" default>
-<AgentLLM/>
-</TabItem>
-<TabItem value="chat_models" label="Chat models">
-
-Agents can also be used with chat models, you can initialize one using `AgentType.CHAT_ZERO_SHOT_REACT_DESCRIPTION` as the agent type.
-
-<AgentChatModel/>
-</TabItem>
-</Tabs>
-
-## Memory
-
-The chains and agents we've looked at so far have been stateless, but for many applications it's necessary to reference past interactions. This is clearly the case with a chatbot for example, where you want it to understand new messages in the context of past messages.
-
-The Memory module gives you a way to maintain application state. The base Memory interface is simple: it lets you update state given the latest run inputs and outputs and it lets you modify (or contextualize) the next input using the stored state.
-
-There are a number of built-in memory systems. The simplest of these is a buffer memory which just prepends the last few inputs/outputs to the current input - we will use this in the example below.
-
-import MemoryLLM from "@snippets/get_started/quickstart/memory_llms.mdx"
-import MemoryChatModel from "@snippets/get_started/quickstart/memory_chat_models.mdx"
-
-<Tabs>
-<TabItem value="llms" label="LLMs" default>
-
-<MemoryLLM/>
-</TabItem>
-<TabItem value="chat_models" label="Chat models">
-
-You can use Memory with chains and agents initialized with chat models. The main difference between this and Memory for LLMs is that rather than trying to condense all previous messages into a string, we can keep them as their own unique memory object.
-
-<MemoryChatModel/>
-
-</TabItem>
-</Tabs>
+- [Dive deeper](/docs/modules/model_io) into LLMs, prompts, and output parsers
+- Learn the other [key components](/docs/modules)
+- Check out our [helpful guides](/docs/guides) for detailed walkthroughs on particular topics
+- Explore [end-to-end use cases](/docs/use_cases)

docs/docs_skeleton/docs/guides/evaluation/comparison/index.mdx

@@ -0,0 +1,24 @@
+---
+sidebar_position: 3 
+---
+# Comparison Evaluators
+
+Comparison evaluators in LangChain help measure two different chain or LLM outputs. These evaluators are helpful for comparative analyses, such as A/B testing between two language models, or comparing different versions of the same model. They can also be useful for things like generating preference scores for ai-assisted reinforcement learning.
+
+These evaluators inherit from the `PairwiseStringEvaluator` class, providing a comparison interface for two strings - typically, the outputs from two different prompts or models, or two versions of the same model. In essence, a comparison evaluator performs an evaluation on a pair of strings and returns a dictionary containing the evaluation score and other relevant details.
+
+To create a custom comparison evaluator, inherit from the `PairwiseStringEvaluator` class and overwrite the `_evaluate_string_pairs` method. If you require asynchronous evaluation, also overwrite the `_aevaluate_string_pairs` method.
+
+Here's a summary of the key methods and properties of a comparison evaluator:
+
+- `evaluate_string_pairs`: Evaluate the output string pairs. This function should be overwritten when creating custom evaluators.
+- `aevaluate_string_pairs`: Asynchronously evaluate the output string pairs. This function should be overwritten for asynchronous evaluation.
+- `requires_input`: This property indicates whether this evaluator requires an input string.
+- `requires_reference`: This property specifies whether this evaluator requires a reference label.
+
+Detailed information about creating custom evaluators and the available built-in comparison evaluators are provided in the following sections.
+
+import DocCardList from "@theme/DocCardList";
+
+<DocCardList />
+

docs/docs_skeleton/docs/guides/evaluation/examples/index.mdx

@@ -0,0 +1,12 @@
+---
+sidebar_position: 5
+---
+# Examples
+
+🚧 _Docs under construction_ 🚧
+
+Below are some examples for inspecting and checking different chains.
+
+import DocCardList from "@theme/DocCardList";
+
+<DocCardList />

docs/docs_skeleton/docs/guides/evaluation/index.mdx

@@ -0,0 +1,31 @@
+---
+sidebar_position: 6
+---
+
+import DocCardList from "@theme/DocCardList";
+
+# Evaluation
+
+Building applications with language models involves many moving parts. One of the most critical components is ensuring that the outcomes produced by your models are reliable and useful across a broad array of inputs, and that they work well with your application's other software components. Ensuring reliability usually boils down to some combination of application design, testing & evaluation, and runtime checks. 
+
+The guides in this section review the APIs and functionality LangChain provides to help yous better evaluate your applications. Evaluation and testing are both critical when thinking about deploying LLM applications, since production environments require repeatable and useful outcomes.
+
+LangChain offers various types of evaluators to help you measure performance and integrity on diverse data, and we hope to encourage the the community to create and share other useful evaluators so everyone can improve. These docs will introduce the evaluator types, how to use them, and provide some examples of their use in real-world scenarios.
+
+Each evaluator type in LangChain comes with ready-to-use implementations and an extensible API that allows for customization according to your unique requirements. Here are some of the types of evaluators we offer:
+
+- [String Evaluators](/docs/guides/evaluation/string/): These evaluators assess the predicted string for a given input, usually comparing it against a reference string.
+- [Trajectory Evaluators](/docs/guides/evaluation/trajectory/): These are used to evaluate the entire trajectory of agent actions.
+- [Comparison Evaluators](/docs/guides/evaluation/comparison/): These evaluators are designed to compare predictions from two runs on a common input.
+
+These evaluators can be used across various scenarios and can be applied to different chain and LLM implementations in the LangChain library.
+
+We also are working to share guides and cookbooks that demonstrate how to use these evaluators in real-world scenarios, such as:
+
+- [Chain Comparisons](/docs/guides/evaluation/examples/comparisons): This example uses a comparison evaluator to predict the preferred output. It reviews ways to measure confidence intervals to select statistically significant differences in aggregate preference scores across different models or prompts.
+
+## Reference Docs
+
+For detailed information on the available evaluators, including how to instantiate, configure, and customize them, check out the [reference documentation](https://api.python.langchain.com/en/latest/api_reference.html#module-langchain.evaluation) directly.
+
+<DocCardList />

docs/docs_skeleton/docs/guides/evaluation/string/index.mdx

@@ -0,0 +1,27 @@
+---
+sidebar_position: 2 
+---
+# String Evaluators
+
+A string evaluator is a component within LangChain designed to assess the performance of a language model by comparing its generated outputs (predictions) to a reference string or an input. This comparison is a crucial step in the evaluation of language models, providing a measure of the accuracy or quality of the generated text.
+
+In practice, string evaluators are typically used to evaluate a predicted string against a given input, such as a question or a prompt. Often, a reference label or context string is provided to define what a correct or ideal response would look like. These evaluators can be customized to tailor the evaluation process to fit your application's specific requirements.
+
+To create a custom string evaluator, inherit from the `StringEvaluator` class and implement the `_evaluate_strings` method. If you require asynchronous support, also implement the `_aevaluate_strings` method.
+
+Here's a summary of the key attributes and methods associated with a string evaluator:
+
+- `evaluation_name`: Specifies the name of the evaluation.
+- `requires_input`: Boolean attribute that indicates whether the evaluator requires an input string. If True, the evaluator will raise an error when the input isn't provided. If False, a warning will be logged if an input _is_ provided, indicating that it will not be considered in the evaluation.
+- `requires_reference`: Boolean attribute specifying whether the evaluator requires a reference label. If True, the evaluator will raise an error when the reference isn't provided. If False, a warning will be logged if a reference _is_ provided, indicating that it will not be considered in the evaluation.
+
+String evaluators also implement the following methods:
+
+- `aevaluate_strings`: Asynchronously evaluates the output of the Chain or Language Model, with support for optional input and label.
+- `evaluate_strings`: Synchronously evaluates the output of the Chain or Language Model, with support for optional input and label.
+
+The following sections provide detailed information on available string evaluator implementations as well as how to create a custom string evaluator.
+
+import DocCardList from "@theme/DocCardList";
+
+<DocCardList />

docs/docs_skeleton/docs/guides/evaluation/trajectory/index.mdx

@@ -0,0 +1,28 @@
+---
+sidebar_position: 4
+---
+# Trajectory Evaluators
+
+Trajectory Evaluators in LangChain provide a more holistic approach to evaluating an agent. These evaluators assess the full sequence of actions taken by an agent and their corresponding responses, which we refer to as the "trajectory". This allows you to better measure an agent's effectiveness and capabilities.
+
+A Trajectory Evaluator implements the `AgentTrajectoryEvaluator` interface, which requires two main methods:
+
+- `evaluate_agent_trajectory`: This method synchronously evaluates an agent's trajectory.
+- `aevaluate_agent_trajectory`: This asynchronous counterpart allows evaluations to be run in parallel for efficiency.
+
+Both methods accept three main parameters:
+
+- `input`: The initial input given to the agent.
+- `prediction`: The final predicted response from the agent.
+- `agent_trajectory`: The intermediate steps taken by the agent, given as a list of tuples.
+
+These methods return a dictionary. It is recommended that custom implementations return a `score` (a float indicating the effectiveness of the agent) and `reasoning` (a string explaining the reasoning behind the score).
+
+You can capture an agent's trajectory by initializing the agent with the `return_intermediate_steps=True` parameter. This lets you collect all intermediate steps without relying on special callbacks.
+
+For a deeper dive into the implementation and use of Trajectory Evaluators, refer to the sections below.
+
+import DocCardList from "@theme/DocCardList";
+
+<DocCardList />
+

docs/docs_skeleton/docs/modules/agents/index.mdx

@@ -3,46 +3,80 @@ sidebar_position: 4
 ---
 # Agents
 
-Some applications require a flexible chain of calls to LLMs and other tools based on user input. The **Agent** interface provides the flexibility for such applications. An agent has access to a suite of tools, and determines which ones to use depending on the user input. Agents can use multiple tools, and use the output of one tool as the input to the next.
+The core idea of agents is to use an LLM to choose a sequence of actions to take.
+In chains, a sequence of actions is hardcoded (in code).
+In agents, a language model is used as a reasoning engine to determine which actions to take and in which order.
 
-There are two main types of agents:
+There are several key components here:
 
-- **Action agents**: at each timestep, decide on the next action using the outputs of all previous actions
-- **Plan-and-execute agents**: decide on the full sequence of actions up front, then execute them all without updating the plan
+## Agent
 
-Action agents are suitable for small tasks, while plan-and-execute agents are better for complex or long-running tasks that require maintaining long-term objectives and focus. Often the best approach is to combine the dynamism of an action agent with the planning abilities of a plan-and-execute agent by letting the plan-and-execute agent use action agents to execute plans.
+This is the class responsible for deciding what step to take next.
+This is powered by a language model and a prompt.
+This prompt can include things like:
 
-For a full list of agent types see [agent types](/docs/modules/agents/agent_types/). Additional abstractions involved in agents are:
-- [**Tools**](/docs/modules/agents/tools/): the actions an agent can take. What tools you give an agent highly depend on what you want the agent to do
-- [**Toolkits**](/docs/modules/agents/toolkits/): wrappers around collections of tools that can be used together a specific use case. For example, in order for an agent to
-  interact with a SQL database it will likely need one tool to execute queries and another to inspect tables
+1. The personality of the agent (useful for having it respond in a certain way)
+2. Background context for the agent (useful for giving it more context on the types of tasks it's being asked to do)
+3. Prompting strategies to invoke better reasoning (the most famous/widely used being [ReAct](https://arxiv.org/abs/2210.03629))
 
-## Action agents
+LangChain provides a few different types of agents to get started.
+Even then, you will likely want to customize those agents with parts (1) and (2).
+For a full list of agent types see [agent types](/docs/modules/agents/agent_types/)
 
-At a high-level an action agent:
-1. Receives user input
-2. Decides which tool, if any, to use and the tool input
-3. Calls the tool and records the output (also known as an "observation")
-4. Decides the next step using the history of tools, tool inputs, and observations
-5. Repeats 3-4 until it determines it can respond directly to the user
+## Tools
 
-Action agents are wrapped in **agent executors**, which are responsible for calling the agent, getting back an action and action input, calling the tool that the action references with the generated input, getting the output of the tool, and then passing all that information back into the agent to get the next action it should take.
+Tools are functions that an agent calls.
+There are two important considerations here:
 
-Although an agent can be constructed in many ways, it typically involves these components:
+1. Giving the agent access to the right tools
+2. Describing the tools in a way that is most helpful to the agent
 
-- **Prompt template**: Responsible for taking the user input and previous steps and constructing a prompt
-  to send to the language model
-- **Language model**: Takes the prompt with use input and action history and decides what to do next
-- **Output parser**: Takes the output of the language model and parses it into the next action or a final answer
+Without both, the agent you are trying to build will not work.
+If you don't give the agent access to a correct set of tools, it will never be able to accomplish the objective.
+If you don't describe the tools properly, the agent won't know how to properly use them.
 
-## Plan-and-execute agents
+LangChain provides a wide set of tools to get started, but also makes it easy to define your own (including custom descriptions).
+For a full list of tools, see [here](/docs/modules/agents/tools/)
 
-At a high-level a plan-and-execute agent:
-1. Receives user input
-2. Plans the full sequence of steps to take
-3. Executes the steps in order, passing the outputs of past steps as inputs to future steps
+## Toolkits
 
-The most typical implementation is to have the planner be a language model, and the executor be an action agent. Read more [here](/docs/modules/agents/agent_types/plan_and_execute.html).
+Often the set of tools an agent has access to is more important than a single tool.
+For this LangChain provides the concept of toolkits - groups of tools needed to accomplish specific objectives.
+There are generally around 3-5 tools in a toolkit.
+
+LangChain provides a wide set of toolkits to get started.
+For a full list of toolkits, see [here](/docs/modules/agents/toolkits/)
+
+## AgentExecutor
+
+The agent executor is the runtime for an agent.
+This is what actually calls the agent and executes the actions it chooses.
+Pseudocode for this runtime is below:
+
+```python
+next_action = agent.get_action(...)
+while next_action != AgentFinish:
+    observation = run(next_action)
+    next_action = agent.get_action(..., next_action, observation)
+return next_action
+```
+
+While this may seem simple, there are several complexities this runtime handles for you, including:
+
+1. Handling cases where the agent selects a non-existent tool
+2. Handling cases where the tool errors
+3. Handling cases where the agent produces output that cannot be parsed into a tool invocation
+4. Logging and observability at all levels (agent decisions, tool calls) either to stdout or [LangSmith](https://smith.langchain.com).
+
+## Other types of agent runtimes
+
+The `AgentExecutor` class is the main agent runtime supported by LangChain.
+However, there are other, more experimental runtimes we also support.
+These include:
+
+- [Plan-and-execute Agent](/docs/modules/agents/agent_types/plan_and_execute.html)
+- [Baby AGI](/docs/use_cases/autonomous_agents/baby_agi.html)
+- [Auto GPT](/docs/use_cases/autonomous_agents/autogpt.html)
 
 ## Get started
 

docs/docs_skeleton/docs/modules/agents/toolkits/index.mdx

@@ -3,8 +3,8 @@ sidebar_position: 3
 ---
 # Toolkits
 
+:::info
+Head to [Integrations](/docs/integrations/toolkits/) for documentation on built-in toolkit integrations.
+:::
+
 Toolkits are collections of tools that are designed to be used together for specific tasks and have convenience loading methods.
-
-import DocCardList from "@theme/DocCardList";
-
-<DocCardList />

docs/docs_skeleton/docs/modules/agents/tools/how_to/_category_.yml

@@ -1,2 +0,0 @@
-label: 'How-to'
-position: 0

docs/docs_skeleton/docs/modules/agents/tools/index.mdx

@@ -3,6 +3,10 @@ sidebar_position: 2
 ---
 # Tools
 
+:::info
+Head to [Integrations](/docs/integrations/tools/) for documentation on built-in tool integrations.
+:::
+
 Tools are interfaces that an agent can use to interact with the world.
 
 ## Get started

docs/docs_skeleton/docs/modules/agents/tools/integrations/_category_.yml

@@ -1 +0,0 @@
-label: 'Integrations'

docs/docs_skeleton/docs/modules/callbacks/how_to/_category_.yml

@@ -1,2 +0,0 @@
-label: 'How-to'
-position: 0

docs/docs_skeleton/docs/modules/callbacks/index.mdx

@@ -3,6 +3,10 @@ sidebar_position: 5
 ---
 # Callbacks
 
+:::info
+Head to [Integrations](/docs/integrations/callbacks/) for documentation on built-in callbacks integrations with 3rd-party tools.
+:::
+
 LangChain provides a callbacks system that allows you to hook into the various stages of your LLM application. This is useful for logging, monitoring, streaming, and other tasks.
 
 import GetStarted from "@snippets/modules/callbacks/get_started.mdx"

docs/docs_skeleton/docs/modules/callbacks/integrations/_category_.yml

@@ -1 +0,0 @@
-label: 'Integrations'

docs/docs_skeleton/docs/modules/data_connection/document_loaders/how_to/_category_.yml

@@ -1,2 +0,0 @@
-label: 'How-to'
-position: 0

docs/docs_skeleton/docs/modules/data_connection/document_loaders/index.mdx

@@ -3,6 +3,10 @@ sidebar_position: 0
 ---
 # Document loaders
 
+:::info
+Head to [Integrations](/docs/integrations/document_loaders/) for documentation on built-in document loader integrations with 3rd-party tools.
+:::
+
 Use document loaders to load data from a source as `Document`'s. A `Document` is a piece of text
 and associated metadata. For example, there are document loaders for loading a simple `.txt` file, for loading the text
 contents of any web page, or even for loading a transcript of a YouTube video.

docs/docs_skeleton/docs/modules/data_connection/document_loaders/integrations/_category_.yml

@@ -1 +0,0 @@
-label: 'Integrations'

docs/docs_skeleton/docs/modules/data_connection/document_transformers/index.mdx

@@ -3,6 +3,10 @@ sidebar_position: 1
 ---
 # Document transformers
 
+:::info
+Head to [Integrations](/docs/integrations/document_transformers/) for documentation on built-in document transformer integrations with 3rd-party tools.
+:::
+
 Once you've loaded documents, you'll often want to transform them to better suit your application. The simplest example
 is you may want to split a long document into smaller chunks that can fit into your model's context window. LangChain
 has a number of built-in document transformers that make it easy to split, combine, filter, and otherwise manipulate documents.

docs/docs_skeleton/docs/modules/data_connection/retrievers/how_to/_category_.yml

@@ -1,2 +0,0 @@
-label: 'How-to'
-position: 0

docs/docs_skeleton/docs/modules/data_connection/retrievers/index.mdx

@@ -3,6 +3,10 @@ sidebar_position: 4
 ---
 # Retrievers
 
+:::info
+Head to [Integrations](/docs/integrations/retrievers/) for documentation on built-in retriever integrations with 3rd-party tools.
+:::
+
 A retriever is an interface that returns documents given an unstructured query. It is more general than a vector store.
 A retriever does not need to be able to store documents, only to return (or retrieve) it. Vector stores can be used
 as the backbone of a retriever, but there are other types of retrievers as well.

docs/docs_skeleton/docs/modules/data_connection/text_embedding/index.mdx

@@ -3,6 +3,10 @@ sidebar_position: 2
 ---
 # Text embedding models
 
+:::info
+Head to [Integrations](/docs/integrations/text_embedding/) for documentation on built-in integrations with text embedding model providers.
+:::
+
 The Embeddings class is a class designed for interfacing with text embedding models. There are lots of embedding model providers (OpenAI, Cohere, Hugging Face, etc) - this class is designed to provide a standard interface for all of them.
 
 Embeddings create a vector representation of a piece of text. This is useful because it means we can think about text in the vector space, and do things like semantic search where we look for pieces of text that are most similar in the vector space.

docs/docs_skeleton/docs/modules/data_connection/text_embedding/integrations/_category_.yml

@@ -1 +0,0 @@
-label: 'Integrations'

docs/docs_skeleton/docs/modules/data_connection/vectorstores/index.mdx

@@ -3,11 +3,17 @@ sidebar_position: 3
 ---
 # Vector stores
 
+:::info
+Head to [Integrations](/docs/integrations/vectorstores/) for documentation on built-in integrations with 3rd-party vector stores.
+:::
+
 One of the most common ways to store and search over unstructured data is to embed it and store the resulting embedding
 vectors, and then at query time to embed the unstructured query and retrieve the embedding vectors that are
 'most similar' to the embedded query. A vector store takes care of storing embedded data and performing vector search
 for you.
 
+![vector store diagram](/img/vector_stores.jpg)
+
 ## Get started
 
 This walkthrough showcases basic functionality related to VectorStores. A key part of working with vector stores is creating the vector to put in them, which is usually created via embeddings. Therefore, it is recommended that you familiarize yourself with the [text embedding model](/docs/modules/data_connection/text_embedding/) interfaces before diving into this.

docs/docs_skeleton/docs/modules/data_connection/vectorstores/integrations/_category_.yml

@@ -1 +0,0 @@
-label: 'Integrations'

docs/docs_skeleton/docs/modules/index.mdx

@@ -17,4 +17,6 @@ Let chains choose which tools to use given high-level directives
 #### [Memory](/docs/modules/memory/)
 Persist application state between runs of a chain
 #### [Callbacks](/docs/modules/callbacks/)
-Log and stream intermediate steps of any chain
+Log and stream intermediate steps of any chain
+#### [Evaluation](/docs/modules/evaluation/)
+Evaluate the performance of a chain.

docs/docs_skeleton/docs/modules/memory/how_to/_category_.yml

@@ -1,2 +0,0 @@
-label: 'How-to'
-position: 0

docs/docs_skeleton/docs/modules/memory/index.mdx

@@ -6,6 +6,10 @@ sidebar_position: 3
 
 🚧 _Docs under construction_ 🚧
 
+:::info
+Head to [Integrations](/docs/integrations/memory/) for documentation on built-in memory integrations with 3rd-party tools.
+:::
+
 By default, Chains and Agents are stateless,
 meaning that they treat each incoming query independently (like the underlying LLMs and chat models themselves).
 In some applications, like chatbots, it is essential

docs/docs_skeleton/docs/modules/memory/integrations/_category_.yml

@@ -1 +0,0 @@
-label: 'Integrations'

docs/docs_skeleton/docs/modules/model_io/models/chat/how_to/_category_.yml

@@ -1,2 +0,0 @@
-label: 'How-to'
-position: 0

docs/docs_skeleton/docs/modules/model_io/models/chat/index.mdx

@@ -3,18 +3,16 @@ sidebar_position: 1
 ---
 # Chat models
 
+:::info
+Head to [Integrations](/docs/integrations/chat/) for documentation on built-in integrations with chat model providers.
+:::
+
 Chat models are a variation on language models.
 While chat models use language models under the hood, the interface they expose is a bit different.
 Rather than expose a "text in, text out" API, they expose an interface where "chat messages" are the inputs and outputs.
 
 Chat model APIs are fairly new, so we are still figuring out the correct abstractions.
 
-The following sections of documentation are provided:
-
-- **How-to guides**: Walkthroughs of core functionality, like streaming, creating chat prompts, etc.
-
-- **Integrations**: How to use different chat model providers (OpenAI, Anthropic, etc).
-
 ## Get started
 
 import GetStarted from "@snippets/modules/model_io/models/chat/get_started.mdx"

docs/docs_skeleton/docs/modules/model_io/models/chat/integrations/_category_.yml

@@ -1 +0,0 @@
-label: 'Integrations'

docs/docs_skeleton/docs/modules/model_io/models/llms/how_to/_category_.yml

@@ -1,2 +0,0 @@
-label: 'How-to'
-position: 0

docs/docs_skeleton/docs/modules/model_io/models/llms/index.mdx

@@ -3,14 +3,12 @@ sidebar_position: 0
 ---
 # LLMs
 
+:::info
+Head to [Integrations](/docs/integrations/llms/) for documentation on built-in integrations with LLM providers.
+:::
+
 Large Language Models (LLMs) are a core component of LangChain.
-LangChain does not serve it's own LLMs, but rather provides a standard interface for interacting with many different LLMs.
-
-For more detailed documentation check out our:
-
-- **How-to guides**: Walkthroughs of core functionality, like streaming, async, etc.
-
-- **Integrations**: How to use different LLM providers (OpenAI, Anthropic, etc.)
+LangChain does not serve its own LLMs, but rather provides a standard interface for interacting with many different LLMs.
 
 ## Get started
 

docs/docs_skeleton/docs/modules/model_io/models/llms/integrations/_category_.yml

@@ -1 +0,0 @@
-label: 'Integrations'

docs/docs_skeleton/docusaurus.config.js

@@ -148,6 +148,33 @@ const config = {
       navbar: {
         title: "🦜️🔗 LangChain",
         items: [
+          {
+            to: "/docs/get_started/introduction",
+            label: "Docs",
+            position: "left",
+          },
+          {
+            type: 'docSidebar',
+            position: 'left',
+            sidebarId: 'use_cases',
+            label: 'Use cases',
+          },
+          {
+            type: 'docSidebar',
+            position: 'left',
+            sidebarId: 'integrations',
+            label: 'Integrations',
+          },
+          {
+            href: "https://api.python.langchain.com",
+            label: "API",
+            position: "left",
+          },
+          {
+            to: "https://smith.langchain.com",
+            label: "LangSmith",
+            position: "right",
+          },
           {
             to: "https://js.langchain.com/docs",
             label: "JS/TS Docs",
@@ -156,8 +183,9 @@ const config = {
           // Please keep GitHub link to the right for consistency.
           {
             href: "https://github.com/hwchase17/langchain",
-            label: "GitHub",
-            position: "right",
+            position: 'right',
+            className: 'header-github-link',
+            'aria-label': 'GitHub repository',
           },
         ],
       },

docs/docs_skeleton/generate_api_reference_links.py

@@ -0,0 +1,150 @@
+import importlib
+import inspect
+import json
+import logging
+import os
+import re
+from pathlib import Path
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+# Base URL for all class documentation
+_BASE_URL = "https://api.python.langchain.com/en/latest/"
+
+# Regular expression to match Python code blocks
+code_block_re = re.compile(r"^(```python\n)(.*?)(```\n)", re.DOTALL | re.MULTILINE)
+# Regular expression to match langchain import lines
+_IMPORT_RE = re.compile(r"(from\s+(langchain\.\w+(\.\w+)*?)\s+import\s+)(\w+)")
+
+_CURRENT_PATH = Path(__file__).parent.absolute()
+# Directory where generated markdown files are stored
+_DOCS_DIR = _CURRENT_PATH / "docs"
+_JSON_PATH = _CURRENT_PATH.parent / "api_reference" / "guide_imports.json"
+
+
+def find_files(path):
+    """Find all MDX files in the given path"""
+    for root, _, files in os.walk(path):
+        for file in files:
+            if file.endswith(".mdx") or file.endswith(".md"):
+                yield os.path.join(root, file)
+
+
+def get_full_module_name(module_path, class_name):
+    """Get full module name using inspect"""
+    module = importlib.import_module(module_path)
+    class_ = getattr(module, class_name)
+    return inspect.getmodule(class_).__name__
+
+
+def main():
+    """Main function"""
+    global_imports = {}
+
+    for file in find_files(_DOCS_DIR):
+        print(f"Adding links for imports in {file}")
+
+        # replace_imports now returns the import information rather than writing it to a file
+        file_imports = replace_imports(file)
+
+        if file_imports:
+            # Use relative file path as key
+            relative_path = os.path.relpath(file, _DOCS_DIR)
+            doc_url = f"https://python.langchain.com/docs/{relative_path.replace('.mdx', '').replace('.md', '')}"
+            for import_info in file_imports:
+                doc_title = import_info["title"]
+                class_name = import_info["imported"]
+                if class_name not in global_imports:
+                    global_imports[class_name] = {}
+                global_imports[class_name][doc_title] = doc_url
+
+    # Write the global imports information to a JSON file
+    with _JSON_PATH.open("w") as f:
+        json.dump(global_imports, f)
+
+
+def _get_doc_title(data: str, file_name: str) -> str:
+    try:
+        return re.findall(r"^#\s+(.*)", data, re.MULTILINE)[0]
+    except IndexError:
+        pass
+    # Parse the rst-style titles
+    try:
+        return re.findall(r"^(.*)\n=+\n", data, re.MULTILINE)[0]
+    except IndexError:
+        return file_name
+
+
+def replace_imports(file):
+    """Replace imports in each Python code block with links to their documentation and append the import info in a comment"""
+    all_imports = []
+    with open(file, "r") as f:
+        data = f.read()
+
+    file_name = os.path.basename(file)
+    _DOC_TITLE = _get_doc_title(data, file_name)
+
+    def replacer(match):
+        # Extract the code block content
+        code = match.group(2)
+        # Replace if any import comment exists
+        # TODO: Use our own custom <code> component rather than this
+        # injection method
+        existing_comment_re = re.compile(r"^<!--IMPORTS:.*?-->\n", re.MULTILINE)
+        code = existing_comment_re.sub("", code)
+
+        # Process imports in the code block
+        imports = []
+        for import_match in _IMPORT_RE.finditer(code):
+            class_name = import_match.group(4)
+            try:
+                module_path = get_full_module_name(import_match.group(2), class_name)
+            except AttributeError as e:
+                logger.warning(f"Could not find module for {class_name}, {e}")
+                continue
+            except ImportError as e:
+                # Some CentOS OpenSSL issues can cause this to fail
+                logger.warning(f"Failed to load for class {class_name}, {e}")
+                continue
+
+            url = (
+                _BASE_URL
+                + "/"
+                + module_path.split(".")[1]
+                + "/"
+                + module_path
+                + "."
+                + class_name
+                + ".html"
+            )
+
+            # Add the import information to our list
+            imports.append(
+                {
+                    "imported": class_name,
+                    "source": import_match.group(2),
+                    "docs": url,
+                    "title": _DOC_TITLE,
+                }
+            )
+
+        if imports:
+            all_imports.extend(imports)
+            # Create a unique comment containing the import information
+            import_comment = f"<!--IMPORTS:{json.dumps(imports)}-->"
+            # Inject the import comment at the start of the code block
+            return match.group(1) + import_comment + "\n" + code + match.group(3)
+        else:
+            # If there are no imports, return the original match
+            return match.group(0)
+
+    # Use re.sub to replace each Python code block
+    data = code_block_re.sub(replacer, data)
+
+    with open(file, "w") as f:
+        f.write(data)
+    return all_imports
+
+
+if __name__ == "__main__":
+    main()

docs/docs_skeleton/package.json

@@ -23,7 +23,7 @@
     "@docusaurus/preset-classic": "2.4.0",
     "@docusaurus/remark-plugin-npm2yarn": "^2.4.0",
     "@mdx-js/react": "^1.6.22",
-    "@mendable/search": "^0.0.122",
+    "@mendable/search": "^0.0.125",
     "clsx": "^1.2.1",
     "json-loader": "^0.5.7",
     "process": "^0.11.10",

docs/docs_skeleton/sidebars.js

@@ -20,7 +20,7 @@
 
 module.exports = {
   // By default, Docusaurus generates a sidebar from the docs folder structure
-  sidebar: [
+  docs: [
     {
       type: "category",
       label: "Get started",
@@ -30,7 +30,7 @@ module.exports = {
       link: {
         type: 'generated-index',
         description: 'Get started with LangChain',
-	    slug: "get_started",
+      slug: "get_started",
       },
     },
     {
@@ -44,17 +44,6 @@ module.exports = {
         id: "modules/index"
       },
     },
-    {
-      type: "category",
-      label: "Use cases",
-      collapsed: true,
-      items: [{ type: "autogenerated", dirName: "use_cases" }],
-      link: {
-        type: 'generated-index',
-        description: 'Walkthroughs of common end-to-end use cases',
-	    slug: "use_cases",
-      },
-    },
     {
       type: "category",
       label: "Guides",
@@ -63,7 +52,7 @@ module.exports = {
       link: {
         type: 'generated-index',
         description: 'Design guides for key parts of the development process',
-	    slug: "guides",
+      slug: "guides",
       },
     },
     {
@@ -73,7 +62,7 @@ module.exports = {
       items: [{ type: "autogenerated", dirName: "ecosystem" }],
       link: {
         type: 'generated-index',
-	    slug: "ecosystem",
+      slug: "ecosystem",
       },
     },
     {
@@ -83,18 +72,32 @@ module.exports = {
       items: [{ type: "autogenerated", dirName: "additional_resources" }, { type: "link", label: "Gallery", href: "https://github.com/kyrolabs/awesome-langchain" }],
       link: {
         type: 'generated-index',
-	    slug: "additional_resources",
+      slug: "additional_resources",
       },
     },
+  ],
+  integrations: [
     {
-      type: "html",
-      value: "<hr>",
-      defaultStyle: true,
+      type: "category",
+      label: "Integrations",
+      collapsible: false,
+      items: [{ type: "autogenerated", dirName: "integrations" }],
+      link: {
+        type: 'generated-index',
+      slug: "integrations",
+      },
     },
+  ],
+  use_cases: [
     {
-       type: "link",
-       href: "https://api.python.langchain.com",
-       label: "API reference",
+      type: "category",
+      label: "Use cases",
+      collapsible: false,
+      items: [{ type: "autogenerated", dirName: "use_cases" }],
+      link: {
+        type: 'generated-index',
+      slug: "use_cases",
+      },
     },
   ],
 };

docs/docs_skeleton/src/css/custom.css

@@ -139,4 +139,22 @@
 
 .hidden {
     display: none !important;
+}
+
+.header-github-link:hover {
+  opacity: 0.6;
+}
+
+.header-github-link::before {
+  content: '';
+  width: 24px;
+  height: 24px;
+  display: flex;
+  background: url("data:image/svg+xml,%3Csvg viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12'/%3E%3C/svg%3E")
+    no-repeat;
+}
+
+[data-theme='dark'] .header-github-link::before {
+  background: url("data:image/svg+xml,%3Csvg viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath fill='white' d='M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12'/%3E%3C/svg%3E")
+    no-repeat;
 }

docs/docs_skeleton/src/theme/CodeBlock/index.js

@@ -21,7 +21,7 @@ function Imports({ imports }) {
       </h4>
       <ul style={{ paddingBottom: "1rem" }}>
         {imports.map(({ imported, source, docs }) => (
-          <li>
+          <li key={imported}>
             <a href={docs}>
               <span>{imported}</span>
             </a>{" "}
@@ -34,14 +34,25 @@ function Imports({ imports }) {
 }
 
 export default function CodeBlockWrapper({ children, ...props }) {
+  // Initialize imports as an empty array
+  let imports = [];
+
+  // Check if children is a string
   if (typeof children === "string") {
-    return <CodeBlock {...props}>{children}</CodeBlock>;
+    // Search for an IMPORTS comment in the code
+    const match = /<!--IMPORTS:(.*?)-->\n/.exec(children);
+    if (match) {
+      imports = JSON.parse(match[1]);
+      children = children.replace(match[0], "");
+    }
+  } else if (children.imports) {
+    imports = children.imports;
   }
 
   return (
     <>
-      <CodeBlock {...props}>{children.content}</CodeBlock>
-      <Imports imports={children.imports} />
+      <CodeBlock {...props}>{children}</CodeBlock>
+      {imports.length > 0 && <Imports imports={imports} />}
     </>
   );
-}
+}

docs/docs_skeleton/vercel_build.sh

@@ -1,10 +1,32 @@
 #!/bin/bash
 
+### See: https://github.com/urllib3/urllib3/issues/2168
+# Requests lib breaks for old SSL versions,
+# which are defaults on Amazon Linux 2 (which Vercel uses for builds)
+yum -y update
+yum remove openssl-devel -y
+yum install gcc bzip2-devel libffi-devel zlib-devel wget tar -y
+yum install openssl11 -y
+yum install openssl11-devel -y
+# Install python 3.11 to connect with openSSL 1.1.1
+wget https://www.python.org/ftp/python/3.11.4/Python-3.11.4.tgz 
+tar xzf Python-3.11.4.tgz 
+cd Python-3.11.4 
+./configure 
+make altinstall
+# Check python version
+echo "Python Version"
+python3.11 --version
 cd ..
-python3 --version
-python3 -m venv .venv
+###
+
+# Install nbdev and generate docs
+cd ..
+python3.11 -m venv .venv
 source .venv/bin/activate
-python3 -m pip install -r requirements.txt
+python3.11 -m pip install --upgrade pip
+python3.11 -m pip install -r vercel_requirements.txt
 cp -r extras/* docs_skeleton/docs
 cd docs_skeleton
 nbdoc_build
+python3.11 generate_api_reference_links.py

docs/extras/_templates/integration.mdx

@@ -31,7 +31,7 @@ There isn't any special setup for it.
 
 ## LLM
 
-See a [usage example](/docs/modules/model_io/models/llms/integrations/INCLUDE_REAL_NAME.html).
+See a [usage example](/docs/integrations/llms/INCLUDE_REAL_NAME).
 
 ```python
 from langchain.llms import integration_class_REPLACE_ME
@@ -40,7 +40,7 @@ from langchain.llms import integration_class_REPLACE_ME
 
 ## Text Embedding Models
 
-See a [usage example](/docs/modules/data_connection/text_embedding/integrations/INCLUDE_REAL_NAME.html)
+See a [usage example](/docs/integrations/text_embedding/INCLUDE_REAL_NAME)
 
 ```python
 from langchain.embeddings import integration_class_REPLACE_ME
@@ -49,7 +49,7 @@ from langchain.embeddings import integration_class_REPLACE_ME
 
 ## Chat Models
 
-See a [usage example](/docs/modules/model_io/models/chat/integrations/INCLUDE_REAL_NAME.html)
+See a [usage example](/docs/integrations/chat/INCLUDE_REAL_NAME)
 
 ```python
 from langchain.chat_models import integration_class_REPLACE_ME
@@ -57,7 +57,7 @@ from langchain.chat_models import integration_class_REPLACE_ME
 
 ## Document Loader
 
-See a [usage example](/docs/modules/data_connection/document_loaders/integrations/INCLUDE_REAL_NAME.html).
+See a [usage example](/docs/integrations/document_loaders/INCLUDE_REAL_NAME).
 
 ```python
 from langchain.document_loaders import integration_class_REPLACE_ME

docs/extras/ecosystem/integrations/modal.mdx

@@ -1,66 +0,0 @@
-# Modal
-
-This page covers how to use the Modal ecosystem within LangChain.
-It is broken into two parts: installation and setup, and then references to specific Modal wrappers.
-
-## Installation and Setup
-- Install with `pip install modal-client`
-- Run `modal token new`
-
-## Define your Modal Functions and Webhooks
-
-You must include a prompt. There is a rigid response structure.
-
-```python
-class Item(BaseModel):
-    prompt: str
-
-@stub.webhook(method="POST")
-def my_webhook(item: Item):
-    return {"prompt": my_function.call(item.prompt)}
-```
-
-An example with GPT2:
-
-```python
-from pydantic import BaseModel
-
-import modal
-
-stub = modal.Stub("example-get-started")
-
-volume = modal.SharedVolume().persist("gpt2_model_vol")
-CACHE_PATH = "/root/model_cache"
-
-@stub.function(
-    gpu="any",
-    image=modal.Image.debian_slim().pip_install(
-        "tokenizers", "transformers", "torch", "accelerate"
-    ),
-    shared_volumes={CACHE_PATH: volume},
-    retries=3,
-)
-def run_gpt2(text: str):
-    from transformers import GPT2Tokenizer, GPT2LMHeadModel
-    tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
-    model = GPT2LMHeadModel.from_pretrained('gpt2')
-    encoded_input = tokenizer(text, return_tensors='pt').input_ids
-    output = model.generate(encoded_input, max_length=50, do_sample=True)
-    return tokenizer.decode(output[0], skip_special_tokens=True)
-
-class Item(BaseModel):
-    prompt: str
-
-@stub.webhook(method="POST")
-def get_text(item: Item):
-    return {"prompt": run_gpt2.call(item.prompt)}
-```
-
-## Wrappers
-
-### LLM
-
-There exists an Modal LLM wrapper, which you can access with 
-```python
-from langchain.llms import Modal
-```

docs/extras/guides/debugging.md

@@ -0,0 +1,661 @@
+# Debugging
+
+If you're building with LLMs, at some point something will break, and you'll need to debug. A model call will fail, or the model output will be misformatted, or there will be some nested model calls and it won't be clear where along the way an incorrect output was created.
+
+Here's a few different tools and functionalities to aid in debugging.
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! Instead, edit the notebook w/the location & name as this file. -->
+
+## Tracing
+
+Platforms with tracing capabilities like [LangSmith](/docs/guides/langsmith/) and [WandB](/docs/ecosystem/integrations/agent_with_wandb_tracing) are the most comprehensive solutions for debugging. These platforms make it easy to not only log and visualize LLM apps, but also to actively debug, test and refine them.
+
+For anyone building production-grade LLM applications, we highly recommend using a platform like this.
+
+![LangSmith run](/img/run_details.png)
+
+## `langchain.debug` and `langchain.verbose`
+
+If you're prototyping in Jupyter Notebooks or running Python scripts, it can be helpful to print out the intermediate steps of a Chain run. 
+
+There's a number of ways to enable printing at varying degrees of verbosity.
+
+Let's suppose we have a simple agent and want to visualize the actions it takes and tool outputs it receives. Without any debugging, here's what we see:
+
+
+```python
+from langchain.agents import AgentType, initialize_agent, load_tools
+from langchain.chat_models import ChatOpenAI
+
+llm = ChatOpenAI(model_name="gpt-4", temperature=0)
+tools = load_tools(["ddg-search", "llm-math"], llm=llm)
+agent = initialize_agent(tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION)
+```
+
+
+```python
+agent.run("Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?")
+```
+
+<CodeOutputBlock lang="python">
+
+```
+    'The director of the 2023 film Oppenheimer is Christopher Nolan and he is approximately 19345 days old in 2023.'
+```
+
+</CodeOutputBlock>
+
+### `langchain.debug = True`
+
+Setting the global `debug` flag will cause all LangChain components with callback support (chains, models, agents, tools, retrievers) to print the inputs they receive and outputs they generate. This is the most verbose setting and will fully log raw inputs and outputs.
+
+
+```python
+import langchain
+
+langchain.debug = True
+
+agent.run("Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?")
+```
+
+<details> <summary>Console output</summary>
+
+<CodeOutputBlock lang="python">
+
+```
+    [chain/start] [1:RunTypeEnum.chain:AgentExecutor] Entering Chain run with input:
+    {
+      "input": "Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?"
+    }
+    [chain/start] [1:RunTypeEnum.chain:AgentExecutor > 2:RunTypeEnum.chain:LLMChain] Entering Chain run with input:
+    {
+      "input": "Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?",
+      "agent_scratchpad": "",
+      "stop": [
+        "\nObservation:",
+        "\n\tObservation:"
+      ]
+    }
+    [llm/start] [1:RunTypeEnum.chain:AgentExecutor > 2:RunTypeEnum.chain:LLMChain > 3:RunTypeEnum.llm:ChatOpenAI] Entering LLM run with input:
+    {
+      "prompts": [
+        "Human: Answer the following questions as best you can. You have access to the following tools:\n\nduckduckgo_search: A wrapper around DuckDuckGo Search. Useful for when you need to answer questions about current events. Input should be a search query.\nCalculator: Useful for when you need to answer questions about math.\n\nUse the following format:\n\nQuestion: the input question you must answer\nThought: you should always think about what to do\nAction: the action to take, should be one of [duckduckgo_search, Calculator]\nAction Input: the input to the action\nObservation: the result of the action\n... (this Thought/Action/Action Input/Observation can repeat N times)\nThought: I now know the final answer\nFinal Answer: the final answer to the original input question\n\nBegin!\n\nQuestion: Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?\nThought:"
+      ]
+    }
+    [llm/end] [1:RunTypeEnum.chain:AgentExecutor > 2:RunTypeEnum.chain:LLMChain > 3:RunTypeEnum.llm:ChatOpenAI] [5.53s] Exiting LLM run with output:
+    {
+      "generations": [
+        [
+          {
+            "text": "I need to find out who directed the 2023 film Oppenheimer and their age. Then, I need to calculate their age in days. I will use DuckDuckGo to find out the director and their age.\nAction: duckduckgo_search\nAction Input: \"Director of the 2023 film Oppenheimer and their age\"",
+            "generation_info": {
+              "finish_reason": "stop"
+            },
+            "message": {
+              "lc": 1,
+              "type": "constructor",
+              "id": [
+                "langchain",
+                "schema",
+                "messages",
+                "AIMessage"
+              ],
+              "kwargs": {
+                "content": "I need to find out who directed the 2023 film Oppenheimer and their age. Then, I need to calculate their age in days. I will use DuckDuckGo to find out the director and their age.\nAction: duckduckgo_search\nAction Input: \"Director of the 2023 film Oppenheimer and their age\"",
+                "additional_kwargs": {}
+              }
+            }
+          }
+        ]
+      ],
+      "llm_output": {
+        "token_usage": {
+          "prompt_tokens": 206,
+          "completion_tokens": 71,
+          "total_tokens": 277
+        },
+        "model_name": "gpt-4"
+      },
+      "run": null
+    }
+    [chain/end] [1:RunTypeEnum.chain:AgentExecutor > 2:RunTypeEnum.chain:LLMChain] [5.53s] Exiting Chain run with output:
+    {
+      "text": "I need to find out who directed the 2023 film Oppenheimer and their age. Then, I need to calculate their age in days. I will use DuckDuckGo to find out the director and their age.\nAction: duckduckgo_search\nAction Input: \"Director of the 2023 film Oppenheimer and their age\""
+    }
+    [tool/start] [1:RunTypeEnum.chain:AgentExecutor > 4:RunTypeEnum.tool:duckduckgo_search] Entering Tool run with input:
+    "Director of the 2023 film Oppenheimer and their age"
+    [tool/end] [1:RunTypeEnum.chain:AgentExecutor > 4:RunTypeEnum.tool:duckduckgo_search] [1.51s] Exiting Tool run with output:
+    "Capturing the mad scramble to build the first atomic bomb required rapid-fire filming, strict set rules and the construction of an entire 1940s western town. By Jada Yuan. July 19, 2023 at 5:00 a ... In Christopher Nolan's new film, "Oppenheimer," Cillian Murphy stars as J. Robert Oppenheimer, the American physicist who oversaw the Manhattan Project in Los Alamos, N.M. Universal Pictures... Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. Christopher Nolan goes deep on 'Oppenheimer,' his most 'extreme' film to date. By Kenneth Turan. July 11, 2023 5 AM PT. For Subscribers. Christopher Nolan is photographed in Los Angeles ... Oppenheimer is a 2023 epic biographical thriller film written and directed by Christopher Nolan.It is based on the 2005 biography American Prometheus by Kai Bird and Martin J. Sherwin about J. Robert Oppenheimer, a theoretical physicist who was pivotal in developing the first nuclear weapons as part of the Manhattan Project and thereby ushering in the Atomic Age."
+    [chain/start] [1:RunTypeEnum.chain:AgentExecutor > 5:RunTypeEnum.chain:LLMChain] Entering Chain run with input:
+    {
+      "input": "Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?",
+      "agent_scratchpad": "I need to find out who directed the 2023 film Oppenheimer and their age. Then, I need to calculate their age in days. I will use DuckDuckGo to find out the director and their age.\nAction: duckduckgo_search\nAction Input: \"Director of the 2023 film Oppenheimer and their age\"\nObservation: Capturing the mad scramble to build the first atomic bomb required rapid-fire filming, strict set rules and the construction of an entire 1940s western town. By Jada Yuan. July 19, 2023 at 5:00 a ... In Christopher Nolan's new film, \"Oppenheimer,\" Cillian Murphy stars as J. Robert Oppenheimer, the American physicist who oversaw the Manhattan Project in Los Alamos, N.M. Universal Pictures... Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. Christopher Nolan goes deep on 'Oppenheimer,' his most 'extreme' film to date. By Kenneth Turan. July 11, 2023 5 AM PT. For Subscribers. Christopher Nolan is photographed in Los Angeles ... Oppenheimer is a 2023 epic biographical thriller film written and directed by Christopher Nolan.It is based on the 2005 biography American Prometheus by Kai Bird and Martin J. Sherwin about J. Robert Oppenheimer, a theoretical physicist who was pivotal in developing the first nuclear weapons as part of the Manhattan Project and thereby ushering in the Atomic Age.\nThought:",
+      "stop": [
+        "\nObservation:",
+        "\n\tObservation:"
+      ]
+    }
+    [llm/start] [1:RunTypeEnum.chain:AgentExecutor > 5:RunTypeEnum.chain:LLMChain > 6:RunTypeEnum.llm:ChatOpenAI] Entering LLM run with input:
+    {
+      "prompts": [
+        "Human: Answer the following questions as best you can. You have access to the following tools:\n\nduckduckgo_search: A wrapper around DuckDuckGo Search. Useful for when you need to answer questions about current events. Input should be a search query.\nCalculator: Useful for when you need to answer questions about math.\n\nUse the following format:\n\nQuestion: the input question you must answer\nThought: you should always think about what to do\nAction: the action to take, should be one of [duckduckgo_search, Calculator]\nAction Input: the input to the action\nObservation: the result of the action\n... (this Thought/Action/Action Input/Observation can repeat N times)\nThought: I now know the final answer\nFinal Answer: the final answer to the original input question\n\nBegin!\n\nQuestion: Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?\nThought:I need to find out who directed the 2023 film Oppenheimer and their age. Then, I need to calculate their age in days. I will use DuckDuckGo to find out the director and their age.\nAction: duckduckgo_search\nAction Input: \"Director of the 2023 film Oppenheimer and their age\"\nObservation: Capturing the mad scramble to build the first atomic bomb required rapid-fire filming, strict set rules and the construction of an entire 1940s western town. By Jada Yuan. July 19, 2023 at 5:00 a ... In Christopher Nolan's new film, \"Oppenheimer,\" Cillian Murphy stars as J. Robert Oppenheimer, the American physicist who oversaw the Manhattan Project in Los Alamos, N.M. Universal Pictures... Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. Christopher Nolan goes deep on 'Oppenheimer,' his most 'extreme' film to date. By Kenneth Turan. July 11, 2023 5 AM PT. For Subscribers. Christopher Nolan is photographed in Los Angeles ... Oppenheimer is a 2023 epic biographical thriller film written and directed by Christopher Nolan.It is based on the 2005 biography American Prometheus by Kai Bird and Martin J. Sherwin about J. Robert Oppenheimer, a theoretical physicist who was pivotal in developing the first nuclear weapons as part of the Manhattan Project and thereby ushering in the Atomic Age.\nThought:"
+      ]
+    }
+    [llm/end] [1:RunTypeEnum.chain:AgentExecutor > 5:RunTypeEnum.chain:LLMChain > 6:RunTypeEnum.llm:ChatOpenAI] [4.46s] Exiting LLM run with output:
+    {
+      "generations": [
+        [
+          {
+            "text": "The director of the 2023 film Oppenheimer is Christopher Nolan. Now I need to find out his age.\nAction: duckduckgo_search\nAction Input: \"Christopher Nolan age\"",
+            "generation_info": {
+              "finish_reason": "stop"
+            },
+            "message": {
+              "lc": 1,
+              "type": "constructor",
+              "id": [
+                "langchain",
+                "schema",
+                "messages",
+                "AIMessage"
+              ],
+              "kwargs": {
+                "content": "The director of the 2023 film Oppenheimer is Christopher Nolan. Now I need to find out his age.\nAction: duckduckgo_search\nAction Input: \"Christopher Nolan age\"",
+                "additional_kwargs": {}
+              }
+            }
+          }
+        ]
+      ],
+      "llm_output": {
+        "token_usage": {
+          "prompt_tokens": 550,
+          "completion_tokens": 39,
+          "total_tokens": 589
+        },
+        "model_name": "gpt-4"
+      },
+      "run": null
+    }
+    [chain/end] [1:RunTypeEnum.chain:AgentExecutor > 5:RunTypeEnum.chain:LLMChain] [4.46s] Exiting Chain run with output:
+    {
+      "text": "The director of the 2023 film Oppenheimer is Christopher Nolan. Now I need to find out his age.\nAction: duckduckgo_search\nAction Input: \"Christopher Nolan age\""
+    }
+    [tool/start] [1:RunTypeEnum.chain:AgentExecutor > 7:RunTypeEnum.tool:duckduckgo_search] Entering Tool run with input:
+    "Christopher Nolan age"
+    [tool/end] [1:RunTypeEnum.chain:AgentExecutor > 7:RunTypeEnum.tool:duckduckgo_search] [1.33s] Exiting Tool run with output:
+    "Christopher Edward Nolan CBE (born 30 July 1970) is a British and American filmmaker. Known for his Hollywood blockbusters with complex storytelling, Nolan is considered a leading filmmaker of the 21st century. His films have grossed $5 billion worldwide. The recipient of many accolades, he has been nominated for five Academy Awards, five BAFTA Awards and six Golden Globe Awards. July 30, 1970 (age 52) London England Notable Works: "Dunkirk" "Tenet" "The Prestige" See all related content → Recent News Jul. 13, 2023, 11:11 AM ET (AP) Cillian Murphy, playing Oppenheimer, finally gets to lead a Christopher Nolan film July 11, 2023 5 AM PT For Subscribers Christopher Nolan is photographed in Los Angeles. (Joe Pugliese / For The Times) This is not the story I was supposed to write. Oppenheimer director Christopher Nolan, Cillian Murphy, Emily Blunt and Matt Damon on the stakes of making a three-hour, CGI-free summer film. Christopher Nolan, the director behind such films as "Dunkirk," "Inception," "Interstellar," and the "Dark Knight" trilogy, has spent the last three years living in Oppenheimer's world, writing ..."
+    [chain/start] [1:RunTypeEnum.chain:AgentExecutor > 8:RunTypeEnum.chain:LLMChain] Entering Chain run with input:
+    {
+      "input": "Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?",
+      "agent_scratchpad": "I need to find out who directed the 2023 film Oppenheimer and their age. Then, I need to calculate their age in days. I will use DuckDuckGo to find out the director and their age.\nAction: duckduckgo_search\nAction Input: \"Director of the 2023 film Oppenheimer and their age\"\nObservation: Capturing the mad scramble to build the first atomic bomb required rapid-fire filming, strict set rules and the construction of an entire 1940s western town. By Jada Yuan. July 19, 2023 at 5:00 a ... In Christopher Nolan's new film, \"Oppenheimer,\" Cillian Murphy stars as J. Robert Oppenheimer, the American physicist who oversaw the Manhattan Project in Los Alamos, N.M. Universal Pictures... Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. Christopher Nolan goes deep on 'Oppenheimer,' his most 'extreme' film to date. By Kenneth Turan. July 11, 2023 5 AM PT. For Subscribers. Christopher Nolan is photographed in Los Angeles ... Oppenheimer is a 2023 epic biographical thriller film written and directed by Christopher Nolan.It is based on the 2005 biography American Prometheus by Kai Bird and Martin J. Sherwin about J. Robert Oppenheimer, a theoretical physicist who was pivotal in developing the first nuclear weapons as part of the Manhattan Project and thereby ushering in the Atomic Age.\nThought:The director of the 2023 film Oppenheimer is Christopher Nolan. Now I need to find out his age.\nAction: duckduckgo_search\nAction Input: \"Christopher Nolan age\"\nObservation: Christopher Edward Nolan CBE (born 30 July 1970) is a British and American filmmaker. Known for his Hollywood blockbusters with complex storytelling, Nolan is considered a leading filmmaker of the 21st century. His films have grossed $5 billion worldwide. The recipient of many accolades, he has been nominated for five Academy Awards, five BAFTA Awards and six Golden Globe Awards. July 30, 1970 (age 52) London England Notable Works: \"Dunkirk\" \"Tenet\" \"The Prestige\" See all related content → Recent News Jul. 13, 2023, 11:11 AM ET (AP) Cillian Murphy, playing Oppenheimer, finally gets to lead a Christopher Nolan film July 11, 2023 5 AM PT For Subscribers Christopher Nolan is photographed in Los Angeles. (Joe Pugliese / For The Times) This is not the story I was supposed to write. Oppenheimer director Christopher Nolan, Cillian Murphy, Emily Blunt and Matt Damon on the stakes of making a three-hour, CGI-free summer film. Christopher Nolan, the director behind such films as \"Dunkirk,\" \"Inception,\" \"Interstellar,\" and the \"Dark Knight\" trilogy, has spent the last three years living in Oppenheimer's world, writing ...\nThought:",
+      "stop": [
+        "\nObservation:",
+        "\n\tObservation:"
+      ]
+    }
+    [llm/start] [1:RunTypeEnum.chain:AgentExecutor > 8:RunTypeEnum.chain:LLMChain > 9:RunTypeEnum.llm:ChatOpenAI] Entering LLM run with input:
+    {
+      "prompts": [
+        "Human: Answer the following questions as best you can. You have access to the following tools:\n\nduckduckgo_search: A wrapper around DuckDuckGo Search. Useful for when you need to answer questions about current events. Input should be a search query.\nCalculator: Useful for when you need to answer questions about math.\n\nUse the following format:\n\nQuestion: the input question you must answer\nThought: you should always think about what to do\nAction: the action to take, should be one of [duckduckgo_search, Calculator]\nAction Input: the input to the action\nObservation: the result of the action\n... (this Thought/Action/Action Input/Observation can repeat N times)\nThought: I now know the final answer\nFinal Answer: the final answer to the original input question\n\nBegin!\n\nQuestion: Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?\nThought:I need to find out who directed the 2023 film Oppenheimer and their age. Then, I need to calculate their age in days. I will use DuckDuckGo to find out the director and their age.\nAction: duckduckgo_search\nAction Input: \"Director of the 2023 film Oppenheimer and their age\"\nObservation: Capturing the mad scramble to build the first atomic bomb required rapid-fire filming, strict set rules and the construction of an entire 1940s western town. By Jada Yuan. July 19, 2023 at 5:00 a ... In Christopher Nolan's new film, \"Oppenheimer,\" Cillian Murphy stars as J. Robert Oppenheimer, the American physicist who oversaw the Manhattan Project in Los Alamos, N.M. Universal Pictures... Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. Christopher Nolan goes deep on 'Oppenheimer,' his most 'extreme' film to date. By Kenneth Turan. July 11, 2023 5 AM PT. For Subscribers. Christopher Nolan is photographed in Los Angeles ... Oppenheimer is a 2023 epic biographical thriller film written and directed by Christopher Nolan.It is based on the 2005 biography American Prometheus by Kai Bird and Martin J. Sherwin about J. Robert Oppenheimer, a theoretical physicist who was pivotal in developing the first nuclear weapons as part of the Manhattan Project and thereby ushering in the Atomic Age.\nThought:The director of the 2023 film Oppenheimer is Christopher Nolan. Now I need to find out his age.\nAction: duckduckgo_search\nAction Input: \"Christopher Nolan age\"\nObservation: Christopher Edward Nolan CBE (born 30 July 1970) is a British and American filmmaker. Known for his Hollywood blockbusters with complex storytelling, Nolan is considered a leading filmmaker of the 21st century. His films have grossed $5 billion worldwide. The recipient of many accolades, he has been nominated for five Academy Awards, five BAFTA Awards and six Golden Globe Awards. July 30, 1970 (age 52) London England Notable Works: \"Dunkirk\" \"Tenet\" \"The Prestige\" See all related content → Recent News Jul. 13, 2023, 11:11 AM ET (AP) Cillian Murphy, playing Oppenheimer, finally gets to lead a Christopher Nolan film July 11, 2023 5 AM PT For Subscribers Christopher Nolan is photographed in Los Angeles. (Joe Pugliese / For The Times) This is not the story I was supposed to write. Oppenheimer director Christopher Nolan, Cillian Murphy, Emily Blunt and Matt Damon on the stakes of making a three-hour, CGI-free summer film. Christopher Nolan, the director behind such films as \"Dunkirk,\" \"Inception,\" \"Interstellar,\" and the \"Dark Knight\" trilogy, has spent the last three years living in Oppenheimer's world, writing ...\nThought:"
+      ]
+    }
+    [llm/end] [1:RunTypeEnum.chain:AgentExecutor > 8:RunTypeEnum.chain:LLMChain > 9:RunTypeEnum.llm:ChatOpenAI] [2.69s] Exiting LLM run with output:
+    {
+      "generations": [
+        [
+          {
+            "text": "Christopher Nolan was born on July 30, 1970, which makes him 52 years old in 2023. Now I need to calculate his age in days.\nAction: Calculator\nAction Input: 52*365",
+            "generation_info": {
+              "finish_reason": "stop"
+            },
+            "message": {
+              "lc": 1,
+              "type": "constructor",
+              "id": [
+                "langchain",
+                "schema",
+                "messages",
+                "AIMessage"
+              ],
+              "kwargs": {
+                "content": "Christopher Nolan was born on July 30, 1970, which makes him 52 years old in 2023. Now I need to calculate his age in days.\nAction: Calculator\nAction Input: 52*365",
+                "additional_kwargs": {}
+              }
+            }
+          }
+        ]
+      ],
+      "llm_output": {
+        "token_usage": {
+          "prompt_tokens": 868,
+          "completion_tokens": 46,
+          "total_tokens": 914
+        },
+        "model_name": "gpt-4"
+      },
+      "run": null
+    }
+    [chain/end] [1:RunTypeEnum.chain:AgentExecutor > 8:RunTypeEnum.chain:LLMChain] [2.69s] Exiting Chain run with output:
+    {
+      "text": "Christopher Nolan was born on July 30, 1970, which makes him 52 years old in 2023. Now I need to calculate his age in days.\nAction: Calculator\nAction Input: 52*365"
+    }
+    [tool/start] [1:RunTypeEnum.chain:AgentExecutor > 10:RunTypeEnum.tool:Calculator] Entering Tool run with input:
+    "52*365"
+    [chain/start] [1:RunTypeEnum.chain:AgentExecutor > 10:RunTypeEnum.tool:Calculator > 11:RunTypeEnum.chain:LLMMathChain] Entering Chain run with input:
+    {
+      "question": "52*365"
+    }
+    [chain/start] [1:RunTypeEnum.chain:AgentExecutor > 10:RunTypeEnum.tool:Calculator > 11:RunTypeEnum.chain:LLMMathChain > 12:RunTypeEnum.chain:LLMChain] Entering Chain run with input:
+    {
+      "question": "52*365",
+      "stop": [
+        "```output"
+      ]
+    }
+    [llm/start] [1:RunTypeEnum.chain:AgentExecutor > 10:RunTypeEnum.tool:Calculator > 11:RunTypeEnum.chain:LLMMathChain > 12:RunTypeEnum.chain:LLMChain > 13:RunTypeEnum.llm:ChatOpenAI] Entering LLM run with input:
+    {
+      "prompts": [
+        "Human: Translate a math problem into a expression that can be executed using Python's numexpr library. Use the output of running this code to answer the question.\n\nQuestion: ${Question with math problem.}\n```text\n${single line mathematical expression that solves the problem}\n```\n...numexpr.evaluate(text)...\n```output\n${Output of running the code}\n```\nAnswer: ${Answer}\n\nBegin.\n\nQuestion: What is 37593 * 67?\n```text\n37593 * 67\n```\n...numexpr.evaluate(\"37593 * 67\")...\n```output\n2518731\n```\nAnswer: 2518731\n\nQuestion: 37593^(1/5)\n```text\n37593**(1/5)\n```\n...numexpr.evaluate(\"37593**(1/5)\")...\n```output\n8.222831614237718\n```\nAnswer: 8.222831614237718\n\nQuestion: 52*365"
+      ]
+    }
+    [llm/end] [1:RunTypeEnum.chain:AgentExecutor > 10:RunTypeEnum.tool:Calculator > 11:RunTypeEnum.chain:LLMMathChain > 12:RunTypeEnum.chain:LLMChain > 13:RunTypeEnum.llm:ChatOpenAI] [2.89s] Exiting LLM run with output:
+    {
+      "generations": [
+        [
+          {
+            "text": "```text\n52*365\n```\n...numexpr.evaluate(\"52*365\")...\n",
+            "generation_info": {
+              "finish_reason": "stop"
+            },
+            "message": {
+              "lc": 1,
+              "type": "constructor",
+              "id": [
+                "langchain",
+                "schema",
+                "messages",
+                "AIMessage"
+              ],
+              "kwargs": {
+                "content": "```text\n52*365\n```\n...numexpr.evaluate(\"52*365\")...\n",
+                "additional_kwargs": {}
+              }
+            }
+          }
+        ]
+      ],
+      "llm_output": {
+        "token_usage": {
+          "prompt_tokens": 203,
+          "completion_tokens": 19,
+          "total_tokens": 222
+        },
+        "model_name": "gpt-4"
+      },
+      "run": null
+    }
+    [chain/end] [1:RunTypeEnum.chain:AgentExecutor > 10:RunTypeEnum.tool:Calculator > 11:RunTypeEnum.chain:LLMMathChain > 12:RunTypeEnum.chain:LLMChain] [2.89s] Exiting Chain run with output:
+    {
+      "text": "```text\n52*365\n```\n...numexpr.evaluate(\"52*365\")...\n"
+    }
+    [chain/end] [1:RunTypeEnum.chain:AgentExecutor > 10:RunTypeEnum.tool:Calculator > 11:RunTypeEnum.chain:LLMMathChain] [2.90s] Exiting Chain run with output:
+    {
+      "answer": "Answer: 18980"
+    }
+    [tool/end] [1:RunTypeEnum.chain:AgentExecutor > 10:RunTypeEnum.tool:Calculator] [2.90s] Exiting Tool run with output:
+    "Answer: 18980"
+    [chain/start] [1:RunTypeEnum.chain:AgentExecutor > 14:RunTypeEnum.chain:LLMChain] Entering Chain run with input:
+    {
+      "input": "Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?",
+      "agent_scratchpad": "I need to find out who directed the 2023 film Oppenheimer and their age. Then, I need to calculate their age in days. I will use DuckDuckGo to find out the director and their age.\nAction: duckduckgo_search\nAction Input: \"Director of the 2023 film Oppenheimer and their age\"\nObservation: Capturing the mad scramble to build the first atomic bomb required rapid-fire filming, strict set rules and the construction of an entire 1940s western town. By Jada Yuan. July 19, 2023 at 5:00 a ... In Christopher Nolan's new film, \"Oppenheimer,\" Cillian Murphy stars as J. Robert Oppenheimer, the American physicist who oversaw the Manhattan Project in Los Alamos, N.M. Universal Pictures... Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. Christopher Nolan goes deep on 'Oppenheimer,' his most 'extreme' film to date. By Kenneth Turan. July 11, 2023 5 AM PT. For Subscribers. Christopher Nolan is photographed in Los Angeles ... Oppenheimer is a 2023 epic biographical thriller film written and directed by Christopher Nolan.It is based on the 2005 biography American Prometheus by Kai Bird and Martin J. Sherwin about J. Robert Oppenheimer, a theoretical physicist who was pivotal in developing the first nuclear weapons as part of the Manhattan Project and thereby ushering in the Atomic Age.\nThought:The director of the 2023 film Oppenheimer is Christopher Nolan. Now I need to find out his age.\nAction: duckduckgo_search\nAction Input: \"Christopher Nolan age\"\nObservation: Christopher Edward Nolan CBE (born 30 July 1970) is a British and American filmmaker. Known for his Hollywood blockbusters with complex storytelling, Nolan is considered a leading filmmaker of the 21st century. His films have grossed $5 billion worldwide. The recipient of many accolades, he has been nominated for five Academy Awards, five BAFTA Awards and six Golden Globe Awards. July 30, 1970 (age 52) London England Notable Works: \"Dunkirk\" \"Tenet\" \"The Prestige\" See all related content → Recent News Jul. 13, 2023, 11:11 AM ET (AP) Cillian Murphy, playing Oppenheimer, finally gets to lead a Christopher Nolan film July 11, 2023 5 AM PT For Subscribers Christopher Nolan is photographed in Los Angeles. (Joe Pugliese / For The Times) This is not the story I was supposed to write. Oppenheimer director Christopher Nolan, Cillian Murphy, Emily Blunt and Matt Damon on the stakes of making a three-hour, CGI-free summer film. Christopher Nolan, the director behind such films as \"Dunkirk,\" \"Inception,\" \"Interstellar,\" and the \"Dark Knight\" trilogy, has spent the last three years living in Oppenheimer's world, writing ...\nThought:Christopher Nolan was born on July 30, 1970, which makes him 52 years old in 2023. Now I need to calculate his age in days.\nAction: Calculator\nAction Input: 52*365\nObservation: Answer: 18980\nThought:",
+      "stop": [
+        "\nObservation:",
+        "\n\tObservation:"
+      ]
+    }
+    [llm/start] [1:RunTypeEnum.chain:AgentExecutor > 14:RunTypeEnum.chain:LLMChain > 15:RunTypeEnum.llm:ChatOpenAI] Entering LLM run with input:
+    {
+      "prompts": [
+        "Human: Answer the following questions as best you can. You have access to the following tools:\n\nduckduckgo_search: A wrapper around DuckDuckGo Search. Useful for when you need to answer questions about current events. Input should be a search query.\nCalculator: Useful for when you need to answer questions about math.\n\nUse the following format:\n\nQuestion: the input question you must answer\nThought: you should always think about what to do\nAction: the action to take, should be one of [duckduckgo_search, Calculator]\nAction Input: the input to the action\nObservation: the result of the action\n... (this Thought/Action/Action Input/Observation can repeat N times)\nThought: I now know the final answer\nFinal Answer: the final answer to the original input question\n\nBegin!\n\nQuestion: Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?\nThought:I need to find out who directed the 2023 film Oppenheimer and their age. Then, I need to calculate their age in days. I will use DuckDuckGo to find out the director and their age.\nAction: duckduckgo_search\nAction Input: \"Director of the 2023 film Oppenheimer and their age\"\nObservation: Capturing the mad scramble to build the first atomic bomb required rapid-fire filming, strict set rules and the construction of an entire 1940s western town. By Jada Yuan. July 19, 2023 at 5:00 a ... In Christopher Nolan's new film, \"Oppenheimer,\" Cillian Murphy stars as J. Robert Oppenheimer, the American physicist who oversaw the Manhattan Project in Los Alamos, N.M. Universal Pictures... Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. Christopher Nolan goes deep on 'Oppenheimer,' his most 'extreme' film to date. By Kenneth Turan. July 11, 2023 5 AM PT. For Subscribers. Christopher Nolan is photographed in Los Angeles ... Oppenheimer is a 2023 epic biographical thriller film written and directed by Christopher Nolan.It is based on the 2005 biography American Prometheus by Kai Bird and Martin J. Sherwin about J. Robert Oppenheimer, a theoretical physicist who was pivotal in developing the first nuclear weapons as part of the Manhattan Project and thereby ushering in the Atomic Age.\nThought:The director of the 2023 film Oppenheimer is Christopher Nolan. Now I need to find out his age.\nAction: duckduckgo_search\nAction Input: \"Christopher Nolan age\"\nObservation: Christopher Edward Nolan CBE (born 30 July 1970) is a British and American filmmaker. Known for his Hollywood blockbusters with complex storytelling, Nolan is considered a leading filmmaker of the 21st century. His films have grossed $5 billion worldwide. The recipient of many accolades, he has been nominated for five Academy Awards, five BAFTA Awards and six Golden Globe Awards. July 30, 1970 (age 52) London England Notable Works: \"Dunkirk\" \"Tenet\" \"The Prestige\" See all related content → Recent News Jul. 13, 2023, 11:11 AM ET (AP) Cillian Murphy, playing Oppenheimer, finally gets to lead a Christopher Nolan film July 11, 2023 5 AM PT For Subscribers Christopher Nolan is photographed in Los Angeles. (Joe Pugliese / For The Times) This is not the story I was supposed to write. Oppenheimer director Christopher Nolan, Cillian Murphy, Emily Blunt and Matt Damon on the stakes of making a three-hour, CGI-free summer film. Christopher Nolan, the director behind such films as \"Dunkirk,\" \"Inception,\" \"Interstellar,\" and the \"Dark Knight\" trilogy, has spent the last three years living in Oppenheimer's world, writing ...\nThought:Christopher Nolan was born on July 30, 1970, which makes him 52 years old in 2023. Now I need to calculate his age in days.\nAction: Calculator\nAction Input: 52*365\nObservation: Answer: 18980\nThought:"
+      ]
+    }
+    [llm/end] [1:RunTypeEnum.chain:AgentExecutor > 14:RunTypeEnum.chain:LLMChain > 15:RunTypeEnum.llm:ChatOpenAI] [3.52s] Exiting LLM run with output:
+    {
+      "generations": [
+        [
+          {
+            "text": "I now know the final answer\nFinal Answer: The director of the 2023 film Oppenheimer is Christopher Nolan and he is 52 years old. His age in days is approximately 18980 days.",
+            "generation_info": {
+              "finish_reason": "stop"
+            },
+            "message": {
+              "lc": 1,
+              "type": "constructor",
+              "id": [
+                "langchain",
+                "schema",
+                "messages",
+                "AIMessage"
+              ],
+              "kwargs": {
+                "content": "I now know the final answer\nFinal Answer: The director of the 2023 film Oppenheimer is Christopher Nolan and he is 52 years old. His age in days is approximately 18980 days.",
+                "additional_kwargs": {}
+              }
+            }
+          }
+        ]
+      ],
+      "llm_output": {
+        "token_usage": {
+          "prompt_tokens": 926,
+          "completion_tokens": 43,
+          "total_tokens": 969
+        },
+        "model_name": "gpt-4"
+      },
+      "run": null
+    }
+    [chain/end] [1:RunTypeEnum.chain:AgentExecutor > 14:RunTypeEnum.chain:LLMChain] [3.52s] Exiting Chain run with output:
+    {
+      "text": "I now know the final answer\nFinal Answer: The director of the 2023 film Oppenheimer is Christopher Nolan and he is 52 years old. His age in days is approximately 18980 days."
+    }
+    [chain/end] [1:RunTypeEnum.chain:AgentExecutor] [21.96s] Exiting Chain run with output:
+    {
+      "output": "The director of the 2023 film Oppenheimer is Christopher Nolan and he is 52 years old. His age in days is approximately 18980 days."
+    }
+
+
+
+
+
+    'The director of the 2023 film Oppenheimer is Christopher Nolan and he is 52 years old. His age in days is approximately 18980 days.'
+```
+
+</CodeOutputBlock>
+
+</details>
+
+### `langchain.verbose = True`
+
+Setting the `verbose` flag will print out inputs and outputs in a slightly more readable format and will skip logging certain raw outputs (like the token usage stats for an LLM call) so that you can focus on application logic.
+
+
+```python
+import langchain
+
+langchain.verbose = True
+
+agent.run("Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?")
+```
+
+<details> <summary>Console output</summary>
+
+<CodeOutputBlock lang="python">
+
+```
+    
+    
+    > Entering new AgentExecutor chain...
+    
+    
+    > Entering new LLMChain chain...
+    Prompt after formatting:
+    Answer the following questions as best you can. You have access to the following tools:
+    
+    duckduckgo_search: A wrapper around DuckDuckGo Search. Useful for when you need to answer questions about current events. Input should be a search query.
+    Calculator: Useful for when you need to answer questions about math.
+    
+    Use the following format:
+    
+    Question: the input question you must answer
+    Thought: you should always think about what to do
+    Action: the action to take, should be one of [duckduckgo_search, Calculator]
+    Action Input: the input to the action
+    Observation: the result of the action
+    ... (this Thought/Action/Action Input/Observation can repeat N times)
+    Thought: I now know the final answer
+    Final Answer: the final answer to the original input question
+    
+    Begin!
+    
+    Question: Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?
+    Thought:
+    
+    > Finished chain.
+    First, I need to find out who directed the film Oppenheimer in 2023 and their birth date to calculate their age.
+    Action: duckduckgo_search
+    Action Input: "Director of the 2023 film Oppenheimer"
+    Observation: Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. In Christopher Nolan's new film, "Oppenheimer," Cillian Murphy stars as J. Robert ... 2023, 12:16 p.m. ET. ... including his role as the director of the Manhattan Engineer District, better ... J Robert Oppenheimer was the director of the secret Los Alamos Laboratory. It was established under US president Franklin D Roosevelt as part of the Manhattan Project to build the first atomic bomb. He oversaw the first atomic bomb detonation in the New Mexico desert in July 1945, code-named "Trinity". In this opening salvo of 2023's Oscar battle, Nolan has enjoined a star-studded cast for a retelling of the brilliant and haunted life of J. Robert Oppenheimer, the American physicist whose... Oppenheimer is a 2023 epic biographical thriller film written and directed by Christopher Nolan.It is based on the 2005 biography American Prometheus by Kai Bird and Martin J. Sherwin about J. Robert Oppenheimer, a theoretical physicist who was pivotal in developing the first nuclear weapons as part of the Manhattan Project and thereby ushering in the Atomic Age.
+    Thought:
+    
+    > Entering new LLMChain chain...
+    Prompt after formatting:
+    Answer the following questions as best you can. You have access to the following tools:
+    
+    duckduckgo_search: A wrapper around DuckDuckGo Search. Useful for when you need to answer questions about current events. Input should be a search query.
+    Calculator: Useful for when you need to answer questions about math.
+    
+    Use the following format:
+    
+    Question: the input question you must answer
+    Thought: you should always think about what to do
+    Action: the action to take, should be one of [duckduckgo_search, Calculator]
+    Action Input: the input to the action
+    Observation: the result of the action
+    ... (this Thought/Action/Action Input/Observation can repeat N times)
+    Thought: I now know the final answer
+    Final Answer: the final answer to the original input question
+    
+    Begin!
+    
+    Question: Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?
+    Thought:First, I need to find out who directed the film Oppenheimer in 2023 and their birth date to calculate their age.
+    Action: duckduckgo_search
+    Action Input: "Director of the 2023 film Oppenheimer"
+    Observation: Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. In Christopher Nolan's new film, "Oppenheimer," Cillian Murphy stars as J. Robert ... 2023, 12:16 p.m. ET. ... including his role as the director of the Manhattan Engineer District, better ... J Robert Oppenheimer was the director of the secret Los Alamos Laboratory. It was established under US president Franklin D Roosevelt as part of the Manhattan Project to build the first atomic bomb. He oversaw the first atomic bomb detonation in the New Mexico desert in July 1945, code-named "Trinity". In this opening salvo of 2023's Oscar battle, Nolan has enjoined a star-studded cast for a retelling of the brilliant and haunted life of J. Robert Oppenheimer, the American physicist whose... Oppenheimer is a 2023 epic biographical thriller film written and directed by Christopher Nolan.It is based on the 2005 biography American Prometheus by Kai Bird and Martin J. Sherwin about J. Robert Oppenheimer, a theoretical physicist who was pivotal in developing the first nuclear weapons as part of the Manhattan Project and thereby ushering in the Atomic Age.
+    Thought:
+    
+    > Finished chain.
+    The director of the 2023 film Oppenheimer is Christopher Nolan. Now I need to find out his birth date to calculate his age.
+    Action: duckduckgo_search
+    Action Input: "Christopher Nolan birth date"
+    Observation: July 30, 1970 (age 52) London England Notable Works: "Dunkirk" "Tenet" "The Prestige" See all related content → Recent News Jul. 13, 2023, 11:11 AM ET (AP) Cillian Murphy, playing Oppenheimer, finally gets to lead a Christopher Nolan film Christopher Edward Nolan CBE (born 30 July 1970) is a British and American filmmaker. Known for his Hollywood blockbusters with complex storytelling, Nolan is considered a leading filmmaker of the 21st century. His films have grossed $5 billion worldwide. The recipient of many accolades, he has been nominated for five Academy Awards, five BAFTA Awards and six Golden Globe Awards. Christopher Nolan is currently 52 according to his birthdate July 30, 1970 Sun Sign Leo Born Place Westminster, London, England, United Kingdom Residence Los Angeles, California, United States Nationality Education Chris attended Haileybury and Imperial Service College, in Hertford Heath, Hertfordshire. Christopher Nolan's next movie will study the man who developed the atomic bomb, J. Robert Oppenheimer. Here's the release date, plot, trailers & more. July 2023 sees the release of Christopher Nolan's new film, Oppenheimer, his first movie since 2020's Tenet and his split from Warner Bros. Billed as an epic thriller about "the man who ...
+    Thought:
+    
+    > Entering new LLMChain chain...
+    Prompt after formatting:
+    Answer the following questions as best you can. You have access to the following tools:
+    
+    duckduckgo_search: A wrapper around DuckDuckGo Search. Useful for when you need to answer questions about current events. Input should be a search query.
+    Calculator: Useful for when you need to answer questions about math.
+    
+    Use the following format:
+    
+    Question: the input question you must answer
+    Thought: you should always think about what to do
+    Action: the action to take, should be one of [duckduckgo_search, Calculator]
+    Action Input: the input to the action
+    Observation: the result of the action
+    ... (this Thought/Action/Action Input/Observation can repeat N times)
+    Thought: I now know the final answer
+    Final Answer: the final answer to the original input question
+    
+    Begin!
+    
+    Question: Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?
+    Thought:First, I need to find out who directed the film Oppenheimer in 2023 and their birth date to calculate their age.
+    Action: duckduckgo_search
+    Action Input: "Director of the 2023 film Oppenheimer"
+    Observation: Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. In Christopher Nolan's new film, "Oppenheimer," Cillian Murphy stars as J. Robert ... 2023, 12:16 p.m. ET. ... including his role as the director of the Manhattan Engineer District, better ... J Robert Oppenheimer was the director of the secret Los Alamos Laboratory. It was established under US president Franklin D Roosevelt as part of the Manhattan Project to build the first atomic bomb. He oversaw the first atomic bomb detonation in the New Mexico desert in July 1945, code-named "Trinity". In this opening salvo of 2023's Oscar battle, Nolan has enjoined a star-studded cast for a retelling of the brilliant and haunted life of J. Robert Oppenheimer, the American physicist whose... Oppenheimer is a 2023 epic biographical thriller film written and directed by Christopher Nolan.It is based on the 2005 biography American Prometheus by Kai Bird and Martin J. Sherwin about J. Robert Oppenheimer, a theoretical physicist who was pivotal in developing the first nuclear weapons as part of the Manhattan Project and thereby ushering in the Atomic Age.
+    Thought:The director of the 2023 film Oppenheimer is Christopher Nolan. Now I need to find out his birth date to calculate his age.
+    Action: duckduckgo_search
+    Action Input: "Christopher Nolan birth date"
+    Observation: July 30, 1970 (age 52) London England Notable Works: "Dunkirk" "Tenet" "The Prestige" See all related content → Recent News Jul. 13, 2023, 11:11 AM ET (AP) Cillian Murphy, playing Oppenheimer, finally gets to lead a Christopher Nolan film Christopher Edward Nolan CBE (born 30 July 1970) is a British and American filmmaker. Known for his Hollywood blockbusters with complex storytelling, Nolan is considered a leading filmmaker of the 21st century. His films have grossed $5 billion worldwide. The recipient of many accolades, he has been nominated for five Academy Awards, five BAFTA Awards and six Golden Globe Awards. Christopher Nolan is currently 52 according to his birthdate July 30, 1970 Sun Sign Leo Born Place Westminster, London, England, United Kingdom Residence Los Angeles, California, United States Nationality Education Chris attended Haileybury and Imperial Service College, in Hertford Heath, Hertfordshire. Christopher Nolan's next movie will study the man who developed the atomic bomb, J. Robert Oppenheimer. Here's the release date, plot, trailers & more. July 2023 sees the release of Christopher Nolan's new film, Oppenheimer, his first movie since 2020's Tenet and his split from Warner Bros. Billed as an epic thriller about "the man who ...
+    Thought:
+    
+    > Finished chain.
+    Christopher Nolan was born on July 30, 1970. Now I need to calculate his age in 2023 and then convert it into days.
+    Action: Calculator
+    Action Input: (2023 - 1970) * 365
+    
+    > Entering new LLMMathChain chain...
+    (2023 - 1970) * 365
+    
+    > Entering new LLMChain chain...
+    Prompt after formatting:
+    Translate a math problem into a expression that can be executed using Python's numexpr library. Use the output of running this code to answer the question.
+    
+    Question: ${Question with math problem.}
+    ```text
+    ${single line mathematical expression that solves the problem}
+    ```
+    ...numexpr.evaluate(text)...
+    ```output
+    ${Output of running the code}
+    ```
+    Answer: ${Answer}
+    
+    Begin.
+    
+    Question: What is 37593 * 67?
+    ```text
+    37593 * 67
+    ```
+    ...numexpr.evaluate("37593 * 67")...
+    ```output
+    2518731
+    ```
+    Answer: 2518731
+    
+    Question: 37593^(1/5)
+    ```text
+    37593**(1/5)
+    ```
+    ...numexpr.evaluate("37593**(1/5)")...
+    ```output
+    8.222831614237718
+    ```
+    Answer: 8.222831614237718
+    
+    Question: (2023 - 1970) * 365
+    
+    
+    > Finished chain.
+    ```text
+    (2023 - 1970) * 365
+    ```
+    ...numexpr.evaluate("(2023 - 1970) * 365")...
+    
+    Answer: 19345
+    > Finished chain.
+    
+    Observation: Answer: 19345
+    Thought:
+    
+    > Entering new LLMChain chain...
+    Prompt after formatting:
+    Answer the following questions as best you can. You have access to the following tools:
+    
+    duckduckgo_search: A wrapper around DuckDuckGo Search. Useful for when you need to answer questions about current events. Input should be a search query.
+    Calculator: Useful for when you need to answer questions about math.
+    
+    Use the following format:
+    
+    Question: the input question you must answer
+    Thought: you should always think about what to do
+    Action: the action to take, should be one of [duckduckgo_search, Calculator]
+    Action Input: the input to the action
+    Observation: the result of the action
+    ... (this Thought/Action/Action Input/Observation can repeat N times)
+    Thought: I now know the final answer
+    Final Answer: the final answer to the original input question
+    
+    Begin!
+    
+    Question: Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?
+    Thought:First, I need to find out who directed the film Oppenheimer in 2023 and their birth date to calculate their age.
+    Action: duckduckgo_search
+    Action Input: "Director of the 2023 film Oppenheimer"
+    Observation: Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. In Christopher Nolan's new film, "Oppenheimer," Cillian Murphy stars as J. Robert ... 2023, 12:16 p.m. ET. ... including his role as the director of the Manhattan Engineer District, better ... J Robert Oppenheimer was the director of the secret Los Alamos Laboratory. It was established under US president Franklin D Roosevelt as part of the Manhattan Project to build the first atomic bomb. He oversaw the first atomic bomb detonation in the New Mexico desert in July 1945, code-named "Trinity". In this opening salvo of 2023's Oscar battle, Nolan has enjoined a star-studded cast for a retelling of the brilliant and haunted life of J. Robert Oppenheimer, the American physicist whose... Oppenheimer is a 2023 epic biographical thriller film written and directed by Christopher Nolan.It is based on the 2005 biography American Prometheus by Kai Bird and Martin J. Sherwin about J. Robert Oppenheimer, a theoretical physicist who was pivotal in developing the first nuclear weapons as part of the Manhattan Project and thereby ushering in the Atomic Age.
+    Thought:The director of the 2023 film Oppenheimer is Christopher Nolan. Now I need to find out his birth date to calculate his age.
+    Action: duckduckgo_search
+    Action Input: "Christopher Nolan birth date"
+    Observation: July 30, 1970 (age 52) London England Notable Works: "Dunkirk" "Tenet" "The Prestige" See all related content → Recent News Jul. 13, 2023, 11:11 AM ET (AP) Cillian Murphy, playing Oppenheimer, finally gets to lead a Christopher Nolan film Christopher Edward Nolan CBE (born 30 July 1970) is a British and American filmmaker. Known for his Hollywood blockbusters with complex storytelling, Nolan is considered a leading filmmaker of the 21st century. His films have grossed $5 billion worldwide. The recipient of many accolades, he has been nominated for five Academy Awards, five BAFTA Awards and six Golden Globe Awards. Christopher Nolan is currently 52 according to his birthdate July 30, 1970 Sun Sign Leo Born Place Westminster, London, England, United Kingdom Residence Los Angeles, California, United States Nationality Education Chris attended Haileybury and Imperial Service College, in Hertford Heath, Hertfordshire. Christopher Nolan's next movie will study the man who developed the atomic bomb, J. Robert Oppenheimer. Here's the release date, plot, trailers & more. July 2023 sees the release of Christopher Nolan's new film, Oppenheimer, his first movie since 2020's Tenet and his split from Warner Bros. Billed as an epic thriller about "the man who ...
+    Thought:Christopher Nolan was born on July 30, 1970. Now I need to calculate his age in 2023 and then convert it into days.
+    Action: Calculator
+    Action Input: (2023 - 1970) * 365
+    Observation: Answer: 19345
+    Thought:
+    
+    > Finished chain.
+    I now know the final answer
+    Final Answer: The director of the 2023 film Oppenheimer is Christopher Nolan and he is 53 years old in 2023. His age in days is 19345 days.
+    
+    > Finished chain.
+
+
+    'The director of the 2023 film Oppenheimer is Christopher Nolan and he is 53 years old in 2023. His age in days is 19345 days.'
+```
+
+</CodeOutputBlock>
+
+</details>
+
+### `Chain(..., verbose=True)`
+
+You can also scope verbosity down to a single object, in which case only the inputs and outputs to that object are printed (along with any additional callbacks calls made specifically by that object).
+
+
+```python
+# Passing verbose=True to initialize_agent will pass that along to the AgentExecutor (which is a Chain).
+agent = initialize_agent(
+    tools, 
+    llm, 
+    agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION,
+    verbose=True,
+)
+
+agent.run("Who directed the 2023 film Oppenheimer and what is their age? What is their age in days (assume 365 days per year)?")
+```
+
+<details> <summary>Console output</summary>
+
+<CodeOutputBlock lang="python">
+
+```
+    > Entering new AgentExecutor chain...
+    First, I need to find out who directed the film Oppenheimer in 2023 and their birth date. Then, I can calculate their age in years and days.
+    Action: duckduckgo_search
+    Action Input: "Director of 2023 film Oppenheimer"
+    Observation: Oppenheimer: Directed by Christopher Nolan. With Cillian Murphy, Emily Blunt, Robert Downey Jr., Alden Ehrenreich. The story of American scientist J. Robert Oppenheimer and his role in the development of the atomic bomb. In Christopher Nolan's new film, "Oppenheimer," Cillian Murphy stars as J. Robert Oppenheimer, the American physicist who oversaw the Manhattan Project in Los Alamos, N.M. Universal Pictures... J Robert Oppenheimer was the director of the secret Los Alamos Laboratory. It was established under US president Franklin D Roosevelt as part of the Manhattan Project to build the first atomic bomb. He oversaw the first atomic bomb detonation in the New Mexico desert in July 1945, code-named "Trinity". A Review of Christopher Nolan's new film 'Oppenheimer' , the story of the man who fathered the Atomic Bomb. Cillian Murphy leads an all star cast ... Release Date: July 21, 2023. Director ... For his new film, "Oppenheimer," starring Cillian Murphy and Emily Blunt, director Christopher Nolan set out to build an entire 1940s western town.
+    Thought:The director of the 2023 film Oppenheimer is Christopher Nolan. Now I need to find out his birth date to calculate his age.
+    Action: duckduckgo_search
+    Action Input: "Christopher Nolan birth date"
+    Observation: July 30, 1970 (age 52) London England Notable Works: "Dunkirk" "Tenet" "The Prestige" See all related content → Recent News Jul. 13, 2023, 11:11 AM ET (AP) Cillian Murphy, playing Oppenheimer, finally gets to lead a Christopher Nolan film Christopher Edward Nolan CBE (born 30 July 1970) is a British and American filmmaker. Known for his Hollywood blockbusters with complex storytelling, Nolan is considered a leading filmmaker of the 21st century. His films have grossed $5 billion worldwide. The recipient of many accolades, he has been nominated for five Academy Awards, five BAFTA Awards and six Golden Globe Awards. Christopher Nolan is currently 52 according to his birthdate July 30, 1970 Sun Sign Leo Born Place Westminster, London, England, United Kingdom Residence Los Angeles, California, United States Nationality Education Chris attended Haileybury and Imperial Service College, in Hertford Heath, Hertfordshire. Christopher Nolan's next movie will study the man who developed the atomic bomb, J. Robert Oppenheimer. Here's the release date, plot, trailers & more. Date of Birth: 30 July 1970 . ... Christopher Nolan is a British-American film director, producer, and screenwriter. His films have grossed more than US$5 billion worldwide, and have garnered 11 Academy Awards from 36 nominations. ...
+    Thought:Christopher Nolan was born on July 30, 1970. Now I can calculate his age in years and then in days.
+    Action: Calculator
+    Action Input: {"operation": "subtract", "operands": [2023, 1970]}
+    Observation: Answer: 53
+    Thought:Christopher Nolan is 53 years old in 2023. Now I need to calculate his age in days.
+    Action: Calculator
+    Action Input: {"operation": "multiply", "operands": [53, 365]}
+    Observation: Answer: 19345
+    Thought:I now know the final answer
+    Final Answer: The director of the 2023 film Oppenheimer is Christopher Nolan. He is 53 years old in 2023, which is approximately 19345 days.
+    
+    > Finished chain.
+
+
+    'The director of the 2023 film Oppenheimer is Christopher Nolan. He is 53 years old in 2023, which is approximately 19345 days.'
+```
+
+</CodeOutputBlock>
+
+</details>
+
+## Other callbacks
+
+`Callbacks` are what we use to execute any functionality within a component outside the primary component logic. All of the above solutions use `Callbacks` under the hood to log intermediate steps of components. There's a number of `Callbacks` relevant for debugging that come with LangChain out of the box, like the [FileCallbackHandler](/docs/modules/callbacks/how_to/filecallbackhandler). You can also implement your own callbacks to execute custom functionality.
+
+See here for more info on [Callbacks](/docs/modules/callbacks/), how to use them, and customize them.

docs/extras/guides/evaluation/agent_benchmarking.ipynb

@@ -1,301 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "984169ca",
-   "metadata": {},
-   "source": [
-    "# Agent Benchmarking: Search + Calculator\n",
-    "\n",
-    "Here we go over how to benchmark performance of an agent on tasks where it has access to a calculator and a search tool.\n",
-    "\n",
-    "It is highly reccomended that you do any evaluation/benchmarking with tracing enabled. See [here](https://python.langchain.com/docs/guides/tracing/) for an explanation of what tracing is and how to set it up."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "46bf9205",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "# Comment this out if you are NOT using tracing\n",
-    "import os\n",
-    "\n",
-    "os.environ[\"LANGCHAIN_HANDLER\"] = \"langchain\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "8a16b75d",
-   "metadata": {},
-   "source": [
-    "## Loading the data\n",
-    "First, let's load the data."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "5b2d5e98",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "from langchain.evaluation.loading import load_dataset\n",
-    "\n",
-    "dataset = load_dataset(\"agent-search-calculator\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4ab6a716",
-   "metadata": {},
-   "source": [
-    "## Setting up a chain\n",
-    "Now we need to load an agent capable of answering these questions."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c18680b5",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "from langchain.llms import OpenAI\n",
-    "from langchain.chains import LLMMathChain\n",
-    "from langchain.agents import initialize_agent, Tool, load_tools\n",
-    "from langchain.agents import AgentType\n",
-    "\n",
-    "tools = load_tools([\"serpapi\", \"llm-math\"], llm=OpenAI(temperature=0))\n",
-    "agent = initialize_agent(\n",
-    "    tools,\n",
-    "    OpenAI(temperature=0),\n",
-    "    agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION,\n",
-    "    verbose=True,\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "68504a8f",
-   "metadata": {},
-   "source": [
-    "## Make a prediction\n",
-    "\n",
-    "First, we can make predictions one datapoint at a time. Doing it at this level of granularity allows use to explore the outputs in detail, and also is a lot cheaper than running over multiple datapoints"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cbcafc92",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "print(dataset[0][\"question\"])\n",
-    "agent.run(dataset[0][\"question\"])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "d0c16cd7",
-   "metadata": {},
-   "source": [
-    "## Make many predictions\n",
-    "Now we can make predictions"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "bbbbb20e",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "agent.run(dataset[4][\"question\"])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "24b4c66e",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "predictions = []\n",
-    "predicted_dataset = []\n",
-    "error_dataset = []\n",
-    "for data in dataset:\n",
-    "    new_data = {\"input\": data[\"question\"], \"answer\": data[\"answer\"]}\n",
-    "    try:\n",
-    "        predictions.append(agent(new_data))\n",
-    "        predicted_dataset.append(new_data)\n",
-    "    except Exception as e:\n",
-    "        predictions.append({\"output\": str(e), **new_data})\n",
-    "        error_dataset.append(new_data)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "49d969fb",
-   "metadata": {},
-   "source": [
-    "## Evaluate performance\n",
-    "Now we can evaluate the predictions. The first thing we can do is look at them by eye."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1d583f03",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "predictions[0]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4783344b",
-   "metadata": {},
-   "source": [
-    "Next, we can use a language model to score them programatically"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d0a9341d",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "from langchain.evaluation.qa import QAEvalChain"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1612dec1",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "llm = OpenAI(temperature=0)\n",
-    "eval_chain = QAEvalChain.from_llm(llm)\n",
-    "graded_outputs = eval_chain.evaluate(\n",
-    "    dataset, predictions, question_key=\"question\", prediction_key=\"output\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "79587806",
-   "metadata": {},
-   "source": [
-    "We can add in the graded output to the `predictions` dict and then get a count of the grades."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2a689df5",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "for i, prediction in enumerate(predictions):\n",
-    "    prediction[\"grade\"] = graded_outputs[i][\"text\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "27b61215",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "from collections import Counter\n",
-    "\n",
-    "Counter([pred[\"grade\"] for pred in predictions])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "12fe30f4",
-   "metadata": {},
-   "source": [
-    "We can also filter the datapoints to the incorrect examples and look at them."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "47c692a1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "incorrect = [pred for pred in predictions if pred[\"grade\"] == \" INCORRECT\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0ef976c1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "incorrect"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3eb948cf-f767-4c87-a12d-275b66eef407",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

docs/extras/guides/evaluation/agent_vectordb_sota_pg.ipynb

@@ -1,524 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "984169ca",
-   "metadata": {},
-   "source": [
-    "# Agent VectorDB Question Answering Benchmarking\n",
-    "\n",
-    "Here we go over how to benchmark performance on a question answering task using an agent to route between multiple vectordatabases.\n",
-    "\n",
-    "It is highly recommended that you do any evaluation/benchmarking with tracing enabled. See [here](https://python.langchain.com/guides/tracing/) for an explanation of what tracing is and how to set it up."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "7b57a50f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Comment this out if you are NOT using tracing\n",
-    "import os\n",
-    "\n",
-    "os.environ[\"LANGCHAIN_HANDLER\"] = \"langchain\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "8a16b75d",
-   "metadata": {},
-   "source": [
-    "## Loading the data\n",
-    "First, let's load the data."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "5b2d5e98",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Found cached dataset json (/Users/qt/.cache/huggingface/datasets/LangChainDatasets___json/LangChainDatasets--agent-vectordb-qa-sota-pg-d3ae24016b514f92/0.0.0/fe5dd6ea2639a6df622901539cb550cf8797e5a6b2dd7af1cf934bed8e233e6e)\n",
-      "100%|██████████| 1/1 [00:00<00:00, 414.42it/s]\n"
-     ]
-    }
-   ],
-   "source": [
-    "from langchain.evaluation.loading import load_dataset\n",
-    "\n",
-    "dataset = load_dataset(\"agent-vectordb-qa-sota-pg\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "61375342",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'question': 'What is the purpose of the NATO Alliance?',\n",
-       " 'answer': 'The purpose of the NATO Alliance is to secure peace and stability in Europe after World War 2.',\n",
-       " 'steps': [{'tool': 'State of Union QA System', 'tool_input': None},\n",
-       "  {'tool': None, 'tool_input': 'What is the purpose of the NATO Alliance?'}]}"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "dataset[0]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "02500304",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'question': 'What is the purpose of YC?',\n",
-       " 'answer': 'The purpose of YC is to cause startups to be founded that would not otherwise have existed.',\n",
-       " 'steps': [{'tool': 'Paul Graham QA System', 'tool_input': None},\n",
-       "  {'tool': None, 'tool_input': 'What is the purpose of YC?'}]}"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "dataset[-1]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4ab6a716",
-   "metadata": {},
-   "source": [
-    "## Setting up a chain\n",
-    "Now we need to create some pipelines for doing question answering. Step one in that is creating indexes over the data in question."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "c18680b5",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain.document_loaders import TextLoader\n",
-    "\n",
-    "loader = TextLoader(\"../../modules/state_of_the_union.txt\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "7f0de2b3",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain.indexes import VectorstoreIndexCreator"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "ef84ff99",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Using embedded DuckDB without persistence: data will be transient\n"
-     ]
-    }
-   ],
-   "source": [
-    "vectorstore_sota = (\n",
-    "    VectorstoreIndexCreator(vectorstore_kwargs={\"collection_name\": \"sota\"})\n",
-    "    .from_loaders([loader])\n",
-    "    .vectorstore\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "f0b5d8f6",
-   "metadata": {},
-   "source": [
-    "Now we can create a question answering chain."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "id": "8843cb0c",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain.chains import RetrievalQA\n",
-    "from langchain.llms import OpenAI"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "id": "573719a0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "chain_sota = RetrievalQA.from_chain_type(\n",
-    "    llm=OpenAI(temperature=0),\n",
-    "    chain_type=\"stuff\",\n",
-    "    retriever=vectorstore_sota.as_retriever(),\n",
-    "    input_key=\"question\",\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "e48b03d8",
-   "metadata": {},
-   "source": [
-    "Now we do the same for the Paul Graham data."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "id": "c2dbb014",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "loader = TextLoader(\"../../modules/paul_graham_essay.txt\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "id": "98d16f08",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Using embedded DuckDB without persistence: data will be transient\n"
-     ]
-    }
-   ],
-   "source": [
-    "vectorstore_pg = (\n",
-    "    VectorstoreIndexCreator(vectorstore_kwargs={\"collection_name\": \"paul_graham\"})\n",
-    "    .from_loaders([loader])\n",
-    "    .vectorstore\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "id": "ec0aab02",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "chain_pg = RetrievalQA.from_chain_type(\n",
-    "    llm=OpenAI(temperature=0),\n",
-    "    chain_type=\"stuff\",\n",
-    "    retriever=vectorstore_pg.as_retriever(),\n",
-    "    input_key=\"question\",\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "76b5f8fb",
-   "metadata": {},
-   "source": [
-    "We can now set up an agent to route between them."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "id": "ade1aafa",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain.agents import initialize_agent, Tool\n",
-    "from langchain.agents import AgentType\n",
-    "\n",
-    "tools = [\n",
-    "    Tool(\n",
-    "        name=\"State of Union QA System\",\n",
-    "        func=chain_sota.run,\n",
-    "        description=\"useful for when you need to answer questions about the most recent state of the union address. Input should be a fully formed question.\",\n",
-    "    ),\n",
-    "    Tool(\n",
-    "        name=\"Paul Graham System\",\n",
-    "        func=chain_pg.run,\n",
-    "        description=\"useful for when you need to answer questions about Paul Graham. Input should be a fully formed question.\",\n",
-    "    ),\n",
-    "]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 34,
-   "id": "104853f8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "agent = initialize_agent(\n",
-    "    tools,\n",
-    "    OpenAI(temperature=0),\n",
-    "    agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION,\n",
-    "    max_iterations=4,\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "7f036641",
-   "metadata": {},
-   "source": [
-    "## Make a prediction\n",
-    "\n",
-    "First, we can make predictions one datapoint at a time. Doing it at this level of granularity allows use to explore the outputs in detail, and also is a lot cheaper than running over multiple datapoints"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 35,
-   "id": "4664e79f",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'The purpose of the NATO Alliance is to secure peace and stability in Europe after World War 2.'"
-      ]
-     },
-     "execution_count": 35,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "agent.run(dataset[0][\"question\"])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "d0c16cd7",
-   "metadata": {},
-   "source": [
-    "## Make many predictions\n",
-    "Now we can make predictions"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 36,
-   "id": "799f6c17",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "predictions = []\n",
-    "predicted_dataset = []\n",
-    "error_dataset = []\n",
-    "for data in dataset:\n",
-    "    new_data = {\"input\": data[\"question\"], \"answer\": data[\"answer\"]}\n",
-    "    try:\n",
-    "        predictions.append(agent(new_data))\n",
-    "        predicted_dataset.append(new_data)\n",
-    "    except Exception:\n",
-    "        error_dataset.append(new_data)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "49d969fb",
-   "metadata": {},
-   "source": [
-    "## Evaluate performance\n",
-    "Now we can evaluate the predictions. The first thing we can do is look at them by eye."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 37,
-   "id": "1d583f03",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'input': 'What is the purpose of the NATO Alliance?',\n",
-       " 'answer': 'The purpose of the NATO Alliance is to secure peace and stability in Europe after World War 2.',\n",
-       " 'output': 'The purpose of the NATO Alliance is to secure peace and stability in Europe after World War 2.'}"
-      ]
-     },
-     "execution_count": 37,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "predictions[0]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4783344b",
-   "metadata": {},
-   "source": [
-    "Next, we can use a language model to score them programatically"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 38,
-   "id": "d0a9341d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain.evaluation.qa import QAEvalChain"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 39,
-   "id": "1612dec1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "llm = OpenAI(temperature=0)\n",
-    "eval_chain = QAEvalChain.from_llm(llm)\n",
-    "graded_outputs = eval_chain.evaluate(\n",
-    "    predicted_dataset, predictions, question_key=\"input\", prediction_key=\"output\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "79587806",
-   "metadata": {},
-   "source": [
-    "We can add in the graded output to the `predictions` dict and then get a count of the grades."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 40,
-   "id": "2a689df5",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "for i, prediction in enumerate(predictions):\n",
-    "    prediction[\"grade\"] = graded_outputs[i][\"text\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 41,
-   "id": "27b61215",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Counter({' CORRECT': 28, ' INCORRECT': 5})"
-      ]
-     },
-     "execution_count": 41,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from collections import Counter\n",
-    "\n",
-    "Counter([pred[\"grade\"] for pred in predictions])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "12fe30f4",
-   "metadata": {},
-   "source": [
-    "We can also filter the datapoints to the incorrect examples and look at them."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 42,
-   "id": "47c692a1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "incorrect = [pred for pred in predictions if pred[\"grade\"] == \" INCORRECT\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 43,
-   "id": "0ef976c1",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'input': 'What are the four common sense steps that the author suggests to move forward safely?',\n",
-       " 'answer': 'The four common sense steps suggested by the author to move forward safely are: stay protected with vaccines and treatments, prepare for new variants, end the shutdown of schools and businesses, and stay vigilant.',\n",
-       " 'output': 'The four common sense steps suggested in the most recent State of the Union address are: cutting the cost of prescription drugs, providing a pathway to citizenship for Dreamers, revising laws so businesses have the workers they need and families don’t wait decades to reunite, and protecting access to health care and preserving a woman’s right to choose.',\n",
-       " 'grade': ' INCORRECT'}"
-      ]
-     },
-     "execution_count": 43,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "incorrect[0]"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.15"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

docs/extras/guides/evaluation/benchmarking_template.ipynb

@@ -1,162 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "a175c650",
-   "metadata": {},
-   "source": [
-    "# Benchmarking Template\n",
-    "\n",
-    "This is an example notebook that can be used to create a benchmarking notebook for a task of your choice. Evaluation is really hard, and so we greatly welcome any contributions that can make it easier for people to experiment"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "984169ca",
-   "metadata": {},
-   "source": [
-    "It is highly reccomended that you do any evaluation/benchmarking with tracing enabled. See [here](https://langchain.readthedocs.io/en/latest/tracing.html) for an explanation of what tracing is and how to set it up."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 28,
-   "id": "9fe4d1b4",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Comment this out if you are NOT using tracing\n",
-    "import os\n",
-    "\n",
-    "os.environ[\"LANGCHAIN_HANDLER\"] = \"langchain\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0f66405e",
-   "metadata": {},
-   "source": [
-    "## Loading the data\n",
-    "\n",
-    "First, let's load the data."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "79402a8f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# This notebook should so how to load the dataset from LangChainDatasets on Hugging Face\n",
-    "\n",
-    "# Please upload your dataset to https://huggingface.co/LangChainDatasets\n",
-    "\n",
-    "# The value passed into `load_dataset` should NOT have the `LangChainDatasets/` prefix\n",
-    "from langchain.evaluation.loading import load_dataset\n",
-    "\n",
-    "dataset = load_dataset(\"TODO\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "8a16b75d",
-   "metadata": {},
-   "source": [
-    "## Setting up a chain\n",
-    "\n",
-    "This next section should have an example of setting up a chain that can be run on this dataset."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a2661ce0",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "id": "6c0062e7",
-   "metadata": {},
-   "source": [
-    "## Make a prediction\n",
-    "\n",
-    "First, we can make predictions one datapoint at a time. Doing it at this level of granularity allows use to explore the outputs in detail, and also is a lot cheaper than running over multiple datapoints"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "d28c5e7d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Example of running the chain on a single datapoint (`dataset[0]`) goes here"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "d0c16cd7",
-   "metadata": {},
-   "source": [
-    "## Make many predictions\n",
-    "Now we can make predictions."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "24b4c66e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Example of running the chain on many predictions goes here\n",
-    "\n",
-    "# Sometimes its as simple as `chain.apply(dataset)`\n",
-    "\n",
-    "# Othertimes you may want to write a for loop to catch errors"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4783344b",
-   "metadata": {},
-   "source": [
-    "## Evaluate performance\n",
-    "\n",
-    "Any guide to evaluating performance in a more systematic manner goes here."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7710401a",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.1"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

docs/extras/guides/evaluation/comparison/custom.ipynb

@@ -0,0 +1,280 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "657d2c8c-54b4-42a3-9f02-bdefa0ed6728",
+   "metadata": {},
+   "source": [
+    "# Custom Pairwise Evaluator\n",
+    "\n",
+    "You can make your own pairwise string evaluators by inheriting from `PairwiseStringEvaluator` class and overwriting the `_evaluate_string_pairs` method (and the `_aevaluate_string_pairs` method if you want to use the evaluator asynchronously).\n",
+    "\n",
+    "In this example, you will make a simple custom evaluator that just returns whether the first prediction has more whitespace tokenized 'words' than the second.\n",
+    "\n",
+    "You can check out the reference docs for the [PairwiseStringEvaluator interface](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.schema.PairwiseStringEvaluator.html#langchain.evaluation.schema.PairwiseStringEvaluator) for more info.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "93f3a653-d198-4291-973c-8d1adba338b2",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from typing import Optional, Any\n",
+    "from langchain.evaluation import PairwiseStringEvaluator\n",
+    "\n",
+    "\n",
+    "class LengthComparisonPairwiseEvalutor(PairwiseStringEvaluator):\n",
+    "    \"\"\"\n",
+    "    Custom evaluator to compare two strings.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def _evaluate_string_pairs(\n",
+    "        self,\n",
+    "        *,\n",
+    "        prediction: str,\n",
+    "        prediction_b: str,\n",
+    "        reference: Optional[str] = None,\n",
+    "        input: Optional[str] = None,\n",
+    "        **kwargs: Any,\n",
+    "    ) -> dict:\n",
+    "        score = int(len(prediction.split()) > len(prediction_b.split()))\n",
+    "        return {\"score\": score}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "7d4a77c3-07a7-4076-8e7f-f9bca0d6c290",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'score': 1}"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "evaluator = LengthComparisonPairwiseEvalutor()\n",
+    "\n",
+    "evaluator.evaluate_string_pairs(\n",
+    "    prediction=\"The quick brown fox jumped over the lazy dog.\",\n",
+    "    prediction_b=\"The quick brown fox jumped over the dog.\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d90f128f-6f49-42a1-b05a-3aea568ee03b",
+   "metadata": {},
+   "source": [
+    "## LLM-Based Example\n",
+    "\n",
+    "That example was simple to illustrate the API, but it wasn't very useful in practice. Below, use an LLM with some custom instructions to form a simple preference scorer similar to the built-in [PairwiseStringEvalChain](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.comparison.eval_chain.PairwiseStringEvalChain.html#langchain.evaluation.comparison.eval_chain.PairwiseStringEvalChain). We will use `ChatAnthropic` for the evaluator chain."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "b4b43098-4d96-417b-a8a9-b3e75779cfe8",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# %pip install anthropic\n",
+    "# %env ANTHROPIC_API_KEY=YOUR_API_KEY"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "b6e978ab-48f1-47ff-9506-e13b1a50be6e",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from typing import Optional, Any\n",
+    "from langchain.evaluation import PairwiseStringEvaluator\n",
+    "from langchain.chat_models import ChatAnthropic\n",
+    "from langchain.chains import LLMChain\n",
+    "\n",
+    "\n",
+    "class CustomPreferenceEvaluator(PairwiseStringEvaluator):\n",
+    "    \"\"\"\n",
+    "    Custom evaluator to compare two strings using a custom LLMChain.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def __init__(self) -> None:\n",
+    "        llm = ChatAnthropic(model=\"claude-2\", temperature=0)\n",
+    "        self.eval_chain = LLMChain.from_string(\n",
+    "            llm,\n",
+    "            \"\"\"Which option is preferred? Do not take order into account. Evaluate based on accuracy and helpfulness. If neither is preferred, respond with C. Provide your reasoning, then finish with Preference: A/B/C\n",
+    "\n",
+    "Input: How do I get the path of the parent directory in python 3.8?\n",
+    "Option A: You can use the following code:\n",
+    "```python\n",
+    "import os\n",
+    "\n",
+    "os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\n",
+    "```\n",
+    "Option B: You can use the following code:\n",
+    "```python\n",
+    "from pathlib import Path\n",
+    "Path(__file__).absolute().parent\n",
+    "```\n",
+    "Reasoning: Both options return the same result. However, since option B is more concise and easily understand, it is preferred.\n",
+    "Preference: B\n",
+    "\n",
+    "Which option is preferred? Do not take order into account. Evaluate based on accuracy and helpfulness. If neither is preferred, respond with C. Provide your reasoning, then finish with Preference: A/B/C\n",
+    "Input: {input}\n",
+    "Option A: {prediction}\n",
+    "Option B: {prediction_b}\n",
+    "Reasoning:\"\"\",\n",
+    "        )\n",
+    "\n",
+    "    @property\n",
+    "    def requires_input(self) -> bool:\n",
+    "        return True\n",
+    "\n",
+    "    @property\n",
+    "    def requires_reference(self) -> bool:\n",
+    "        return False\n",
+    "\n",
+    "    def _evaluate_string_pairs(\n",
+    "        self,\n",
+    "        *,\n",
+    "        prediction: str,\n",
+    "        prediction_b: str,\n",
+    "        reference: Optional[str] = None,\n",
+    "        input: Optional[str] = None,\n",
+    "        **kwargs: Any,\n",
+    "    ) -> dict:\n",
+    "        result = self.eval_chain(\n",
+    "            {\n",
+    "                \"input\": input,\n",
+    "                \"prediction\": prediction,\n",
+    "                \"prediction_b\": prediction_b,\n",
+    "                \"stop\": [\"Which option is preferred?\"],\n",
+    "            },\n",
+    "            **kwargs,\n",
+    "        )\n",
+    "\n",
+    "        response_text = result[\"text\"]\n",
+    "        reasoning, preference = response_text.split(\"Preference:\", maxsplit=1)\n",
+    "        preference = preference.strip()\n",
+    "        score = 1.0 if preference == \"A\" else (0.0 if preference == \"B\" else None)\n",
+    "        return {\"reasoning\": reasoning.strip(), \"value\": preference, \"score\": score}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "5cbd8b1d-2cb0-4f05-b435-a1a00074d94a",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "evaluator = CustomPreferenceEvaluator()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "2c0a7fb7-b976-4443-9f0e-e707a6dfbdf7",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'reasoning': 'Option B is preferred over option A for importing from a relative directory, because it is more straightforward and concise.\\n\\nOption A uses the importlib module, which allows importing a module by specifying the full name as a string. While this works, it is less clear compared to option B.\\n\\nOption B directly imports from the relative path using dot notation, which clearly shows that it is a relative import. This is the recommended way to do relative imports in Python.\\n\\nIn summary, option B is more accurate and helpful as it uses the standard Python relative import syntax.',\n",
+       " 'value': 'B',\n",
+       " 'score': 0.0}"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "evaluator.evaluate_string_pairs(\n",
+    "    input=\"How do I import from a relative directory?\",\n",
+    "    prediction=\"use importlib! importlib.import_module('.my_package', '.')\",\n",
+    "    prediction_b=\"from .sibling import foo\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "f13a1346-7dbe-451d-b3a3-99e8fc7b753b",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CustomPreferenceEvaluator requires an input string.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Setting requires_input to return True adds additional validation to avoid returning a grade when insufficient data is provided to the chain.\n",
+    "\n",
+    "try:\n",
+    "    evaluator.evaluate_string_pairs(\n",
+    "        prediction=\"use importlib! importlib.import_module('.my_package', '.')\",\n",
+    "        prediction_b=\"from .sibling import foo\",\n",
+    "    )\n",
+    "except ValueError as e:\n",
+    "    print(e)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e7829cc3-ebd1-4628-ae97-15166202e9cc",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}