key

- Description: fix the Loader 'BiliBiliLoader'
  - Issue: the API response was changed

![image](https://github.com/langchain-ai/langchain/assets/2113954/91216793-82f8-4c82-a018-d49f36f5f6aa)
The previously used API no longer returns the "subtitle_url" property.

![image](https://github.com/langchain-ai/langchain/assets/2113954/a8ec2a7a-f40d-4c2a-b7d0-0ccdf2b327cc)
We should use another API to get `subtitle_url` property. 
The `subtitle_url` returned by this API does not include the http schema
and needs to be added.

  - Dependencies: Nope
  - Tag maintainer: @rlancemartin

Makefile

@@ -43,7 +43,12 @@ spell_fix:
 
 help:
 	@echo '----'
-	@echo 'coverage                     - run unit tests and generate coverage report'
+	@echo 'clean                        - run docs_clean and api_docs_clean'
 	@echo 'docs_build                   - build the documentation'
 	@echo 'docs_clean                   - clean the documentation build artifacts'
 	@echo 'docs_linkcheck               - run linkchecker on the documentation'
+	@echo 'api_docs_build               - build the API Reference documentation'
+	@echo 'api_docs_clean               - clean the API Reference documentation build artifacts'
+	@echo 'api_docs_linkcheck           - run linkchecker on the API Reference documentation'
+	@echo 'spell_check               	- run codespell on the project'
+	@echo 'spell_fix               		- run codespell on the project and fix the errors'

README.md

@@ -18,8 +18,8 @@
 
 Looking for the JS/TS version? Check out [LangChain.js](https://github.com/hwchase17/langchainjs).
 
-**Production Support:** As you move your LangChains into production, we'd love to offer more comprehensive support.
-Please fill out [this form](https://6w1pwbss0py.typeform.com/to/rrbrdTH2) and we'll set up a dedicated support Slack channel.
+**Production Support:** As you move your LangChains into production, we'd love to offer more hands-on support.
+Fill out [this form](https://airtable.com/appwQzlErAS2qiP0L/shrGtGaVBVAz7NcV2) to share more about what you're building, and our team will get in touch.
 
 ## 🚨Breaking Changes for select chains (SQLDatabase) on 7/28
 

docs/api_reference/conf.py

@@ -100,6 +100,9 @@ extensions = [
 ]
 source_suffix = [".rst"]
 
+# some autodoc pydantic options are repeated in the actual template.
+# potentially user error, but there may be bugs in the sphinx extension
+# with options not being passed through correctly (from either the location in the code)
 autodoc_pydantic_model_show_json = False
 autodoc_pydantic_field_list_validators = False
 autodoc_pydantic_config_members = False
@@ -112,13 +115,6 @@ autodoc_member_order = "groupwise"
 autoclass_content = "both"
 autodoc_typehints_format = "short"
 
-autodoc_default_options = {
-    "members": True,
-    "show-inheritance": True,
-    "inherited-members": "BaseModel",
-    "undoc-members": True,
-    "special-members": "__call__",
-}
 # autodoc_typehints = "description"
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["templates"]

docs/api_reference/create_api_rst.py

@@ -1,49 +1,209 @@
-"""Script for auto-generating api_reference.rst"""
-import glob
-import re
+"""Script for auto-generating api_reference.rst."""
+import importlib
+import inspect
+import typing
 from pathlib import Path
+from typing import TypedDict, Sequence, List, Dict, Literal, Union
+from enum import Enum
+
+from pydantic import BaseModel
 
 ROOT_DIR = Path(__file__).parents[2].absolute()
+HERE = Path(__file__).parent
+
 PKG_DIR = ROOT_DIR / "libs" / "langchain" / "langchain"
 EXP_DIR = ROOT_DIR / "libs" / "experimental" / "langchain_experimental"
-WRITE_FILE = Path(__file__).parent / "api_reference.rst"
-EXP_WRITE_FILE = Path(__file__).parent / "experimental_api_reference.rst"
+WRITE_FILE = HERE / "api_reference.rst"
+EXP_WRITE_FILE = HERE / "experimental_api_reference.rst"
 
 
-def load_members(dir: Path) -> dict:
-    members: dict = {}
-    for py in glob.glob(str(dir) + "/**/*.py", recursive=True):
-        module = py[len(str(dir)) + 1 :].replace(".py", "").replace("/", ".")
-        top_level = module.split(".")[0]
-        if top_level not in members:
-            members[top_level] = {"classes": [], "functions": []}
-        with open(py, "r") as f:
-            for line in f.readlines():
-                cls = re.findall(r"^class ([^_].*)\(", line)
-                members[top_level]["classes"].extend([module + "." + c for c in cls])
-                func = re.findall(r"^def ([^_].*)\(", line)
-                afunc = re.findall(r"^async def ([^_].*)\(", line)
-                func_strings = [module + "." + f for f in func + afunc]
-                members[top_level]["functions"].extend(func_strings)
-    return members
+ClassKind = Literal["TypedDict", "Regular", "Pydantic", "enum"]
 
 
-def construct_doc(pkg: str, members: dict) -> str:
+class ClassInfo(TypedDict):
+    """Information about a class."""
+
+    name: str
+    """The name of the class."""
+    qualified_name: str
+    """The fully qualified name of the class."""
+    kind: ClassKind
+    """The kind of the class."""
+    is_public: bool
+    """Whether the class is public or not."""
+
+
+class FunctionInfo(TypedDict):
+    """Information about a function."""
+
+    name: str
+    """The name of the function."""
+    qualified_name: str
+    """The fully qualified name of the function."""
+    is_public: bool
+    """Whether the function is public or not."""
+
+
+class ModuleMembers(TypedDict):
+    """A dictionary of module members."""
+
+    classes_: Sequence[ClassInfo]
+    functions: Sequence[FunctionInfo]
+
+
+def _load_module_members(module_path: str, namespace: str) -> ModuleMembers:
+    """Load all members of a module.
+
+    Args:
+        module_path: Path to the module.
+        namespace: the namespace of the module.
+
+    Returns:
+        list: A list of loaded module objects.
+    """
+    classes_: List[ClassInfo] = []
+    functions: List[FunctionInfo] = []
+    module = importlib.import_module(module_path)
+    for name, type_ in inspect.getmembers(module):
+        if not hasattr(type_, "__module__"):
+            continue
+        if type_.__module__ != module_path:
+            continue
+
+        if inspect.isclass(type_):
+            if type(type_) == typing._TypedDictMeta:  # type: ignore
+                kind: ClassKind = "TypedDict"
+            elif issubclass(type_, Enum):
+                kind = "enum"
+            elif issubclass(type_, BaseModel):
+                kind = "Pydantic"
+            else:
+                kind = "Regular"
+
+            classes_.append(
+                ClassInfo(
+                    name=name,
+                    qualified_name=f"{namespace}.{name}",
+                    kind=kind,
+                    is_public=not name.startswith("_"),
+                )
+            )
+        elif inspect.isfunction(type_):
+            functions.append(
+                FunctionInfo(
+                    name=name,
+                    qualified_name=f"{namespace}.{name}",
+                    is_public=not name.startswith("_"),
+                )
+            )
+        else:
+            continue
+
+    return ModuleMembers(
+        classes_=classes_,
+        functions=functions,
+    )
+
+
+def _merge_module_members(
+    module_members: Sequence[ModuleMembers],
+) -> ModuleMembers:
+    """Merge module members."""
+    classes_: List[ClassInfo] = []
+    functions: List[FunctionInfo] = []
+    for module in module_members:
+        classes_.extend(module["classes_"])
+        functions.extend(module["functions"])
+
+    return ModuleMembers(
+        classes_=classes_,
+        functions=functions,
+    )
+
+
+def _load_package_modules(
+    package_directory: Union[str, Path]
+) -> Dict[str, ModuleMembers]:
+    """Recursively load modules of a package based on the file system.
+
+    Traversal based on the file system makes it easy to determine which
+    of the modules/packages are part of the package vs. 3rd party or built-in.
+
+    Parameters:
+        package_directory: Path to the package directory.
+
+    Returns:
+        list: A list of loaded module objects.
+    """
+    package_path = (
+        Path(package_directory)
+        if isinstance(package_directory, str)
+        else package_directory
+    )
+    modules_by_namespace = {}
+
+    package_name = package_path.name
+
+    for file_path in package_path.rglob("*.py"):
+        if not file_path.name.startswith("__"):
+            relative_module_name = file_path.relative_to(package_path)
+            # Get the full namespace of the module
+            namespace = str(relative_module_name).replace(".py", "").replace("/", ".")
+            # Keep only the top level namespace
+            top_namespace = namespace.split(".")[0]
+
+            try:
+                module_members = _load_module_members(
+                    f"{package_name}.{namespace}", namespace
+                )
+                # Merge module members if the namespace already exists
+                if top_namespace in modules_by_namespace:
+                    existing_module_members = modules_by_namespace[top_namespace]
+                    _module_members = _merge_module_members(
+                        [existing_module_members, module_members]
+                    )
+                else:
+                    _module_members = module_members
+
+                modules_by_namespace[top_namespace] = _module_members
+
+            except ImportError as e:
+                print(f"Error: Unable to import module '{namespace}' with error: {e}")
+
+    return modules_by_namespace
+
+
+def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) -> str:
+    """Construct the contents of the reference.rst file for the given package.
+
+    Args:
+        pkg: The package name
+        members_by_namespace: The members of the package, dict organized by top level
+                              module contains a list of classes and functions
+                              inside of the top level namespace.
+
+    Returns:
+        The contents of the reference.rst file.
+    """
     full_doc = f"""\
-=============
+=======================
 ``{pkg}`` API Reference
-=============
+=======================
 
 """
-    for module, _members in sorted(members.items(), key=lambda kv: kv[0]):
-        classes = _members["classes"]
+    namespaces = sorted(members_by_namespace)
+
+    for module in namespaces:
+        _members = members_by_namespace[module]
+        classes = _members["classes_"]
         functions = _members["functions"]
         if not (classes or functions):
             continue
         section = f":mod:`{pkg}.{module}`"
+        underline = "=" * (len(section) + 1)
         full_doc += f"""\
 {section}
-{'=' * (len(section) + 1)}
+{underline}
 
 .. automodule:: {pkg}.{module}
     :no-members:
@@ -52,7 +212,6 @@ def construct_doc(pkg: str, members: dict) -> str:
 """
 
         if classes:
-            cstring = "\n    ".join(sorted(classes))
             full_doc += f"""\
 Classes
 --------------
@@ -60,13 +219,31 @@ Classes
 
 .. autosummary::
     :toctree: {module}
-    :template: class.rst
-
-    {cstring}
-
 """
+
+            for class_ in classes:
+                if not class_['is_public']:
+                    continue
+                    
+                if class_["kind"] == "TypedDict":
+                    template = "typeddict.rst"
+                elif class_["kind"] == "enum":
+                    template = "enum.rst"
+                elif class_["kind"] == "Pydantic":
+                    template = "pydantic.rst"
+                else:
+                    template = "class.rst"
+
+                full_doc += f"""\
+    :template: {template}
+    
+    {class_["qualified_name"]}
+    
+"""
+
         if functions:
-            fstring = "\n    ".join(sorted(functions))
+            _functions = [f["qualified_name"] for f in functions if f["is_public"]]
+            fstring = "\n    ".join(sorted(_functions))
             full_doc += f"""\
 Functions
 --------------
@@ -83,12 +260,15 @@ Functions
 
 
 def main() -> None:
-    lc_members = load_members(PKG_DIR)
-    lc_doc = ".. _api_reference:\n\n" + construct_doc("langchain", lc_members)
+    """Generate the reference.rst file for each package."""
+    lc_members = _load_package_modules(PKG_DIR)
+    lc_doc = ".. _api_reference:\n\n" + _construct_doc("langchain", lc_members)
     with open(WRITE_FILE, "w") as f:
         f.write(lc_doc)
-    exp_members = load_members(EXP_DIR)
-    exp_doc = ".. _experimental_api_reference:\n\n" + construct_doc("langchain_experimental", exp_members)
+    exp_members = _load_package_modules(EXP_DIR)
+    exp_doc = ".. _experimental_api_reference:\n\n" + _construct_doc(
+        "langchain_experimental", exp_members
+    )
     with open(EXP_WRITE_FILE, "w") as f:
         f.write(exp_doc)
 

docs/api_reference/requirements.txt

@@ -1,4 +1,5 @@
 -e libs/langchain
+-e libs/experimental
 autodoc_pydantic==1.8.0
 myst_parser
 nbsphinx==0.8.9
@@ -10,4 +11,4 @@ sphinx-panels
 toml
 myst_nb
 sphinx_copybutton
-pydata-sphinx-theme==0.13.1
+pydata-sphinx-theme==0.13.1

docs/api_reference/templates/class.rst

@@ -5,17 +5,6 @@
 
 .. autoclass:: {{ objname }}
 
-   {% block methods %}
-   {% if methods %}
-   .. rubric:: {{ _('Methods') }}
-
-   .. autosummary::
-   {% for item in methods %}
-      ~{{ name }}.{{ item }}
-   {%- endfor %}
-   {% endif %}
-   {% endblock %}
-
    {% block attributes %}
    {% if attributes %}
    .. rubric:: {{ _('Attributes') }}
@@ -27,4 +16,21 @@
    {% endif %}
    {% endblock %}
 
+   {% block methods %}
+   {% if methods %}
+   .. rubric:: {{ _('Methods') }}
+
+   .. autosummary::
+   {% for item in methods %}
+      ~{{ name }}.{{ item }}
+   {%- endfor %}
+
+   {% for item in methods %}
+   .. automethod:: {{ name }}.{{ item }}
+   {%- endfor %}
+
+   {% endif %}
+   {% endblock %}
+
+
 .. example_links:: {{ objname }}

docs/api_reference/templates/enum.rst

@@ -0,0 +1,14 @@
+:mod:`{{module}}`.{{objname}}
+{{ underline }}==============
+
+.. currentmodule:: {{ module }}
+
+.. autoclass:: {{ objname }}
+
+    {% block attributes %}
+    {% for item in attributes %}
+    .. autoattribute:: {{ item }}
+    {% endfor %}
+    {% endblock %}
+
+.. example_links:: {{ objname }}

docs/api_reference/templates/pydantic.rst

@@ -0,0 +1,22 @@
+:mod:`{{module}}`.{{objname}}
+{{ underline }}==============
+
+.. currentmodule:: {{ module }}
+
+.. autopydantic_model:: {{ objname }}
+    :model-show-json: False
+    :model-show-config-summary: False
+    :model-show-validator-members: False
+    :model-show-field-summary: False
+    :field-signature-prefix: param
+    :members:
+    :undoc-members:
+    :inherited-members:
+    :member-order: groupwise
+    :show-inheritance: True
+    :special-members: __call__
+
+    {% block attributes %}
+    {% endblock %}
+
+.. example_links:: {{ objname }}

docs/api_reference/templates/typeddict.rst

@@ -0,0 +1,14 @@
+:mod:`{{module}}`.{{objname}}
+{{ underline }}==============
+
+.. currentmodule:: {{ module }}
+
+.. autoclass:: {{ objname }}
+
+    {% block attributes %}
+   {% for item in attributes %}
+  .. autoattribute:: {{ item }}
+   {% endfor %}
+   {% endblock %}
+
+.. example_links:: {{ objname }}

docs/api_reference/themes/scikit-learn-modern/layout.html

@@ -19,7 +19,7 @@
   {% block htmltitle %}
   <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
   {% endblock %}
-  <link rel="canonical" href="http://scikit-learn.org/stable/{{pagename}}.html" />
+  <link rel="canonical" href="https://api.python.langchain.com/en/latest/{{pagename}}.html" />
 
   {% if favicon_url %}
   <link rel="shortcut icon" href="{{ favicon_url|e }}"/>

docs/api_reference/themes/scikit-learn-modern/nav.html

@@ -6,17 +6,6 @@
   {%- set top_container_cls = "sk-landing-container" %}
 {%- endif %}
 
-{% if theme_link_to_live_contributing_page|tobool %}
-{# Link to development page for live builds #}
-  {%- set development_link = "https://scikit-learn.org/dev/developers/index.html" %}
-{# Open on a new development page in new window/tab for live builds #}
-  {%- set development_attrs = 'target="_blank" rel="noopener noreferrer"' %}
-{%- else %}
-  {%- set development_link = pathto('developers/index') %}
-  {%- set development_attrs = '' %}
-{%- endif %}
-
-
 <nav id="navbar" class="{{ nav_bar_class }} navbar navbar-expand-md navbar-light bg-light py-0">
   <div class="container-fluid {{ top_container_cls }} px-0">
     {%- if logo_url %}

docs/docs_skeleton/docs/modules/agents/agent_types/index.mdx

@@ -28,7 +28,7 @@ navigating around a browser.
 ### [OpenAI Functions](/docs/modules/agents/agent_types/openai_functions_agent.html)
 
 Certain OpenAI models (like gpt-3.5-turbo-0613 and gpt-4-0613) have been explicitly fine-tuned to detect when a
-function should to be called and respond with the inputs that should be passed to the function.
+function should be called and respond with the inputs that should be passed to the function.
 The OpenAI Functions Agent is designed to work with these models.
 
 ### [Conversational](/docs/modules/agents/agent_types/chat_conversation_agent.html)

docs/docs_skeleton/docs/modules/agents/agent_types/openai_functions_agent.mdx

@@ -1,6 +1,6 @@
 # OpenAI functions
 
-Certain OpenAI models (like gpt-3.5-turbo-0613 and gpt-4-0613) have been fine-tuned to detect when a function should to be called and respond with the inputs that should be passed to the function.
+Certain OpenAI models (like gpt-3.5-turbo-0613 and gpt-4-0613) have been fine-tuned to detect when a function should be called and respond with the inputs that should be passed to the function.
 In an API call, you can describe functions and have the model intelligently choose to output a JSON object containing arguments to call those functions.
 The goal of the OpenAI Function APIs is to more reliably return valid and useful function calls than a generic text completion or chat API.
 

docs/docs_skeleton/docs/modules/model_io/prompts/index.mdx

@@ -3,10 +3,12 @@ sidebar_position: 0
 ---
 # Prompts
 
-The new way of programming models is through prompts.
-A **prompt** refers to the input to the model.
-This input is often constructed from multiple components.
-LangChain provides several classes and functions to make constructing and working with prompts easy.
+A prompt for a language model is a set of instructions or input provided by a user to
+guide the model's response, helping it understand the context and generate relevant
+and coherent language-based output, such as answering questions, completing sentences,
+or engaging in a conversation.
 
-- [Prompt templates](/docs/modules/model_io/prompts/prompt_templates/): Parametrize model inputs
+LangChain provides several classes and functions to help construct and work with prompts.
+
+- [Prompt templates](/docs/modules/model_io/prompts/prompt_templates/): Parametrized model inputs
 - [Example selectors](/docs/modules/model_io/prompts/example_selectors/): Dynamically select examples to include in prompts

docs/docs_skeleton/docs/modules/model_io/prompts/prompt_templates/index.mdx

@@ -4,18 +4,15 @@ sidebar_position: 0
 
 # Prompt templates
 
-Language models take text as input - that text is commonly referred to as a prompt.
-Typically this is not simply a hardcoded string but rather a combination of a template, some examples, and user input.
-LangChain provides several classes and functions to make constructing and working with prompts easy.
+Prompt templates are pre-defined recipes for generating prompts for language models.
 
-## What is a prompt template?
+A template may include instructions, few shot examples, and specific context and
+questions appropriate for a given task.
 
-A prompt template refers to a reproducible way to generate a prompt. It contains a text string ("the template"), that can take in a set of parameters from the end user and generates a prompt.
+LangChain provides tooling to create and work with prompt templates.
 
-A prompt template can contain:
-- instructions to the language model,
-- a set of few shot examples to help the language model generate a better response,
-- a question to the language model.
+LangChain strives to create model agnostic templates to make it easy to reuse
+existing templates across different language models.
 
 import GetStarted from "@snippets/modules/model_io/prompts/prompt_templates/get_started.mdx"
 

docs/docs_skeleton/docs/use_cases/summarization/summarize.mdx

@@ -1,8 +0,0 @@
-# Summarization
-
-A summarization chain can be used to summarize multiple documents. One way is to input multiple smaller documents, after they have been divided into chunks, and operate over them with a MapReduceDocumentsChain. You can also choose instead for the chain that does summarization to be a StuffDocumentsChain, or a RefineDocumentsChain.
-
-import Example from "@snippets/modules/chains/popular/summarize.mdx"
-
-<Example/>
-

docs/docs_skeleton/package-lock.json

@@ -8,6 +8,7 @@
       "name": "docs",
       "version": "0.0.0",
       "dependencies": {
+        "@docsly/react": "^1.8.6",
         "@docusaurus/core": "2.4.0",
         "@docusaurus/preset-classic": "2.4.0",
         "@docusaurus/remark-plugin-npm2yarn": "^2.4.0",
@@ -2041,6 +2042,23 @@
         }
       }
     },
+    "node_modules/@docsly/react": {
+      "version": "1.8.6",
+      "resolved": "https://registry.npmjs.org/@docsly/react/-/react-1.8.6.tgz",
+      "integrity": "sha512-FwcVeTYsZnOMIFYgW0R9/0xaWKWfzvW4eKA9QjJApXKbfGFHgzm0tWEmVf9TyZgsA3qoI+0qaUuJ3JlQrdONXg==",
+      "dependencies": {
+        "@heroicons/react": "^2.0.18",
+        "clsx": "^1.2.1",
+        "dayjs": "^1.11.7",
+        "react-error-boundary": "^3.1.4",
+        "react-hot-toast": "^2.4.0",
+        "swr": "^2.1.0"
+      },
+      "peerDependencies": {
+        "react": ">=17",
+        "react-dom": ">=17"
+      }
+    },
     "node_modules/@docusaurus/core": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/@docusaurus/core/-/core-2.4.0.tgz",
@@ -2900,6 +2918,14 @@
         "@hapi/hoek": "^9.0.0"
       }
     },
+    "node_modules/@heroicons/react": {
+      "version": "2.0.18",
+      "resolved": "https://registry.npmjs.org/@heroicons/react/-/react-2.0.18.tgz",
+      "integrity": "sha512-7TyMjRrZZMBPa+/5Y8lN0iyvUU/01PeMGX2+RE7cQWpEUIcb4QotzUObFkJDejj/HUH4qjP/eQ0gzzKs2f+6Yw==",
+      "peerDependencies": {
+        "react": ">= 16"
+      }
+    },
     "node_modules/@humanwhocodes/config-array": {
       "version": "0.11.10",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz",
@@ -5925,6 +5951,11 @@
       "integrity": "sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==",
       "dev": true
     },
+    "node_modules/dayjs": {
+      "version": "1.11.9",
+      "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.9.tgz",
+      "integrity": "sha512-QvzAURSbQ0pKdIye2txOzNaHmxtUBXerpY0FJsFXUMKbIZeFm5ht1LS/jFsrncjnmtv8HsG0W2g6c0zUjZWmpA=="
+    },
     "node_modules/debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -7977,6 +8008,14 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/goober": {
+      "version": "2.1.13",
+      "resolved": "https://registry.npmjs.org/goober/-/goober-2.1.13.tgz",
+      "integrity": "sha512-jFj3BQeleOoy7t93E9rZ2de+ScC4lQICLwiAQmKMg9F6roKGaLSHoCDYKkWlSafg138jejvq/mTdvmnwDQgqoQ==",
+      "peerDependencies": {
+        "csstype": "^3.0.10"
+      }
+    },
     "node_modules/gopd": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
@@ -11683,6 +11722,21 @@
         "react": "17.0.2"
       }
     },
+    "node_modules/react-error-boundary": {
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/react-error-boundary/-/react-error-boundary-3.1.4.tgz",
+      "integrity": "sha512-uM9uPzZJTF6wRQORmSrvOIgt4lJ9MC1sNgEOj2XGsDTRE4kmpWxg7ENK9EWNKJRMAOY9z0MuF4yIfl6gp4sotA==",
+      "dependencies": {
+        "@babel/runtime": "^7.12.5"
+      },
+      "engines": {
+        "node": ">=10",
+        "npm": ">=6"
+      },
+      "peerDependencies": {
+        "react": ">=16.13.1"
+      }
+    },
     "node_modules/react-error-overlay": {
       "version": "6.0.11",
       "resolved": "https://registry.npmjs.org/react-error-overlay/-/react-error-overlay-6.0.11.tgz",
@@ -11709,6 +11763,21 @@
         "react-dom": "^16.6.0 || ^17.0.0 || ^18.0.0"
       }
     },
+    "node_modules/react-hot-toast": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/react-hot-toast/-/react-hot-toast-2.4.1.tgz",
+      "integrity": "sha512-j8z+cQbWIM5LY37pR6uZR6D4LfseplqnuAO4co4u8917hBUvXlEqyP1ZzqVLcqoyUesZZv/ImreoCeHVDpE5pQ==",
+      "dependencies": {
+        "goober": "^2.1.10"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "react": ">=16",
+        "react-dom": ">=16"
+      }
+    },
     "node_modules/react-is": {
       "version": "16.13.1",
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
@@ -13268,6 +13337,17 @@
         "url": "https://github.com/fb55/entities?sponsor=1"
       }
     },
+    "node_modules/swr": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/swr/-/swr-2.2.0.tgz",
+      "integrity": "sha512-AjqHOv2lAhkuUdIiBu9xbuettzAzWXmCEcLONNKJRba87WAefz8Ca9d6ds/SzrPc235n1IxWYdhJ2zF3MNUaoQ==",
+      "dependencies": {
+        "use-sync-external-store": "^1.2.0"
+      },
+      "peerDependencies": {
+        "react": "^16.11.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
     "node_modules/tapable": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",

docs/docs_skeleton/package.json

@@ -19,6 +19,7 @@
     "format:check": "prettier --check \"**/*.{js,jsx,ts,tsx,md,mdx}\""
   },
   "dependencies": {
+    "@docsly/react": "^1.8.6",
     "@docusaurus/core": "2.4.0",
     "@docusaurus/preset-classic": "2.4.0",
     "@docusaurus/remark-plugin-npm2yarn": "^2.4.0",

docs/docs_skeleton/src/theme/Footer/index.js

@@ -0,0 +1,15 @@
+import React from "react";
+import Footer from "@theme-original/Footer";
+import Docsly from "@docsly/react";
+import "@docsly/react/styles.css";
+import { useLocation } from "@docusaurus/router";
+ 
+export default function FooterWrapper(props) {
+  const { pathname } = useLocation();
+  return (
+    <>
+      <Footer {...props} />
+      <Docsly publicId="public_q4K8GfsRTcz0rrnpMqnzJBLQV6cKvUP64viWjtqNoNhyei1hLVQDjK9NUVvqj4vs" pathname={pathname} />
+    </>
+  );
+}

docs/docs_skeleton/vercel.json

@@ -3951,6 +3951,10 @@
     {
       "source": "/docs/modules/chains/additional/tagging",
       "destination": "/docs/use_cases/tagging"
+    },
+    {
+      "source": "docs/integrations/providers/agent_with_wandb_tracing",
+      "destination": "docs/integrations/providers/wandb_tracing"
     }
   ]
 }

docs/extras/guides/expression_language/interface.ipynb

@@ -19,9 +19,22 @@
     "- `ainvoke`: call the chain on an input async\n",
     "- `abatch`: call the chain on a list of inputs async\n",
     "\n",
-    "The type of the input varies by component. For a prompt it is a dictionary, for a retriever it is a single string, for a model either a single string, a list of chat messages, or a PromptValue.\n",
+    "The type of the input varies by component:\n",
     "\n",
-    "The output type also varies by component. For an LLM it is a string, for a ChatModel it's a ChatMessage, for a prompt it's a PromptValue, for a retriever it's a list of documents.\n",
+    "| Component | Input Type |\n",
+    "| --- | --- |\n",
+    "|Prompt|Dictionary|\n",
+    "|Retriever|Single string|\n",
+    "|Model| Single string, list of chat messages or a PromptValue|\n",
+    "\n",
+    "The output type also varies by component:\n",
+    "\n",
+    "| Component | Output Type |\n",
+    "| --- | --- |\n",
+    "| LLM | String |\n",
+    "| ChatModel | ChatMessage |\n",
+    "| Prompt | PromptValue |\n",
+    "| Retriever | List of documents |\n",
     "\n",
     "Let's take a look at these methods! To do so, we'll create a super simple PromptTemplate + ChatModel chain."
    ]

docs/extras/integrations/callbacks/streamlit.md

@@ -71,3 +71,6 @@ or any other local ENV management tool.
 
 Currently `StreamlitCallbackHandler` is geared towards use with a LangChain Agent Executor. Support for additional agent types,
 use directly with Chains, etc will be added in the future.
+
+You may also be interested in using
+[StreamlitChatMessageHistory](/docs/integrations/memory/streamlit_chat_message_history) for LangChain.

docs/extras/integrations/document_loaders/example_data/sample_rss_feeds.opml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<opml version="1.0">
+    <head>
+        <title>Sample RSS feed subscriptions</title>
+    </head>
+    <body>
+        <outline text="Tech" title="Tech">
+            <outline type="rss" text="Engadget" title="Engadget" xmlUrl="http://www.engadget.com/rss-full.xml" htmlUrl="http://www.engadget.com"/>
+            <outline type="rss" text="Ars Technica - All content" title="Ars Technica - All content" xmlUrl="http://feeds.arstechnica.com/arstechnica/index/" htmlUrl="https://arstechnica.com"/>
+        </outline>
+    </body>
+</opml>

docs/extras/integrations/document_loaders/huawei_obs_directory.ipynb

@@ -0,0 +1,178 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "c83b6a4c",
+   "metadata": {},
+   "source": [
+    "# Huawei OBS Directory\n",
+    "The following code demonstrates how to load objects from the Huawei OBS (Object Storage Service) as documents."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c2191935",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Install the required package\n",
+    "# pip install esdk-obs-python"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "55fca3b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import OBSDirectoryLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "c3ed419f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "endpoint = \"your-endpoint\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "3428fd4e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Configure your access credentials\\n\n",
+    "config = {\n",
+    "    \"ak\": \"your-access-key\",\n",
+    "    \"sk\": \"your-secret-key\"\n",
+    "}\n",
+    "loader = OBSDirectoryLoader(\"your-bucket-name\", endpoint=endpoint, config=config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9beede9f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader.load()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1e20a839",
+   "metadata": {},
+   "source": [
+    "## Specify a Prefix for Loading\n",
+    "If you want to load objects with a specific prefix from the bucket, you can use the following code:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "125f311d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = OBSDirectoryLoader(\"your-bucket-name\", endpoint=endpoint, config=config, prefix=\"test_prefix\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b3488037",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader.load()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "84c82c0a",
+   "metadata": {},
+   "source": [
+    "## Get Authentication Information from ECS\n",
+    "If your langchain is deployed on Huawei Cloud ECS and [Agency is set up](https://support.huaweicloud.com/intl/en-us/usermanual-ecs/ecs_03_0166.html#section7), the loader can directly get the security token from ECS without needing access key and secret key. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "1db99969",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config = {\"get_token_from_ecs\": True}\n",
+    "loader = OBSDirectoryLoader(\"your-bucket-name\", endpoint=endpoint, config=config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "57dd9f35",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader.load()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "30205d25",
+   "metadata": {},
+   "source": [
+    "## Use a Public Bucket\n",
+    "If your bucket's bucket policy allows anonymous access (anonymous users have `listBucket` and `GetObject` permissions), you can directly load the objects without configuring the `config` parameter."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "4dfa2ef0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = OBSDirectoryLoader(\"your-bucket-name\", endpoint=endpoint)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67d4c1d0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader.load()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/integrations/document_loaders/huawei_obs_file.ipynb

@@ -0,0 +1,180 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "4394a872",
+   "metadata": {},
+   "source": [
+    "# Huawei OBS File\n",
+    "The following code demonstrates how to load an object from the Huawei OBS (Object Storage Service) as document."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c43d811b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Install the required package\n",
+    "# pip install esdk-obs-python"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "5e16bae6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders.obs_file import OBSFileLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "75cc7e7c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "endpoint = \"your-endpoint\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "f9816984",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from obs import ObsClient\n",
+    "obs_client = ObsClient(access_key_id=\"your-access-key\", secret_access_key=\"your-secret-key\", server=endpoint)\n",
+    "loader = OBSFileLoader(\"your-bucket-name\", \"your-object-key\", client=obs_client)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6143b39b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader.load()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "633e05ca",
+   "metadata": {},
+   "source": [
+    "## Each Loader with Separate Authentication Information\n",
+    "If you don't need to reuse OBS connections between different loaders, you can directly configure the `config`. The loader will use the config information to initialize its own OBS client."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a5dd6a5d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Configure your access credentials\\n\n",
+    "config = {\n",
+    "    \"ak\": \"your-access-key\",\n",
+    "    \"sk\": \"your-secret-key\"\n",
+    "}\n",
+    "loader = OBSFileLoader(\"your-bucket-name\", \"your-object-key\",endpoint=endpoint, config=config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9a741f1c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader.load()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1e2e611c",
+   "metadata": {},
+   "source": [
+    "## Get Authentication Information from ECS\n",
+    "If your langchain is deployed on Huawei Cloud ECS and [Agency is set up](https://support.huaweicloud.com/intl/en-us/usermanual-ecs/ecs_03_0166.html#section7), the loader can directly get the security token from ECS without needing access key and secret key. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "338fafef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config = {\"get_token_from_ecs\": True}\n",
+    "loader = OBSFileLoader(\"your-bucket-name\", \"your-object-key\", endpoint=endpoint, config=config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73976c55",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader.load()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b77aa18c",
+   "metadata": {},
+   "source": [
+    "## Access a Publicly Accessible Object\n",
+    "If the object you want to access allows anonymous user access (anonymous users have `GetObject` permission), you can directly load the object without configuring the `config` parameter."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "df83d121",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = OBSFileLoader(\"your-bucket-name\", \"your-object-key\", endpoint=endpoint)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "82a844ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader.load()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/integrations/document_loaders/news.ipynb

@@ -0,0 +1,192 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "2dfc4698",
+   "metadata": {},
+   "source": [
+    "# News URL\n",
+    "\n",
+    "This covers how to load HTML news articles from a list of URLs into a document format that we can use downstream."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "16c3699e",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-08-02T21:18:18.886031400Z",
+     "start_time": "2023-08-02T21:18:17.682345Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import NewsURLLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "836fbac1",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-08-02T21:18:18.895539800Z",
+     "start_time": "2023-08-02T21:18:18.895539800Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "urls = [\n",
+    "    \"https://www.bbc.com/news/world-us-canada-66388172\",\n",
+    "    \"https://www.bbc.com/news/entertainment-arts-66384971\",\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "33089aba-ff74-4d00-8f40-9449c29587cc",
+   "metadata": {},
+   "source": [
+    "Pass in urls to load them into Documents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "00f46fda",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-08-02T21:18:19.227074500Z",
+     "start_time": "2023-08-02T21:18:18.895539800Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "First article:  page_content='In testimony to the congressional committee examining the 6 January riot, Mrs Powell said she did not review all of the many claims of election fraud she made, telling them that \"no reasonable person\" would view her claims as fact. Neither she nor her representatives have commented.' metadata={'title': 'Donald Trump indictment: What do we know about the six co-conspirators?', 'link': 'https://www.bbc.com/news/world-us-canada-66388172', 'authors': [], 'language': 'en', 'description': 'Six people accused of helping Mr Trump undermine the election have been described by prosecutors.', 'publish_date': None}\n",
+      "\n",
+      "Second article:  page_content='Ms Williams added: \"If there\\'s anything that I can do in my power to ensure that dancers or singers or whoever decides to work with her don\\'t have to go through that same experience, I\\'m going to do that.\"' metadata={'title': \"Lizzo dancers Arianna Davis and Crystal Williams: 'No one speaks out, they are scared'\", 'link': 'https://www.bbc.com/news/entertainment-arts-66384971', 'authors': [], 'language': 'en', 'description': 'The US pop star is being sued for sexual harassment and fat-shaming but has yet to comment.', 'publish_date': None}\n"
+     ]
+    }
+   ],
+   "source": [
+    "loader = NewsURLLoader(urls=urls)\n",
+    "data = loader.load()\n",
+    "print(\"First article: \", data[0])\n",
+    "print(\"\\nSecond article: \", data[1])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "Use nlp=True to run nlp analysis and generate keywords + summary"
+   ],
+   "metadata": {
+    "collapsed": false
+   },
+   "id": "98ac26c488315bff"
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "b68a26b3",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-08-02T21:18:19.585758200Z",
+     "start_time": "2023-08-02T21:18:19.227074500Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "First article:  page_content='In testimony to the congressional committee examining the 6 January riot, Mrs Powell said she did not review all of the many claims of election fraud she made, telling them that \"no reasonable person\" would view her claims as fact. Neither she nor her representatives have commented.' metadata={'title': 'Donald Trump indictment: What do we know about the six co-conspirators?', 'link': 'https://www.bbc.com/news/world-us-canada-66388172', 'authors': [], 'language': 'en', 'description': 'Six people accused of helping Mr Trump undermine the election have been described by prosecutors.', 'publish_date': None, 'keywords': ['powell', 'know', 'donald', 'trump', 'review', 'indictment', 'telling', 'view', 'reasonable', 'person', 'testimony', 'coconspirators', 'riot', 'representatives', 'claims'], 'summary': 'In testimony to the congressional committee examining the 6 January riot, Mrs Powell said she did not review all of the many claims of election fraud she made, telling them that \"no reasonable person\" would view her claims as fact.\\nNeither she nor her representatives have commented.'}\n",
+      "\n",
+      "Second article:  page_content='Ms Williams added: \"If there\\'s anything that I can do in my power to ensure that dancers or singers or whoever decides to work with her don\\'t have to go through that same experience, I\\'m going to do that.\"' metadata={'title': \"Lizzo dancers Arianna Davis and Crystal Williams: 'No one speaks out, they are scared'\", 'link': 'https://www.bbc.com/news/entertainment-arts-66384971', 'authors': [], 'language': 'en', 'description': 'The US pop star is being sued for sexual harassment and fat-shaming but has yet to comment.', 'publish_date': None, 'keywords': ['davis', 'lizzo', 'singers', 'experience', 'crystal', 'ensure', 'arianna', 'theres', 'williams', 'power', 'going', 'dancers', 'im', 'speaks', 'work', 'ms', 'scared'], 'summary': 'Ms Williams added: \"If there\\'s anything that I can do in my power to ensure that dancers or singers or whoever decides to work with her don\\'t have to go through that same experience, I\\'m going to do that.\"'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "loader = NewsURLLoader(urls=urls, nlp=True)\n",
+    "data = loader.load()\n",
+    "print(\"First article: \", data[0])\n",
+    "print(\"\\nSecond article: \", data[1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "outputs": [
+    {
+     "data": {
+      "text/plain": "['powell',\n 'know',\n 'donald',\n 'trump',\n 'review',\n 'indictment',\n 'telling',\n 'view',\n 'reasonable',\n 'person',\n 'testimony',\n 'coconspirators',\n 'riot',\n 'representatives',\n 'claims']"
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data[0].metadata['keywords']"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2023-08-02T21:18:19.585758200Z",
+     "start_time": "2023-08-02T21:18:19.585758200Z"
+    }
+   },
+   "id": "ae37e004e0284b1d"
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "outputs": [
+    {
+     "data": {
+      "text/plain": "'In testimony to the congressional committee examining the 6 January riot, Mrs Powell said she did not review all of the many claims of election fraud she made, telling them that \"no reasonable person\" would view her claims as fact.\\nNeither she nor her representatives have commented.'"
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data[0].metadata['summary']"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2023-08-02T21:18:19.598966800Z",
+     "start_time": "2023-08-02T21:18:19.594950200Z"
+    }
+   },
+   "id": "7676155fb175e53e"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/integrations/document_loaders/rss.ipynb

@@ -0,0 +1,311 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "2dfc4698",
+   "metadata": {},
+   "source": [
+    "# RSS Feeds\n",
+    "\n",
+    "This covers how to load HTML news articles from a list of RSS feed URLs into a document format that we can use downstream."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e7c2cd52-c1f7-4a06-8539-b0117da91fba",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "!pip install feedparser newspaper3k listparser"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "16c3699e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import RSSFeedLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "836fbac1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "urls = [\"https://news.ycombinator.com/rss\"]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "33089aba-ff74-4d00-8f40-9449c29587cc",
+   "metadata": {},
+   "source": [
+    "Pass in urls to load them into Documents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "00f46fda",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = RSSFeedLoader(urls=urls)\n",
+    "data = loader.load()\n",
+    "print(len(data))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "id": "b447468cc42266d0",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(next Rich)\n",
+      "\n",
+      "04 August 2023\n",
+      "\n",
+      "Rich Hickey\n",
+      "\n",
+      "It is with a mixture of heartache and optimism that I announce today my (long planned) retirement from commercial software development, and my employment at Nubank. It’s been thrilling to see Clojure and Datomic successfully applied at scale.\n",
+      "\n",
+      "I look forward to continuing to lead ongoing work maintaining and enhancing Clojure with Alex, Stu, Fogus and many others, as an independent developer once again. We have many useful things planned for 1.12 and beyond. The community remains friendly, mature and productive, and is taking Clojure into many interesting new domains.\n",
+      "\n",
+      "I want to highlight and thank Nubank for their ongoing sponsorship of Alex, Fogus and the core team, as well as the Clojure community at large.\n",
+      "\n",
+      "Stu will continue to lead the development of Datomic at Nubank, where the Datomic team grows and thrives. I’m particularly excited to see where the new free availability of Datomic will lead.\n",
+      "\n",
+      "My time with Cognitect remains the highlight of my career. I have learned from absolutely everyone on our team, and am forever grateful to all for our interactions. There are too many people to thank here, but I must extend my sincerest appreciation and love to Stu and Justin for (repeatedly) taking a risk on me and my ideas, and for being the best of partners and friends, at all times fully embodying the notion of integrity. And of course to Alex Miller - who possesses in abundance many skills I lack, and without whose indomitable spirit, positivity and friendship Clojure would not have become what it did.\n",
+      "\n",
+      "I have made many friends through Clojure and Cognitect, and I hope to nurture those friendships moving forward.\n",
+      "\n",
+      "Retirement returns me to the freedom and independence I had when originally developing Clojure. The journey continues!\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(data[0].page_content)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c36d3b0d329faf2a",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "source": [
+    "You can pass arguments to the NewsURLLoader which it uses to load articles."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "id": "5fdada62470d3019",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Error fetching or processing https://twitter.com/andrewmccalip/status/1687405505604734978, exception: You must `parse()` an article first!\n",
+      "Error processing entry https://twitter.com/andrewmccalip/status/1687405505604734978, exception: list index out of range\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "13\n"
+     ]
+    }
+   ],
+   "source": [
+    "loader = RSSFeedLoader(urls=urls, nlp=True)\n",
+    "data = loader.load()\n",
+    "print(len(data))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "id": "11d71963f7735c1d",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['nubank',\n",
+       " 'alex',\n",
+       " 'stu',\n",
+       " 'taking',\n",
+       " 'team',\n",
+       " 'remains',\n",
+       " 'rich',\n",
+       " 'clojure',\n",
+       " 'thank',\n",
+       " 'planned',\n",
+       " 'datomic']"
+      ]
+     },
+     "execution_count": 37,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data[0].metadata['keywords']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "9fb64ba0e8780966",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'It’s been thrilling to see Clojure and Datomic successfully applied at scale.\\nI look forward to continuing to lead ongoing work maintaining and enhancing Clojure with Alex, Stu, Fogus and many others, as an independent developer once again.\\nThe community remains friendly, mature and productive, and is taking Clojure into many interesting new domains.\\nI want to highlight and thank Nubank for their ongoing sponsorship of Alex, Fogus and the core team, as well as the Clojure community at large.\\nStu will continue to lead the development of Datomic at Nubank, where the Datomic team grows and thrives.'"
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data[0].metadata['summary']"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "98ac26c488315bff",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "source": [
+    "You can also use an OPML file such as a Feedly export.  Pass in either a URL or the OPML contents."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "id": "8b6f07ae526a897c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Error fetching http://www.engadget.com/rss-full.xml, exception: Error fetching http://www.engadget.com/rss-full.xml, exception: document declared as us-ascii, but parsed as utf-8\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "20\n"
+     ]
+    }
+   ],
+   "source": [
+    "with open(\"example_data/sample_rss_feeds.opml\", \"r\") as f:\n",
+    "    loader = RSSFeedLoader(opml=f.read())\n",
+    "data = loader.load()\n",
+    "print(len(data))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "id": "b68a26b3",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'The electric vehicle startup Fisker made a splash in Huntington Beach last night, showing off a range of new EVs it plans to build alongside the Fisker Ocean, which is slowly beginning deliveries in Europe and the US. With shades of Lotus circa 2010, it seems there\\'s something for most tastes, with a powerful four-door GT, a versatile pickup truck, and an affordable electric city car.\\n\\n\"We want the world to know that we have big plans and intend to move into several different segments, redefining each with our unique blend of design, innovation, and sustainability,\" said CEO Henrik Fisker.\\n\\nStarting with the cheapest, the Fisker PEAR—a cutesy acronym for \"Personal Electric Automotive Revolution\"—is said to use 35 percent fewer parts than other small EVs. Although it\\'s a smaller car, the PEAR seats six thanks to front and rear bench seats. Oh, and it has a frunk, which the company is calling the \"froot,\" something that will satisfy some British English speakers like Ars\\' friend and motoring journalist Jonny Smith.\\n\\nBut most exciting is the price—starting at $29,900 and scheduled for 2025. Fisker plans to contract with Foxconn to build the PEAR in Lordstown, Ohio, meaning it would be eligible for federal tax incentives.\\n\\nAdvertisement\\n\\nThe Fisker Alaska is the company\\'s pickup truck, built on a modified version of the platform used by the Ocean. It has an extendable cargo bed, which can be as little as 4.5 feet (1,371 mm) or as much as 9.2 feet (2,804 mm) long. Fisker claims it will be both the lightest EV pickup on sale and the most sustainable pickup truck in the world. Range will be an estimated 230–240 miles (370–386 km).\\n\\nThis, too, is slated for 2025, and also at a relatively affordable price, starting at $45,400. Fisker hopes to build this car in North America as well, although it isn\\'t saying where that might take place.\\n\\nFinally, there\\'s the Ronin, a four-door GT that bears more than a passing resemblance to the Fisker Karma, Henrik Fisker\\'s 2012 creation. There\\'s no price for this one, but Fisker says its all-wheel drive powertrain will boast 1,000 hp (745 kW) and will hit 60 mph from a standing start in two seconds—just about as fast as modern tires will allow. Expect a massive battery in this one, as Fisker says it\\'s targeting a 600-mile (956 km) range.\\n\\n\"Innovation and sustainability, along with design, are our three brand values. By 2027, we intend to produce the world’s first climate-neutral vehicle, and as our customers reinvent their relationships with mobility, we want to be a leader in software-defined transportation,\" Fisker said.'"
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data[0].page_content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d5a0cbe8-18a6-4af2-b447-7abb8b734451",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "poetry-venv",
+   "language": "python",
+   "name": "poetry-venv"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/integrations/document_loaders/unstructured_file.ipynb

@@ -18,8 +18,7 @@
    "outputs": [],
    "source": [
     "# # Install package\n",
-    "!pip install \"unstructured[local-inference]\"\n",
-    "!pip install layoutparser[layoutmodels,tesseract]"
+    "!pip install \"unstructured[all-docs]\"\n"
    ]
   },
   {

docs/extras/integrations/document_loaders/youtube_audio.ipynb

@@ -1,6 +1,7 @@
 {
  "cells": [
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "e48afb8d",
    "metadata": {},
@@ -11,7 +12,8 @@
     "\n",
     "Below we show how to easily go from a YouTube url to text to chat!\n",
     "\n",
-    "We wil use the `OpenAIWhisperParser`, which will use the OpenAI Whisper API to transcribe audio to text.\n",
+    "We wil use the `OpenAIWhisperParser`, which will use the OpenAI Whisper API to transcribe audio to text, \n",
+    "and the  `OpenAIWhisperParserLocal` for local support and running on private clouds or on premise.\n",
     "\n",
     "Note: You will need to have an `OPENAI_API_KEY` supplied."
    ]
@@ -24,7 +26,7 @@
    "outputs": [],
    "source": [
     "from langchain.document_loaders.generic import GenericLoader\n",
-    "from langchain.document_loaders.parsers import OpenAIWhisperParser\n",
+    "from langchain.document_loaders.parsers import OpenAIWhisperParser, OpenAIWhisperParserLocal\n",
     "from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader"
    ]
   },
@@ -46,7 +48,8 @@
    "outputs": [],
    "source": [
     "! pip install yt_dlp\n",
-    "! pip install pydub"
+    "! pip install pydub\n",
+    "! pip install librosa"
    ]
   },
   {
@@ -63,6 +66,18 @@
     "Let's take the first lecture of Andrej Karpathy's YouTube course as an example! "
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8682f256",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# set a flag to switch between local and remote parsing\n",
+    "# change this to True if you want to use local parsing\n",
+    "local = False"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 2,
@@ -102,7 +117,10 @@
     "save_dir = \"~/Downloads/YouTube\"\n",
     "\n",
     "# Transcribe the videos to text\n",
-    "loader = GenericLoader(YoutubeAudioLoader(urls, save_dir), OpenAIWhisperParser())\n",
+    "if local:\n",
+    "    loader = GenericLoader(YoutubeAudioLoader(urls, save_dir), OpenAIWhisperParserLocal())\n",
+    "else:\n",
+    "    loader = GenericLoader(YoutubeAudioLoader(urls, save_dir), OpenAIWhisperParser())\n",
     "docs = loader.load()"
    ]
   },
@@ -275,7 +293,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
@@ -289,7 +307,12 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.10.11"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "97cc609b13305c559618ec78a438abc56230b9381f827f22d070313b9a1f3777"
+   }
   }
  },
  "nbformat": 4,

docs/extras/integrations/llms/Fireworks.ipynb

@@ -0,0 +1,230 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cc6caafa",
+   "metadata": {},
+   "source": [
+    "# Fireworks\n",
+    "\n",
+    ">[Fireworks](https://app.fireworks.ai/) accelerates product development on generative AI by creating an innovative AI experiment and production platform. \n",
+    "\n",
+    "This example goes over how to use LangChain to interact with `Fireworks` models."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "60b6dbb2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.llms.fireworks import Fireworks, FireworksChat\n",
+    "from langchain import PromptTemplate, LLMChain\n",
+    "from langchain.prompts.chat import (\n",
+    "    ChatPromptTemplate,\n",
+    "    HumanMessagePromptTemplate,\n",
+    ")\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ccff689e",
+   "metadata": {},
+   "source": [
+    "# Setup\n",
+    "\n",
+    "Contact Fireworks AI for the an API Key to access our models\n",
+    "\n",
+    "Set up your model using a model id. If the model is not set, the default model is fireworks-llama-v2-7b-chat."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "9ca87a2e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialize a Fireworks LLM\n",
+    "os.environ['FIREWORKS_API_KEY'] = \"\" #change this to your own API KEY\n",
+    "llm = Fireworks(model_id=\"accounts/fireworks/models/fireworks-llama-v2-13b-chat\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "43a11ba8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create LLM chain\n",
+    "llm_chain = LLMChain(prompt=prompt, llm=llm)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "acc24d0c",
+   "metadata": {},
+   "source": [
+    "# Calling the Model\n",
+    "\n",
+    "You can use the LLMs to call the model for specified prompt(s). \n",
+    "\n",
+    "Current Specified Models: \n",
+    "* accounts/fireworks/models/fireworks-falcon-7b, accounts/fireworks/models/fireworks-falcon-40b-w8a16\n",
+    "* accounts/fireworks/models/fireworks-starcoder-1b-w8a16-1gpu, accounts/fireworks/models/fireworks-starcoder-3b-w8a16-1gpu, accounts/fireworks/models/fireworks-starcoder-7b-w8a16-1gpu, accounts/fireworks/models/fireworks-starcoder-16b-w8a16 \n",
+    "* accounts/fireworks/models/fireworks-llama-v2-13b, accounts/fireworks/models/fireworks-llama-v2-13b-chat, accounts/fireworks/models/fireworks-llama-v2-13b-w8a16, accounts/fireworks/models/fireworks-llama-v2-13b-chat-w8a16\n",
+    "* accounts/fireworks/models/fireworks-llama-v2-7b, accounts/fireworks/models/fireworks-llama-v2-7b-chat, accounts/fireworks/models/fireworks-llama-v2-7b-w8a16, accounts/fireworks/models/fireworks-llama-v2-7b-chat-w8a16, accounts/fireworks/models/fireworks-llama-v2-70b-chat-4gpu"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "bf0a425c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "It's a question that has been debated for years, with different analysts and fans making their cases for various signal-callers. Here are some of the top contenders for the title of best quarterback in the NFL:\n",
+      "\n",
+      "1. Tom Brady: The New England Patriots legend has won six Super Bowls and has been named Super Bowl MVP four times. He's known for his precision passing, pocket presence, and ability to read defenses.\n",
+      "2. Aaron Rodgers: The Green Bay Packers quarterback has won two Super Bowls and has been named NFL MVP twice. He's known for his quick release, accuracy, and ability to extend plays with his feet.\n",
+      "3. Drew Brees: The New Orleans Saints quarterback has won a Super Bowl and has been named NFL MVP once. He's known for his accuracy, pocket presence, and ability to read defenses.\n",
+      "4. Patrick Mahomes: The Kansas City Chiefs quarterback has won a Super Bowl and has been named NFL MVP twice. He's known for his arm strength, athleticism, and ability to make plays outside of the pocket.\n",
+      "5. Russell Wilson: The Seattle Seahawks quarterback has won a Super Bowl and has been named NFL MVP once. He's known for his mobility, accuracy, and ability to extend plays with his feet.\n",
+      "\n",
+      "Of course, there are other talented quarterbacks in the league, such as Lamar Jackson, Deshaun Watson, and Carson Wentz, who could also be considered among the best. Ultimately, the answer to the question of who's the best quarterback in the NFL is subjective and can vary depending on individual perspectives and criteria.\n"
+     ]
+    }
+   ],
+   "source": [
+    "#single prompt\n",
+    "output = llm(\"Who's the best quarterback in the NFL?\")\n",
+    "print(output)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "afc7de6f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "generations=[[Generation(text=\"\\nWho is the best cricket player in the world in 2016?\\nThe best cricket player in the world in 2016 is Virat Kohli. The Indian captain has had a fabulous year, scoring heavily in all formats of the game, leading India to several victories, and breaking several records. In Test cricket, Kohli has scored 1215 runs at an average of 75.33 with 6 centuries and 4 fifties, which is the highest number of runs scored by any player in a calendar year. In ODI cricket, he has scored 1143 runs at an average of 83.42 with 7 centuries and 6 fifties, which is also the highest number of runs scored by any player in a calendar year. Additionally, Kohli has led India to the number one ranking in Test cricket, and has been named the ICC Test Player of the Year and the ICC ODI Player of the Year.\\nVirat Kohli has been in incredible form in 2016, and his performances have made him the standout player of the year. Other players who have had a great year include Steve Smith, Joe Root, and Kane Williamson, but Kohli's consistency and dominance in all formats of the game make him the best cricket player in the world in 2016.\", generation_info=None)], [Generation(text=\"\\n\\nA: LeBron James.\\n\\nB: Kevin Durant.\\n\\nC: Steph Curry.\\n\\nD: James Harden.\\n\\nE: Other (please specify).\\n\\nWhat's your answer?\", generation_info=None)]] llm_output={'token_usage': {}, 'model_id': 'fireworks-llama-v2-13b-chat'} run=[RunInfo(run_id=UUID('d14b6bee-7692-46ad-8798-acb6f72fc7fb')), RunInfo(run_id=UUID('b9f5b3b5-9e62-4eaf-b269-ecf0cbbcfb82'))]\n"
+     ]
+    }
+   ],
+   "source": [
+    "#calling multiple prompts\n",
+    "output = llm.generate([\"Who's the best cricket player in 2016?\", \"Who's the best basketball player in the league?\"])\n",
+    "print(output)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "b801c20d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Kansas City in December can be quite chilly, with average high\n"
+     ]
+    }
+   ],
+   "source": [
+    "#setting parameters: model_id, temperature, max_tokens, top_p\n",
+    "llm = Fireworks(model_id=\"accounts/fireworks/models/fireworks-llama-v2-13b-chat\", temperature=0.7, max_tokens=15, top_p=1.0)\n",
+    "print(llm(\"What's the weather like in Kansas City in December?\"))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "137662a6",
+   "metadata": {},
+   "source": [
+    "# Create and Run Chain\n",
+    "\n",
+    "Create a prompt template to be used with the LLM Chain. Once this prompt template is created, initialize the chain with the LLM and prompt template, and run the chain with the specified prompts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "fd2c6bc1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "(Note: I'm just an AI and not a branding expert, so take this as a starting point for your own research and brainstorming.)\n",
+      "A good name for a company that makes football helmets could be:\n",
+      "\n",
+      "1. Helix Pro: This name plays off the idea of a helix, or spiral, shape that is commonly associated with football helmets. \"Pro\" implies a professional-level product.\n",
+      "2. Gridiron Gear: \"Gridiron\" is a term used to describe a football field, and \"gear\" highlights the company's focus on producing high-quality football helmets.\n",
+      "3. Linebacker Lab: \"Linebacker\" is a position on the football field, and \"Lab\" suggests a focus on research and development.\n",
+      "4. Helmet Hut: This name is simple and easy to remember, and it immediately conveys the company's focus on football helmets.\n",
+      "5. Tackle Tech: \"Tackle\" is a term used in football to describe a hit or collision, and \"Tech\" implies a focus on advanced technology and innovation.\n",
+      "6. Victory Vest: \"Victory\" implies a focus on winning and success, and \"Vest\" could suggest a protective or armored design.\n",
+      "7. Pigskin Pro: \"Pigskin\" is a term used to describe a football, and \"Pro\" implies a professional-level product.\n",
+      "8. Football Fusion: This name could suggest a combination of different materials or technologies to create a high-quality football helmet.\n",
+      "9. Endzone Edge: \"Endzone\" is the area of the football field where a team scores a touchdown, and \"Edge\" implies a competitive advantage.\n",
+      "10. MVP Masks: \"MVP\" stands for \"Most Valuable Player,\" and \"Masks\" highlights the protective nature of the company's football helmets.\n",
+      "\n",
+      "Remember, the name you choose for your company should be memorable, easy to pronounce and spell, and convey a sense of quality and professionalism. It's also important to check that the name isn't already in use by another company, and to consider any potential trademark issues.\n"
+     ]
+    }
+   ],
+   "source": [
+    "human_message_prompt = HumanMessagePromptTemplate(\n",
+    "    prompt=PromptTemplate(\n",
+    "        template=\"What is a good name for a company that makes {product}?\",\n",
+    "        input_variables=[\"product\"],\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "chat_prompt_template = ChatPromptTemplate.from_messages([human_message_prompt])\n",
+    "chat = Fireworks()\n",
+    "chain = LLMChain(llm=chat, prompt=chat_prompt_template)\n",
+    "output = chain.run(\"football helmets\")\n",
+    "\n",
+    "print(output)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/integrations/llms/llamacpp.ipynb

@@ -4,12 +4,12 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Llama-cpp\n",
+    "# Llama.cpp\n",
     "\n",
-    "[llama-cpp](https://github.com/abetlen/llama-cpp-python) is a Python binding for [llama.cpp](https://github.com/ggerganov/llama.cpp). \n",
+    "[llama-cpp-python](https://github.com/abetlen/llama-cpp-python) is a Python binding for [llama.cpp](https://github.com/ggerganov/llama.cpp). \n",
     "It supports [several LLMs](https://github.com/ggerganov/llama.cpp).\n",
     "\n",
-    "This notebook goes over how to run `llama-cpp` within LangChain."
+    "This notebook goes over how to run `llama-cpp-python` within LangChain."
    ]
   },
   {
@@ -18,7 +18,7 @@
    "source": [
     "## Installation\n",
     "\n",
-    "There is a bunch of options how to install the llama-cpp package: \n",
+    "There are different options on how to install the llama-cpp package: \n",
     "- only CPU usage\n",
     "- CPU + GPU (using one of many BLAS backends)\n",
     "- Metal GPU (MacOS with Apple Silicon Chip) \n",
@@ -61,7 +61,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "**IMPORTANT**: If you have already installed a cpu only version of the package, you need to reinstall it from scratch: consider the following command: "
+    "**IMPORTANT**: If you have already installed the CPU only version of the package, you need to reinstall it from scratch. Consider the following command: "
    ]
   },
   {
@@ -79,7 +79,7 @@
    "source": [
     "### Installation with Metal\n",
     "\n",
-    "`lama.cpp` supports Apple silicon first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks. Use the `FORCE_CMAKE=1` environment variable to force the use of cmake and install the pip package for the Metal support ([source](https://github.com/abetlen/llama-cpp-python/blob/main/docs/install/macos.md)).\n",
+    "`llama.cpp` supports Apple silicon first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks. Use the `FORCE_CMAKE=1` environment variable to force the use of cmake and install the pip package for the Metal support ([source](https://github.com/abetlen/llama-cpp-python/blob/main/docs/install/macos.md)).\n",
     "\n",
     "Example installation with Metal Support:"
    ]
@@ -143,7 +143,7 @@
     "\n",
     "#### Compiling and installing\n",
     "\n",
-    "In the same command prompt (anaconda prompt) you set the variables, you can cd into `llama-cpp-python` directory and run the following commands.\n",
+    "In the same command prompt (anaconda prompt) you set the variables, you can `cd` into `llama-cpp-python` directory and run the following commands.\n",
     "\n",
     "```\n",
     "python setup.py clean\n",
@@ -164,7 +164,9 @@
    "source": [
     "Make sure you are following all instructions to [install all necessary model files](https://github.com/ggerganov/llama.cpp).\n",
     "\n",
-    "You don't need an `API_TOKEN`!"
+    "You don't need an `API_TOKEN` as you will run the LLM locally.\n",
+    "\n",
+    "It is worth understanding which models are suitable to be used on the desired machine."
    ]
   },
   {
@@ -227,7 +229,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "`Llama-v2`"
+    "Example using a LLaMA 2 7B model"
    ]
   },
   {
@@ -304,7 +306,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "`Llama-v1`"
+    "Example using a LLaMA v1 model"
    ]
   },
   {
@@ -381,7 +383,7 @@
    "source": [
     "### GPU\n",
     "\n",
-    "If the installation with BLAS backend was correct, you will see an `BLAS = 1` indicator in model properties.\n",
+    "If the installation with BLAS backend was correct, you will see a `BLAS = 1` indicator in model properties.\n",
     "\n",
     "Two of the most important parameters for use with GPU are:\n",
     "\n",
@@ -473,22 +475,15 @@
     "llm_chain.run(question)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "### Metal\n",
     "\n",
-    "If the installation with Metal was correct, you will see an `NEON = 1` indicator in model properties.\n",
+    "If the installation with Metal was correct, you will see a `NEON = 1` indicator in model properties.\n",
     "\n",
-    "Two of the most important parameters for use with GPU are:\n",
+    "Two of the most important GPU parameters are:\n",
     "\n",
     "- `n_gpu_layers` - determines how many layers of the model are offloaded to your Metal GPU, in the most case, set it to `1` is enough for Metal\n",
     "- `n_batch` - how many tokens are processed in parallel, default is 8, set to bigger number.\n",
@@ -522,7 +517,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The rest are almost same as GPU, the console log will show the following log to indicate the Metal was enable properly.\n",
+    "The console log will show the following log to indicate Metal was enable properly.\n",
     "\n",
     "```\n",
     "ggml_metal_init: allocating\n",
@@ -530,7 +525,9 @@
     "...\n",
     "```\n",
     "\n",
-    "You also could check the `Activity Monitor` by watching the % GPU of the process, the % CPU will drop dramatically after turn on `n_gpu_layers=1`. Also for the first time call LLM, the performance might be slow due to the model compilation in Metal GPU."
+    "You also could check `Activity Monitor` by watching the GPU usage of the process, the CPU usage will drop dramatically after turn on `n_gpu_layers=1`. \n",
+    "\n",
+    "For the first call to the LLM, the performance may be slow due to the model compilation in Metal GPU."
    ]
   }
  ],

docs/extras/integrations/memory/streamlit_chat_message_history.ipynb

@@ -0,0 +1,154 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "91c6a7ef",
+   "metadata": {},
+   "source": [
+    "# Streamlit Chat Message History\n",
+    "\n",
+    "This notebook goes over how to store and use chat message history in a Streamlit app. StreamlitChatMessageHistory will store messages in\n",
+    "[Streamlit session state](https://docs.streamlit.io/library/api-reference/session-state)\n",
+    "at the specified `key=`. The default key is `\"langchain_messages\"`.\n",
+    "\n",
+    "- Note, StreamlitChatMessageHistory only works when run in a Streamlit app.\n",
+    "- You may also be interested in [StreamlitCallbackHandler](/docs/integrations/callbacks/streamlit) for LangChain.\n",
+    "- For more on Streamlit check out their\n",
+    "[getting started documentation](https://docs.streamlit.io/library/get-started).\n",
+    "\n",
+    "You can see the [full app example running here](https://langchain-st-memory.streamlit.app/), and more examples in\n",
+    "[github.com/langchain-ai/streamlit-agent](https://github.com/langchain-ai/streamlit-agent)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d15e3302",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.memory import StreamlitChatMessageHistory\n",
+    "\n",
+    "history = StreamlitChatMessageHistory(key=\"chat_messages\")\n",
+    "\n",
+    "history.add_user_message(\"hi!\")\n",
+    "history.add_ai_message(\"whats up?\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64fc465e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "history.messages"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b60dc735",
+   "metadata": {},
+   "source": [
+    "You can integrate StreamlitChatMessageHistory into ConversationBufferMemory and chains or agents as usual. The history will be persisted across re-runs of the Streamlit app within a given user session. A given StreamlitChatMessageHistory will NOT be persisted or shared across user sessions."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "42ab5bf3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "from langchain.memory import ConversationBufferMemory\n",
+    "from langchain.memory.chat_message_histories import StreamlitChatMessageHistory\n",
+    "\n",
+    "# Optionally, specify your own session_state key for storing messages\n",
+    "msgs = StreamlitChatMessageHistory(key=\"special_app_key\")\n",
+    "\n",
+    "memory = ConversationBufferMemory(memory_key=\"history\", chat_memory=msgs)\n",
+    "if len(msgs.messages) == 0:\n",
+    "    msgs.add_ai_message(\"How can I help you?\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a29252de",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chains import LLMChain\n",
+    "from langchain.llms import OpenAI\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "template = \"\"\"You are an AI chatbot having a conversation with a human.\n",
+    "\n",
+    "{history}\n",
+    "Human: {human_input}\n",
+    "AI: \"\"\"\n",
+    "prompt = PromptTemplate(input_variables=[\"history\", \"human_input\"], template=template)\n",
+    "\n",
+    "# Add the memory to an LLMChain as usual\n",
+    "llm_chain = LLMChain(llm=OpenAI(), prompt=prompt, memory=memory)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7cd99b4b",
+   "metadata": {},
+   "source": [
+    "Conversational Streamlit apps will often re-draw each previous chat message on every re-run. This is easy to do by iterating through `StreamlitChatMessageHistory.messages`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3bdb637b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import streamlit as st\n",
+    "\n",
+    "for msg in msgs.messages:\n",
+    "    st.chat_message(msg.type).write(msg.content)\n",
+    "\n",
+    "if prompt := st.chat_input():\n",
+    "    st.chat_message(\"human\").write(prompt)\n",
+    "\n",
+    "    # As usual, new messages are added to StreamlitChatMessageHistory when the Chain is called.\n",
+    "    response = llm_chain.run(prompt)\n",
+    "    st.chat_message(\"ai\").write(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7adaf3d6",
+   "metadata": {},
+   "source": [
+    "**[View the final app](https://langchain-st-memory.streamlit.app/).**"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "poetry-venv",
+   "language": "python",
+   "name": "poetry-venv"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/integrations/providers/fireworks.md

@@ -0,0 +1,22 @@
+# Fireworks
+
+This page covers how to use the Fireworks models within Langchain.
+
+## Installation and Setup
+
+- To use the Fireworks model, you need to have a Fireworks API key. To generate one, sign up at platform.fireworks.ai
+- Authenticate by setting the FIREWORKS_API_KEY environment variable.
+
+## LLM
+
+Fireworks integrates with Langchain through the LLM module, which allows for standardized usage of any models deployed on the Fireworks models.
+
+In this example, we'll work the llama-v2-13b. 
+```python
+from langchain.llms.fireworks import Fireworks 
+
+llm = Fireworks(model="fireworks-llama-v2-13b-chat", max_tokens=256, temperature=0.4)
+llm("Name 3 sports.")
+```
+
+For a more detailed walkthrough, see [here](/docs/integrations/llms/Fireworks).

docs/extras/integrations/providers/mlflow_ai_gateway.mdx

@@ -1,6 +1,6 @@
 # MLflow AI Gateway
 
-The MLflow AI Gateway service is a powerful tool designed to streamline the usage and management of various large language model (LLM) providers, such as OpenAI and Anthropic, within an organization. It offers a high-level interface that simplifies the interaction with these services by providing a unified endpoint to handle specific LLM related requests. See [the MLflow AI Gateway documentation](https://mlflow.org/docs/latest/gateway/index.html) for more details.
+>`The MLflow AI Gateway` service is a powerful tool designed to streamline the usage and management of various large language model (LLM) providers, such as OpenAI and Anthropic, within an organization. It offers a high-level interface that simplifies the interaction with these services by providing a unified endpoint to handle specific LLM related requests. See [the MLflow AI Gateway documentation](https://mlflow.org/docs/latest/gateway/index.html) for more details.
 
 ## Installation and Setup
 

docs/extras/integrations/providers/mlflow_tracking.ipynb

@@ -1,19 +1,49 @@
 {
  "cells": [
   {
-   "attachments": {},
    "cell_type": "markdown",
+   "id": "5d184f91",
    "metadata": {},
    "source": [
     "# MLflow\n",
     "\n",
-    "This notebook goes over how to track your LangChain experiments into your MLflow Server"
-   ],
-   "id": "5d184f91"
+    ">[MLflow](https://www.mlflow.org/docs/latest/what-is-mlflow.html) is a versatile, expandable, open-source platform for managing workflows and artifacts across the machine learning lifecycle. It has built-in integrations with many popular ML libraries, but can be used with any library, algorithm, or deployment tool. It is designed to be extensible, so you can write plugins to support new workflows, libraries, and tools.\n",
+    "\n",
+    "This notebook goes over how to track your LangChain experiments into your `MLflow Server`"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ea73efae-7182-4a89-a492-c865b1fcf981",
+   "metadata": {},
+   "source": [
+    "## External examples"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "97361a84-4e8f-45ba-b291-814cf73cd8f2",
+   "metadata": {},
+   "source": [
+    "`MLflow` provides [several examples](https://github.com/mlflow/mlflow/tree/master/examples/langchain) for the `LangChain` integration:\n",
+    "- [simple_chain](https://github.com/mlflow/mlflow/blob/master/examples/langchain/simple_chain.py)\n",
+    "- [simple_agent](https://github.com/mlflow/mlflow/blob/master/examples/langchain/simple_agent.py)\n",
+    "- [retriever_chain](https://github.com/mlflow/mlflow/blob/master/examples/langchain/retriever_chain.py)\n",
+    "- [retrieval_qa_chain](https://github.com/mlflow/mlflow/blob/master/examples/langchain/retrieval_qa_chain.py)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e0cbd74b-1542-45a4-a72b-b2eedeffd2e0",
+   "metadata": {},
+   "source": [
+    "## Example"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "ca7bd72f",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -24,12 +54,12 @@
     "!pip install openai\n",
     "!pip install google-search-results\n",
     "!python -m spacy download en_core_web_sm"
-   ],
-   "id": "ca7bd72f"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "bf8e1f5c",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -38,23 +68,23 @@
     "os.environ[\"MLFLOW_TRACKING_URI\"] = \"\"\n",
     "os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
     "os.environ[\"SERPAPI_API_KEY\"] = \"\""
-   ],
-   "id": "bf8e1f5c"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "fd49fd45",
    "metadata": {},
    "outputs": [],
    "source": [
     "from langchain.callbacks import MlflowCallbackHandler\n",
     "from langchain.llms import OpenAI"
-   ],
-   "id": "fd49fd45"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "578cac8c",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -70,12 +100,12 @@
     "llm = OpenAI(\n",
     "    model_name=\"gpt-3.5-turbo\", temperature=0, callbacks=[mlflow_callback], verbose=True\n",
     ")"
-   ],
-   "id": "578cac8c"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "9b20acae",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -83,23 +113,23 @@
     "llm_result = llm.generate([\"Tell me a joke\"])\n",
     "\n",
     "mlflow_callback.flush_tracker(llm)"
-   ],
-   "id": "9b20acae"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "8b872046",
    "metadata": {},
    "outputs": [],
    "source": [
     "from langchain.prompts import PromptTemplate\n",
     "from langchain.chains import LLMChain"
-   ],
-   "id": "8b872046"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "1b2627ef",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -117,12 +147,12 @@
     "]\n",
     "synopsis_chain.apply(test_prompts)\n",
     "mlflow_callback.flush_tracker(synopsis_chain)"
-   ],
-   "id": "1b2627ef"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "e002823a",
    "metadata": {
     "id": "_jN73xcPVEpI"
    },
@@ -130,12 +160,12 @@
    "source": [
     "from langchain.agents import initialize_agent, load_tools\n",
     "from langchain.agents import AgentType"
-   ],
-   "id": "e002823a"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "655bd47e",
    "metadata": {
     "id": "Gpq4rk6VT9cu"
    },
@@ -154,8 +184,7 @@
     "    \"Who is Leo DiCaprio's girlfriend? What is her current age raised to the 0.43 power?\"\n",
     ")\n",
     "mlflow_callback.flush_tracker(agent, finish=True)"
-   ],
-   "id": "655bd47e"
+   ]
   }
  ],
  "metadata": {
@@ -177,9 +206,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

docs/extras/integrations/providers/sagemaker_tracking.ipynb

@@ -0,0 +1,916 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "ef3909cf-72ca-4841-85c6-ef4e0eae3aaf",
+   "metadata": {},
+   "source": [
+    "# SageMaker Tracking\n",
+    "\n",
+    "This notebook shows how LangChain Callback can be used to log and track prompts and other LLM hyperparameters into SageMaker Experiments. Here, we use different scenarios to showcase the capability:\n",
+    "* **Scenario 1**: *Single LLM* - A case where a single LLM model is used to generate output based on a given prompt.\n",
+    "* **Scenario 2**: *Sequential Chain* - A case where a sequential chain of two LLM models is used.\n",
+    "* **Scenario 3**: *Agent with Tools (Chain of Thought)* - A case where multiple tools (search and math) are used in addition to an LLM.\n",
+    "\n",
+    "[Amazon SageMaker](https://aws.amazon.com/sagemaker/) is a fully managed service that is used to quickly and easily build, train and deploy machine learning (ML) models. \n",
+    "\n",
+    "[Amazon SageMaker Experiments](https://docs.aws.amazon.com/sagemaker/latest/dg/experiments.html) is a capability of Amazon SageMaker that lets you organize, track, compare and evaluate ML experiments and model versions.\n",
+    "\n",
+    "In this notebook, we will create a single experiment to log the prompts from each scenario."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "94c22cb4-3b1c-432b-b3be-0235eec79c5c",
+   "metadata": {},
+   "source": [
+    "## Installation and Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2353436d-17fe-4f58-a2f9-c299d56393fd",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "!pip install sagemaker\n",
+    "!pip install openai\n",
+    "!pip install google-search-results"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "65dcf62e-7a38-4119-adb9-d9e884e82499",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "First, setup the required API keys\n",
+    "* OpenAI: https://platform.openai.com/account/api-keys (For OpenAI LLM model)\n",
+    "* Google SERP API: https://serpapi.com/manage-api-key (For Google Search Tool)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5ec2b898-0cfc-4308-8e86-569cd7b7cf41",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "## Add your API keys below\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"<ADD-KEY-HERE>\"\n",
+    "os.environ[\"SERPAPI_API_KEY\"] = \"<ADD-KEY-HERE>\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "80968ebf-519f-46de-8703-97532ac39e3e",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.llms import OpenAI\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "from langchain.chains import LLMChain, SimpleSequentialChain\n",
+    "from langchain.agents import initialize_agent, load_tools\n",
+    "from langchain.agents import Tool\n",
+    "from langchain.callbacks import SageMakerCallbackHandler\n",
+    "\n",
+    "from sagemaker.analytics import ExperimentAnalytics\n",
+    "from sagemaker.session import Session\n",
+    "from sagemaker.experiments.run import Run"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b67d031f-a01f-4009-ad29-c80ab8ad50ea",
+   "metadata": {},
+   "source": [
+    "## LLM Prompt Tracking"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "da2d70ee-173b-469d-a718-54c33d862844",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "#LLM Hyperparameters\n",
+    "HPARAMS = {\n",
+    "    \"temperature\": 0.1,\n",
+    "    \"model_name\": \"text-davinci-003\",\n",
+    "}\n",
+    "\n",
+    "#Bucket used to save prompt logs (Use `None` is used to save the default bucket or otherwise change it)\n",
+    "BUCKET_NAME = None\n",
+    "\n",
+    "#Experiment name\n",
+    "EXPERIMENT_NAME = \"langchain-sagemaker-tracker\"\n",
+    "\n",
+    "#Create SageMaker Session with the given bucket\n",
+    "session = Session(default_bucket=BUCKET_NAME)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7239a39a-08d8-43cb-8922-81abdd5d9ebf",
+   "metadata": {},
+   "source": [
+    "### Scenario 1 - LLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "abc00335-50c8-4119-adb8-4c4ab8522e23",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "RUN_NAME = \"run-scenario-1\"\n",
+    "PROMPT_TEMPLATE = \"tell me a joke about {topic}\"\n",
+    "INPUT_VARIABLES = {\"topic\": \"fish\"}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4a3a3cbe-db85-4255-8d8b-eaafdca8c6e2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with Run(experiment_name=EXPERIMENT_NAME, run_name=RUN_NAME, sagemaker_session=session) as run:\n",
+    "\n",
+    "    # Create SageMaker Callback\n",
+    "    sagemaker_callback = SageMakerCallbackHandler(run)\n",
+    "\n",
+    "    # Define LLM model with callback\n",
+    "    llm = OpenAI(callbacks=[sagemaker_callback], **HPARAMS)\n",
+    "\n",
+    "    # Create prompt template\n",
+    "    prompt = PromptTemplate.from_template(template=PROMPT_TEMPLATE)\n",
+    "\n",
+    "    # Create LLM Chain\n",
+    "    chain = LLMChain(llm=llm, prompt=prompt, callbacks=[sagemaker_callback])\n",
+    "\n",
+    "    # Run chain\n",
+    "    chain.run(**INPUT_VARIABLES)\n",
+    "\n",
+    "    # Reset the callback\n",
+    "    sagemaker_callback.flush_tracker()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7dc69934-9f42-40b7-9931-36a3371a38da",
+   "metadata": {},
+   "source": [
+    "### Scenario 2 - Sequential Chain"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "50b75ef9-9825-4ccc-8414-4cd7525a1b68",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "RUN_NAME = \"run-scenario-2\"\n",
+    "\n",
+    "PROMPT_TEMPLATE_1 = \"\"\"You are a playwright. Given the title of play, it is your job to write a synopsis for that title.\n",
+    "Title: {title}\n",
+    "Playwright: This is a synopsis for the above play:\"\"\"\n",
+    "PROMPT_TEMPLATE_2 = \"\"\"You are a play critic from the New York Times. Given the synopsis of play, it is your job to write a review for that play.\n",
+    "Play Synopsis: {synopsis}\n",
+    "Review from a New York Times play critic of the above play:\"\"\"\n",
+    "\n",
+    "INPUT_VARIABLES = {\n",
+    "    \"input\": \"documentary about good video games that push the boundary of game design\"\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fb7fff5f-e89f-40e2-96b4-3641a0b6e9b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with Run(experiment_name=EXPERIMENT_NAME, run_name=RUN_NAME, sagemaker_session=session) as run:\n",
+    "\n",
+    "    # Create SageMaker Callback\n",
+    "    sagemaker_callback = SageMakerCallbackHandler(run)\n",
+    "\n",
+    "    # Create prompt templates for the chain\n",
+    "    prompt_template1 = PromptTemplate.from_template(template=PROMPT_TEMPLATE_1)\n",
+    "    prompt_template2 = PromptTemplate.from_template(template=PROMPT_TEMPLATE_2)\n",
+    "\n",
+    "    # Define LLM model with callback\n",
+    "    llm = OpenAI(callbacks=[sagemaker_callback], **HPARAMS)\n",
+    "\n",
+    "    # Create chain1\n",
+    "    chain1 = LLMChain(llm=llm, prompt=prompt_template1, callbacks=[sagemaker_callback])\n",
+    "\n",
+    "    # Create chain2\n",
+    "    chain2 = LLMChain(llm=llm, prompt=prompt_template2, callbacks=[sagemaker_callback])\n",
+    "\n",
+    "    # Create Sequential chain\n",
+    "    overall_chain = SimpleSequentialChain(chains=[chain1, chain2], callbacks=[sagemaker_callback])\n",
+    "\n",
+    "    # Run overall sequential chain\n",
+    "    overall_chain.run(**INPUT_VARIABLES)\n",
+    "\n",
+    "    # Reset the callback\n",
+    "    sagemaker_callback.flush_tracker()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6b82bd0e-c626-4797-bb06-c1983f176315",
+   "metadata": {},
+   "source": [
+    "### Scenario 3 - Agent with Tools"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b5066f03-49dc-4868-be8e-d21ce22063fe",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "RUN_NAME = \"run-scenario-3\"\n",
+    "PROMPT_TEMPLATE = \"Who is the oldest person alive? And what is their current age raised to the power of 1.51?\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "98385c42-9e44-4b03-b76d-007cb4797864",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "with Run(experiment_name=EXPERIMENT_NAME, run_name=RUN_NAME, sagemaker_session=session) as run:\n",
+    "\n",
+    "    # Create SageMaker Callback\n",
+    "    sagemaker_callback = SageMakerCallbackHandler(run)\n",
+    "\n",
+    "    # Define LLM model with callback\n",
+    "    llm = OpenAI(callbacks=[sagemaker_callback], **HPARAMS)\n",
+    "\n",
+    "    # Define tools\n",
+    "    tools = load_tools([\"serpapi\", \"llm-math\"], llm=llm, callbacks=[sagemaker_callback])\n",
+    "\n",
+    "    # Initialize agent with all the tools\n",
+    "    agent = initialize_agent(tools, llm, agent=\"zero-shot-react-description\", callbacks=[sagemaker_callback])\n",
+    "\n",
+    "    # Run agent\n",
+    "    agent.run(input=PROMPT_TEMPLATE)\n",
+    "\n",
+    "    # Reset the callback\n",
+    "    sagemaker_callback.flush_tracker()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c306a1c9-99f8-476d-96db-347746f5cfe0",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "## Load Log Data\n",
+    "\n",
+    "Once the prompts are logged, we can easily load and convert them to Pandas DataFrame as follows."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ec7b4af2-e01d-4f6c-9de5-70d2b4acb9e6",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "#Load\n",
+    "logs = ExperimentAnalytics(experiment_name=EXPERIMENT_NAME)\n",
+    "\n",
+    "#Convert as pandas dataframe\n",
+    "df = logs.dataframe(force_refresh=True)\n",
+    "\n",
+    "print(df.shape)\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "29991c75-f9cf-4c36-abfd-903c09fb170d",
+   "metadata": {},
+   "source": [
+    "As can be seen above, there are three runs (rows) in the experiment corresponding to each scenario. Each run logs the prompts and related LLM settings/hyperparameters as json and are saved in s3 bucket. Feel free to load and explore the log data from each json path."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "61a695d6-0aef-4284-9e12-eea8bc143dbd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "availableInstances": [
+   {
+    "_defaultOrder": 0,
+    "_isFastLaunch": true,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 4,
+    "name": "ml.t3.medium",
+    "vcpuNum": 2
+   },
+   {
+    "_defaultOrder": 1,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 8,
+    "name": "ml.t3.large",
+    "vcpuNum": 2
+   },
+   {
+    "_defaultOrder": 2,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 16,
+    "name": "ml.t3.xlarge",
+    "vcpuNum": 4
+   },
+   {
+    "_defaultOrder": 3,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 32,
+    "name": "ml.t3.2xlarge",
+    "vcpuNum": 8
+   },
+   {
+    "_defaultOrder": 4,
+    "_isFastLaunch": true,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 8,
+    "name": "ml.m5.large",
+    "vcpuNum": 2
+   },
+   {
+    "_defaultOrder": 5,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 16,
+    "name": "ml.m5.xlarge",
+    "vcpuNum": 4
+   },
+   {
+    "_defaultOrder": 6,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 32,
+    "name": "ml.m5.2xlarge",
+    "vcpuNum": 8
+   },
+   {
+    "_defaultOrder": 7,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 64,
+    "name": "ml.m5.4xlarge",
+    "vcpuNum": 16
+   },
+   {
+    "_defaultOrder": 8,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 128,
+    "name": "ml.m5.8xlarge",
+    "vcpuNum": 32
+   },
+   {
+    "_defaultOrder": 9,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 192,
+    "name": "ml.m5.12xlarge",
+    "vcpuNum": 48
+   },
+   {
+    "_defaultOrder": 10,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 256,
+    "name": "ml.m5.16xlarge",
+    "vcpuNum": 64
+   },
+   {
+    "_defaultOrder": 11,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 384,
+    "name": "ml.m5.24xlarge",
+    "vcpuNum": 96
+   },
+   {
+    "_defaultOrder": 12,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 8,
+    "name": "ml.m5d.large",
+    "vcpuNum": 2
+   },
+   {
+    "_defaultOrder": 13,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 16,
+    "name": "ml.m5d.xlarge",
+    "vcpuNum": 4
+   },
+   {
+    "_defaultOrder": 14,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 32,
+    "name": "ml.m5d.2xlarge",
+    "vcpuNum": 8
+   },
+   {
+    "_defaultOrder": 15,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 64,
+    "name": "ml.m5d.4xlarge",
+    "vcpuNum": 16
+   },
+   {
+    "_defaultOrder": 16,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 128,
+    "name": "ml.m5d.8xlarge",
+    "vcpuNum": 32
+   },
+   {
+    "_defaultOrder": 17,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 192,
+    "name": "ml.m5d.12xlarge",
+    "vcpuNum": 48
+   },
+   {
+    "_defaultOrder": 18,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 256,
+    "name": "ml.m5d.16xlarge",
+    "vcpuNum": 64
+   },
+   {
+    "_defaultOrder": 19,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 384,
+    "name": "ml.m5d.24xlarge",
+    "vcpuNum": 96
+   },
+   {
+    "_defaultOrder": 20,
+    "_isFastLaunch": false,
+    "category": "General purpose",
+    "gpuNum": 0,
+    "hideHardwareSpecs": true,
+    "memoryGiB": 0,
+    "name": "ml.geospatial.interactive",
+    "supportedImageNames": [
+     "sagemaker-geospatial-v1-0"
+    ],
+    "vcpuNum": 0
+   },
+   {
+    "_defaultOrder": 21,
+    "_isFastLaunch": true,
+    "category": "Compute optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 4,
+    "name": "ml.c5.large",
+    "vcpuNum": 2
+   },
+   {
+    "_defaultOrder": 22,
+    "_isFastLaunch": false,
+    "category": "Compute optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 8,
+    "name": "ml.c5.xlarge",
+    "vcpuNum": 4
+   },
+   {
+    "_defaultOrder": 23,
+    "_isFastLaunch": false,
+    "category": "Compute optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 16,
+    "name": "ml.c5.2xlarge",
+    "vcpuNum": 8
+   },
+   {
+    "_defaultOrder": 24,
+    "_isFastLaunch": false,
+    "category": "Compute optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 32,
+    "name": "ml.c5.4xlarge",
+    "vcpuNum": 16
+   },
+   {
+    "_defaultOrder": 25,
+    "_isFastLaunch": false,
+    "category": "Compute optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 72,
+    "name": "ml.c5.9xlarge",
+    "vcpuNum": 36
+   },
+   {
+    "_defaultOrder": 26,
+    "_isFastLaunch": false,
+    "category": "Compute optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 96,
+    "name": "ml.c5.12xlarge",
+    "vcpuNum": 48
+   },
+   {
+    "_defaultOrder": 27,
+    "_isFastLaunch": false,
+    "category": "Compute optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 144,
+    "name": "ml.c5.18xlarge",
+    "vcpuNum": 72
+   },
+   {
+    "_defaultOrder": 28,
+    "_isFastLaunch": false,
+    "category": "Compute optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 192,
+    "name": "ml.c5.24xlarge",
+    "vcpuNum": 96
+   },
+   {
+    "_defaultOrder": 29,
+    "_isFastLaunch": true,
+    "category": "Accelerated computing",
+    "gpuNum": 1,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 16,
+    "name": "ml.g4dn.xlarge",
+    "vcpuNum": 4
+   },
+   {
+    "_defaultOrder": 30,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 1,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 32,
+    "name": "ml.g4dn.2xlarge",
+    "vcpuNum": 8
+   },
+   {
+    "_defaultOrder": 31,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 1,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 64,
+    "name": "ml.g4dn.4xlarge",
+    "vcpuNum": 16
+   },
+   {
+    "_defaultOrder": 32,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 1,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 128,
+    "name": "ml.g4dn.8xlarge",
+    "vcpuNum": 32
+   },
+   {
+    "_defaultOrder": 33,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 4,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 192,
+    "name": "ml.g4dn.12xlarge",
+    "vcpuNum": 48
+   },
+   {
+    "_defaultOrder": 34,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 1,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 256,
+    "name": "ml.g4dn.16xlarge",
+    "vcpuNum": 64
+   },
+   {
+    "_defaultOrder": 35,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 1,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 61,
+    "name": "ml.p3.2xlarge",
+    "vcpuNum": 8
+   },
+   {
+    "_defaultOrder": 36,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 4,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 244,
+    "name": "ml.p3.8xlarge",
+    "vcpuNum": 32
+   },
+   {
+    "_defaultOrder": 37,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 8,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 488,
+    "name": "ml.p3.16xlarge",
+    "vcpuNum": 64
+   },
+   {
+    "_defaultOrder": 38,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 8,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 768,
+    "name": "ml.p3dn.24xlarge",
+    "vcpuNum": 96
+   },
+   {
+    "_defaultOrder": 39,
+    "_isFastLaunch": false,
+    "category": "Memory Optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 16,
+    "name": "ml.r5.large",
+    "vcpuNum": 2
+   },
+   {
+    "_defaultOrder": 40,
+    "_isFastLaunch": false,
+    "category": "Memory Optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 32,
+    "name": "ml.r5.xlarge",
+    "vcpuNum": 4
+   },
+   {
+    "_defaultOrder": 41,
+    "_isFastLaunch": false,
+    "category": "Memory Optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 64,
+    "name": "ml.r5.2xlarge",
+    "vcpuNum": 8
+   },
+   {
+    "_defaultOrder": 42,
+    "_isFastLaunch": false,
+    "category": "Memory Optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 128,
+    "name": "ml.r5.4xlarge",
+    "vcpuNum": 16
+   },
+   {
+    "_defaultOrder": 43,
+    "_isFastLaunch": false,
+    "category": "Memory Optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 256,
+    "name": "ml.r5.8xlarge",
+    "vcpuNum": 32
+   },
+   {
+    "_defaultOrder": 44,
+    "_isFastLaunch": false,
+    "category": "Memory Optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 384,
+    "name": "ml.r5.12xlarge",
+    "vcpuNum": 48
+   },
+   {
+    "_defaultOrder": 45,
+    "_isFastLaunch": false,
+    "category": "Memory Optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 512,
+    "name": "ml.r5.16xlarge",
+    "vcpuNum": 64
+   },
+   {
+    "_defaultOrder": 46,
+    "_isFastLaunch": false,
+    "category": "Memory Optimized",
+    "gpuNum": 0,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 768,
+    "name": "ml.r5.24xlarge",
+    "vcpuNum": 96
+   },
+   {
+    "_defaultOrder": 47,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 1,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 16,
+    "name": "ml.g5.xlarge",
+    "vcpuNum": 4
+   },
+   {
+    "_defaultOrder": 48,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 1,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 32,
+    "name": "ml.g5.2xlarge",
+    "vcpuNum": 8
+   },
+   {
+    "_defaultOrder": 49,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 1,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 64,
+    "name": "ml.g5.4xlarge",
+    "vcpuNum": 16
+   },
+   {
+    "_defaultOrder": 50,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 1,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 128,
+    "name": "ml.g5.8xlarge",
+    "vcpuNum": 32
+   },
+   {
+    "_defaultOrder": 51,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 1,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 256,
+    "name": "ml.g5.16xlarge",
+    "vcpuNum": 64
+   },
+   {
+    "_defaultOrder": 52,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 4,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 192,
+    "name": "ml.g5.12xlarge",
+    "vcpuNum": 48
+   },
+   {
+    "_defaultOrder": 53,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 4,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 384,
+    "name": "ml.g5.24xlarge",
+    "vcpuNum": 96
+   },
+   {
+    "_defaultOrder": 54,
+    "_isFastLaunch": false,
+    "category": "Accelerated computing",
+    "gpuNum": 8,
+    "hideHardwareSpecs": false,
+    "memoryGiB": 768,
+    "name": "ml.g5.48xlarge",
+    "vcpuNum": 192
+   }
+  ],
+  "instance_type": "ml.t3.large",
+  "kernelspec": {
+   "display_name": "conda_pytorch_p310",
+   "language": "python",
+   "name": "conda_pytorch_p310"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/integrations/providers/unstructured.mdx

@@ -11,7 +11,9 @@ ecosystem within LangChain.
 If you are using a loader that runs locally, use the following steps to get `unstructured` and
 its dependencies running locally.
 
-- Install the Python SDK with `pip install "unstructured[local-inference]"`
+- Install the Python SDK with `pip install unstructured`.
+    - You can install document specific dependencies with extras, i.e. `pip install "unstructured[docx]"`.
+    - To install the dependencies for all document types, use `pip install "unstructured[all-docs]"`.
 - Install the following system dependencies if they are not already available on your system.
   Depending on what document types you're parsing, you may not need all of these.
     - `libmagic-dev` (filetype detection)

docs/extras/integrations/providers/wandb_tracing.ipynb

@@ -177,7 +177,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.9.1"
   }
  },
  "nbformat": 4,

docs/extras/integrations/retrievers/re_phrase.ipynb

@@ -0,0 +1,222 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e8624be2",
+   "metadata": {},
+   "source": [
+    "# RePhraseQueryRetriever\n",
+    "\n",
+    "Simple retriever that applies an LLM between the user input and the query pass the to retriever.\n",
+    "\n",
+    "It can be used to pre-process the user input in any way.\n",
+    "\n",
+    "The default prompt used in the `from_llm` classmethod:\n",
+    "\n",
+    "```\n",
+    "DEFAULT_TEMPLATE = \"\"\"You are an assistant tasked with taking a natural language \\\n",
+    "query from a user and converting it into a query for a vectorstore. \\\n",
+    "In this process, you strip out information that is not relevant for \\\n",
+    "the retrieval task. Here is the user query: {question}\"\"\"\n",
+    "```\n",
+    "\n",
+    "Create a vectorstore."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "1bfa6834",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import WebBaseLoader\n",
+    "\n",
+    "loader = WebBaseLoader(\"https://lilianweng.github.io/posts/2023-06-23-agent/\")\n",
+    "data = loader.load()\n",
+    "\n",
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "\n",
+    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0)\n",
+    "all_splits = text_splitter.split_documents(data)\n",
+    "\n",
+    "from langchain.vectorstores import Chroma\n",
+    "from langchain.embeddings import OpenAIEmbeddings\n",
+    "\n",
+    "vectorstore = Chroma.from_documents(documents=all_splits, embedding=OpenAIEmbeddings())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "d0b51556",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import logging\n",
+    "\n",
+    "logging.basicConfig()\n",
+    "logging.getLogger(\"langchain.retrievers.re_phraser\").setLevel(logging.INFO)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "20e1e787",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.retrievers import RePhraseQueryRetriever"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "88c0a972",
+   "metadata": {},
+   "source": [
+    "## Using the default prompt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "503994bd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = ChatOpenAI(temperature=0)\n",
+    "retriever_from_llm = RePhraseQueryRetriever.from_llm(\n",
+    "    retriever=vectorstore.as_retriever(), llm=llm\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "8d17ecc9",
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:langchain.retrievers.re_phraser:Re-phrased question: The user query can be converted into a query for a vectorstore as follows:\n",
+      "\n",
+      "\"approaches to Task Decomposition\"\n"
+     ]
+    }
+   ],
+   "source": [
+    "docs = retriever_from_llm.get_relevant_documents(\n",
+    "    \"Hi I'm Lance. What are the approaches to Task Decomposition?\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "76d54f1a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:langchain.retrievers.re_phraser:Re-phrased question: Query for vectorstore: \"Types of Memory\"\n"
+     ]
+    }
+   ],
+   "source": [
+    "docs = retriever_from_llm.get_relevant_documents(\n",
+    "    \"I live in San Francisco. What are the Types of Memory?\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0513a6e2",
+   "metadata": {},
+   "source": [
+    "## Supply a prompt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "410d6a64",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain import LLMChain\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "\n",
+    "QUERY_PROMPT = PromptTemplate(\n",
+    "    input_variables=[\"question\"],\n",
+    "    template=\"\"\"You are an assistant tasked with taking a natural languge query from a user\n",
+    "    and converting it into a query for a vectorstore. In the process, strip out all \n",
+    "    information that is not relevant for the retrieval task and return a new, simplified\n",
+    "    question for vectorstore retrieval. The new user query should be in pirate speech.\n",
+    "    Here is the user query: {question} \"\"\",\n",
+    ")\n",
+    "llm = ChatOpenAI(temperature=0)\n",
+    "llm_chain = LLMChain(llm=llm, prompt=QUERY_PROMPT)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "2dbffdd3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "retriever_from_llm_chain = RePhraseQueryRetriever(\n",
+    "    retriever=vectorstore.as_retriever(), llm_chain=llm_chain\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "103b4be3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:langchain.retrievers.re_phraser:Re-phrased question: Ahoy matey! What be Maximum Inner Product Search, ye scurvy dog?\n"
+     ]
+    }
+   ],
+   "source": [
+    "docs = retriever_from_llm_chain.get_relevant_documents(\n",
+    "    \"Hi I'm Lance. What is Maximum Inner Product Search?\"\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/integrations/retrievers/tf_idf.ipynb

@@ -16,7 +16,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "id": "a801b57c",
    "metadata": {},
    "outputs": [],
@@ -26,7 +26,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "393ac030",
    "metadata": {
     "tags": []
@@ -46,7 +46,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "id": "98b1c017",
    "metadata": {
     "tags": []
@@ -133,6 +133,68 @@
    "source": [
     "result"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "363f3c04",
+   "metadata": {},
+   "source": [
+    "## Save and load\n",
+    "\n",
+    "You can easily save and load this retriever, making it handy for local development!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "10c90d03",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "retriever.save_local(\"testing.pkl\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "fb3b153c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "retriever_copy = TFIDFRetriever.load_local(\"testing.pkl\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "c03ff3c7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='foo', metadata={}),\n",
+       " Document(page_content='foo bar', metadata={}),\n",
+       " Document(page_content='hello', metadata={}),\n",
+       " Document(page_content='world', metadata={})]"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "retriever_copy.get_relevant_documents(\"foo\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2d7c5728",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -151,7 +213,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.10.1"
   }
  },
  "nbformat": 4,

docs/extras/integrations/text_embedding/aleph_alpha.ipynb

@@ -20,7 +20,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "id": "8a920a89",
    "metadata": {},
    "outputs": [],
@@ -30,7 +30,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "id": "f2d04da3",
    "metadata": {},
    "outputs": [],
@@ -41,17 +41,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "e6ecde96",
    "metadata": {},
    "outputs": [],
    "source": [
-    "embeddings = AlephAlphaAsymmetricSemanticEmbedding()"
+    "embeddings = AlephAlphaAsymmetricSemanticEmbedding(normalize=True, compress_to_size=128)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "90e68411",
    "metadata": {},
    "outputs": [],
@@ -61,7 +61,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "55903233",
    "metadata": {},
    "outputs": [],
@@ -79,7 +79,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "eabb763a",
    "metadata": {},
    "outputs": [],
@@ -89,7 +89,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 7,
    "id": "0ad799f7",
    "metadata": {},
    "outputs": [],
@@ -99,17 +99,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "id": "af86dc10",
    "metadata": {},
    "outputs": [],
    "source": [
-    "embeddings = AlephAlphaSymmetricSemanticEmbedding()"
+    "embeddings = AlephAlphaSymmetricSemanticEmbedding(normalize=True, compress_to_size=128)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "d292536f",
    "metadata": {},
    "outputs": [],
@@ -119,7 +119,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "id": "c704a7cf",
    "metadata": {},
    "outputs": [],
@@ -130,7 +130,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "33492471",
+   "id": "5d999f8f",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -152,7 +152,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.1"
+   "version": "3.9.13"
   },
   "vscode": {
    "interpreter": {

docs/extras/integrations/text_embedding/edenai.ipynb

@@ -0,0 +1,169 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# EDEN AI"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Eden AI is an AI consulting company that was founded to use its resources to empower people and create impactful products that use AI to improve the quality of life for individuals, businesses and societies at large."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This example goes over how to use LangChain to interact with Eden AI embedding models\n",
+    "\n",
+    "-----------------------------------------------------------------------------------\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Accessing the EDENAI's API requires an API key, \n",
+    "\n",
+    "which you can get by creating an account https://app.edenai.run/user/register  and heading here https://app.edenai.run/admin/account/settings\n",
+    "\n",
+    "Once we have a key we'll want to set it as an environment variable by running:\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "export EDENAI_API_KEY=\"...\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you'd prefer not to set an environment variable you can pass the key in directly via the edenai_api_key named parameter\n",
+    "\n",
+    " when initiating the EdenAI embedding class:\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.embeddings.edenai import EdenAiEmbeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "embeddings = EdenAiEmbeddings(edenai_api_key=\"...\",provider=\"...\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Calling a model\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The EdenAI API brings together various providers.\n",
+    "\n",
+    "To access a specific model, you can simply use the \"provider\" when calling.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "embeddings = EdenAiEmbeddings(provider=\"openai\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "docs = [\"It's raining right now\", \"cats are cute\"]\n",
+    "document_result = embeddings.embed_documents(docs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = \"my umbrella is broken\"\n",
+    "query_result = embeddings.embed_query(query)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Cosine similarity between \"It's raining right now\" and query: 0.849261496107252\n",
+      "Cosine similarity between \"cats are cute\" and query: 0.7525900655705218\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "query_numpy = np.array(query_result)\n",
+    "for doc_res, doc in zip(document_result, docs):\n",
+    "    document_numpy = np.array(doc_res)\n",
+    "    similarity = np.dot(query_numpy, document_numpy) / (\n",
+    "        np.linalg.norm(query_numpy) * np.linalg.norm(document_numpy)\n",
+    "    )\n",
+    "    print(f'Cosine similarity between \"{doc}\" and query: {similarity}')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

docs/extras/integrations/vectorstores/faiss.ipynb

@@ -249,6 +249,36 @@
     "docs[0]"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "30c8f57b",
+   "metadata": {},
+   "source": [
+    "# Serializing and De-Serializing to bytes\n",
+    "\n",
+    "you can pickle the FAISS Index by these functions. If you use embeddings model which is of 90 mb (sentence-transformers/all-MiniLM-L6-v2 or any other model), the resultant pickle size would be more than 90 mb. the size of the model is also included in the overall size. To overcome this, use the below functions. These functions only serializes FAISS index and size would be much lesser. this can be helpful if you wish to store the index in database like sql."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d8faead5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pkl = db.serialize_to_bytes() # serializes the faiss index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e36e220b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db = FAISS.deserialize_from_bytes(pkl) # Load the index"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "57da60d4",
@@ -491,7 +521,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.9"
+   "version": "3.9.17"
   }
  },
  "nbformat": 4,

docs/extras/integrations/vectorstores/opensearch.ipynb

@@ -1,436 +1,440 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "683953b3",
-   "metadata": {},
-   "source": [
-    "# OpenSearch\n",
-    "\n",
-    "> [OpenSearch](https://opensearch.org/) is a scalable, flexible, and extensible open-source software suite for search, analytics, and observability applications licensed under Apache 2.0. `OpenSearch` is a distributed search and analytics engine based on `Apache Lucene`.\n",
-    "\n",
-    "\n",
-    "This notebook shows how to use functionality related to the `OpenSearch` database.\n",
-    "\n",
-    "To run, you should have an OpenSearch instance up and running: [see here for an easy Docker installation](https://hub.docker.com/r/opensearchproject/opensearch).\n",
-    "\n",
-    "`similarity_search` by default performs the Approximate k-NN Search which uses one of the several algorithms like lucene, nmslib, faiss recommended for\n",
-    "large datasets. To perform brute force search we have other search methods known as Script Scoring and Painless Scripting.\n",
-    "Check [this](https://opensearch.org/docs/latest/search-plugins/knn/index/) for more details."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "94963977-9dfc-48b7-872a-53f2947f46c6",
-   "metadata": {},
-   "source": [
-    "## Installation\n",
-    "Install the Python client."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6e606066-9386-4427-8a87-1b93f435c57e",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "!pip install opensearch-py"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b1fa637e-4fbf-4d5a-9188-2cad826a193e",
-   "metadata": {},
-   "source": [
-    "We want to use OpenAIEmbeddings so we have to get the OpenAI API Key."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "28e5455e-322d-4010-9e3b-491d522ef5db",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import getpass\n",
-    "\n",
-    "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"OpenAI API Key:\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "aac9563e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
-    "from langchain.text_splitter import CharacterTextSplitter\n",
-    "from langchain.vectorstores import OpenSearchVectorSearch\n",
-    "from langchain.document_loaders import TextLoader"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a3c3999a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain.document_loaders import TextLoader\n",
-    "\n",
-    "loader = TextLoader(\"../../../state_of_the_union.txt\")\n",
-    "documents = loader.load()\n",
-    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
-    "docs = text_splitter.split_documents(documents)\n",
-    "\n",
-    "embeddings = OpenAIEmbeddings()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "01a9a035",
-   "metadata": {},
-   "source": [
-    "## similarity_search using Approximate k-NN\n",
-    "\n",
-    "`similarity_search` using `Approximate k-NN` Search with Custom Parameters"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "803fe12b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "docsearch = OpenSearchVectorSearch.from_documents(\n",
-    "    docs, embeddings, opensearch_url=\"http://localhost:9200\"\n",
-    ")\n",
-    "\n",
-    "# If using the default Docker installation, use this instantiation instead:\n",
-    "# docsearch = OpenSearchVectorSearch.from_documents(\n",
-    "#     docs,\n",
-    "#     embeddings,\n",
-    "#     opensearch_url=\"https://localhost:9200\",\n",
-    "#     http_auth=(\"admin\", \"admin\"),\n",
-    "#     use_ssl = False,\n",
-    "#     verify_certs = False,\n",
-    "#     ssl_assert_hostname = False,\n",
-    "#     ssl_show_warn = False,\n",
-    "# )"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "db3fa309",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "query = \"What did the president say about Ketanji Brown Jackson\"\n",
-    "docs = docsearch.similarity_search(query, k=10)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c160d5bb",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(docs[0].page_content)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "96215c90",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "docsearch = OpenSearchVectorSearch.from_documents(\n",
-    "    docs,\n",
-    "    embeddings,\n",
-    "    opensearch_url=\"http://localhost:9200\",\n",
-    "    engine=\"faiss\",\n",
-    "    space_type=\"innerproduct\",\n",
-    "    ef_construction=256,\n",
-    "    m=48,\n",
-    ")\n",
-    "\n",
-    "query = \"What did the president say about Ketanji Brown Jackson\"\n",
-    "docs = docsearch.similarity_search(query)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "62a7cea0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(docs[0].page_content)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0d0cd877",
-   "metadata": {},
-   "source": [
-    "## similarity_search using Script Scoring\n",
-    "\n",
-    "`similarity_search` using `Script Scoring` with Custom Parameters"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0a8e3c0e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "docsearch = OpenSearchVectorSearch.from_documents(\n",
-    "    docs, embeddings, opensearch_url=\"http://localhost:9200\", is_appx_search=False\n",
-    ")\n",
-    "\n",
-    "query = \"What did the president say about Ketanji Brown Jackson\"\n",
-    "docs = docsearch.similarity_search(\n",
-    "    \"What did the president say about Ketanji Brown Jackson\",\n",
-    "    k=1,\n",
-    "    search_type=\"script_scoring\",\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "92bc40db",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(docs[0].page_content)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "a4af96cc",
-   "metadata": {},
-   "source": [
-    "## similarity_search using Painless Scripting\n",
-    "\n",
-    "`similarity_search` using `Painless Scripting` with Custom Parameters"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6d9f436e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "docsearch = OpenSearchVectorSearch.from_documents(\n",
-    "    docs, embeddings, opensearch_url=\"http://localhost:9200\", is_appx_search=False\n",
-    ")\n",
-    "filter = {\"bool\": {\"filter\": {\"term\": {\"text\": \"smuggling\"}}}}\n",
-    "query = \"What did the president say about Ketanji Brown Jackson\"\n",
-    "docs = docsearch.similarity_search(\n",
-    "    \"What did the president say about Ketanji Brown Jackson\",\n",
-    "    search_type=\"painless_scripting\",\n",
-    "    space_type=\"cosineSimilarity\",\n",
-    "    pre_filter=filter,\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8ca50bce",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(docs[0].page_content)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4f8fb0d0",
-   "metadata": {},
-   "source": [
-    "## Maximum marginal relevance search (MMR)\n",
-    "If you’d like to look up for some similar documents, but you’d also like to receive diverse results, MMR is method you should consider. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ba85e092",
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "id": "683953b3",
+      "metadata": {},
+      "source": [
+        "# OpenSearch\n",
+        "\n",
+        "> [OpenSearch](https://opensearch.org/) is a scalable, flexible, and extensible open-source software suite for search, analytics, and observability applications licensed under Apache 2.0. `OpenSearch` is a distributed search and analytics engine based on `Apache Lucene`.\n",
+        "\n",
+        "\n",
+        "This notebook shows how to use functionality related to the `OpenSearch` database.\n",
+        "\n",
+        "To run, you should have an OpenSearch instance up and running: [see here for an easy Docker installation](https://hub.docker.com/r/opensearchproject/opensearch).\n",
+        "\n",
+        "`similarity_search` by default performs the Approximate k-NN Search which uses one of the several algorithms like lucene, nmslib, faiss recommended for\n",
+        "large datasets. To perform brute force search we have other search methods known as Script Scoring and Painless Scripting.\n",
+        "Check [this](https://opensearch.org/docs/latest/search-plugins/knn/index/) for more details."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "94963977-9dfc-48b7-872a-53f2947f46c6",
+      "metadata": {},
+      "source": [
+        "## Installation\n",
+        "Install the Python client."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "6e606066-9386-4427-8a87-1b93f435c57e",
+      "metadata": {
+        "tags": []
+      },
+      "outputs": [],
+      "source": [
+        "!pip install opensearch-py"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "b1fa637e-4fbf-4d5a-9188-2cad826a193e",
+      "metadata": {},
+      "source": [
+        "We want to use OpenAIEmbeddings so we have to get the OpenAI API Key."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "28e5455e-322d-4010-9e3b-491d522ef5db",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import getpass\n",
+        "\n",
+        "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"OpenAI API Key:\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "aac9563e",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
+        "from langchain.text_splitter import CharacterTextSplitter\n",
+        "from langchain.vectorstores import OpenSearchVectorSearch\n",
+        "from langchain.document_loaders import TextLoader"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "a3c3999a",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from langchain.document_loaders import TextLoader\n",
+        "\n",
+        "loader = TextLoader(\"../../../state_of_the_union.txt\")\n",
+        "documents = loader.load()\n",
+        "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
+        "docs = text_splitter.split_documents(documents)\n",
+        "\n",
+        "embeddings = OpenAIEmbeddings()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "01a9a035",
+      "metadata": {},
+      "source": [
+        "## similarity_search using Approximate k-NN\n",
+        "\n",
+        "`similarity_search` using `Approximate k-NN` Search with Custom Parameters"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "803fe12b",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "docsearch = OpenSearchVectorSearch.from_documents(\n",
+        "    docs, embeddings, opensearch_url=\"http://localhost:9200\"\n",
+        ")\n",
+        "\n",
+        "# If using the default Docker installation, use this instantiation instead:\n",
+        "# docsearch = OpenSearchVectorSearch.from_documents(\n",
+        "#     docs,\n",
+        "#     embeddings,\n",
+        "#     opensearch_url=\"https://localhost:9200\",\n",
+        "#     http_auth=(\"admin\", \"admin\"),\n",
+        "#     use_ssl = False,\n",
+        "#     verify_certs = False,\n",
+        "#     ssl_assert_hostname = False,\n",
+        "#     ssl_show_warn = False,\n",
+        "# )"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "db3fa309",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "query = \"What did the president say about Ketanji Brown Jackson\"\n",
+        "docs = docsearch.similarity_search(query, k=10)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "c160d5bb",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "print(docs[0].page_content)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "96215c90",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "docsearch = OpenSearchVectorSearch.from_documents(\n",
+        "    docs,\n",
+        "    embeddings,\n",
+        "    opensearch_url=\"http://localhost:9200\",\n",
+        "    engine=\"faiss\",\n",
+        "    space_type=\"innerproduct\",\n",
+        "    ef_construction=256,\n",
+        "    m=48,\n",
+        ")\n",
+        "\n",
+        "query = \"What did the president say about Ketanji Brown Jackson\"\n",
+        "docs = docsearch.similarity_search(query)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "62a7cea0",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "print(docs[0].page_content)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "0d0cd877",
+      "metadata": {},
+      "source": [
+        "## similarity_search using Script Scoring\n",
+        "\n",
+        "`similarity_search` using `Script Scoring` with Custom Parameters"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "0a8e3c0e",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "docsearch = OpenSearchVectorSearch.from_documents(\n",
+        "    docs, embeddings, opensearch_url=\"http://localhost:9200\", is_appx_search=False\n",
+        ")\n",
+        "\n",
+        "query = \"What did the president say about Ketanji Brown Jackson\"\n",
+        "docs = docsearch.similarity_search(\n",
+        "    \"What did the president say about Ketanji Brown Jackson\",\n",
+        "    k=1,\n",
+        "    search_type=\"script_scoring\",\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "92bc40db",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "print(docs[0].page_content)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "a4af96cc",
+      "metadata": {},
+      "source": [
+        "## similarity_search using Painless Scripting\n",
+        "\n",
+        "`similarity_search` using `Painless Scripting` with Custom Parameters"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "6d9f436e",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "docsearch = OpenSearchVectorSearch.from_documents(\n",
+        "    docs, embeddings, opensearch_url=\"http://localhost:9200\", is_appx_search=False\n",
+        ")\n",
+        "filter = {\"bool\": {\"filter\": {\"term\": {\"text\": \"smuggling\"}}}}\n",
+        "query = \"What did the president say about Ketanji Brown Jackson\"\n",
+        "docs = docsearch.similarity_search(\n",
+        "    \"What did the president say about Ketanji Brown Jackson\",\n",
+        "    search_type=\"painless_scripting\",\n",
+        "    space_type=\"cosineSimilarity\",\n",
+        "    pre_filter=filter,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "8ca50bce",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "print(docs[0].page_content)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "4f8fb0d0",
+      "metadata": {},
+      "source": [
+        "## Maximum marginal relevance search (MMR)\n",
+        "If you\u2019d like to look up for some similar documents, but you\u2019d also like to receive diverse results, MMR is method you should consider. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "ba85e092",
+      "metadata": {
+        "collapsed": false,
+        "jupyter": {
+          "outputs_hidden": false
+        }
+      },
+      "outputs": [],
+      "source": [
+        "query = \"What did the president say about Ketanji Brown Jackson\"\n",
+        "docs = docsearch.max_marginal_relevance_search(query, k=2, fetch_k=10, lambda_param=0.5)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "73264864",
+      "metadata": {},
+      "source": [
+        "## Using a preexisting OpenSearch instance\n",
+        "\n",
+        "It's also possible to use a preexisting OpenSearch instance with documents that already have vectors present."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "82a23440",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# this is just an example, you would need to change these values to point to another opensearch instance\n",
+        "docsearch = OpenSearchVectorSearch(\n",
+        "    index_name=\"index-*\",\n",
+        "    embedding_function=embeddings,\n",
+        "    opensearch_url=\"http://localhost:9200\",\n",
+        ")\n",
+        "\n",
+        "# you can specify custom field names to match the fields you're using to store your embedding, document text value, and metadata\n",
+        "docs = docsearch.similarity_search(\n",
+        "    \"Who was asking about getting lunch today?\",\n",
+        "    search_type=\"script_scoring\",\n",
+        "    space_type=\"cosinesimil\",\n",
+        "    vector_field=\"message_embedding\",\n",
+        "    text_field=\"message\",\n",
+        "    metadata_field=\"message_metadata\",\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Using AOSS (Amazon OpenSearch Service Serverless)"
+      ],
+      "metadata": {
+        "collapsed": false,
+        "pycharm": {
+          "name": "#%% md\n"
+        }
+      },
+      "id": "5f590d35"
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# This is just an example to show how to use AOSS with faiss engine and efficient_filter, you need to set proper values.\n",
+        "\n",
+        "service = 'aoss' # must set the service as 'aoss'\n",
+        "region = 'us-east-2'\n",
+        "credentials = boto3.Session(aws_access_key_id='xxxxxx',aws_secret_access_key='xxxxx').get_credentials()\n",
+        "awsauth = AWS4Auth('xxxxx', 'xxxxxx', region,service, session_token=credentials.token)\n",
+        "\n",
+        "docsearch = OpenSearchVectorSearch.from_documents(\n",
+        "    docs,\n",
+        "    embeddings,\n",
+        "    opensearch_url=\"host url\",\n",
+        "    http_auth=awsauth,\n",
+        "    timeout = 300,\n",
+        "    use_ssl = True,\n",
+        "    verify_certs = True,\n",
+        "    connection_class = RequestsHttpConnection,\n",
+        "    index_name=\"test-index-using-aoss\",\n",
+        "    engine=\"faiss\",\n",
+        ")\n",
+        "\n",
+        "docs = docsearch.similarity_search(\n",
+        "    \"What is feature selection\",\n",
+        "     efficient_filter=filter,\n",
+        "     k=200,\n",
+        ")"
+      ],
+      "metadata": {
+        "collapsed": false,
+        "pycharm": {
+          "name": "#%%\n"
+        }
+      },
+      "id": "de397be7"
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Using AOS (Amazon OpenSearch Service)"
+      ],
+      "metadata": {
+        "collapsed": false
+      },
+      "id": "0aa012c8"
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# This is just an example to show how to use AOS , you need to set proper values.\n",
+        "\n",
+        "service = 'es' # must set the service as 'es'\n",
+        "region = 'us-east-2'\n",
+        "credentials = boto3.Session(aws_access_key_id='xxxxxx',aws_secret_access_key='xxxxx').get_credentials()\n",
+        "awsauth = AWS4Auth('xxxxx', 'xxxxxx', region,service, session_token=credentials.token)\n",
+        "\n",
+        "docsearch = OpenSearchVectorSearch.from_documents(\n",
+        "    docs,\n",
+        "    embeddings,\n",
+        "    opensearch_url=\"host url\",\n",
+        "    http_auth=awsauth,\n",
+        "    timeout = 300,\n",
+        "    use_ssl = True,\n",
+        "    verify_certs = True,\n",
+        "    connection_class = RequestsHttpConnection,\n",
+        "    index_name=\"test-index\",\n",
+        ")\n",
+        "\n",
+        "docs = docsearch.similarity_search(\n",
+        "    \"What is feature selection\",\n",
+        "     k=200,\n",
+        ")"
+      ],
+      "metadata": {
+        "collapsed": false,
+        "pycharm": {
+          "name": "#%%\n"
+        }
+      },
+      "id": "2c47e408"
     }
-   },
-   "outputs": [],
-   "source": [
-    "query = \"What did the president say about Ketanji Brown Jackson\"\n",
-    "docs = docsearch.max_marginal_relevance_search(query, k=2, fetch_k=10, lambda_param=0.5)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "73264864",
-   "metadata": {},
-   "source": [
-    "## Using a preexisting OpenSearch instance\n",
-    "\n",
-    "It's also possible to use a preexisting OpenSearch instance with documents that already have vectors present."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "82a23440",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# this is just an example, you would need to change these values to point to another opensearch instance\n",
-    "docsearch = OpenSearchVectorSearch(\n",
-    "    index_name=\"index-*\",\n",
-    "    embedding_function=embeddings,\n",
-    "    opensearch_url=\"http://localhost:9200\",\n",
-    ")\n",
-    "\n",
-    "# you can specify custom field names to match the fields you're using to store your embedding, document text value, and metadata\n",
-    "docs = docsearch.similarity_search(\n",
-    "    \"Who was asking about getting lunch today?\",\n",
-    "    search_type=\"script_scoring\",\n",
-    "    space_type=\"cosinesimil\",\n",
-    "    vector_field=\"message_embedding\",\n",
-    "    text_field=\"message\",\n",
-    "    metadata_field=\"message_metadata\",\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Using AOSS (Amazon OpenSearch Service Serverless)"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3 (ipykernel)",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.6"
     }
-   }
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "# This is just an example to show how to use AOSS with faiss engine and efficient_filter, you need to set proper values.\n",
-    "\n",
-    "service = 'aoss' # must set the service as 'aoss'\n",
-    "region = 'us-east-2'\n",
-    "credentials = boto3.Session(aws_access_key_id='xxxxxx',aws_secret_access_key='xxxxx').get_credentials()\n",
-    "awsauth = AWS4Auth('xxxxx', 'xxxxxx', region,service, session_token=credentials.token)\n",
-    "\n",
-    "docsearch = OpenSearchVectorSearch.from_documents(\n",
-    "    docs,\n",
-    "    embeddings,\n",
-    "    opensearch_url=\"host url\",\n",
-    "    http_auth=awsauth,\n",
-    "    timeout = 300,\n",
-    "    use_ssl = True,\n",
-    "    verify_certs = True,\n",
-    "    connection_class = RequestsHttpConnection,\n",
-    "    index_name=\"test-index-using-aoss\",\n",
-    "    engine=\"faiss\",\n",
-    ")\n",
-    "\n",
-    "docs = docsearch.similarity_search(\n",
-    "    \"What is feature selection\",\n",
-    "     efficient_filter=filter,\n",
-    "     k=200,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Using AOS (Amazon OpenSearch Service)"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "# This is just an example to show how to use AOS , you need to set proper values.\n",
-    "\n",
-    "service = 'es' # must set the service as 'es'\n",
-    "region = 'us-east-2'\n",
-    "credentials = boto3.Session(aws_access_key_id='xxxxxx',aws_secret_access_key='xxxxx').get_credentials()\n",
-    "awsauth = AWS4Auth('xxxxx', 'xxxxxx', region,service, session_token=credentials.token)\n",
-    "\n",
-    "docsearch = OpenSearchVectorSearch.from_documents(\n",
-    "    docs,\n",
-    "    embeddings,\n",
-    "    opensearch_url=\"host url\",\n",
-    "    http_auth=awsauth,\n",
-    "    timeout = 300,\n",
-    "    use_ssl = True,\n",
-    "    verify_certs = True,\n",
-    "    connection_class = RequestsHttpConnection,\n",
-    "    index_name=\"test-index\",\n",
-    ")\n",
-    "\n",
-    "docs = docsearch.similarity_search(\n",
-    "    \"What is feature selection\",\n",
-    "     k=200,\n",
-    ")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
+  "nbformat": 4,
+  "nbformat_minor": 5
 }

docs/extras/integrations/vectorstores/scann.ipynb

@@ -0,0 +1,190 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e4afbbb6",
+   "metadata": {},
+   "source": [
+    "# ScaNN\n",
+    "\n",
+    "ScaNN (Scalable Nearest Neighbors) is a method for efficient vector similarity search at scale.\n",
+    "\n",
+    "ScaNN includes search space pruning and quantization for Maximum Inner Product Search and also supports other distance functions such as Euclidean distance. The implementation is optimized for x86 processors with AVX2 support. See its [Google Research github](https://github.com/google-research/google-research/tree/master/scann) for more details."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "082f593e",
+   "metadata": {},
+   "source": [
+    "## Installation\n",
+    "Install ScaNN through pip. Alternatively, you can follow instructions on the [ScaNN Website](https://github.com/google-research/google-research/tree/master/scann#building-from-source) to install from source."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a35e4f09",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install scann"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "44bf38a8",
+   "metadata": {},
+   "source": [
+    "## Retrieval Demo\n",
+    "\n",
+    "Below we show how to use ScaNN in conjunction with Huggingface Embeddings."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "377bc723",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Document(page_content='Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \\n\\nTonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \\n\\nOne of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \\n\\nAnd I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.', metadata={'source': 'state_of_the_union.txt'})"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain.embeddings import HuggingFaceEmbeddings\n",
+    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain.vectorstores import ScaNN\n",
+    "from langchain.document_loaders import TextLoader\n",
+    "\n",
+    "loader = TextLoader(\"state_of_the_union.txt\")\n",
+    "documents = loader.load()\n",
+    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
+    "docs = text_splitter.split_documents(documents)\n",
+    "\n",
+    "from langchain.embeddings import TensorflowHubEmbeddings\n",
+    "embeddings = HuggingFaceEmbeddings()\n",
+    "\n",
+    "db = ScaNN.from_documents(docs, embeddings)\n",
+    "query = \"What did the president say about Ketanji Brown Jackson\"\n",
+    "docs = db.similarity_search(query)\n",
+    "\n",
+    "docs[0]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9ad5b151",
+   "metadata": {},
+   "source": [
+    "## RetrievalQA Demo\n",
+    "\n",
+    "Next, we demonstrate using ScaNN in conjunction with Google PaLM API.\n",
+    "\n",
+    "You can obtain an API key from https://developers.generativeai.google/tutorials/setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "fc27ad51",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chains import RetrievalQA\n",
+    "from langchain.chat_models import google_palm\n",
+    "\n",
+    "palm_client = google_palm.ChatGooglePalm(google_api_key='YOUR_GOOGLE_PALM_API_KEY')\n",
+    "\n",
+    "qa = RetrievalQA.from_chain_type(\n",
+    "    llm=palm_client,\n",
+    "    chain_type=\"stuff\",\n",
+    "    retriever=db.as_retriever(search_kwargs={'k': 10})\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "5b77f919",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The president said that Ketanji Brown Jackson is one of our nation's top legal minds, who will continue Justice Breyer's legacy of excellence.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(qa.run('What did the president say about Ketanji Brown Jackson?'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "0c6deec6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The president did not mention Michael Phelps in his speech.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(qa.run('What did the president say about Michael Phelps?'))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8a49f4a6",
+   "metadata": {},
+   "source": [
+    "## Save and loading local retrieval index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "6b7496b9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db.save_local('/tmp/db', 'state_of_union')\n",
+    "restored_db = ScaNN.load_local('/tmp/db', embeddings, index_name='state_of_union')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/integrations/vectorstores/vectara.ipynb

@@ -8,32 +8,55 @@
    "source": [
     "# Vectara\n",
     "\n",
-    ">[Vectara](https://vectara.com/) is a API platform for building LLM-powered applications. It provides a simple to use API for document indexing and query that is managed by Vectara and is optimized for performance and accuracy. \n",
+    ">[Vectara](https://vectara.com/) is a API platform for building GenAI applications. It provides an easy-to-use API for document indexing and query that is managed by Vectara and is optimized for performance and accuracy. \n",
+    "See the [Vectara API documentation ](https://docs.vectara.com/docs/) for more information on how to use the API.\n",
     "\n",
+    "This notebook shows how to use functionality related to the `Vectara`'s integration with langchain.\n",
+    "Note that unlike many other integrations in this category, Vectara provides an end-to-end managed service for [Grounded Generation](https://vectara.com/grounded-generation/) (aka retrieval agumented generation), which includes:\n",
+    "1. A way to extract text from document files and chunk them into sentences.\n",
+    "2. Its own embeddings model and vector store - each text segment is encoded into a vector embedding and stored in the Vectara internal vector store\n",
+    "3. A query service that automatically encodes the query into embedding, and retrieves the most relevant text segments (including support for [Hybrid Search](https://docs.vectara.com/docs/api-reference/search-apis/lexical-matching))\n",
     "\n",
-    "This notebook shows how to use functionality related to the `Vectara` vector database or the `Vectara` retriever. \n",
-    "\n",
-    "See the [Vectara API documentation ](https://docs.vectara.com/docs/) for more information on how to use the API."
+    "All of these are supported in this LangChain integration."
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "aac9563e",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-04-04T10:51:22.282884Z",
-     "start_time": "2023-04-04T10:51:21.408077Z"
-    },
-    "tags": []
-   },
-   "outputs": [],
+   "cell_type": "markdown",
+   "id": "dc0f4344",
+   "metadata": {},
    "source": [
+    "# Setup\n",
+    "\n",
+    "You will need a Vectara account to use Vectara with LangChain. To get started, use the following steps:\n",
+    "1. [Sign up](https://console.vectara.com/signup) for a Vectara account if you don't already have one. Once you have completed your sign up you will have a Vectara customer ID. You can find your customer ID by clicking on your name, on the top-right of the Vectara console window.\n",
+    "2. Within your account you can create one or more corpora. Each corpus represents an area that stores text data upon ingest from input documents. To create a corpus, use the **\"Create Corpus\"** button. You then provide a name to your corpus as well as a description. Optionally you can define filtering attributes and apply some advanced options. If you click on your created corpus, you can see its name and corpus ID right on the top.\n",
+    "3. Next you'll need to create API keys to access the corpus. Click on the **\"Authorization\"** tab in the corpus view and then the **\"Create API Key\"** button. Give your key a name, and choose whether you want query only or query+index for your key. Click \"Create\" and you now have an active API key. Keep this key confidential. \n",
+    "\n",
+    "To use LangChain with Vectara, you'll need to have these three values: customer ID, corpus ID and api_key.\n",
+    "You can provide those to LangChain in two ways:\n",
+    "\n",
+    "1. Include in your environment these three variables: `VECTARA_CUSTOMER_ID`, `VECTARA_CORPUS_ID` and `VECTARA_API_KEY`.\n",
+    "\n",
+    "> For example, you can set these variables using os.environ and getpass as follows:\n",
+    "\n",
+    "```python\n",
     "import os\n",
-    "from langchain.embeddings import FakeEmbeddings\n",
-    "from langchain.text_splitter import CharacterTextSplitter\n",
-    "from langchain.vectorstores import Vectara\n",
-    "from langchain.document_loaders import TextLoader"
+    "import getpass\n",
+    "\n",
+    "os.environ[\"VECTARA_CUSTOMER_ID\"] = getpass.getpass(\"Vectara Customer ID:\")\n",
+    "os.environ[\"VECTARA_CORPUS_ID\"] = getpass.getpass(\"Vectara Corpus ID:\")\n",
+    "os.environ[\"VECTARA_API_KEY\"] = getpass.getpass(\"Vectara API Key:\")\n",
+    "```\n",
+    "\n",
+    "2. Add them to the Vectara vectorstore constructor:\n",
+    "\n",
+    "```python\n",
+    "vectorstore = Vectara(\n",
+    "                vectara_customer_id=vectara_customer_id,\n",
+    "                vectara_corpus_id=vectara_corpus_id,\n",
+    "                vectara_api_key=vectara_api_key\n",
+    "            )\n",
+    "```"
    ]
   },
   {
@@ -44,8 +67,21 @@
    "source": [
     "## Connecting to Vectara from LangChain\n",
     "\n",
-    "The Vectara API provides simple API endpoints for indexing and querying, which is encapsulated in the Vectara integration.\n",
-    "First let's ingest the documents using the from_documents() method:"
+    "To get started, let's ingest the documents using the from_documents() method.\n",
+    "We assume here that you've added your VECTARA_CUSTOMER_ID, VECTARA_CORPUS_ID and query+indexing VECTARA_API_KEY as environment variables."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "04a1f1a0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.embeddings import FakeEmbeddings\n",
+    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain.vectorstores import Vectara\n",
+    "from langchain.document_loaders import TextLoader"
    ]
   },
   {
@@ -88,7 +124,7 @@
    "metadata": {},
    "source": [
     "Vectara's indexing API provides a file upload API where the file is handled directly by Vectara - pre-processed, chunked optimally and added to the Vectara vector store.\n",
-    "To use this, we added the add_files() method (and from_files()). \n",
+    "To use this, we added the add_files() method (as well as from_files()). \n",
     "\n",
     "Let's see this in action. We pick two PDF documents to upload: \n",
     "1. The \"I have a dream\" speech by Dr. King\n",
@@ -296,7 +332,7 @@
    "source": [
     "## Vectara as a Retriever\n",
     "\n",
-    "Vectara, as all the other vector stores, can be used also as a LangChain Retriever:"
+    "Vectara, as all the other LangChain vectorstores, is most often used as a LangChain Retriever:"
    ]
   },
   {
@@ -378,7 +414,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.9"
+   "version": "3.9.1"
   }
  },
  "nbformat": 4,

docs/extras/use_cases/chatbots.ipynb

@@ -0,0 +1,727 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "ee7f95e4",
+   "metadata": {},
+   "source": [
+    "# Chatbots\n",
+    "\n",
+    "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain/blob/master/docs/extras/use_cases/chatbots/chatbots.ipynb)\n",
+    "\n",
+    "## Use case\n",
+    "\n",
+    "Chatbots are one of the central LLM use-cases. The core features of chatbots are that they can have long-running conversations and have access to information that users want to know about.\n",
+    "\n",
+    "Aside from basic prompting and LLMs, memory and retrieval are the core components of a chatbot. Memory allows a chatbot to remember past interactions, and retrieval provides a chatbot with up-to-date, domain-specific information."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "56615b45",
+   "metadata": {},
+   "source": [
+    "![Image description](/img/chat_use_case.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ff48f490",
+   "metadata": {},
+   "source": [
+    "## Overview\n",
+    "\n",
+    "The chat model interface is based around messages rather than raw text. Several components are important to consider for chat:\n",
+    "\n",
+    "* `chat model`: See [here](/docs/integrations/chat) for a list of chat model integrations and [here](/docs/modules/model_io/models/chat) for documentation on the chat model interface in LangChain. You can use `LLMs` (see [here](/docs/modules/model_io/models/llms)) for chatbots as well, but chat models have a more conversational tone and natively support a message interface.\n",
+    "* `prompt template`: Prompt templates make it easy to assemble prompts that combine default messages, user input, chat history, and (optionally) additional retrieved context.\n",
+    "* `memory`: [See here](/docs/modules/memory/) for in-depth documentation on memory types\n",
+    "* `retriever` (optional): [See here](/docs/modules/data_connection/retrievers) for in-depth documentation on retrieval systems. These are useful if you want to build a chatbot with domain-specific knowledge.\n",
+    "\n",
+    "## Quickstart\n",
+    "\n",
+    "Here's a quick preview of how we can create chatbot interfaces. First let's install some dependencies and set the required credentials:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5070a1fd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install langchain openai \n",
+    "\n",
+    "# Set env var OPENAI_API_KEY or load from a .env file:\n",
+    "# import dotenv\n",
+    "# dotenv.load_env()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "88197b95",
+   "metadata": {},
+   "source": [
+    "With a plain chat model, we can get chat completions by [passing one or more messages](/docs/modules/model_io/models/chat) to the model.\n",
+    "\n",
+    "The chat model will respond with a message."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "5b0d84ae",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AIMessage(content=\"J'adore la programmation.\", additional_kwargs={}, example=False)"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain.schema import (\n",
+    "    AIMessage,\n",
+    "    HumanMessage,\n",
+    "    SystemMessage\n",
+    ")\n",
+    "from langchain.chat_models import ChatOpenAI\n",
+    "\n",
+    "chat = ChatOpenAI()\n",
+    "chat([HumanMessage(content=\"Translate this sentence from English to French: I love programming.\")])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7935d9a5",
+   "metadata": {},
+   "source": [
+    "And if we pass in a list of messages:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "afd27a9f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AIMessage(content=\"J'adore la programmation.\", additional_kwargs={}, example=False)"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "messages = [\n",
+    "    SystemMessage(content=\"You are a helpful assistant that translates English to French.\"),\n",
+    "    HumanMessage(content=\"I love programming.\")\n",
+    "]\n",
+    "chat(messages)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c7a1d169",
+   "metadata": {},
+   "source": [
+    "We can then wrap our chat model in a `ConversationChain`, which has built-in memory for remembering past user inputs and model outputs."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "fdb05d74",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Je adore la programmation.'"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain.chains import ConversationChain  \n",
+    "  \n",
+    "conversation = ConversationChain(llm=chat)  \n",
+    "conversation.run(\"Translate this sentence from English to French: I love programming.\")  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "d801a173",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Ich liebe Programmieren.'"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "conversation.run(\"Translate it to German.\")  "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9e86788c",
+   "metadata": {},
+   "source": [
+    "## Memory \n",
+    "\n",
+    "As we mentioned above, the core component of chatbots is the memory system. One of the simplest and most commonly used forms of memory is `ConversationBufferMemory`:\n",
+    "* This memory allows for storing of messages in a `buffer`\n",
+    "* When called in a chain, it returns all of the messages it has stored\n",
+    "\n",
+    "LangChain comes with many other types of memory, too. [See here](/docs/modules/memory/) for in-depth documentation on memory types.\n",
+    "\n",
+    "For now let's take a quick look at ConversationBufferMemory. We can manually add a few chat messages to the memory like so:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "1380a4ea",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.memory import ConversationBufferMemory\n",
+    "\n",
+    "memory = ConversationBufferMemory()\n",
+    "memory.chat_memory.add_user_message(\"hi!\")\n",
+    "memory.chat_memory.add_ai_message(\"whats up?\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a3d5d1f8",
+   "metadata": {},
+   "source": [
+    "And now we can load from our memory. The key method exposed by all `Memory` classes is `load_memory_variables`. This takes in any initial chain input and returns a list of memory variables which are added to the chain input. \n",
+    "\n",
+    "Since this simple memory type doesn't actually take into account the chain input when loading memory, we can pass in an empty input for now:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "982467e7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'history': 'Human: hi!\\nAI: whats up?'}"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "memory.load_memory_variables({})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7c1b20d4",
+   "metadata": {},
+   "source": [
+    "We can also keep a sliding window of the most recent `k` interactions using `ConversationBufferWindowMemory`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "f72b9ff7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'history': 'Human: not much you\\nAI: not much'}"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain.memory import ConversationBufferWindowMemory\n",
+    "\n",
+    "memory = ConversationBufferWindowMemory(k=1)\n",
+    "memory.save_context({\"input\": \"hi\"}, {\"output\": \"whats up\"})\n",
+    "memory.save_context({\"input\": \"not much you\"}, {\"output\": \"not much\"})\n",
+    "memory.load_memory_variables({})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7b84f90a",
+   "metadata": {},
+   "source": [
+    "`ConversationSummaryMemory` is an extension of this theme.\n",
+    "\n",
+    "It creates a summary of the conversation over time. \n",
+    "\n",
+    "This memory is most useful for longer conversations where the full message history would consume many tokens."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "ca2596ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.llms import OpenAI\n",
+    "from langchain.memory import ConversationSummaryMemory\n",
+    "\n",
+    "llm = OpenAI(temperature=0)\n",
+    "memory = ConversationSummaryMemory(llm=llm)\n",
+    "memory.save_context({\"input\": \"hi\"},{\"output\": \"whats up\"})\n",
+    "memory.save_context({\"input\": \"im working on better docs for chatbots\"},{\"output\": \"oh, that sounds like a lot of work\"})\n",
+    "memory.save_context({\"input\": \"yes, but it's worth the effort\"},{\"output\": \"agreed, good docs are important!\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "060f69b7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'history': '\\nThe human greets the AI, to which the AI responds. The human then mentions they are working on better docs for chatbots, to which the AI responds that it sounds like a lot of work. The human agrees that it is worth the effort, and the AI agrees that good docs are important.'}"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "memory.load_memory_variables({})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4bf036f6",
+   "metadata": {},
+   "source": [
+    "`ConversationSummaryBufferMemory` extends this a bit further:\n",
+    "\n",
+    "It uses token length rather than number of interactions to determine when to flush interactions."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "38b42728",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.memory import ConversationSummaryBufferMemory\n",
+    "memory = ConversationSummaryBufferMemory(llm=llm, max_token_limit=10)\n",
+    "memory.save_context({\"input\": \"hi\"}, {\"output\": \"whats up\"})\n",
+    "memory.save_context({\"input\": \"not much you\"}, {\"output\": \"not much\"})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ff0db09f",
+   "metadata": {},
+   "source": [
+    "## Conversation \n",
+    "\n",
+    "We can unpack what goes under the hood with `ConversationChain`. \n",
+    "\n",
+    "We can specify our memory, `ConversationSummaryMemory` and we can specify the prompt. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "fccd6995",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
+      "Prompt after formatting:\n",
+      "\u001b[32;1m\u001b[1;3mSystem: You are a nice chatbot having a conversation with a human.\n",
+      "Human: hi\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'question': 'hi',\n",
+       " 'chat_history': [HumanMessage(content='hi', additional_kwargs={}, example=False),\n",
+       "  AIMessage(content='Hello! How can I assist you today?', additional_kwargs={}, example=False)],\n",
+       " 'text': 'Hello! How can I assist you today?'}"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain.prompts import (\n",
+    "    ChatPromptTemplate,\n",
+    "    MessagesPlaceholder,\n",
+    "    SystemMessagePromptTemplate,\n",
+    "    HumanMessagePromptTemplate,\n",
+    ")\n",
+    "from langchain.chains import LLMChain\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOpenAI()\n",
+    "\n",
+    "# Prompt \n",
+    "prompt = ChatPromptTemplate(\n",
+    "    messages=[\n",
+    "        SystemMessagePromptTemplate.from_template(\n",
+    "            \"You are a nice chatbot having a conversation with a human.\"\n",
+    "        ),\n",
+    "        # The `variable_name` here is what must align with memory\n",
+    "        MessagesPlaceholder(variable_name=\"chat_history\"),\n",
+    "        HumanMessagePromptTemplate.from_template(\"{question}\")\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "# Notice that we `return_messages=True` to fit into the MessagesPlaceholder\n",
+    "# Notice that `\"chat_history\"` aligns with the MessagesPlaceholder name\n",
+    "memory = ConversationBufferMemory(memory_key=\"chat_history\",return_messages=True)\n",
+    "conversation = LLMChain(\n",
+    "    llm=llm,\n",
+    "    prompt=prompt,\n",
+    "    verbose=True,\n",
+    "    memory=memory\n",
+    ")\n",
+    "\n",
+    "# Notice that we just pass in the `question` variables - `chat_history` gets populated by memory\n",
+    "conversation({\"question\": \"hi\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "eb0cadfd",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
+      "Prompt after formatting:\n",
+      "\u001b[32;1m\u001b[1;3mSystem: You are a nice chatbot having a conversation with a human.\n",
+      "Human: hi\n",
+      "AI: Hello! How can I assist you today?\n",
+      "Human: Translate this sentence from English to French: I love programming.\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'question': 'Translate this sentence from English to French: I love programming.',\n",
+       " 'chat_history': [HumanMessage(content='hi', additional_kwargs={}, example=False),\n",
+       "  AIMessage(content='Hello! How can I assist you today?', additional_kwargs={}, example=False),\n",
+       "  HumanMessage(content='Translate this sentence from English to French: I love programming.', additional_kwargs={}, example=False),\n",
+       "  AIMessage(content='Sure! The translation of \"I love programming\" from English to French is \"J\\'adore programmer.\"', additional_kwargs={}, example=False)],\n",
+       " 'text': 'Sure! The translation of \"I love programming\" from English to French is \"J\\'adore programmer.\"'}"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "conversation({\"question\": \"Translate this sentence from English to French: I love programming.\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "c56d6219",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
+      "Prompt after formatting:\n",
+      "\u001b[32;1m\u001b[1;3mSystem: You are a nice chatbot having a conversation with a human.\n",
+      "Human: hi\n",
+      "AI: Hello! How can I assist you today?\n",
+      "Human: Translate this sentence from English to French: I love programming.\n",
+      "AI: Sure! The translation of \"I love programming\" from English to French is \"J'adore programmer.\"\n",
+      "Human: Now translate the sentence to German.\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'question': 'Now translate the sentence to German.',\n",
+       " 'chat_history': [HumanMessage(content='hi', additional_kwargs={}, example=False),\n",
+       "  AIMessage(content='Hello! How can I assist you today?', additional_kwargs={}, example=False),\n",
+       "  HumanMessage(content='Translate this sentence from English to French: I love programming.', additional_kwargs={}, example=False),\n",
+       "  AIMessage(content='Sure! The translation of \"I love programming\" from English to French is \"J\\'adore programmer.\"', additional_kwargs={}, example=False),\n",
+       "  HumanMessage(content='Now translate the sentence to German.', additional_kwargs={}, example=False),\n",
+       "  AIMessage(content='Certainly! The translation of \"I love programming\" from English to German is \"Ich liebe das Programmieren.\"', additional_kwargs={}, example=False)],\n",
+       " 'text': 'Certainly! The translation of \"I love programming\" from English to German is \"Ich liebe das Programmieren.\"'}"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "conversation({\"question\": \"Now translate the sentence to German.\"})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "43858489",
+   "metadata": {},
+   "source": [
+    "We can see the chat history preserved in the prompt using the [LangSmith trace](https://smith.langchain.com/public/dce34c57-21ca-4283-9020-a8e0d78a59de/r).\n",
+    "\n",
+    "![Image description](/img/chat_use_case_2.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3f35cc16",
+   "metadata": {},
+   "source": [
+    "## Chat Retrieval\n",
+    "\n",
+    "Now, suppose we want to [chat with documents](https://twitter.com/mayowaoshin/status/1640385062708424708?s=20) or some other source of knowledge.\n",
+    "\n",
+    "This is popular use case, combining chat with [document retrieval](/docs/use_cases/question_answering).\n",
+    "\n",
+    "It allows us to chat with specific information that the model was not trained on."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1a01e7b5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install tiktoken chromadb"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "88e220de",
+   "metadata": {},
+   "source": [
+    "Load a blog post."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "1b99b36c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import WebBaseLoader\n",
+    "\n",
+    "loader = WebBaseLoader(\"https://lilianweng.github.io/posts/2023-06-23-agent/\")\n",
+    "data = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3662ce79",
+   "metadata": {},
+   "source": [
+    "Split and store this in a vector."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "058f1541",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "\n",
+    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0)\n",
+    "all_splits = text_splitter.split_documents(data)\n",
+    "\n",
+    "from langchain.embeddings import OpenAIEmbeddings\n",
+    "from langchain.vectorstores import Chroma\n",
+    "\n",
+    "vectorstore = Chroma.from_documents(documents=all_splits, embedding=OpenAIEmbeddings())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "603d9441",
+   "metadata": {},
+   "source": [
+    "Create our memory, as before, but's let's use `ConversationSummaryMemory`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "id": "f89fd3f5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "memory = ConversationSummaryMemory(llm=llm,memory_key=\"chat_history\",return_messages=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "28503423",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.chains import ConversationalRetrievalChain\n",
+    "\n",
+    "llm = ChatOpenAI()\n",
+    "retriever = vectorstore.as_retriever()\n",
+    "qa = ConversationalRetrievalChain.from_llm(llm, retriever=retriever, memory=memory)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "id": "a9c3bd5e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'question': 'How do agents use Task decomposition?',\n",
+       " 'chat_history': [SystemMessage(content='', additional_kwargs={})],\n",
+       " 'answer': 'Agents can use task decomposition in several ways:\\n\\n1. Simple prompting: Agents can use Language Model based prompting to break down tasks into subgoals. For example, by providing prompts like \"Steps for XYZ\" or \"What are the subgoals for achieving XYZ?\", the agent can generate a sequence of smaller steps that lead to the completion of the overall task.\\n\\n2. Task-specific instructions: Agents can be given task-specific instructions to guide their planning process. For example, if the task is to write a novel, the agent can be instructed to \"Write a story outline.\" This provides a high-level structure for the task and helps in breaking it down into smaller components.\\n\\n3. Human inputs: Agents can also take inputs from humans to decompose tasks. This can be done through direct communication or by leveraging human expertise. Humans can provide guidance and insights to help the agent break down complex tasks into manageable subgoals.\\n\\nOverall, task decomposition allows agents to break down large tasks into smaller, more manageable subgoals, enabling them to plan and execute complex tasks efficiently.'}"
+      ]
+     },
+     "execution_count": 39,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "qa(\"How do agents use Task decomposition?\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "id": "a29a7713",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'question': 'What are the various ways to implemet memory to support it?',\n",
+       " 'chat_history': [SystemMessage(content='The human asks how agents use task decomposition. The AI explains that agents can use task decomposition in several ways, including simple prompting, task-specific instructions, and human inputs. Task decomposition allows agents to break down large tasks into smaller, more manageable subgoals, enabling them to plan and execute complex tasks efficiently.', additional_kwargs={})],\n",
+       " 'answer': 'There are several ways to implement memory to support task decomposition:\\n\\n1. Long-Term Memory Management: This involves storing and organizing information in a long-term memory system. The agent can retrieve past experiences, knowledge, and learned strategies to guide the task decomposition process.\\n\\n2. Internet Access: The agent can use internet access to search for relevant information and gather resources to aid in task decomposition. This allows the agent to access a vast amount of information and utilize it in the decomposition process.\\n\\n3. GPT-3.5 Powered Agents: The agent can delegate simple tasks to GPT-3.5 powered agents. These agents can perform specific tasks or provide assistance in task decomposition, allowing the main agent to focus on higher-level planning and decision-making.\\n\\n4. File Output: The agent can store the results of task decomposition in files or documents. This allows for easy retrieval and reference during the execution of the task.\\n\\nThese memory resources help the agent in organizing and managing information, making informed decisions, and effectively decomposing complex tasks into smaller, manageable subgoals.'}"
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "qa(\"What are the various ways to implemet memory to support it?\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d5e8d5f4",
+   "metadata": {},
+   "source": [
+    "Again, we can use the [LangSmith trace](https://smith.langchain.com/public/18460363-0c70-4c72-81c7-3b57253bb58c/r) to explore the prompt structure.\n",
+    "\n",
+    "### Going deeper \n",
+    "\n",
+    "* Agents, such as the [conversational retrieval agent](/docs/use_cases/question_answering/how_to/conversational_retrieval_agents), can be used for retrieval when necessary while also holding a conversation.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1ff8925f-4c21-4680-a9cd-3670ad4852b3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/use_cases/chatbots/index.mdx

@@ -1,20 +0,0 @@
----
-sidebar_position: 4
----
-
-# Chatbots
-
-Since language models are good at producing text, that makes them ideal for creating chatbots.
-Aside from the base prompts/LLMs, an important concept to know for Chatbots is `memory`.
-Most chat based applications rely on remembering what happened in previous interactions, which `memory` is designed to help with.
-
-The following resources exist:
-- [ChatGPT Clone](/docs/modules/agents/how_to/chatgpt_clone.html): A notebook walking through how to recreate a ChatGPT-like experience with LangChain.
-- [Conversation Agent](/docs/modules/agents/agent_types/chat_conversation_agent.html): A notebook walking through how to create an agent optimized for conversation.
-
-
-Additional related resources include:
-- [Memory concepts and examples](/docs/modules/memory/): Explanation of key concepts related to memory along with how-to's and examples.
-
-More end-to-end examples include:
-- [Voice Assistant](./voice_assistant.html): A notebook walking through how to create a voice assistant using LangChain.

docs/extras/use_cases/chatbots/voice_assistant.ipynb

@@ -1,482 +0,0 @@
-{
- "cells": [
-  {
-   "attachments": {},
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Voice Assistant\n",
-    "\n",
-    "This chain creates a clone of ChatGPT with a few modifications to make it a voice assistant. \n",
-    "It uses the `pyttsx3` and `speech_recognition` libraries to convert text to speech and speech to text respectively. The prompt template is also changed to make it more suitable for voice assistant use."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain import OpenAI, LLMChain, PromptTemplate\n",
-    "from langchain.memory import ConversationBufferWindowMemory\n",
-    "\n",
-    "\n",
-    "template = \"\"\"Assistant is a large language model trained by OpenAI.\n",
-    "\n",
-    "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-    "\n",
-    "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-    "\n",
-    "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-    "\n",
-    "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-    "\n",
-    "{history}\n",
-    "Human: {human_input}\n",
-    "Assistant:\"\"\"\n",
-    "\n",
-    "prompt = PromptTemplate(input_variables=[\"history\", \"human_input\"], template=template)\n",
-    "\n",
-    "\n",
-    "chatgpt_chain = LLMChain(\n",
-    "    llm=OpenAI(temperature=0),\n",
-    "    prompt=prompt,\n",
-    "    verbose=True,\n",
-    "    memory=ConversationBufferWindowMemory(k=2),\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import speech_recognition as sr\n",
-    "import pyttsx3\n",
-    "\n",
-    "engine = pyttsx3.init()\n",
-    "\n",
-    "\n",
-    "def listen():\n",
-    "    r = sr.Recognizer()\n",
-    "    with sr.Microphone() as source:\n",
-    "        print(\"Calibrating...\")\n",
-    "        r.adjust_for_ambient_noise(source, duration=5)\n",
-    "        # optional parameters to adjust microphone sensitivity\n",
-    "        # r.energy_threshold = 200\n",
-    "        # r.pause_threshold=0.5\n",
-    "\n",
-    "        print(\"Okay, go!\")\n",
-    "        while 1:\n",
-    "            text = \"\"\n",
-    "            print(\"listening now...\")\n",
-    "            try:\n",
-    "                audio = r.listen(source, timeout=5, phrase_time_limit=30)\n",
-    "                print(\"Recognizing...\")\n",
-    "                # whisper model options are found here: https://github.com/openai/whisper#available-models-and-languages\n",
-    "                # other speech recognition models are also available.\n",
-    "                text = r.recognize_whisper(\n",
-    "                    audio,\n",
-    "                    model=\"medium.en\",\n",
-    "                    show_dict=True,\n",
-    "                )[\"text\"]\n",
-    "            except Exception as e:\n",
-    "                unrecognized_speech_text = (\n",
-    "                    f\"Sorry, I didn't catch that. Exception was: {e}s\"\n",
-    "                )\n",
-    "                text = unrecognized_speech_text\n",
-    "            print(text)\n",
-    "\n",
-    "            response_text = chatgpt_chain.predict(human_input=text)\n",
-    "            print(response_text)\n",
-    "            engine.say(response_text)\n",
-    "            engine.runAndWait()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Calibrating...\n",
-      "Okay, go!\n",
-      "listening now...\n",
-      "Recognizing...\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "C:\\Users\\jaden\\AppData\\Roaming\\Python\\Python310\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      " Hello, Assistant. What's going on?\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "\n",
-      "Human:  Hello, Assistant. What's going on?\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      " Hi there! It's great to hear from you. I'm doing well. How can I help you today?\n",
-      "listening now...\n",
-      "Recognizing...\n",
-      " That's cool. Isn't that neat? Yeah, I'm doing great.\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "Human:  Hello, Assistant. What's going on?\n",
-      "AI:  Hi there! It's great to hear from you. I'm doing well. How can I help you today?\n",
-      "Human:  That's cool. Isn't that neat? Yeah, I'm doing great.\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      "  That's great to hear! What can I do for you today?\n",
-      "listening now...\n",
-      "Recognizing...\n",
-      " Thank you.\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "Human:  Hello, Assistant. What's going on?\n",
-      "AI:  Hi there! It's great to hear from you. I'm doing well. How can I help you today?\n",
-      "Human:  That's cool. Isn't that neat? Yeah, I'm doing great.\n",
-      "AI:   That's great to hear! What can I do for you today?\n",
-      "Human:  Thank you.\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      " You're welcome! Is there anything else I can help you with?\n",
-      "listening now...\n",
-      "Recognizing...\n",
-      " I'd like to learn more about neural networks.\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "Human:  That's cool. Isn't that neat? Yeah, I'm doing great.\n",
-      "AI:   That's great to hear! What can I do for you today?\n",
-      "Human:  Thank you.\n",
-      "AI:  You're welcome! Is there anything else I can help you with?\n",
-      "Human:  I'd like to learn more about neural networks.\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      " Sure! Neural networks are a type of artificial intelligence that use a network of interconnected nodes to process data and make decisions. They are used in a variety of applications, from image recognition to natural language processing. Neural networks are often used to solve complex problems that are too difficult for traditional algorithms.\n",
-      "listening now...\n",
-      "Recognizing...\n",
-      " Tell me a fun fact about neural networks.\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "Human:  Thank you.\n",
-      "AI:  You're welcome! Is there anything else I can help you with?\n",
-      "Human:  I'd like to learn more about neural networks.\n",
-      "AI:  Sure! Neural networks are a type of artificial intelligence that use a network of interconnected nodes to process data and make decisions. They are used in a variety of applications, from image recognition to natural language processing. Neural networks are often used to solve complex problems that are too difficult for traditional algorithms.\n",
-      "Human:  Tell me a fun fact about neural networks.\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      " Neural networks are inspired by the way the human brain works. They are composed of interconnected nodes that process data and make decisions, just like neurons in the brain. Neural networks can learn from their mistakes and improve their performance over time, just like humans do.\n",
-      "listening now...\n",
-      "Recognizing...\n",
-      " Tell me about a brand new discovered bird species.\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "Human:  I'd like to learn more about neural networks.\n",
-      "AI:  Sure! Neural networks are a type of artificial intelligence that use a network of interconnected nodes to process data and make decisions. They are used in a variety of applications, from image recognition to natural language processing. Neural networks are often used to solve complex problems that are too difficult for traditional algorithms.\n",
-      "Human:  Tell me a fun fact about neural networks.\n",
-      "AI:  Neural networks are inspired by the way the human brain works. They are composed of interconnected nodes that process data and make decisions, just like neurons in the brain. Neural networks can learn from their mistakes and improve their performance over time, just like humans do.\n",
-      "Human:  Tell me about a brand new discovered bird species.\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      " A new species of bird was recently discovered in the Amazon rainforest. The species, called the Spix's Macaw, is a small, blue parrot that is believed to be extinct in the wild. It is the first new species of bird to be discovered in the Amazon in over 100 years.\n",
-      "listening now...\n",
-      "Recognizing...\n",
-      " Tell me a children's story about the importance of honesty and trust.\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "Human:  Tell me a fun fact about neural networks.\n",
-      "AI:  Neural networks are inspired by the way the human brain works. They are composed of interconnected nodes that process data and make decisions, just like neurons in the brain. Neural networks can learn from their mistakes and improve their performance over time, just like humans do.\n",
-      "Human:  Tell me about a brand new discovered bird species.\n",
-      "AI:  A new species of bird was recently discovered in the Amazon rainforest. The species, called the Spix's Macaw, is a small, blue parrot that is believed to be extinct in the wild. It is the first new species of bird to be discovered in the Amazon in over 100 years.\n",
-      "Human:  Tell me a children's story about the importance of honesty and trust.\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      " Once upon a time, there was a young boy named Jack who lived in a small village. Jack was always honest and trustworthy, and his friends and family knew they could always count on him. One day, Jack was walking through the forest when he stumbled upon a magical tree. The tree told Jack that if he was honest and trustworthy, he would be rewarded with a special gift. Jack was so excited, and he promised to always be honest and trustworthy. Sure enough, the tree rewarded Jack with a beautiful golden apple. From that day forward, Jack was always honest and trustworthy, and he was rewarded with many more magical gifts. The moral of the story is that honesty and trust are the most important things in life.\n",
-      "listening now...\n",
-      "Recognizing...\n",
-      " Wow, Assistant, that was a really good story. Congratulations!\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "Human:  Tell me about a brand new discovered bird species.\n",
-      "AI:  A new species of bird was recently discovered in the Amazon rainforest. The species, called the Spix's Macaw, is a small, blue parrot that is believed to be extinct in the wild. It is the first new species of bird to be discovered in the Amazon in over 100 years.\n",
-      "Human:  Tell me a children's story about the importance of honesty and trust.\n",
-      "AI:  Once upon a time, there was a young boy named Jack who lived in a small village. Jack was always honest and trustworthy, and his friends and family knew they could always count on him. One day, Jack was walking through the forest when he stumbled upon a magical tree. The tree told Jack that if he was honest and trustworthy, he would be rewarded with a special gift. Jack was so excited, and he promised to always be honest and trustworthy. Sure enough, the tree rewarded Jack with a beautiful golden apple. From that day forward, Jack was always honest and trustworthy, and he was rewarded with many more magical gifts. The moral of the story is that honesty and trust are the most important things in life.\n",
-      "Human:  Wow, Assistant, that was a really good story. Congratulations!\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      " Thank you! I'm glad you enjoyed it.\n",
-      "listening now...\n",
-      "Recognizing...\n",
-      " Thank you.\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "Human:  Tell me a children's story about the importance of honesty and trust.\n",
-      "AI:  Once upon a time, there was a young boy named Jack who lived in a small village. Jack was always honest and trustworthy, and his friends and family knew they could always count on him. One day, Jack was walking through the forest when he stumbled upon a magical tree. The tree told Jack that if he was honest and trustworthy, he would be rewarded with a special gift. Jack was so excited, and he promised to always be honest and trustworthy. Sure enough, the tree rewarded Jack with a beautiful golden apple. From that day forward, Jack was always honest and trustworthy, and he was rewarded with many more magical gifts. The moral of the story is that honesty and trust are the most important things in life.\n",
-      "Human:  Wow, Assistant, that was a really good story. Congratulations!\n",
-      "AI:  Thank you! I'm glad you enjoyed it.\n",
-      "Human:  Thank you.\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      " You're welcome!\n",
-      "listening now...\n",
-      "Recognizing...\n",
-      " Do you know of online brands like Photoshop and Freq that you don't have to download in some sort of way? Do you know of online brands like Photoshop and Freq that you don't have to download in some sort of way?\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "Human:  Wow, Assistant, that was a really good story. Congratulations!\n",
-      "AI:  Thank you! I'm glad you enjoyed it.\n",
-      "Human:  Thank you.\n",
-      "AI:  You're welcome!\n",
-      "Human:  Do you know of online brands like Photoshop and Freq that you don't have to download in some sort of way? Do you know of online brands like Photoshop and Freq that you don't have to download in some sort of way?\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      " Yes, there are several online brands that offer photo editing and other creative tools without the need to download any software. Adobe Photoshop Express, Pixlr, and Fotor are some of the most popular online photo editing tools. Freq is an online music production platform that allows users to create and share music without downloading any software.\n",
-      "listening now...\n",
-      "Recognizing...\n",
-      " Our whole process of awesome is free.\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "Human:  Thank you.\n",
-      "AI:  You're welcome!\n",
-      "Human:  Do you know of online brands like Photoshop and Freq that you don't have to download in some sort of way? Do you know of online brands like Photoshop and Freq that you don't have to download in some sort of way?\n",
-      "AI:  Yes, there are several online brands that offer photo editing and other creative tools without the need to download any software. Adobe Photoshop Express, Pixlr, and Fotor are some of the most popular online photo editing tools. Freq is an online music production platform that allows users to create and share music without downloading any software.\n",
-      "Human:  Our whole process of awesome is free.\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      " That's great! It's always nice to have access to free tools and resources.\n",
-      "listening now...\n",
-      "Recognizing...\n",
-      " No, I meant to ask, are those options that you mentioned free? No, I meant to ask, are those options that you mentioned free?\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n",
-      "Prompt after formatting:\n",
-      "\u001b[32;1m\u001b[1;3mAssistant is a large language model trained by OpenAI.\n",
-      "\n",
-      "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
-      "\n",
-      "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
-      "\n",
-      "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
-      "\n",
-      "Assistant is aware that human input is being transcribed from audio and as such there may be some errors in the transcription. It will attempt to account for some words being swapped with similar-sounding words or phrases. Assistant will also keep responses concise, because human attention spans are more limited over the audio channel since it takes time to listen to a response.\n",
-      "\n",
-      "Human:  Do you know of online brands like Photoshop and Freq that you don't have to download in some sort of way? Do you know of online brands like Photoshop and Freq that you don't have to download in some sort of way?\n",
-      "AI:  Yes, there are several online brands that offer photo editing and other creative tools without the need to download any software. Adobe Photoshop Express, Pixlr, and Fotor are some of the most popular online photo editing tools. Freq is an online music production platform that allows users to create and share music without downloading any software.\n",
-      "Human:  Our whole process of awesome is free.\n",
-      "AI:  That's great! It's always nice to have access to free tools and resources.\n",
-      "Human:  No, I meant to ask, are those options that you mentioned free? No, I meant to ask, are those options that you mentioned free?\n",
-      "Assistant:\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n",
-      " Yes, the online brands I mentioned are all free to use. Adobe Photoshop Express, Pixlr, and Fotor are all free to use, and Freq is a free music production platform.\n",
-      "listening now...\n"
-     ]
-    },
-    {
-     "ename": "KeyboardInterrupt",
-     "evalue": "",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
-      "Cell \u001b[1;32mIn[6], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m listen(\u001b[39mNone\u001b[39;49;00m)\n",
-      "Cell \u001b[1;32mIn[5], line 20\u001b[0m, in \u001b[0;36mlisten\u001b[1;34m(command_queue)\u001b[0m\n\u001b[0;32m     18\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m'\u001b[39m\u001b[39mlistening now...\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[0;32m     19\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m---> 20\u001b[0m     audio \u001b[39m=\u001b[39m r\u001b[39m.\u001b[39;49mlisten(source, timeout\u001b[39m=\u001b[39;49m\u001b[39m5\u001b[39;49m, phrase_time_limit\u001b[39m=\u001b[39;49m\u001b[39m30\u001b[39;49m)\n\u001b[0;32m     21\u001b[0m     \u001b[39m# audio = r.record(source,duration = 5)\u001b[39;00m\n\u001b[0;32m     22\u001b[0m     \u001b[39mprint\u001b[39m(\u001b[39m'\u001b[39m\u001b[39mRecognizing...\u001b[39m\u001b[39m'\u001b[39m)\n",
-      "File \u001b[1;32mc:\\ProgramData\\miniconda3\\envs\\lang\\lib\\site-packages\\speech_recognition\\__init__.py:523\u001b[0m, in \u001b[0;36mRecognizer.listen\u001b[1;34m(self, source, timeout, phrase_time_limit, snowboy_configuration)\u001b[0m\n\u001b[0;32m    520\u001b[0m \u001b[39mif\u001b[39;00m phrase_time_limit \u001b[39mand\u001b[39;00m elapsed_time \u001b[39m-\u001b[39m phrase_start_time \u001b[39m>\u001b[39m phrase_time_limit:\n\u001b[0;32m    521\u001b[0m     \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m--> 523\u001b[0m buffer \u001b[39m=\u001b[39m source\u001b[39m.\u001b[39;49mstream\u001b[39m.\u001b[39;49mread(source\u001b[39m.\u001b[39;49mCHUNK)\n\u001b[0;32m    524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(buffer) \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m: \u001b[39mbreak\u001b[39;00m  \u001b[39m# reached end of the stream\u001b[39;00m\n\u001b[0;32m    525\u001b[0m frames\u001b[39m.\u001b[39mappend(buffer)\n",
-      "File \u001b[1;32mc:\\ProgramData\\miniconda3\\envs\\lang\\lib\\site-packages\\speech_recognition\\__init__.py:199\u001b[0m, in \u001b[0;36mMicrophone.MicrophoneStream.read\u001b[1;34m(self, size)\u001b[0m\n\u001b[0;32m    198\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mread\u001b[39m(\u001b[39mself\u001b[39m, size):\n\u001b[1;32m--> 199\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpyaudio_stream\u001b[39m.\u001b[39;49mread(size, exception_on_overflow\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\n",
-      "File \u001b[1;32mc:\\ProgramData\\miniconda3\\envs\\lang\\lib\\site-packages\\pyaudio\\__init__.py:570\u001b[0m, in \u001b[0;36mPyAudio.Stream.read\u001b[1;34m(self, num_frames, exception_on_overflow)\u001b[0m\n\u001b[0;32m    567\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_is_input:\n\u001b[0;32m    568\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mIOError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mNot input stream\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[0;32m    569\u001b[0m                   paCanNotReadFromAnOutputOnlyStream)\n\u001b[1;32m--> 570\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39;49mread_stream(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_stream, num_frames,\n\u001b[0;32m    571\u001b[0m                       exception_on_overflow)\n",
-      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
-     ]
-    }
-   ],
-   "source": [
-    "listen(None)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "lang",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.10"
-  },
-  "orig_nbformat": 4
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

docs/extras/use_cases/question_answering/how_to/conversational_retrieval_agents.ipynb

@@ -0,0 +1,583 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "839f3c76",
+   "metadata": {},
+   "source": [
+    "# Conversational Retrieval Agent\n",
+    "\n",
+    "This is an agent specifically optimized for doing retrieval when necessary and also holding a conversation.\n",
+    "\n",
+    "To start, we will set up the retriever we want to use, and then turn it into a retriever tool. Next, we will use the high level constructor for this type of agent. Finally, we will walk through how to construct a conversational retrieval agent from components."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dc66a262",
+   "metadata": {},
+   "source": [
+    "## The Retriever\n",
+    "\n",
+    "To start, we need a retriever to use! The code here is mostly just example code. Feel free to use your own retriever and skip to the section on creating a retriever tool."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "22533f46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import TextLoader\n",
+    "loader = TextLoader('../../../../../docs/extras/modules/state_of_the_union.txt')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "c252c7e9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain.vectorstores import FAISS\n",
+    "from langchain.embeddings import OpenAIEmbeddings\n",
+    "\n",
+    "documents = loader.load()\n",
+    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
+    "texts = text_splitter.split_documents(documents)\n",
+    "embeddings = OpenAIEmbeddings()\n",
+    "db = FAISS.from_documents(texts, embeddings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "5b8a404f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "retriever = db.as_retriever()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9cd528f5",
+   "metadata": {},
+   "source": [
+    "## Retriever Tool\n",
+    "\n",
+    "Now we need to create a tool for our retriever. The main things we need to pass in are a name for the retriever as well as a description. These will both be used by the language model, so they should be informative."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "9a82f72a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.agents.agent_toolkits import create_retriever_tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "dfbd92a2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tool = create_retriever_tool(\n",
+    "    retriever, \n",
+    "    \"search_state_of_union\",\n",
+    "    \"Searches and returns documents regarding the state-of-the-union.\"\n",
+    ")\n",
+    "tools = [tool]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6d9b7210",
+   "metadata": {},
+   "source": [
+    "## Agent Constructor\n",
+    "\n",
+    "Here, we will use the high level `create_conversational_retrieval_agent` API to construct the agent.\n",
+    "\n",
+    "Notice that beside the list of tools, the only thing we need to pass in is a language model to use.\n",
+    "Under the hood, this agent is using the OpenAIFunctionsAgent, so we need to use an ChatOpenAI model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0cd147eb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.agents.agent_toolkits import create_conversational_retrieval_agent "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "9fa4661b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatOpenAI\n",
+    "llm = ChatOpenAI(temperature = 0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "da2ad764",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "agent_executor = create_conversational_retrieval_agent(llm, tools, verbose=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "85ee7677",
+   "metadata": {},
+   "source": [
+    "We can now try it out!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "03059322",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
+      "\u001b[32;1m\u001b[1;3mHello Bob! How can I assist you today?\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "result = agent_executor({\"input\": \"hi, im bob\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "33073ff0",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Hello Bob! How can I assist you today?'"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result[\"output\"]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5f1f0b79",
+   "metadata": {},
+   "source": [
+    "Notice that it remembers your name"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "4ad92bc7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
+      "\u001b[32;1m\u001b[1;3mYour name is Bob.\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "result = agent_executor({\"input\": \"whats my name?\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "7ae62ecd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Your name is Bob.'"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result[\"output\"]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8e258ade",
+   "metadata": {},
+   "source": [
+    "Notice that it now does retrieval"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "6cd17d67",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
+      "\u001b[32;1m\u001b[1;3m\n",
+      "Invoking: `search_state_of_union` with `{'query': 'Kentaji Brown Jackson'}`\n",
+      "\n",
+      "\n",
+      "\u001b[0m\u001b[36;1m\u001b[1;3m[Document(page_content='Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \\n\\nTonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \\n\\nOne of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \\n\\nAnd I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.', metadata={'source': '../../../../../docs/extras/modules/state_of_the_union.txt'}), Document(page_content='One was stationed at bases and breathing in toxic smoke from “burn pits” that incinerated wastes of war—medical and hazard material, jet fuel, and more. \\n\\nWhen they came home, many of the world’s fittest and best trained warriors were never the same. \\n\\nHeadaches. Numbness. Dizziness. \\n\\nA cancer that would put them in a flag-draped coffin. \\n\\nI know. \\n\\nOne of those soldiers was my son Major Beau Biden. \\n\\nWe don’t know for sure if a burn pit was the cause of his brain cancer, or the diseases of so many of our troops. \\n\\nBut I’m committed to finding out everything we can. \\n\\nCommitted to military families like Danielle Robinson from Ohio. \\n\\nThe widow of Sergeant First Class Heath Robinson.  \\n\\nHe was born a soldier. Army National Guard. Combat medic in Kosovo and Iraq. \\n\\nStationed near Baghdad, just yards from burn pits the size of football fields. \\n\\nHeath’s widow Danielle is here with us tonight. They loved going to Ohio State football games. He loved building Legos with their daughter.', metadata={'source': '../../../../../docs/extras/modules/state_of_the_union.txt'}), Document(page_content='A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder. Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. \\n\\nAnd if we are to advance liberty and justice, we need to secure the Border and fix the immigration system. \\n\\nWe can do both. At our border, we’ve installed new technology like cutting-edge scanners to better detect drug smuggling.  \\n\\nWe’ve set up joint patrols with Mexico and Guatemala to catch more human traffickers.  \\n\\nWe’re putting in place dedicated immigration judges so families fleeing persecution and violence can have their cases heard faster. \\n\\nWe’re securing commitments and supporting partners in South and Central America to host more refugees and secure their own borders.', metadata={'source': '../../../../../docs/extras/modules/state_of_the_union.txt'}), Document(page_content='We can’t change how divided we’ve been. But we can change how we move forward—on COVID-19 and other issues we must face together. \\n\\nI recently visited the New York City Police Department days after the funerals of Officer Wilbert Mora and his partner, Officer Jason Rivera. \\n\\nThey were responding to a 9-1-1 call when a man shot and killed them with a stolen gun. \\n\\nOfficer Mora was 27 years old. \\n\\nOfficer Rivera was 22. \\n\\nBoth Dominican Americans who’d grown up on the same streets they later chose to patrol as police officers. \\n\\nI spoke with their families and told them that we are forever in debt for their sacrifice, and we will carry on their mission to restore the trust and safety every community deserves. \\n\\nI’ve worked on these issues a long time. \\n\\nI know what works: Investing in crime preventionand community police officers who’ll walk the beat, who’ll know the neighborhood, and who can restore trust and safety.', metadata={'source': '../../../../../docs/extras/modules/state_of_the_union.txt'})]\u001b[0m\u001b[32;1m\u001b[1;3mIn the most recent state of the union, the President mentioned Kentaji Brown Jackson. The President nominated Circuit Court of Appeals Judge Ketanji Brown Jackson to serve on the United States Supreme Court. The President described Judge Ketanji Brown Jackson as one of our nation's top legal minds who will continue Justice Breyer's legacy of excellence.\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "result = agent_executor({\"input\": \"what did the president say about kentaji brown jackson in the most recent state of the union?\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "c51de037",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\"In the most recent state of the union, the President mentioned Kentaji Brown Jackson. The President nominated Circuit Court of Appeals Judge Ketanji Brown Jackson to serve on the United States Supreme Court. The President described Judge Ketanji Brown Jackson as one of our nation's top legal minds who will continue Justice Breyer's legacy of excellence.\""
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result[\"output\"]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d3bad540",
+   "metadata": {},
+   "source": [
+    "Notice that the follow up question asks about information previously retrieved, so no need to do another retrieval"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "527ea5fd",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
+      "\u001b[32;1m\u001b[1;3mThe President nominated Judge Ketanji Brown Jackson four days ago.\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "result = agent_executor({\"input\": \"how long ago did he nominate her?\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "a34d20fd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'The President nominated Judge Ketanji Brown Jackson four days ago.'"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result[\"output\"]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e599dbd3",
+   "metadata": {},
+   "source": [
+    "## Creating from components\n",
+    "\n",
+    "What actually is going on underneath the hood? Let's take a look so we can understand how to modify going forward.\n",
+    "\n",
+    "There are a few components:\n",
+    "\n",
+    "- The memory\n",
+    "- The prompt template\n",
+    "- The agent\n",
+    "- The agent executor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "1b21be1d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This is needed for both the memory and the prompt\n",
+    "memory_key = \"history\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f827f95f",
+   "metadata": {},
+   "source": [
+    "### The Memory\n",
+    "\n",
+    "In this example, we want the agent to remember not only previous conversations, but also previous intermediate steps. For that, we can use `AgentTokenBufferMemory`. Note that if you want to change whether the agent remembers intermediate steps, or how the long the buffer is, or anything like that you should change this part."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "138b0675",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.agents.openai_functions_agent.agent_token_buffer_memory import AgentTokenBufferMemory\n",
+    "\n",
+    "memory = AgentTokenBufferMemory(memory_key=memory_key, llm=llm)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4827993f",
+   "metadata": {},
+   "source": [
+    "## The Prompt Template\n",
+    "\n",
+    "For the prompt template, we will use the `OpenAIFunctionsAgent` default way of creating one, but pass in a system prompt and a placeholder for memory."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "779272dd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.agents.openai_functions_agent.base import OpenAIFunctionsAgent\n",
+    "from langchain.schema.messages import SystemMessage\n",
+    "from langchain.prompts import MessagesPlaceholder"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "824bc74e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "system_message = SystemMessage(\n",
+    "        content=(\n",
+    "            \"Do your best to answer the questions. \"\n",
+    "            \"Feel free to use any tools available to look up \"\n",
+    "            \"relevant information, only if neccessary\"\n",
+    "        )\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "07e41722",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = OpenAIFunctionsAgent.create_prompt(\n",
+    "        system_message=system_message,\n",
+    "        extra_prompt_messages=[MessagesPlaceholder(variable_name=memory_key)]\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "001e455e",
+   "metadata": {},
+   "source": [
+    "## The Agent\n",
+    "\n",
+    "We will use the OpenAIFunctionsAgent"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "adf4b5b3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "agent = OpenAIFunctionsAgent(llm=llm, tools=tools, prompt=prompt)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c5c321e",
+   "metadata": {},
+   "source": [
+    "## The Agent Executor\n",
+    "\n",
+    "Importantly, we pass in `return_intermediate_steps=True` since we are recording that with our memory object"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "2e7ffe96",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.agents import AgentExecutor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "e39a095f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "agent_executor = AgentExecutor(agent=agent, tools=tools, memory=memory, verbose=True,\n",
+    "                                   return_intermediate_steps=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "96136958",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
+      "\u001b[32;1m\u001b[1;3mHello Bob! How can I assist you today?\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "result = agent_executor({\"input\": \"hi, im bob\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "8de674cb",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
+      "\u001b[32;1m\u001b[1;3mYour name is Bob.\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "result = agent_executor({\"input\": \"whats my name\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bf655a48",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/use_cases/summarization.ipynb

@@ -0,0 +1,511 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cf13f702",
+   "metadata": {},
+   "source": [
+    "# Summarization\n",
+    "\n",
+    "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain/blob/master/docs/extras/use_cases/summarization.ipynb)\n",
+    "\n",
+    "## Use case\n",
+    "\n",
+    "Suppose you have a set of documents (PDFs, Notion pages, customer questions, etc.) and you want to summarize the content. \n",
+    "\n",
+    "LLMs are a great tool for this given their proficiency in understanding and synthesizing text.\n",
+    "\n",
+    "In this walkthrough we'll go over how to perform document summarization using LLMs."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8e233997",
+   "metadata": {},
+   "source": [
+    "![Image description](/img/summarization_use_case_1.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4715b4ff",
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true
+   },
+   "source": [
+    "## Overview\n",
+    "\n",
+    "A central question for building a summarizer is how to pass your documents into the LLM's context window. Two common approaches for this are:\n",
+    "\n",
+    "1. `Stuff`: Simply \"stuff\" all your documents into a single prompt. This is the simplest approach (see [here](/docs/modules/chains/document/stuff) for more on the `StuffDocumentsChains`, which is used for this method).\n",
+    "\n",
+    "2. `Map-reduce`: Summarize each document on it's own in a \"map\" step and then \"reduce\" the summaries into a final summary (see [here](/docs/modules/chains/document/map_reduce) for more on the `MapReduceDocumentsChain`, which is used for this method)."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "08ec66bc",
+   "metadata": {},
+   "source": [
+    "![Image description](/img/summarization_use_case_2.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bea785ac",
+   "metadata": {},
+   "source": [
+    "## Quickstart\n",
+    "\n",
+    "To give you a sneak preview, either pipeline can be wrapped in a single object: `load_summarize_chain`. \n",
+    "\n",
+    "Suppose we want to summarize a blog post. We can create this in a few lines of code.\n",
+    "\n",
+    "First set environment variables and install packages:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "578d6a90",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install openai tiktoken chromadb langchain\n",
+    "\n",
+    "# Set env var OPENAI_API_KEY or load from a .env file\n",
+    "# import dotenv\n",
+    "\n",
+    "# dotenv.load_env()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "36138740",
+   "metadata": {},
+   "source": [
+    "We can use `chain_type=\"stuff\"`, especially if using larger context window models such as:\n",
+    "\n",
+    "* 16k token OpenAI `gpt-3.5-turbo-16k` \n",
+    "* 100k token Anthropic [Claude-2](https://www.anthropic.com/index/claude-2)\n",
+    "\n",
+    "We can also supply `chain_type=\"map_reduce\"` or `chain_type=\"refine\"` (read more [here](/docs/modules/chains/document/refine))."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "fd271681",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'The article discusses the concept of building autonomous agents powered by large language models (LLMs). It explores the components of such agents, including planning, memory, and tool use. The article provides case studies and proof-of-concept examples of LLM-powered agents in various domains. It also highlights the challenges and limitations of using LLMs in agent systems.'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.document_loaders import WebBaseLoader\n",
+    "from langchain.chains.summarize import load_summarize_chain\n",
+    "\n",
+    "loader = WebBaseLoader(\"https://lilianweng.github.io/posts/2023-06-23-agent/\")\n",
+    "docs = loader.load()\n",
+    "\n",
+    "llm = ChatOpenAI(temperature=0, model_name=\"gpt-3.5-turbo-16k\")\n",
+    "chain = load_summarize_chain(llm, chain_type=\"stuff\")\n",
+    "\n",
+    "chain.run(docs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "615b36e1",
+   "metadata": {},
+   "source": [
+    "## Option 1. Stuff\n",
+    "\n",
+    "When we use `load_summarize_chain` with `chain_type=\"stuff\"`, we will use the [StuffDocumentsChain](/docs/modules/chains/document/stuff).\n",
+    "\n",
+    "The chain will take a list of documents, inserts them all into a prompt, and passes that prompt to an LLM:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "ef45585d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The article discusses the concept of building autonomous agents powered by large language models (LLMs). It explores the components of such agents, including planning, memory, and tool use. The article provides case studies and examples of proof-of-concept demos, highlighting the challenges and limitations of LLM-powered agents. It also includes references to related research papers and provides a citation for the article.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from langchain.chains.llm import LLMChain\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "from langchain.chains.combine_documents.stuff import StuffDocumentsChain\n",
+    "\n",
+    "# Define prompt\n",
+    "prompt_template = \"\"\"Write a concise summary of the following:\n",
+    "\"{text}\"\n",
+    "CONCISE SUMMARY:\"\"\"\n",
+    "prompt = PromptTemplate.from_template(prompt_template)\n",
+    "\n",
+    "# Define LLM chain\n",
+    "llm = ChatOpenAI(temperature=0, model_name=\"gpt-3.5-turbo-16k\")\n",
+    "llm_chain = LLMChain(llm=llm, prompt=prompt)\n",
+    "\n",
+    "# Define StuffDocumentsChain\n",
+    "stuff_chain = StuffDocumentsChain(\n",
+    "    llm_chain=llm_chain, document_variable_name=\"text\"\n",
+    ")\n",
+    "\n",
+    "docs = loader.load()\n",
+    "print(stuff_chain.run(docs))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4e4e4a43",
+   "metadata": {},
+   "source": [
+    "Great! We can see that we reproduce the earlier result using the `load_summarize_chain`.\n",
+    "\n",
+    "### Go deeper\n",
+    "\n",
+    "* You can easily customize the prompt. \n",
+    "* You can easily try different LLMs, (e.g., [Claude](/docs/integrations/chat/anthropic)) via the `llm` parameter."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ad6cabee",
+   "metadata": {},
+   "source": [
+    "## Option 2. Map-Reduce\n",
+    "\n",
+    "Let's unpack the map reduce approach. For this, we'll first map each document to an individual summary using an `LLMChain`. Then we'll use a `ReduceDocumentsChain` to combine those summaries into a single global summary.\n",
+    " \n",
+    "First, we specfy the LLMChain to use for mapping each document to an individual summary:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a1e6773c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chains.mapreduce import MapReduceChain\n",
+    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain.chains import ReduceDocumentsChain, MapReduceDocumentsChain\n",
+    "\n",
+    "llm = ChatOpenAI(temperature=0)\n",
+    "\n",
+    "# Map\n",
+    "map_template = \"\"\"The following is a set of documents\n",
+    "{docs}\n",
+    "Based on this list of docs, please identify the main themes \n",
+    "Helpful Answer:\"\"\"\n",
+    "map_prompt = PromptTemplate.from_template(map_template)\n",
+    "map_chain = LLMChain(llm=llm, prompt=map_prompt)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bee3c331",
+   "metadata": {},
+   "source": [
+    "The `ReduceDocumentsChain` handles taking the document mapping results and reducing them into a single output. It wraps a generic `CombineDocumentsChain` (like `StuffDocumentsChain`) but adds the ability to collapse documents before passing it to the `CombineDocumentsChain` if their cumulative size exceeds `token_max`. In this example, we can actually re-use our chain for combining our docs to also collapse our docs.\n",
+    "\n",
+    "So if the cumulative number of tokens in our mapped documents exceeds 4000 tokens, then we'll recursively pass in the documents in batches of < 4000 tokens to our `StuffDocumentsChain` to create batched summaries. And once those batched summaries are cumulatively less than 4000 tokens, we'll pass them all one last time to the `StuffDocumentsChain` to create the final summary."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "1edb1b0d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Reduce\n",
+    "reduce_template = \"\"\"The following is set of summaries:\n",
+    "{doc_summaries}\n",
+    "Take these and distill it into a final, consolidated summary of the main themes. \n",
+    "Helpful Answer:\"\"\"\n",
+    "reduce_prompt = PromptTemplate.from_template(reduce_template)\n",
+    "reduce_chain = LLMChain(llm=llm, prompt=reduce_prompt)\n",
+    "\n",
+    "# Takes a list of documents, combines them into a single string, and passes this to an LLMChain\n",
+    "combine_documents_chain = StuffDocumentsChain(\n",
+    "    llm_chain=reduce_chain, document_variable_name=\"doc_summaries\"\n",
+    ")\n",
+    "\n",
+    "# Combines and iteravely reduces the mapped documents\n",
+    "reduce_documents_chain = ReduceDocumentsChain(\n",
+    "    # This is final chain that is called.\n",
+    "    combine_documents_chain=combine_documents_chain,\n",
+    "    # If documents exceed context for `StuffDocumentsChain`\n",
+    "    collapse_documents_chain=combine_documents_chain,\n",
+    "    # The maximum number of tokens to group documents into.\n",
+    "    token_max=4000,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fdb5ae1a",
+   "metadata": {},
+   "source": [
+    "Combining our map and reduce chains into one:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "22f1cdc2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Created a chunk of size 1003, which is longer than the specified 1000\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Combining documents by mapping a chain over them, then combining results\n",
+    "map_reduce_chain = MapReduceDocumentsChain(\n",
+    "    # Map chain\n",
+    "    llm_chain=map_chain,\n",
+    "    # Reduce chain\n",
+    "    reduce_documents_chain=reduce_documents_chain,\n",
+    "    # The variable name in the llm_chain to put the documents in\n",
+    "    document_variable_name=\"docs\",\n",
+    "    # Return the results of the map steps in the output\n",
+    "    return_intermediate_steps=False,\n",
+    ")\n",
+    "\n",
+    "text_splitter = CharacterTextSplitter.from_tiktoken_encoder(\n",
+    "    chunk_size=1000, chunk_overlap=0\n",
+    ")\n",
+    "split_docs = text_splitter.split_documents(docs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "c7afb8c3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The main themes identified in the provided set of documents are:\n",
+      "\n",
+      "1. LLM-powered autonomous agent systems: The documents discuss the concept of building autonomous agents with large language models (LLMs) as the core controller. They explore the potential of LLMs beyond content generation and present them as powerful problem solvers.\n",
+      "\n",
+      "2. Components of the agent system: The documents outline the key components of LLM-powered agent systems, including planning, memory, and tool use. Each component is described in detail, highlighting its role in enhancing the agent's capabilities.\n",
+      "\n",
+      "3. Planning and task decomposition: The planning component focuses on task decomposition and self-reflection. The agent breaks down complex tasks into smaller subgoals and learns from past actions to improve future results.\n",
+      "\n",
+      "4. Memory and learning: The memory component includes short-term memory for in-context learning and long-term memory for retaining and recalling information over extended periods. The use of external vector stores for fast retrieval is also mentioned.\n",
+      "\n",
+      "5. Tool use and external APIs: The agent learns to utilize external APIs for accessing additional information, code execution, and proprietary sources. This enhances the agent's knowledge and problem-solving abilities.\n",
+      "\n",
+      "6. Case studies and proof-of-concept examples: The documents provide case studies and examples to demonstrate the application of LLM-powered agents in scientific discovery, generative simulations, and other domains. These examples serve as proof-of-concept for the effectiveness of the agent system.\n",
+      "\n",
+      "7. Challenges and limitations: The documents mention challenges associated with building LLM-powered autonomous agents, such as the limitations of finite context length, difficulties in long-term planning, and reliability issues with natural language interfaces.\n",
+      "\n",
+      "8. Citation and references: The documents include a citation and reference section for acknowledging the sources and inspirations for the concepts discussed.\n",
+      "\n",
+      "Overall, the main themes revolve around the development and capabilities of LLM-powered autonomous agent systems, including their components, planning and task decomposition, memory and learning mechanisms, tool use and external APIs, case studies and proof-of-concept examples, challenges and limitations, and the importance of proper citation and references.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(map_reduce_chain.run(split_docs))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e62c21cf",
+   "metadata": {},
+   "source": [
+    "### Go deeper\n",
+    " \n",
+    "**Customization** \n",
+    "\n",
+    "* As shown above, you can customize the LLMs and prompts for map and reduce stages.\n",
+    "\n",
+    "**Real-world use-case**\n",
+    "\n",
+    "* See [this blog post](https://blog.langchain.dev/llms-to-improve-documentation/) case-study on analyzing user interactions (questions about LangChain documentation)!  \n",
+    "* The blog post and associated [repo](https://github.com/mendableai/QA_clustering) also introduce clustering as a means of summarization.\n",
+    "* This opens up a third path beyond the `stuff` or `map-reduce` approaches that is worth considering.\n",
+    "\n",
+    "![Image description](/img/summarization_use_case_3.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f08ff365",
+   "metadata": {},
+   "source": [
+    "## Option 3. Refine\n",
+    " \n",
+    "[Refine](/docs/modules/chains/document/refine) is similar to map-reduce:\n",
+    "\n",
+    "> The refine documents chain constructs a response by looping over the input documents and iteratively updating its answer. For each document, it passes all non-document inputs, the current document, and the latest intermediate answer to an LLM chain to get a new answer.\n",
+    "\n",
+    "This can be easily run with the `chain_type=\"refine\"` specified."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "de1dc10e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'The GPT-Engineer project aims to create a repository of code for specific tasks specified in natural language. It involves breaking down tasks into smaller components and seeking clarification from the user when needed. The project emphasizes the importance of implementing every detail of the architecture as code and provides guidelines for file organization, code structure, and dependencies. However, there are challenges in long-term planning and task decomposition, as well as the reliability of the natural language interface. The system has limited communication bandwidth and struggles to adjust plans when faced with unexpected errors. The reliability of model outputs is questionable, as formatting errors and rebellious behavior can occur. The conversation also includes instructions for writing the code, including laying out the core classes, functions, and methods, and providing the code in a markdown code block format. The user is reminded to ensure that the code is fully functional and follows best practices for file naming, imports, and types. The project is powered by LLM (Large Language Models) and incorporates prompting techniques from various research papers.'"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chain = load_summarize_chain(llm, chain_type=\"refine\")\n",
+    "chain.run(split_docs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5b46f44d",
+   "metadata": {},
+   "source": [
+    "It's also possible to supply a prompt and return intermediate steps."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "f86c8072",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt_template = \"\"\"Write a concise summary of the following:\n",
+    "{text}\n",
+    "CONCISE SUMMARY:\"\"\"\n",
+    "prompt = PromptTemplate.from_template(prompt_template)\n",
+    "\n",
+    "refine_template = (\n",
+    "    \"Your job is to produce a final summary\\n\"\n",
+    "    \"We have provided an existing summary up to a certain point: {existing_answer}\\n\"\n",
+    "    \"We have the opportunity to refine the existing summary\"\n",
+    "    \"(only if needed) with some more context below.\\n\"\n",
+    "    \"------------\\n\"\n",
+    "    \"{text}\\n\"\n",
+    "    \"------------\\n\"\n",
+    "    \"Given the new context, refine the original summary in Italian\"\n",
+    "    \"If the context isn't useful, return the original summary.\"\n",
+    ")\n",
+    "refine_prompt = PromptTemplate.from_template(refine_template)\n",
+    "chain = load_summarize_chain(\n",
+    "    llm=llm,\n",
+    "    chain_type=\"refine\",\n",
+    "    question_prompt=prompt,\n",
+    "    refine_prompt=refine_prompt,\n",
+    "    return_intermediate_steps=True,\n",
+    "    input_key=\"input_documents\",\n",
+    "    output_key=\"output_text\",\n",
+    ")\n",
+    "result = chain({\"input_documents\": split_docs}, return_only_outputs=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "d9600b67-79d4-4f85-aba2-9fe81fa29f49",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "L'articolo discute il concetto di costruire agenti autonomi utilizzando LLM (large language model) come controller principale. Esplora i diversi componenti di un sistema di agenti alimentato da LLM, inclusa la pianificazione, la memoria e l'uso di strumenti. Dimostrazioni di concetto come AutoGPT mostrano la possibilità di creare agenti autonomi con LLM come controller principale. Approcci come Chain of Thought, Tree of Thoughts, LLM+P, ReAct e Reflexion consentono agli agenti autonomi di pianificare, riflettere su se stessi e migliorare iterativamente. Tuttavia, ci sono sfide legate alla lunghezza del contesto, alla pianificazione a lungo termine e alla decomposizione delle attività. Inoltre, l'affidabilità dell'interfaccia di linguaggio naturale tra LLM e componenti esterni come la memoria e gli strumenti è incerta. Nonostante ciò, l'uso di LLM come router per indirizzare le richieste ai moduli esperti più adatti è stato proposto come architettura neuro-simbolica per agenti autonomi nel sistema MRKL. L'articolo fa riferimento a diverse pubblicazioni che approfondiscono l'argomento, tra cui Chain of Thought, Tree of Thoughts, LLM+P, ReAct, Reflexion, e MRKL Systems.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(result[\"output_text\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "5f91a8eb-daa5-4191-ace4-01765801db3e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "This article discusses the concept of building autonomous agents using LLM (large language model) as the core controller. The article explores the different components of an LLM-powered agent system, including planning, memory, and tool use. It also provides examples of proof-of-concept demos and highlights the potential of LLM as a general problem solver.\n",
+      "\n",
+      "Questo articolo discute del concetto di costruire agenti autonomi utilizzando LLM (large language model) come controller principale. L'articolo esplora i diversi componenti di un sistema di agenti alimentato da LLM, inclusa la pianificazione, la memoria e l'uso degli strumenti. Vengono anche forniti esempi di dimostrazioni di proof-of-concept e si evidenzia il potenziale di LLM come risolutore generale di problemi. Inoltre, vengono presentati approcci come Chain of Thought, Tree of Thoughts, LLM+P, ReAct e Reflexion che consentono agli agenti autonomi di pianificare, riflettere su se stessi e migliorare iterativamente.\n",
+      "\n",
+      "Questo articolo discute del concetto di costruire agenti autonomi utilizzando LLM (large language model) come controller principale. L'articolo esplora i diversi componenti di un sistema di agenti alimentato da LLM, inclusa la pianificazione, la memoria e l'uso degli strumenti. Vengono anche forniti esempi di dimostrazioni di proof-of-concept e si evidenzia il potenziale di LLM come risolutore generale di problemi. Inoltre, vengono presentati approcci come Chain of Thought, Tree of Thoughts, LLM+P, ReAct e Reflexion che consentono agli agenti autonomi di pianificare, riflettere su se stessi e migliorare iterativamente. Il nuovo contesto riguarda l'approccio Chain of Hindsight (CoH) che permette al modello di migliorare autonomamente i propri output attraverso un processo di apprendimento supervisionato. Viene anche presentato l'approccio Algorithm Distillation (AD) che applica lo stesso concetto alle traiettorie di apprendimento per compiti di reinforcement learning.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"\\n\\n\".join(result[\"intermediate_steps\"][:3]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ddd522e-30dc-4f6a-b993-c4f97e656c4f",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/extras/use_cases/summarization/index.mdx

@@ -1,22 +0,0 @@
----
-sidebar_position: 5
----
-
-# Summarization
-
-Summarization involves creating a smaller summary of multiple longer documents.
-This can be useful for distilling long documents into the core pieces of information.
-
-The recommended way to get started using a summarization chain is:
-
-```python
-from langchain.chains.summarize import load_summarize_chain
-chain = load_summarize_chain(llm, chain_type="map_reduce")
-chain.run(docs)
-```
-
-The following resources exist:
-- [Summarization notebook](/docs/use_cases/summarization/summarize.html): A notebook walking through how to accomplish this task.
-
-Additional related resources include:
-- [Modules for working with documents](/docs/modules/data_connection): Core components for working with documents.

docs/snippets/modules/chains/popular/summarize.mdx

@@ -1,384 +0,0 @@
-## Prepare Data
-First we prepare the data. For this example we create multiple documents from one long one, but these documents could be fetched in any manner (the point of this notebook to highlight what to do AFTER you fetch the documents).
-
-```python
-from langchain import OpenAI, PromptTemplate, LLMChain
-from langchain.text_splitter import CharacterTextSplitter
-from langchain.chains.mapreduce import MapReduceChain
-from langchain.prompts import PromptTemplate
-
-llm = OpenAI(temperature=0)
-
-text_splitter = CharacterTextSplitter()
-```
-
-
-```python
-with open("../../state_of_the_union.txt") as f:
-    state_of_the_union = f.read()
-texts = text_splitter.split_text(state_of_the_union)
-```
-
-
-```python
-from langchain.docstore.document import Document
-
-docs = [Document(page_content=t) for t in texts[:3]]
-```
-
-## Quickstart
-If you just want to get started as quickly as possible, this is the recommended way to do it:
-
-
-```python
-from langchain.chains.summarize import load_summarize_chain
-```
-
-
-```python
-chain = load_summarize_chain(llm, chain_type="map_reduce")
-chain.run(docs)
-```
-
-<CodeOutputBlock lang="python">
-
-```
-    ' In response to Russian aggression in Ukraine, the United States and its allies are taking action to hold Putin accountable, including economic sanctions, asset seizures, and military assistance. The US is also providing economic and humanitarian aid to Ukraine, and has passed the American Rescue Plan and the Bipartisan Infrastructure Law to help struggling families and create jobs. The US remains unified and determined to protect Ukraine and the free world.'
-```
-
-</CodeOutputBlock>
-
-If you want more control and understanding over what is happening, please see the information below.
-
-## The `stuff` Chain
-
-This sections shows results of using the `stuff` Chain to do summarization.
-
-
-```python
-chain = load_summarize_chain(llm, chain_type="stuff")
-```
-
-
-```python
-chain.run(docs)
-```
-
-<CodeOutputBlock lang="python">
-
-```
-    ' In his speech, President Biden addressed the crisis in Ukraine, the American Rescue Plan, and the Bipartisan Infrastructure Law. He discussed the need to invest in America, educate Americans, and build the economy from the bottom up. He also announced the release of 60 million barrels of oil from reserves around the world, and the creation of a dedicated task force to go after the crimes of Russian oligarchs. He concluded by emphasizing the need to Buy American and use taxpayer dollars to rebuild America.'
-```
-
-</CodeOutputBlock>
-
-**Custom Prompts**
-
-You can also use your own prompts with this chain. In this example, we will respond in Italian.
-
-
-```python
-prompt_template = """Write a concise summary of the following:
-
-
-{text}
-
-
-CONCISE SUMMARY IN ITALIAN:"""
-PROMPT = PromptTemplate(template=prompt_template, input_variables=["text"])
-chain = load_summarize_chain(llm, chain_type="stuff", prompt=PROMPT)
-chain.run(docs)
-```
-
-<CodeOutputBlock lang="python">
-
-```
-    "\n\nIn questa serata, il Presidente degli Stati Uniti ha annunciato una serie di misure per affrontare la crisi in Ucraina, causata dall'aggressione di Putin. Ha anche annunciato l'invio di aiuti economici, militari e umanitari all'Ucraina. Ha anche annunciato che gli Stati Uniti e i loro alleati stanno imponendo sanzioni economiche a Putin e stanno rilasciando 60 milioni di barili di petrolio dalle riserve di tutto il mondo. Inoltre, ha annunciato che il Dipartimento di Giustizia degli Stati Uniti sta creando una task force dedicata ai crimini degli oligarchi russi. Il Presidente ha anche annunciato l'approvazione della legge bipartitica sull'infrastruttura, che prevede investimenti per la ricostruzione dell'America. Questo porterà a creare posti"
-```
-
-</CodeOutputBlock>
-
-## The `map_reduce` Chain
-
-This sections shows results of using the `map_reduce` Chain to do summarization.
-
-
-```python
-chain = load_summarize_chain(llm, chain_type="map_reduce")
-```
-
-
-```python
-chain.run(docs)
-```
-
-<CodeOutputBlock lang="python">
-
-```
-    " In response to Russia's aggression in Ukraine, the United States and its allies have imposed economic sanctions and are taking other measures to hold Putin accountable. The US is also providing economic and military assistance to Ukraine, protecting NATO countries, and releasing oil from its Strategic Petroleum Reserve. President Biden and Vice President Harris have passed legislation to help struggling families and rebuild America's infrastructure."
-```
-
-</CodeOutputBlock>
-
-**Intermediate Steps**
-
-We can also return the intermediate steps for `map_reduce` chains, should we want to inspect them. This is done with the `return_map_steps` variable.
-
-
-```python
-chain = load_summarize_chain(OpenAI(temperature=0), chain_type="map_reduce", return_intermediate_steps=True)
-```
-
-
-```python
-chain({"input_documents": docs}, return_only_outputs=True)
-```
-
-<CodeOutputBlock lang="python">
-
-```
-    {'map_steps': [" In response to Russia's aggression in Ukraine, the United States has united with other freedom-loving nations to impose economic sanctions and hold Putin accountable. The U.S. Department of Justice is also assembling a task force to go after the crimes of Russian oligarchs and seize their ill-gotten gains.",
-      ' The United States and its European allies are taking action to punish Russia for its invasion of Ukraine, including seizing assets, closing off airspace, and providing economic and military assistance to Ukraine. The US is also mobilizing forces to protect NATO countries and has released 30 million barrels of oil from its Strategic Petroleum Reserve to help blunt gas prices. The world is uniting in support of Ukraine and democracy, and the US stands with its Ukrainian-American citizens.',
-      " President Biden and Vice President Harris ran for office with a new economic vision for America, and have since passed the American Rescue Plan and the Bipartisan Infrastructure Law to help struggling families and rebuild America's infrastructure. This includes creating jobs, modernizing roads, airports, ports, and waterways, replacing lead pipes, providing affordable high-speed internet, and investing in American products to support American jobs."],
-     'output_text': " In response to Russia's aggression in Ukraine, the United States and its allies have imposed economic sanctions and are taking other measures to hold Putin accountable. The US is also providing economic and military assistance to Ukraine, protecting NATO countries, and passing legislation to help struggling families and rebuild America's infrastructure. The world is uniting in support of Ukraine and democracy, and the US stands with its Ukrainian-American citizens."}
-```
-
-</CodeOutputBlock>
-
-**Custom Prompts**
-
-You can also use your own prompts with this chain. In this example, we will respond in Italian.
-
-
-```python
-prompt_template = """Write a concise summary of the following:
-
-
-{text}
-
-
-CONCISE SUMMARY IN ITALIAN:"""
-PROMPT = PromptTemplate(template=prompt_template, input_variables=["text"])
-chain = load_summarize_chain(OpenAI(temperature=0), chain_type="map_reduce", return_intermediate_steps=True, map_prompt=PROMPT, combine_prompt=PROMPT)
-chain({"input_documents": docs}, return_only_outputs=True)
-```
-
-<CodeOutputBlock lang="python">
-
-```
-    {'intermediate_steps': ["\n\nQuesta sera, ci incontriamo come democratici, repubblicani e indipendenti, ma soprattutto come americani. La Russia di Putin ha cercato di scuotere le fondamenta del mondo libero, ma ha sottovalutato la forza della gente ucraina. Gli Stati Uniti e i loro alleati stanno ora imponendo sanzioni economiche a Putin e stanno tagliando l'accesso della Russia alla tecnologia. Il Dipartimento di Giustizia degli Stati Uniti sta anche creando una task force dedicata per andare dopo i crimini degli oligarchi russi.",
-      "\n\nStiamo unendo le nostre forze con quelle dei nostri alleati europei per sequestrare yacht, appartamenti di lusso e jet privati di Putin. Abbiamo chiuso lo spazio aereo americano ai voli russi e stiamo fornendo più di un miliardo di dollari in assistenza all'Ucraina. Abbiamo anche mobilitato le nostre forze terrestri, aeree e navali per proteggere i paesi della NATO. Abbiamo anche rilasciato 60 milioni di barili di petrolio dalle riserve di tutto il mondo, di cui 30 milioni dalla nostra riserva strategica di petrolio. Stiamo affrontando una prova reale e ci vorrà del tempo, ma alla fine Putin non riuscirà a spegnere l'amore dei popoli per la libertà.",
-      "\n\nIl Presidente Biden ha lottato per passare l'American Rescue Plan per aiutare le persone che soffrivano a causa della pandemia. Il piano ha fornito sollievo economico immediato a milioni di americani, ha aiutato a mettere cibo sulla loro tavola, a mantenere un tetto sopra le loro teste e a ridurre il costo dell'assicurazione sanitaria. Il piano ha anche creato più di 6,5 milioni di nuovi posti di lavoro, il più alto numero di posti di lavoro creati in un anno nella storia degli Stati Uniti. Il Presidente Biden ha anche firmato la legge bipartitica sull'infrastruttura, la più ampia iniziativa di ricostruzione della storia degli Stati Uniti. Il piano prevede di modernizzare le strade, gli aeroporti, i porti e le vie navigabili in"],
-     'output_text': "\n\nIl Presidente Biden sta lavorando per aiutare le persone che soffrono a causa della pandemia attraverso l'American Rescue Plan e la legge bipartitica sull'infrastruttura. Gli Stati Uniti e i loro alleati stanno anche imponendo sanzioni economiche a Putin e tagliando l'accesso della Russia alla tecnologia. Stanno anche sequestrando yacht, appartamenti di lusso e jet privati di Putin e fornendo più di un miliardo di dollari in assistenza all'Ucraina. Alla fine, Putin non riuscirà a spegnere l'amore dei popoli per la libertà."}
-```
-
-</CodeOutputBlock>
-
-## The custom `MapReduceChain`
-
-**Multi input prompt**
-
-You can also use prompt with multi input. In this example, we will use a MapReduce chain to answer specific question about our code.
-
-
-```python
-from langchain.chains.combine_documents.map_reduce import MapReduceDocumentsChain
-from langchain.chains.combine_documents.stuff import StuffDocumentsChain
-
-map_template_string = """Give the following python code information, generate a description that explains what the code does and also mention the time complexity.
-Code:
-{code}
-
-Return the the description in the following format:
-name of the function: description of the function
-"""
-
-
-reduce_template_string = """Given the following python function names and descriptions, answer the following question
-{code_description}
-Question: {question}
-Answer:
-"""
-
-# Prompt to use in map and reduce stages 
-MAP_PROMPT = PromptTemplate(input_variables=["code"], template=map_template_string)
-REDUCE_PROMPT = PromptTemplate(input_variables=["code_description", "question"], template=reduce_template_string)
-
-# LLM to use in map and reduce stages 
-llm = OpenAI()
-map_llm_chain = LLMChain(llm=llm, prompt=MAP_PROMPT)
-reduce_llm_chain = LLMChain(llm=llm, prompt=REDUCE_PROMPT)
-
-# Takes a list of documents and combines them into a single string
-combine_documents_chain = StuffDocumentsChain(
-    llm_chain=reduce_llm_chain,
-    document_variable_name="code_description",
-)
-
-# Combines and iteravely reduces the mapped documents 
-reduce_documents_chain = ReduceDocumentsChain(
-        # This is final chain that is called.
-        combine_documents_chain=combine_documents_chain,
-        # If documents exceed context for `combine_documents_chain`
-        collapse_documents_chain=combine_documents_chain,
-        # The maximum number of tokens to group documents into
-        token_max=3000)
-
-# Combining documents by mapping a chain over them, then combining results with reduce chain
-combine_documents = MapReduceDocumentsChain(
-    # Map chain
-    llm_chain=map_llm_chain,
-     # Reduce chain
-    reduce_documents_chain=reduce_documents_chain,
-    # The variable name in the llm_chain to put the documents in
-    document_variable_name="code",
-)
-
-map_reduce = MapReduceChain(
-    combine_documents_chain=combine_documents,
-    text_splitter=CharacterTextSplitter(separator="\n##\n", chunk_size=100, chunk_overlap=0),
-)
-```
-
-
-```python
-code = """
-def bubblesort(list):
-   for iter_num in range(len(list)-1,0,-1):
-      for idx in range(iter_num):
-         if list[idx]>list[idx+1]:
-            temp = list[idx]
-            list[idx] = list[idx+1]
-            list[idx+1] = temp
-    return list
-##
-def insertion_sort(InputList):
-   for i in range(1, len(InputList)):
-      j = i-1
-      nxt_element = InputList[i]
-   while (InputList[j] > nxt_element) and (j >= 0):
-      InputList[j+1] = InputList[j]
-      j=j-1
-   InputList[j+1] = nxt_element
-   return InputList
-##
-def shellSort(input_list):
-   gap = len(input_list) // 2
-   while gap > 0:
-      for i in range(gap, len(input_list)):
-         temp = input_list[i]
-         j = i
-   while j >= gap and input_list[j - gap] > temp:
-      input_list[j] = input_list[j - gap]
-      j = j-gap
-      input_list[j] = temp
-   gap = gap//2
-   return input_list
-
-"""
-```
-
-
-```python
-map_reduce.run(input_text=code, question="Which function has a better time complexity?")
-```
-
-<CodeOutputBlock lang="python">
-
-```
-    Created a chunk of size 247, which is longer than the specified 100
-    Created a chunk of size 267, which is longer than the specified 100
-
-
-
-
-
-    'shellSort has a better time complexity than both bubblesort and insertion_sort, as it has a time complexity of O(n^2), while the other two have a time complexity of O(n^2).'
-```
-
-</CodeOutputBlock>
-
-## The `refine` Chain
-
-This sections shows results of using the `refine` Chain to do summarization.
-
-
-```python
-chain = load_summarize_chain(llm, chain_type="refine")
-
-chain.run(docs)
-```
-
-<CodeOutputBlock lang="python">
-
-```
-    "\n\nIn response to Russia's aggression in Ukraine, the United States has united with other freedom-loving nations to impose economic sanctions and hold Putin accountable. The U.S. Department of Justice is also assembling a task force to go after the crimes of Russian oligarchs and seize their ill-gotten gains. We are joining with our European allies to find and seize the assets of Russian oligarchs, including yachts, luxury apartments, and private jets. The U.S. is also closing off American airspace to all Russian flights, further isolating Russia and adding an additional squeeze on their economy. The U.S. and its allies are providing support to the Ukrainians in their fight for freedom, including military, economic, and humanitarian assistance. The U.S. is also mobilizing ground forces, air squadrons, and ship deployments to protect NATO countries. The U.S. and its allies are also releasing 60 million barrels of oil from reserves around the world, with the U.S. contributing 30 million barrels from its own Strategic Petroleum Reserve. In addition, the U.S. has passed the American Rescue Plan to provide immediate economic relief for tens of millions of Americans, and the Bipartisan Infrastructure Law to rebuild America and create jobs. This investment will"
-```
-
-</CodeOutputBlock>
-
-**Intermediate Steps**
-
-We can also return the intermediate steps for `refine` chains, should we want to inspect them. This is done with the `return_refine_steps` variable.
-
-
-```python
-chain = load_summarize_chain(OpenAI(temperature=0), chain_type="refine", return_intermediate_steps=True)
-
-chain({"input_documents": docs}, return_only_outputs=True)
-```
-
-<CodeOutputBlock lang="python">
-
-```
-    {'refine_steps': [" In response to Russia's aggression in Ukraine, the United States has united with other freedom-loving nations to impose economic sanctions and hold Putin accountable. The U.S. Department of Justice is also assembling a task force to go after the crimes of Russian oligarchs and seize their ill-gotten gains.",
-      "\n\nIn response to Russia's aggression in Ukraine, the United States has united with other freedom-loving nations to impose economic sanctions and hold Putin accountable. The U.S. Department of Justice is also assembling a task force to go after the crimes of Russian oligarchs and seize their ill-gotten gains. We are joining with our European allies to find and seize the assets of Russian oligarchs, including yachts, luxury apartments, and private jets. The U.S. is also closing off American airspace to all Russian flights, further isolating Russia and adding an additional squeeze on their economy. The U.S. and its allies are providing support to the Ukrainians in their fight for freedom, including military, economic, and humanitarian assistance. The U.S. is also mobilizing ground forces, air squadrons, and ship deployments to protect NATO countries. The U.S. and its allies are also releasing 60 million barrels of oil from reserves around the world, with the U.S. contributing 30 million barrels from its own Strategic Petroleum Reserve. Putin's war on Ukraine has left Russia weaker and the rest of the world stronger, with the world uniting in support of democracy and peace.",
-      "\n\nIn response to Russia's aggression in Ukraine, the United States has united with other freedom-loving nations to impose economic sanctions and hold Putin accountable. The U.S. Department of Justice is also assembling a task force to go after the crimes of Russian oligarchs and seize their ill-gotten gains. We are joining with our European allies to find and seize the assets of Russian oligarchs, including yachts, luxury apartments, and private jets. The U.S. is also closing off American airspace to all Russian flights, further isolating Russia and adding an additional squeeze on their economy. The U.S. and its allies are providing support to the Ukrainians in their fight for freedom, including military, economic, and humanitarian assistance. The U.S. is also mobilizing ground forces, air squadrons, and ship deployments to protect NATO countries. The U.S. and its allies are also releasing 60 million barrels of oil from reserves around the world, with the U.S. contributing 30 million barrels from its own Strategic Petroleum Reserve. In addition, the U.S. has passed the American Rescue Plan to provide immediate economic relief for tens of millions of Americans, and the Bipartisan Infrastructure Law to rebuild America and create jobs. This includes investing"],
-     'output_text': "\n\nIn response to Russia's aggression in Ukraine, the United States has united with other freedom-loving nations to impose economic sanctions and hold Putin accountable. The U.S. Department of Justice is also assembling a task force to go after the crimes of Russian oligarchs and seize their ill-gotten gains. We are joining with our European allies to find and seize the assets of Russian oligarchs, including yachts, luxury apartments, and private jets. The U.S. is also closing off American airspace to all Russian flights, further isolating Russia and adding an additional squeeze on their economy. The U.S. and its allies are providing support to the Ukrainians in their fight for freedom, including military, economic, and humanitarian assistance. The U.S. is also mobilizing ground forces, air squadrons, and ship deployments to protect NATO countries. The U.S. and its allies are also releasing 60 million barrels of oil from reserves around the world, with the U.S. contributing 30 million barrels from its own Strategic Petroleum Reserve. In addition, the U.S. has passed the American Rescue Plan to provide immediate economic relief for tens of millions of Americans, and the Bipartisan Infrastructure Law to rebuild America and create jobs. This includes investing"}
-```
-
-</CodeOutputBlock>
-
-**Custom Prompts**
-
-You can also use your own prompts with this chain. In this example, we will respond in Italian.
-
-
-```python
-prompt_template = """Write a concise summary of the following:
-
-
-{text}
-
-
-CONCISE SUMMARY IN ITALIAN:"""
-PROMPT = PromptTemplate(template=prompt_template, input_variables=["text"])
-refine_template = (
-    "Your job is to produce a final summary\n"
-    "We have provided an existing summary up to a certain point: {existing_answer}\n"
-    "We have the opportunity to refine the existing summary"
-    "(only if needed) with some more context below.\n"
-    "------------\n"
-    "{text}\n"
-    "------------\n"
-    "Given the new context, refine the original summary in Italian"
-    "If the context isn't useful, return the original summary."
-)
-refine_prompt = PromptTemplate(
-    input_variables=["existing_answer", "text"],
-    template=refine_template,
-)
-chain = load_summarize_chain(OpenAI(temperature=0), chain_type="refine", return_intermediate_steps=True, question_prompt=PROMPT, refine_prompt=refine_prompt)
-chain({"input_documents": docs}, return_only_outputs=True)
-```
-
-<CodeOutputBlock lang="python">
-
-```
-    {'intermediate_steps': ["\n\nQuesta sera, ci incontriamo come democratici, repubblicani e indipendenti, ma soprattutto come americani. La Russia di Putin ha cercato di scuotere le fondamenta del mondo libero, ma ha sottovalutato la forza della gente ucraina. Insieme ai nostri alleati, stiamo imponendo sanzioni economiche, tagliando l'accesso della Russia alla tecnologia e bloccando i suoi più grandi istituti bancari dal sistema finanziario internazionale. Il Dipartimento di Giustizia degli Stati Uniti sta anche assemblando una task force dedicata per andare dopo i crimini degli oligarchi russi.",
-      "\n\nQuesta sera, ci incontriamo come democratici, repubblicani e indipendenti, ma soprattutto come americani. La Russia di Putin ha cercato di scuotere le fondamenta del mondo libero, ma ha sottovalutato la forza della gente ucraina. Insieme ai nostri alleati, stiamo imponendo sanzioni economiche, tagliando l'accesso della Russia alla tecnologia, bloccando i suoi più grandi istituti bancari dal sistema finanziario internazionale e chiudendo lo spazio aereo americano a tutti i voli russi. Il Dipartimento di Giustizia degli Stati Uniti sta anche assemblando una task force dedicata per andare dopo i crimini degli oligarchi russi. Stiamo fornendo più di un miliardo di dollari in assistenza diretta all'Ucraina e fornendo assistenza militare,",
-      "\n\nQuesta sera, ci incontriamo come democratici, repubblicani e indipendenti, ma soprattutto come americani. La Russia di Putin ha cercato di scuotere le fondamenta del mondo libero, ma ha sottovalutato la forza della gente ucraina. Insieme ai nostri alleati, stiamo imponendo sanzioni economiche, tagliando l'accesso della Russia alla tecnologia, bloccando i suoi più grandi istituti bancari dal sistema finanziario internazionale e chiudendo lo spazio aereo americano a tutti i voli russi. Il Dipartimento di Giustizia degli Stati Uniti sta anche assemblando una task force dedicata per andare dopo i crimini degli oligarchi russi. Stiamo fornendo più di un miliardo di dollari in assistenza diretta all'Ucraina e fornendo assistenza militare."],
-     'output_text': "\n\nQuesta sera, ci incontriamo come democratici, repubblicani e indipendenti, ma soprattutto come americani. La Russia di Putin ha cercato di scuotere le fondamenta del mondo libero, ma ha sottovalutato la forza della gente ucraina. Insieme ai nostri alleati, stiamo imponendo sanzioni economiche, tagliando l'accesso della Russia alla tecnologia, bloccando i suoi più grandi istituti bancari dal sistema finanziario internazionale e chiudendo lo spazio aereo americano a tutti i voli russi. Il Dipartimento di Giustizia degli Stati Uniti sta anche assemblando una task force dedicata per andare dopo i crimini degli oligarchi russi. Stiamo fornendo più di un miliardo di dollari in assistenza diretta all'Ucraina e fornendo assistenza militare."}
-```
-
-</CodeOutputBlock>

docs/snippets/modules/data_connection/document_transformers/text_splitters/character_text_splitter.mdx

@@ -12,6 +12,7 @@ text_splitter = CharacterTextSplitter(
     chunk_size = 1000,
     chunk_overlap  = 200,
     length_function = len,
+    is_separator_regex = False,
 )
 ```
 

docs/snippets/modules/data_connection/document_transformers/text_splitters/recursive_text_splitter.mdx

@@ -16,6 +16,7 @@ text_splitter = RecursiveCharacterTextSplitter(
     chunk_size = 100,
     chunk_overlap  = 20,
     length_function = len,
+    is_separator_regex = False,
 )
 ```
 

docs/snippets/modules/data_connection/retrievers/self_query/get_started.mdx

@@ -3,7 +3,7 @@ We'll use a Pinecone vector store in this example.
 
 First we'll want to create a `Pinecone` VectorStore and seed it with some data. We've created a small demo set of documents that contain summaries of movies.
 
-To use Pinecone, you to have `pinecone` package installed and you must have an API key and an Environment. Here are the [installation instructions](https://docs.pinecone.io/docs/quickstart).
+To use Pinecone, you need to have `pinecone` package installed and you must have an API key and an Environment. Here are the [installation instructions](https://docs.pinecone.io/docs/quickstart).
 
 NOTE: The self-query retriever requires you to have `lark` package installed.
 
@@ -85,7 +85,7 @@ retriever = SelfQueryRetriever.from_llm(llm, vectorstore, document_content_descr
 ```
 
 ## Testing it out
-And now we can try actually using our retriever!
+And now we can actually try using our retriever!
 
 
 ```python
@@ -198,4 +198,4 @@ retriever = SelfQueryRetriever.from_llm(
 ```python
 # This example only specifies a relevant query
 retriever.get_relevant_documents("What are two movies about dinosaurs")
-```
+```

docs/snippets/modules/model_io/prompts/prompt_templates/get_started.mdx

@@ -1,140 +1,115 @@
-Here's the simplest example:
+Typically, language models expect the prompt to either be a string or else a list of chat messages.
+
+## Prompt template
+
+Use `PromptTemplate` to create a template for a string prompt.
+
+By default, `PromptTemplate` uses [Python's str.format](https://docs.python.org/3/library/stdtypes.html#str.format)
+syntax for templating; however other templating syntax is available (e.g., `jinja2`).
 
 ```python
 from langchain import PromptTemplate
 
-
-template = """\
-You are a naming consultant for new companies.
-What is a good name for a company that makes {product}?
-"""
-
-prompt = PromptTemplate.from_template(template)
-prompt.format(product="colorful socks")
+prompt_template = PromptTemplate.from_template(
+    "Tell me a {adjective} joke about {content}."
+)
+prompt_template.format(adjective="funny", content="chickens")
 ```
 
 <CodeOutputBlock lang="python">
 
 ```
-You are a naming consultant for new companies.
-What is a good name for a company that makes colorful socks?
+"Tell me a funny joke about chickens."
 ```
 
 </CodeOutputBlock>
 
-
-## Create a prompt template
-
-You can create simple hardcoded prompts using the `PromptTemplate` class. Prompt templates can take any number of input variables, and can be formatted to generate a prompt.
-
+The template supports any number of variables, including no variables:
 
 ```python
 from langchain import PromptTemplate
 
-# An example prompt with no input variables
-no_input_prompt = PromptTemplate(input_variables=[], template="Tell me a joke.")
-no_input_prompt.format()
-# -> "Tell me a joke."
-
-# An example prompt with one input variable
-one_input_prompt = PromptTemplate(input_variables=["adjective"], template="Tell me a {adjective} joke.")
-one_input_prompt.format(adjective="funny")
-# -> "Tell me a funny joke."
-
-# An example prompt with multiple input variables
-multiple_input_prompt = PromptTemplate(
-    input_variables=["adjective", "content"], 
-    template="Tell me a {adjective} joke about {content}."
+prompt_template = PromptTemplate.from_template(
+"Tell me a joke"
 )
-multiple_input_prompt.format(adjective="funny", content="chickens")
-# -> "Tell me a funny joke about chickens."
+prompt_template.format()
 ```
 
-If you do not wish to specify `input_variables` manually, you can also create a `PromptTemplate` using `from_template` class method. `langchain` will automatically infer the `input_variables` based on the `template` passed.
+For additional validation, specify `input_variables` explicitly. These variables
+will be compared against the variables present in the template string during instantiation, raising an exception if
+there is a mismatch; for example,
 
 ```python
-template = "Tell me a {adjective} joke about {content}."
+from langchain import PromptTemplate
 
-prompt_template = PromptTemplate.from_template(template)
-prompt_template.input_variables
-# -> ['adjective', 'content']
-prompt_template.format(adjective="funny", content="chickens")
-# -> Tell me a funny joke about chickens.
+invalid_prompt = PromptTemplate(
+    input_variables=["adjective"],
+    template="Tell me a {adjective} joke about {content}."
+)
 ```
 
-You can create custom prompt templates that format the prompt in any way you want. For more information, see [Custom Prompt Templates](./custom_prompt_template.html).
-
+You can create custom prompt templates that format the prompt in any way you want.
+For more information, see [Custom Prompt Templates](./custom_prompt_template.html).
 
 <!-- TODO(shreya): Add link to Jinja -->
 
 ## Chat prompt template
 
-[Chat Models](../models/chat) take a list of chat messages as input - this list commonly referred to as a `prompt`.
-These chat messages differ from raw string (which you would pass into a [LLM](/docs/modules/model_io/models/llms) model) in that every message is associated with a `role`.
-
-For example, in OpenAI [Chat Completion API](https://platform.openai.com/docs/guides/chat/introduction), a chat message can be associated with the AI, human or system role. The model is supposed to follow instruction from system chat message more closely.
-
-LangChain provides several prompt templates to make constructing and working with prompts easy. You are encouraged to use these chat related prompt templates instead of `PromptTemplate` when querying chat models to fully utilize the potential of the underlying chat model.
-
-
+The prompt to [Chat Models](../models/chat) is a list of chat messages.
 
+Each chat message is associated with content, and an additional parameter called `role`.
+For example, in the OpenAI [Chat Completions API](https://platform.openai.com/docs/guides/chat/introduction), a chat message can be associated with an AI assistant, a human or a system role.
 
+Create a chat prompt template like this:
 
 ```python
-from langchain.prompts import (
-    ChatPromptTemplate,
-    PromptTemplate,
-    SystemMessagePromptTemplate,
-    AIMessagePromptTemplate,
-    HumanMessagePromptTemplate,
-)
-from langchain.schema import (
-    AIMessage,
-    HumanMessage,
-    SystemMessage
+from langchain.prompts import ChatPromptTemplate
+
+template = ChatPromptTemplate.from_messages([
+    ("system", "You are a helpful AI bot. Your name is {name}."),
+    ("human", "Hello, how are you doing?"),
+    ("ai", "I'm doing well, thanks!"),
+    ("human", "{user_input}"),
+])
+
+messages = template.format_messages(
+    name="Bob",
+    user_input="What is your name?"
 )
 ```
 
-To create a message template associated with a role, you use `MessagePromptTemplate`.
-
-For convenience, there is a `from_template` method exposed on the template. If you were to use this template, this is what it would look like:
+`ChatPromptTemplate.from_messages` accepts a variety of message representations.
 
+For example, in addition to using the 2-tuple representation of (type, content) used
+above, you could pass in an instance of `MessagePromptTemplate` or `BaseMessage`.
 
 ```python
-template="You are a helpful assistant that translates {input_language} to {output_language}."
-system_message_prompt = SystemMessagePromptTemplate.from_template(template)
-human_template="{text}"
-human_message_prompt = HumanMessagePromptTemplate.from_template(human_template)
-```
+from langchain.prompts import ChatPromptTemplate
+from langchain.prompts.chat import SystemMessage, HumanMessagePromptTemplate
 
-If you wanted to construct the `MessagePromptTemplate` more directly, you could create a PromptTemplate outside and then pass it in, eg:
-
-
-```python
-prompt=PromptTemplate(
-    template="You are a helpful assistant that translates {input_language} to {output_language}.",
-    input_variables=["input_language", "output_language"],
+template = ChatPromptTemplate.from_messages(
+    [
+        SystemMessage(
+            content=(
+                "You are a helpful assistant that re-writes the user's text to "
+                "sound more upbeat."
+            )
+        ),
+        HumanMessagePromptTemplate.from_template("{text}"),
+    ]
 )
-system_message_prompt_2 = SystemMessagePromptTemplate(prompt=prompt)
 
-assert system_message_prompt == system_message_prompt_2
-```
+from langchain.chat_models import ChatOpenAI
 
-After that, you can build a `ChatPromptTemplate` from one or more `MessagePromptTemplates`. You can use `ChatPromptTemplate`'s `format_prompt` -- this returns a `PromptValue`, which you can convert to a string or Message object, depending on whether you want to use the formatted value as input to an llm or chat model.
-
-
-```python
-chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])
-
-# get a chat completion from the formatted messages
-chat_prompt.format_prompt(input_language="English", output_language="French", text="I love programming.").to_messages()
+llm = ChatOpenAI()
+llm(template.format_messages(text='i dont like eating tasty things.'))
 ```
 
 <CodeOutputBlock lang="python">
-
 ```
-    [SystemMessage(content='You are a helpful assistant that translates English to French.', additional_kwargs={}),
-     HumanMessage(content='I love programming.', additional_kwargs={})]
+AIMessage(content='I absolutely adore indulging in delicious treats!', additional_kwargs={}, example=False)
 ```
-
 </CodeOutputBlock>
+
+This provides you with a lot of flexibility in how you construct your chat prompts.
+

libs/langchain/Makefile

@@ -92,15 +92,24 @@ spell_fix:
 ######################
 
 help:
-	@echo '----'
-	@echo 'coverage                     - run unit tests and generate coverage report'
+	@echo '===================='
+	@echo '-- DOCUMENTATION --'
+	@echo 'clean                        - run docs_clean and api_docs_clean'
 	@echo 'docs_build                   - build the documentation'
 	@echo 'docs_clean                   - clean the documentation build artifacts'
 	@echo 'docs_linkcheck               - run linkchecker on the documentation'
+	@echo 'api_docs_build               - build the API Reference documentation'
+	@echo 'api_docs_clean               - clean the API Reference documentation build artifacts'
+	@echo 'api_docs_linkcheck           - run linkchecker on the API Reference documentation'
+	@echo '-- LINTING --'
 	@echo 'format                       - run code formatters'
 	@echo 'lint                         - run linters'
+	@echo 'spell_check               	- run codespell on the project'
+	@echo 'spell_fix               		- run codespell on the project and fix the errors'
+	@echo '-- TESTS --'
+	@echo 'coverage                     - run unit tests and generate coverage report'
 	@echo 'test                         - run unit tests'
-	@echo 'tests                        - run unit tests'
+	@echo 'tests                        - run unit tests (alias for "make test")'
 	@echo 'test TEST_FILE=<test_file>   - run all tests in file'
 	@echo 'extended_tests               - run only extended unit tests'
 	@echo 'test_watch                   - run unit tests in watch mode'

libs/langchain/README.md

@@ -18,8 +18,8 @@
 
 Looking for the JS/TS version? Check out [LangChain.js](https://github.com/hwchase17/langchainjs).
 
-**Production Support:** As you move your LangChains into production, we'd love to offer more comprehensive support.
-Please fill out [this form](https://6w1pwbss0py.typeform.com/to/rrbrdTH2) and we'll set up a dedicated support Slack channel.
+**Production Support:** As you move your LangChains into production, we'd love to offer more hands-on support.
+Fill out [this form](https://airtable.com/appwQzlErAS2qiP0L/shrGtGaVBVAz7NcV2) to share more about what you're building, and our team will get in touch.
 
 ## Quick Install
 

libs/langchain/langchain/agents/__init__.py

@@ -1,4 +1,33 @@
-"""Interface for agents."""
+"""
+**Agent** is a class that uses an LLM to choose a sequence of actions to take.
+
+In Chains, a sequence of actions is hardcoded. In Agents,
+a language model is used as a reasoning engine to determine which actions
+to take and in which order.
+
+Agents select and use **Tools** and **Toolkits** for actions.
+
+**Class hierarchy:**
+
+.. code-block::
+
+    BaseSingleActionAgent --> LLMSingleActionAgent
+                              OpenAIFunctionsAgent
+                              XMLAgent
+                              Agent --> <name>Agent  # Examples: ZeroShotAgent, ChatAgent
+                                        
+
+    BaseMultiActionAgent  --> OpenAIMultiFunctionsAgent
+    
+    
+**Main helpers:**
+
+.. code-block::
+
+    AgentType, AgentExecutor, AgentOutputParser, AgentExecutorIterator,
+    AgentAction, AgentFinish
+    
+"""  # noqa: E501
 from langchain.agents.agent import (
     Agent,
     AgentExecutor,

libs/langchain/langchain/agents/agent_toolkits/__init__.py

@@ -3,6 +3,12 @@ from langchain.agents.agent_toolkits.amadeus.toolkit import AmadeusToolkit
 from langchain.agents.agent_toolkits.azure_cognitive_services import (
     AzureCognitiveServicesToolkit,
 )
+from langchain.agents.agent_toolkits.conversational_retrieval.openai_functions import (
+    create_conversational_retrieval_agent,
+)
+from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
+    create_retriever_tool,
+)
 from langchain.agents.agent_toolkits.csv.base import create_csv_agent
 from langchain.agents.agent_toolkits.file_management.toolkit import (
     FileManagementToolkit,
@@ -59,16 +65,18 @@ __all__ = [
     "ZapierToolkit",
     "create_csv_agent",
     "create_json_agent",
+    "create_multion_agent",
     "create_openapi_agent",
     "create_pandas_dataframe_agent",
     "create_pbi_agent",
     "create_pbi_chat_agent",
     "create_python_agent",
-    "create_multion_agent",
+    "create_retriever_tool",
     "create_spark_dataframe_agent",
     "create_spark_sql_agent",
     "create_sql_agent",
     "create_vectorstore_agent",
     "create_vectorstore_router_agent",
     "create_xorbits_agent",
+    "create_conversational_retrieval_agent",
 ]

libs/langchain/langchain/agents/agent_toolkits/conversational_retrieval/openai_functions.py

@@ -0,0 +1,87 @@
+from typing import Any, List, Optional
+
+from langchain.agents.agent import AgentExecutor
+from langchain.agents.openai_functions_agent.agent_token_buffer_memory import (
+    AgentTokenBufferMemory,
+)
+from langchain.agents.openai_functions_agent.base import OpenAIFunctionsAgent
+from langchain.chat_models.openai import ChatOpenAI
+from langchain.memory.token_buffer import ConversationTokenBufferMemory
+from langchain.prompts.chat import MessagesPlaceholder
+from langchain.schema.language_model import BaseLanguageModel
+from langchain.schema.memory import BaseMemory
+from langchain.schema.messages import SystemMessage
+from langchain.tools.base import BaseTool
+
+
+def _get_default_system_message() -> SystemMessage:
+    return SystemMessage(
+        content=(
+            "Do your best to answer the questions. "
+            "Feel free to use any tools available to look up "
+            "relevant information, only if necessary"
+        )
+    )
+
+
+def create_conversational_retrieval_agent(
+    llm: BaseLanguageModel,
+    tools: List[BaseTool],
+    remember_intermediate_steps: bool = True,
+    memory_key: str = "chat_history",
+    system_message: Optional[SystemMessage] = None,
+    verbose: bool = False,
+    max_token_limit: int = 2000,
+    **kwargs: Any
+) -> AgentExecutor:
+    """A convenience method for creating a conversational retrieval agent.
+
+    Args:
+        llm: The language model to use, should be ChatOpenAI
+        tools: A list of tools the agent has access to
+        remember_intermediate_steps: Whether the agent should remember intermediate
+            steps or not. Intermediate steps refer to prior action/observation
+            pairs from previous questions. The benefit of remembering these is if
+            there is relevant information in there, the agent can use it to answer
+            follow up questions. The downside is it will take up more tokens.
+        memory_key: The name of the memory key in the prompt.
+        system_message: The system message to use. By default, a basic one will
+            be used.
+        verbose: Whether or not the final AgentExecutor should be verbose or not,
+            defaults to False.
+        max_token_limit: The max number of tokens to keep around in memory.
+            Defaults to 2000.
+
+    Returns:
+        An agent executor initialized appropriately
+    """
+
+    if not isinstance(llm, ChatOpenAI):
+        raise ValueError("Only supported with ChatOpenAI models.")
+    if remember_intermediate_steps:
+        memory: BaseMemory = AgentTokenBufferMemory(
+            memory_key=memory_key, llm=llm, max_token_limit=max_token_limit
+        )
+    else:
+        memory = ConversationTokenBufferMemory(
+            memory_key=memory_key,
+            return_messages=True,
+            output_key="output",
+            llm=llm,
+            max_token_limit=max_token_limit,
+        )
+
+    _system_message = system_message or _get_default_system_message()
+    prompt = OpenAIFunctionsAgent.create_prompt(
+        system_message=_system_message,
+        extra_prompt_messages=[MessagesPlaceholder(variable_name=memory_key)],
+    )
+    agent = OpenAIFunctionsAgent(llm=llm, tools=tools, prompt=prompt)
+    return AgentExecutor(
+        agent=agent,
+        tools=tools,
+        memory=memory,
+        verbose=verbose,
+        return_intermediate_steps=remember_intermediate_steps,
+        **kwargs
+    )

libs/langchain/langchain/agents/agent_toolkits/conversational_retrieval/tool.py

@@ -0,0 +1,22 @@
+from langchain.schema import BaseRetriever
+from langchain.tools import Tool
+
+
+def create_retriever_tool(
+    retriever: BaseRetriever, name: str, description: str
+) -> Tool:
+    """Create a tool to do retrieval of documents.
+
+    Args:
+        retriever: The retriever to use for the retrieval
+        name: The name for the tool. This will be passed to the language model,
+            so should be unique and somewhat descriptive.
+        description: The description for the tool. This will be passed to the language
+            model, so should be descriptive.
+
+    Returns:
+        Tool class to pass to an agent
+    """
+    return Tool(
+        name=name, description=description, func=retriever.get_relevant_documents
+    )

libs/langchain/langchain/agents/agent_toolkits/csv/base.py

@@ -15,7 +15,7 @@ def create_csv_agent(
     try:
         import pandas as pd
     except ImportError:
-        raise ValueError(
+        raise ImportError(
             "pandas package not found, please install with `pip install pandas`"
         )
 

libs/langchain/langchain/agents/agent_toolkits/pandas/base.py

@@ -121,7 +121,7 @@ def _get_prompt_and_tools(
 
         pd.set_option("display.max_columns", None)
     except ImportError:
-        raise ValueError(
+        raise ImportError(
             "pandas package not found, please install with `pip install pandas`"
         )
 
@@ -232,7 +232,7 @@ def _get_functions_prompt_and_tools(
 
         pd.set_option("display.max_columns", None)
     except ImportError:
-        raise ValueError(
+        raise ImportError(
             "pandas package not found, please install with `pip install pandas`"
         )
     if input_variables is not None:

libs/langchain/langchain/agents/agent_toolkits/spark/base.py

@@ -46,7 +46,7 @@ def create_spark_dataframe_agent(
     """Construct a Spark agent from an LLM and dataframe."""
 
     if not _validate_spark_df(df) and not _validate_spark_connect_df(df):
-        raise ValueError("Spark is not installed. run `pip install pyspark`.")
+        raise ImportError("Spark is not installed. run `pip install pyspark`.")
 
     if input_variables is None:
         input_variables = ["df", "input", "agent_scratchpad"]

libs/langchain/langchain/agents/openai_functions_agent/agent_token_buffer_memory.py

@@ -0,0 +1,63 @@
+"""Memory used to save agent output AND intermediate steps."""
+from typing import Any, Dict, List
+
+from langchain.agents.openai_functions_agent.base import _format_intermediate_steps
+from langchain.memory.chat_memory import BaseChatMemory
+from langchain.schema.language_model import BaseLanguageModel
+from langchain.schema.messages import BaseMessage, get_buffer_string
+
+
+class AgentTokenBufferMemory(BaseChatMemory):
+    """Memory used to save agent output AND intermediate steps."""
+
+    human_prefix: str = "Human"
+    ai_prefix: str = "AI"
+    llm: BaseLanguageModel
+    memory_key: str = "history"
+    max_token_limit: int = 12000
+    """The max number of tokens to keep in the buffer. 
+    Once the buffer exceeds this many tokens, the oldest messages will be pruned."""
+    return_messages: bool = True
+    output_key = "output"
+    intermediate_steps_key = "intermediate_steps"
+
+    @property
+    def buffer(self) -> List[BaseMessage]:
+        """String buffer of memory."""
+        return self.chat_memory.messages
+
+    @property
+    def memory_variables(self) -> List[str]:
+        """Will always return list of memory variables.
+
+        :meta private:
+        """
+        return [self.memory_key]
+
+    def load_memory_variables(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        """Return history buffer."""
+        if self.return_messages:
+            final_buffer: Any = self.buffer
+        else:
+            final_buffer = get_buffer_string(
+                self.buffer,
+                human_prefix=self.human_prefix,
+                ai_prefix=self.ai_prefix,
+            )
+        return {self.memory_key: final_buffer}
+
+    def save_context(self, inputs: Dict[str, Any], outputs: Dict[str, Any]) -> None:
+        """Save context from this conversation to buffer. Pruned."""
+        input_str, output_str = self._get_input_output(inputs, outputs)
+        self.chat_memory.add_user_message(input_str)
+        steps = _format_intermediate_steps(outputs[self.intermediate_steps_key])
+        for msg in steps:
+            self.chat_memory.add_message(msg)
+        self.chat_memory.add_ai_message(output_str)
+        # Prune buffer if it exceeds max token limit
+        buffer = self.chat_memory.messages
+        curr_buffer_length = self.llm.get_num_tokens_from_messages(buffer)
+        if curr_buffer_length > self.max_token_limit:
+            while curr_buffer_length > self.max_token_limit:
+                buffer.pop(0)
+                curr_buffer_length = self.llm.get_num_tokens_from_messages(buffer)

libs/langchain/langchain/cache.py

@@ -1,4 +1,24 @@
-"""Beta Feature: base interface for cache."""
+"""
+.. warning::
+  Beta Feature!
+
+**Cache** provides an optional caching layer for LLMs.
+
+Cache is useful for two reasons:
+
+- It can save you money by reducing the number of API calls you make to the LLM
+  provider if you're often requesting the same completion multiple times.
+- It can speed up your application by reducing the number of API calls you make
+  to the LLM provider.
+
+Cache directly competes with Memory. See documentation for Pros and Cons.
+
+**Class hierarchy:**
+
+.. code-block::
+
+    BaseCache --> <name>Cache  # Examples: InMemoryCache, RedisCache, GPTCache
+"""
 from __future__ import annotations
 
 import hashlib

libs/langchain/langchain/callbacks/__init__.py

@@ -1,4 +1,11 @@
-"""Callback handlers that allow listening to events in LangChain."""
+"""**Callback handlers** allow listening to events in LangChain.
+
+**Class hierarchy:**
+
+.. code-block::
+
+    BaseCallbackHandler --> <name>CallbackHandler  # Example: AimCallbackHandler
+"""
 
 from langchain.callbacks.aim_callback import AimCallbackHandler
 from langchain.callbacks.argilla_callback import ArgillaCallbackHandler
@@ -20,6 +27,7 @@ from langchain.callbacks.manager import (
 from langchain.callbacks.mlflow_callback import MlflowCallbackHandler
 from langchain.callbacks.openai_info import OpenAICallbackHandler
 from langchain.callbacks.promptlayer_callback import PromptLayerCallbackHandler
+from langchain.callbacks.sagemaker_callback import SageMakerCallbackHandler
 from langchain.callbacks.stdout import StdOutCallbackHandler
 from langchain.callbacks.streaming_aiter import AsyncIteratorCallbackHandler
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
@@ -59,4 +67,5 @@ __all__ = [
     "tracing_v2_enabled",
     "wandb_tracing_enabled",
     "FlyteCallbackHandler",
+    "SageMakerCallbackHandler",
 ]

libs/langchain/langchain/callbacks/manager.py

@@ -730,7 +730,7 @@ class CallbackManagerForChainRun(ParentRunManager, ChainManagerMixin):
 
     def on_chain_error(
         self,
-        error: Union[Exception, KeyboardInterrupt],
+        error: BaseException,
         **kwargs: Any,
     ) -> None:
         """Run when chain errors.
@@ -812,7 +812,7 @@ class AsyncCallbackManagerForChainRun(AsyncParentRunManager, ChainManagerMixin):
 
     async def on_chain_error(
         self,
-        error: Union[Exception, KeyboardInterrupt],
+        error: BaseException,
         **kwargs: Any,
     ) -> None:
         """Run when chain errors.

libs/langchain/langchain/callbacks/sagemaker_callback.py

@@ -0,0 +1,280 @@
+import json
+import os
+import shutil
+import tempfile
+from copy import deepcopy
+from typing import Any, Dict, List, Optional, Union
+
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.callbacks.utils import (
+    flatten_dict,
+)
+from langchain.schema import AgentAction, AgentFinish, LLMResult
+
+
+def save_json(data: dict, file_path: str) -> None:
+    """Save dict to local file path.
+
+    Parameters:
+        data (dict): The dictionary to be saved.
+        file_path (str): Local file path.
+    """
+    with open(file_path, "w") as outfile:
+        json.dump(data, outfile)
+
+
+class SageMakerCallbackHandler(BaseCallbackHandler):
+    """Callback Handler that logs prompt artifacts and metrics to SageMaker Experiments.
+
+    Parameters:
+        run (sagemaker.experiments.run.Run): Run object where the experiment is logged.
+    """
+
+    def __init__(self, run: Any) -> None:
+        """Initialize callback handler."""
+        super().__init__()
+
+        self.run = run
+
+        self.metrics = {
+            "step": 0,
+            "starts": 0,
+            "ends": 0,
+            "errors": 0,
+            "text_ctr": 0,
+            "chain_starts": 0,
+            "chain_ends": 0,
+            "llm_starts": 0,
+            "llm_ends": 0,
+            "llm_streams": 0,
+            "tool_starts": 0,
+            "tool_ends": 0,
+            "agent_ends": 0,
+        }
+
+        # Create a temporary directory
+        self.temp_dir = tempfile.mkdtemp()
+
+    def _reset(self) -> None:
+        for k, v in self.metrics.items():
+            self.metrics[k] = 0
+
+    def on_llm_start(
+        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
+    ) -> None:
+        """Run when LLM starts."""
+        self.metrics["step"] += 1
+        self.metrics["llm_starts"] += 1
+        self.metrics["starts"] += 1
+
+        llm_starts = self.metrics["llm_starts"]
+
+        resp: Dict[str, Any] = {}
+        resp.update({"action": "on_llm_start"})
+        resp.update(flatten_dict(serialized))
+        resp.update(self.metrics)
+
+        for idx, prompt in enumerate(prompts):
+            prompt_resp = deepcopy(resp)
+            prompt_resp["prompt"] = prompt
+            self.jsonf(
+                prompt_resp,
+                self.temp_dir,
+                f"llm_start_{llm_starts}_prompt_{idx}",
+            )
+
+    def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
+        """Run when LLM generates a new token."""
+        self.metrics["step"] += 1
+        self.metrics["llm_streams"] += 1
+
+        llm_streams = self.metrics["llm_streams"]
+
+        resp: Dict[str, Any] = {}
+        resp.update({"action": "on_llm_new_token", "token": token})
+        resp.update(self.metrics)
+
+        self.jsonf(resp, self.temp_dir, f"llm_new_tokens_{llm_streams}")
+
+    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        """Run when LLM ends running."""
+        self.metrics["step"] += 1
+        self.metrics["llm_ends"] += 1
+        self.metrics["ends"] += 1
+
+        llm_ends = self.metrics["llm_ends"]
+
+        resp: Dict[str, Any] = {}
+        resp.update({"action": "on_llm_end"})
+        resp.update(flatten_dict(response.llm_output or {}))
+
+        resp.update(self.metrics)
+
+        for generations in response.generations:
+            for idx, generation in enumerate(generations):
+                generation_resp = deepcopy(resp)
+                generation_resp.update(flatten_dict(generation.dict()))
+
+                self.jsonf(
+                    resp,
+                    self.temp_dir,
+                    f"llm_end_{llm_ends}_generation_{idx}",
+                )
+
+    def on_llm_error(
+        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
+    ) -> None:
+        """Run when LLM errors."""
+        self.metrics["step"] += 1
+        self.metrics["errors"] += 1
+
+    def on_chain_start(
+        self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any
+    ) -> None:
+        """Run when chain starts running."""
+        self.metrics["step"] += 1
+        self.metrics["chain_starts"] += 1
+        self.metrics["starts"] += 1
+
+        chain_starts = self.metrics["chain_starts"]
+
+        resp: Dict[str, Any] = {}
+        resp.update({"action": "on_chain_start"})
+        resp.update(flatten_dict(serialized))
+        resp.update(self.metrics)
+
+        chain_input = ",".join([f"{k}={v}" for k, v in inputs.items()])
+        input_resp = deepcopy(resp)
+        input_resp["inputs"] = chain_input
+
+        self.jsonf(input_resp, self.temp_dir, f"chain_start_{chain_starts}")
+
+    def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
+        """Run when chain ends running."""
+        self.metrics["step"] += 1
+        self.metrics["chain_ends"] += 1
+        self.metrics["ends"] += 1
+
+        chain_ends = self.metrics["chain_ends"]
+
+        resp: Dict[str, Any] = {}
+        chain_output = ",".join([f"{k}={v}" for k, v in outputs.items()])
+        resp.update({"action": "on_chain_end", "outputs": chain_output})
+        resp.update(self.metrics)
+
+        self.jsonf(resp, self.temp_dir, f"chain_end_{chain_ends}")
+
+    def on_chain_error(
+        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
+    ) -> None:
+        """Run when chain errors."""
+        self.metrics["step"] += 1
+        self.metrics["errors"] += 1
+
+    def on_tool_start(
+        self, serialized: Dict[str, Any], input_str: str, **kwargs: Any
+    ) -> None:
+        """Run when tool starts running."""
+        self.metrics["step"] += 1
+        self.metrics["tool_starts"] += 1
+        self.metrics["starts"] += 1
+
+        tool_starts = self.metrics["tool_starts"]
+
+        resp: Dict[str, Any] = {}
+        resp.update({"action": "on_tool_start", "input_str": input_str})
+        resp.update(flatten_dict(serialized))
+        resp.update(self.metrics)
+
+        self.jsonf(resp, self.temp_dir, f"tool_start_{tool_starts}")
+
+    def on_tool_end(self, output: str, **kwargs: Any) -> None:
+        """Run when tool ends running."""
+        self.metrics["step"] += 1
+        self.metrics["tool_ends"] += 1
+        self.metrics["ends"] += 1
+
+        tool_ends = self.metrics["tool_ends"]
+
+        resp: Dict[str, Any] = {}
+        resp.update({"action": "on_tool_end", "output": output})
+        resp.update(self.metrics)
+
+        self.jsonf(resp, self.temp_dir, f"tool_end_{tool_ends}")
+
+    def on_tool_error(
+        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
+    ) -> None:
+        """Run when tool errors."""
+        self.metrics["step"] += 1
+        self.metrics["errors"] += 1
+
+    def on_text(self, text: str, **kwargs: Any) -> None:
+        """
+        Run when agent is ending.
+        """
+        self.metrics["step"] += 1
+        self.metrics["text_ctr"] += 1
+
+        text_ctr = self.metrics["text_ctr"]
+
+        resp: Dict[str, Any] = {}
+        resp.update({"action": "on_text", "text": text})
+        resp.update(self.metrics)
+
+        self.jsonf(resp, self.temp_dir, f"on_text_{text_ctr}")
+
+    def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> None:
+        """Run when agent ends running."""
+        self.metrics["step"] += 1
+        self.metrics["agent_ends"] += 1
+        self.metrics["ends"] += 1
+
+        agent_ends = self.metrics["agent_ends"]
+        resp: Dict[str, Any] = {}
+        resp.update(
+            {
+                "action": "on_agent_finish",
+                "output": finish.return_values["output"],
+                "log": finish.log,
+            }
+        )
+        resp.update(self.metrics)
+
+        self.jsonf(resp, self.temp_dir, f"agent_finish_{agent_ends}")
+
+    def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any:
+        """Run on agent action."""
+        self.metrics["step"] += 1
+        self.metrics["tool_starts"] += 1
+        self.metrics["starts"] += 1
+
+        tool_starts = self.metrics["tool_starts"]
+        resp: Dict[str, Any] = {}
+        resp.update(
+            {
+                "action": "on_agent_action",
+                "tool": action.tool,
+                "tool_input": action.tool_input,
+                "log": action.log,
+            }
+        )
+        resp.update(self.metrics)
+        self.jsonf(resp, self.temp_dir, f"agent_action_{tool_starts}")
+
+    def jsonf(
+        self,
+        data: Dict[str, Any],
+        data_dir: str,
+        filename: str,
+        is_output: Optional[bool] = True,
+    ) -> None:
+        """To log the input data as json file artifact."""
+        file_path = os.path.join(data_dir, f"{filename}.json")
+        save_json(data, file_path)
+        self.run.log_file(file_path, name=filename, is_output=is_output)
+
+    def flush_tracker(self) -> None:
+        """Reset the steps and delete the temporary local directory."""
+        self._reset()
+        shutil.rmtree(self.temp_dir)

libs/langchain/langchain/callbacks/tracers/base.py

@@ -227,6 +227,7 @@ class BaseTracer(BaseCallbackHandler, ABC):
         tags: Optional[List[str]] = None,
         parent_run_id: Optional[UUID] = None,
         metadata: Optional[Dict[str, Any]] = None,
+        run_type: Optional[str] = None,
         **kwargs: Any,
     ) -> None:
         """Start a trace for a chain run."""
@@ -246,7 +247,7 @@ class BaseTracer(BaseCallbackHandler, ABC):
             execution_order=execution_order,
             child_execution_order=execution_order,
             child_runs=[],
-            run_type="chain",
+            run_type=run_type or "chain",
             tags=tags or [],
         )
         self._start_trace(chain_run)
@@ -259,7 +260,7 @@ class BaseTracer(BaseCallbackHandler, ABC):
         if not run_id:
             raise TracerException("No run_id provided for on_chain_end callback.")
         chain_run = self.run_map.get(str(run_id))
-        if chain_run is None or chain_run.run_type != "chain":
+        if chain_run is None:
             raise TracerException("No chain Run found to be traced")
 
         chain_run.outputs = outputs
@@ -279,7 +280,7 @@ class BaseTracer(BaseCallbackHandler, ABC):
         if not run_id:
             raise TracerException("No run_id provided for on_chain_error callback.")
         chain_run = self.run_map.get(str(run_id))
-        if chain_run is None or chain_run.run_type != "chain":
+        if chain_run is None:
             raise TracerException("No chain Run found to be traced")
 
         chain_run.error = repr(error)

libs/langchain/langchain/chains/__init__.py

@@ -1,16 +1,21 @@
-"""Chains are easily reusable components which can be linked together.
+"""**Chains** are easily reusable components linked together.
 
-    Chains should be used to encode a sequence of calls to components like
-    models, document retrievers, other chains, etc., and provide a simple interface
-    to this sequence.
+Chains encode a sequence of calls to components like models, document retrievers,
+other Chains, etc., and provide a simple interface to this sequence.
 
-    The Chain interface makes it easy to create apps that are:
-        - Stateful: add Memory to any Chain to give it state,
-        - Observable: pass Callbacks to a Chain to execute additional functionality,
-            like logging, outside the main sequence of component calls,
-        - Composable: the Chain API is flexible enough that it is easy to combine
-            Chains with other components, including other Chains.
-    """
+The Chain interface makes it easy to create apps that are:
+
+    - **Stateful:** add Memory to any Chain to give it state,
+    - **Observable:** pass Callbacks to a Chain to execute additional functionality,
+      like logging, outside the main sequence of component calls,
+    - **Composable:** combine Chains with other components, including other Chains.
+
+**Class hierarchy:**
+
+.. code-block::
+
+    Chain --> <name>Chain  # Examples: LLMChain, MapReduceChain, RouterChain
+"""
 
 from langchain.chains.api.base import APIChain
 from langchain.chains.api.openapi.chain import OpenAPIEndpointChain

libs/langchain/langchain/chains/combine_documents/refine.py

@@ -53,7 +53,7 @@ class RefineDocumentsChain(BaseCombineDocumentsChain):
             prompt = PromptTemplate.from_template(
                 "Summarize this content: {context}"
             )
-            llm_chain = LLMChain(llm=llm, prompt=prompt)
+            initial_llm_chain = LLMChain(llm=llm, prompt=prompt)
             initial_response_name = "prev_response"
             # The prompt here should take as an input variable the
             # `document_variable_name` as well as `initial_response_name`
@@ -61,7 +61,7 @@ class RefineDocumentsChain(BaseCombineDocumentsChain):
                 "Here's your first summary: {prev_response}. "
                 "Now add to it based on the following context: {context}"
             )
-            llm_chain_refine = LLMChain(llm=llm, prompt=prompt_refine)
+            refine_llm_chain = LLMChain(llm=llm, prompt=prompt_refine)
             chain = RefineDocumentsChain(
                 initial_llm_chain=initial_llm_chain,
                 refine_llm_chain=refine_llm_chain,

libs/langchain/langchain/chains/loading.py

@@ -114,17 +114,55 @@ def _load_map_reduce_documents_chain(
     if not isinstance(llm_chain, LLMChain):
         raise ValueError(f"Expected LLMChain, got {llm_chain}")
 
-    if "combine_document_chain" in config:
+    if "reduce_documents_chain" in config:
+        reduce_documents_chain = load_chain_from_config(
+            config.pop("reduce_documents_chain")
+        )
+    elif "reduce_documents_chain_path" in config:
+        reduce_documents_chain = load_chain(config.pop("reduce_documents_chain_path"))
+    else:
+        reduce_documents_chain = _load_reduce_documents_chain(config)
+
+    return MapReduceDocumentsChain(
+        llm_chain=llm_chain,
+        reduce_documents_chain=reduce_documents_chain,
+        **config,
+    )
+
+
+def _load_reduce_documents_chain(config: dict, **kwargs: Any) -> ReduceDocumentsChain:
+    combine_documents_chain = None
+    collapse_documents_chain = None
+
+    if "combine_documents_chain" in config:
+        combine_document_chain_config = config.pop("combine_documents_chain")
+        combine_documents_chain = load_chain_from_config(combine_document_chain_config)
+    elif "combine_document_chain" in config:
         combine_document_chain_config = config.pop("combine_document_chain")
         combine_documents_chain = load_chain_from_config(combine_document_chain_config)
+    elif "combine_documents_chain_path" in config:
+        combine_documents_chain = load_chain(config.pop("combine_documents_chain_path"))
     elif "combine_document_chain_path" in config:
         combine_documents_chain = load_chain(config.pop("combine_document_chain_path"))
     else:
         raise ValueError(
-            "One of `combine_document_chain` or "
-            "`combine_document_chain_path` must be present."
+            "One of `combine_documents_chain` or "
+            "`combine_documents_chain_path` must be present."
         )
-    if "collapse_document_chain" in config:
+
+    if "collapse_documents_chain" in config:
+        collapse_document_chain_config = config.pop("collapse_documents_chain")
+        if collapse_document_chain_config is None:
+            collapse_documents_chain = None
+        else:
+            collapse_documents_chain = load_chain_from_config(
+                collapse_document_chain_config
+            )
+    elif "collapse_documents_chain_path" in config:
+        collapse_documents_chain = load_chain(
+            config.pop("collapse_documents_chain_path")
+        )
+    elif "collapse_document_chain" in config:
         collapse_document_chain_config = config.pop("collapse_document_chain")
         if collapse_document_chain_config is None:
             collapse_documents_chain = None
@@ -136,15 +174,10 @@ def _load_map_reduce_documents_chain(
         collapse_documents_chain = load_chain(
             config.pop("collapse_document_chain_path")
         )
-    else:
-        collapse_documents_chain = None
-    reduce_documents_chain = ReduceDocumentsChain(
+
+    return ReduceDocumentsChain(
         combine_documents_chain=combine_documents_chain,
         collapse_documents_chain=collapse_documents_chain,
-    )
-    return MapReduceDocumentsChain(
-        llm_chain=llm_chain,
-        reduce_documents_chain=reduce_documents_chain,
         **config,
     )
 
@@ -497,6 +530,7 @@ type_to_loader_dict = {
     "qa_with_sources_chain": _load_qa_with_sources_chain,
     "stuff_documents_chain": _load_stuff_documents_chain,
     "map_reduce_documents_chain": _load_map_reduce_documents_chain,
+    "reduce_documents_chain": _load_reduce_documents_chain,
     "map_rerank_documents_chain": _load_map_rerank_documents_chain,
     "refine_documents_chain": _load_refine_documents_chain,
     "sql_database_chain": _load_sql_database_chain,

libs/langchain/langchain/chains/retrieval_qa/base.py

@@ -11,6 +11,7 @@ from pydantic import Extra, Field, root_validator
 from langchain.callbacks.manager import (
     AsyncCallbackManagerForChainRun,
     CallbackManagerForChainRun,
+    Callbacks,
 )
 from langchain.chains.base import Chain
 from langchain.chains.combine_documents.base import BaseCombineDocumentsChain
@@ -65,11 +66,12 @@ class BaseRetrievalQA(Chain):
         cls,
         llm: BaseLanguageModel,
         prompt: Optional[PromptTemplate] = None,
+        callbacks: Callbacks = None,
         **kwargs: Any,
     ) -> BaseRetrievalQA:
         """Initialize from LLM."""
         _prompt = prompt or PROMPT_SELECTOR.get_prompt(llm)
-        llm_chain = LLMChain(llm=llm, prompt=_prompt)
+        llm_chain = LLMChain(llm=llm, prompt=_prompt, callbacks=callbacks)
         document_prompt = PromptTemplate(
             input_variables=["page_content"], template="Context:\n{page_content}"
         )
@@ -77,9 +79,14 @@ class BaseRetrievalQA(Chain):
             llm_chain=llm_chain,
             document_variable_name="context",
             document_prompt=document_prompt,
+            callbacks=callbacks,
         )
 
-        return cls(combine_documents_chain=combine_documents_chain, **kwargs)
+        return cls(
+            combine_documents_chain=combine_documents_chain,
+            callbacks=callbacks,
+            **kwargs,
+        )
 
     @classmethod
     def from_chain_type(

libs/langchain/langchain/chains/router/multi_prompt_prompt.py

@@ -27,5 +27,5 @@ modifications are needed.
 << INPUT >>
 {{input}}
 
-<< OUTPUT >>
+<< OUTPUT (must include ```json at the start of the response) >>
 """

libs/langchain/langchain/chains/summarize/__init__.py

@@ -1,6 +1,7 @@
 """Load summarizing chains."""
 from typing import Any, Mapping, Optional, Protocol
 
+from langchain.callbacks.manager import Callbacks
 from langchain.chains.combine_documents.base import BaseCombineDocumentsChain
 from langchain.chains.combine_documents.map_reduce import MapReduceDocumentsChain
 from langchain.chains.combine_documents.reduce import ReduceDocumentsChain
@@ -49,16 +50,22 @@ def _load_map_reduce_chain(
     collapse_llm: Optional[BaseLanguageModel] = None,
     verbose: Optional[bool] = None,
     token_max: int = 3000,
+    callbacks: Callbacks = None,
     **kwargs: Any,
 ) -> MapReduceDocumentsChain:
-    map_chain = LLMChain(llm=llm, prompt=map_prompt, verbose=verbose)
+    map_chain = LLMChain(
+        llm=llm, prompt=map_prompt, verbose=verbose, callbacks=callbacks
+    )
     _reduce_llm = reduce_llm or llm
-    reduce_chain = LLMChain(llm=_reduce_llm, prompt=combine_prompt, verbose=verbose)
+    reduce_chain = LLMChain(
+        llm=_reduce_llm, prompt=combine_prompt, verbose=verbose, callbacks=callbacks
+    )
     # TODO: document prompt
     combine_documents_chain = StuffDocumentsChain(
         llm_chain=reduce_chain,
         document_variable_name=combine_document_variable_name,
         verbose=verbose,
+        callbacks=callbacks,
     )
     if collapse_prompt is None:
         collapse_chain = None
@@ -74,6 +81,7 @@ def _load_map_reduce_chain(
                 llm=_collapse_llm,
                 prompt=collapse_prompt,
                 verbose=verbose,
+                callbacks=callbacks,
             ),
             document_variable_name=combine_document_variable_name,
         )
@@ -82,12 +90,14 @@ def _load_map_reduce_chain(
         collapse_documents_chain=collapse_chain,
         token_max=token_max,
         verbose=verbose,
+        callbacks=callbacks,
     )
     return MapReduceDocumentsChain(
         llm_chain=map_chain,
         reduce_documents_chain=reduce_documents_chain,
         document_variable_name=map_reduce_document_variable_name,
         verbose=verbose,
+        callbacks=callbacks,
         **kwargs,
     )
 

libs/langchain/langchain/chat_models/__init__.py

@@ -1,3 +1,22 @@
+"""**Chat Models** are a variation on language models.
+
+While Chat Models use language models under the hood, the interface they expose
+is a bit different. Rather than expose a "text in, text out" API, they expose
+an interface where "chat messages" are the inputs and outputs.
+
+**Class hierarchy:**
+
+.. code-block::
+
+    BaseLanguageModel --> BaseChatModel --> <name>  # Examples: ChatOpenAI, ChatGooglePalm
+
+**Main helpers:**
+
+.. code-block::
+
+    AIMessage, BaseMessage, HumanMessage
+"""  # noqa: E501
+
 from langchain.chat_models.anthropic import ChatAnthropic
 from langchain.chat_models.azure_openai import AzureChatOpenAI
 from langchain.chat_models.fake import FakeListChatModel

libs/langchain/langchain/chat_models/azure_openai.py

@@ -45,7 +45,7 @@ class AzureChatOpenAI(ChatOpenAI):
     """
 
     deployment_name: str = ""
-    openai_api_type: str = "azure"
+    openai_api_type: str = ""
     openai_api_base: str = ""
     openai_api_version: str = ""
     openai_api_key: str = ""
@@ -71,9 +71,7 @@ class AzureChatOpenAI(ChatOpenAI):
             "OPENAI_API_VERSION",
         )
         values["openai_api_type"] = get_from_dict_or_env(
-            values,
-            "openai_api_type",
-            "OPENAI_API_TYPE",
+            values, "openai_api_type", "OPENAI_API_TYPE", default="azure"
         )
         values["openai_organization"] = get_from_dict_or_env(
             values,

libs/langchain/langchain/chat_models/openai.py

@@ -570,7 +570,9 @@ class ChatOpenAI(BaseChatModel):
         for message in messages_dict:
             num_tokens += tokens_per_message
             for key, value in message.items():
-                num_tokens += len(encoding.encode(value))
+                # Cast str(value) in case the message value is not a string
+                # This occurs with function messages
+                num_tokens += len(encoding.encode(str(value)))
                 if key == "name":
                     num_tokens += tokens_per_name
         # every reply is primed with <im_start>assistant

libs/langchain/langchain/chat_models/vertexai.py

@@ -111,7 +111,7 @@ class ChatVertexAI(_VertexAICommon, BaseChatModel):
 
                 values["client"] = ChatModel.from_pretrained(values["model_name"])
         except ImportError:
-            raise_vertex_import_error()
+            raise_vertex_import_error(minimum_expected_version="1.28.0")
         return values
 
     def _generate(

libs/langchain/langchain/docstore/__init__.py

@@ -1,4 +1,19 @@
-"""Wrappers on top of docstores."""
+"""**Docstores** are classes to store and load Documents.
+
+The **Docstore** is a simplified version of the Document Loader.
+
+**Class hierarchy:**
+
+.. code-block::
+
+    Docstore --> <name> # Examples: InMemoryDocstore, Wikipedia
+
+**Main helpers:**
+
+.. code-block::
+
+    Document, AddableMixin
+"""
 from langchain.docstore.arbitrary_fn import DocstoreFn
 from langchain.docstore.in_memory import InMemoryDocstore
 from langchain.docstore.wikipedia import Wikipedia

libs/langchain/langchain/document_loaders/__init__.py

@@ -1,4 +1,19 @@
-"""All different types of document loaders."""
+"""**Document Loaders**  are classes to load Documents.
+
+**Document Loaders** are usually used to load a lot of Documents in a single run.
+
+**Class hierarchy:**
+
+.. code-block::
+
+    BaseLoader --> <name>Loader  # Examples: TextLoader, UnstructuredFileLoader
+
+**Main helpers:**
+
+.. code-block::
+
+    Document, <name>TextSplitter
+"""
 
 from langchain.document_loaders.acreom import AcreomLoader
 from langchain.document_loaders.airbyte_json import AirbyteJSONLoader
@@ -80,9 +95,12 @@ from langchain.document_loaders.mediawikidump import MWDumpLoader
 from langchain.document_loaders.merge import MergedDataLoader
 from langchain.document_loaders.mhtml import MHTMLLoader
 from langchain.document_loaders.modern_treasury import ModernTreasuryLoader
+from langchain.document_loaders.news import NewsURLLoader
 from langchain.document_loaders.notebook import NotebookLoader
 from langchain.document_loaders.notion import NotionDirectoryLoader
 from langchain.document_loaders.notiondb import NotionDBLoader
+from langchain.document_loaders.obs_directory import OBSDirectoryLoader
+from langchain.document_loaders.obs_file import OBSFileLoader
 from langchain.document_loaders.obsidian import ObsidianLoader
 from langchain.document_loaders.odt import UnstructuredODTLoader
 from langchain.document_loaders.onedrive import OneDriveLoader
@@ -90,6 +108,7 @@ from langchain.document_loaders.onedrive_file import OneDriveFileLoader
 from langchain.document_loaders.open_city_data import OpenCityDataLoader
 from langchain.document_loaders.org_mode import UnstructuredOrgModeLoader
 from langchain.document_loaders.pdf import (
+    AmazonTextractPDFLoader,
     MathpixPDFLoader,
     OnlinePDFLoader,
     PDFMinerLoader,
@@ -110,6 +129,7 @@ from langchain.document_loaders.recursive_url_loader import RecursiveUrlLoader
 from langchain.document_loaders.reddit import RedditPostsLoader
 from langchain.document_loaders.roam import RoamLoader
 from langchain.document_loaders.rocksetdb import RocksetLoader
+from langchain.document_loaders.rss import RSSFeedLoader
 from langchain.document_loaders.rst import UnstructuredRSTLoader
 from langchain.document_loaders.rtf import UnstructuredRTFLoader
 from langchain.document_loaders.s3_directory import S3DirectoryLoader
@@ -233,9 +253,12 @@ __all__ = [
     "MergedDataLoader",
     "MHTMLLoader",
     "ModernTreasuryLoader",
+    "NewsURLLoader",
     "NotebookLoader",
     "NotionDBLoader",
     "NotionDirectoryLoader",
+    "OBSDirectoryLoader",
+    "OBSFileLoader",
     "ObsidianLoader",
     "OneDriveFileLoader",
     "OneDriveLoader",
@@ -259,6 +282,7 @@ __all__ = [
     "RedditPostsLoader",
     "RoamLoader",
     "RocksetLoader",
+    "RSSFeedLoader",
     "S3DirectoryLoader",
     "S3FileLoader",
     "SRTLoader",
@@ -307,4 +331,5 @@ __all__ = [
     "YoutubeAudioLoader",
     "YoutubeLoader",
     "ConcurrentLoader",
+    "AmazonTextractPDFLoader",
 ]

libs/langchain/langchain/document_loaders/bilibili.py

@@ -54,12 +54,14 @@ class BiliBiliLoader(BaseLoader):
 
         video_info = sync(v.get_info())
         video_info.update({"url": url})
+        sub = sync(v.get_subtitle(video_info["cid"]))
 
         # Get subtitle url
-        subtitle = video_info.pop("subtitle")
-        sub_list = subtitle["list"]
+        sub_list = sub["subtitles"]
         if sub_list:
             sub_url = sub_list[0]["subtitle_url"]
+            if not sub_url.startswith("http"):
+                sub_url = "https:" + sub_url
             result = requests.get(sub_url)
             raw_sub_titles = json.loads(result.content)["body"]
             raw_transcript = " ".join([c["content"] for c in raw_sub_titles])

libs/langchain/langchain/document_loaders/blob_loaders/youtube_audio.py

@@ -21,7 +21,7 @@ class YoutubeAudioLoader(BlobLoader):
         try:
             import yt_dlp
         except ImportError:
-            raise ValueError(
+            raise ImportError(
                 "yt_dlp package not found, please install it with "
                 "`pip install yt_dlp`"
             )

libs/langchain/langchain/document_loaders/directory.py

@@ -121,7 +121,11 @@ class DirectoryLoader(BaseLoader):
                 if self.silent_errors:
                     logger.warning(e)
                 else:
-                    raise e
+                    raise ImportError(
+                        "To log the progress of DirectoryLoader "
+                        "you need to install tqdm, "
+                        "`pip install tqdm`"
+                    )
 
         if self.use_multithreading:
             with concurrent.futures.ThreadPoolExecutor(

libs/langchain/langchain/document_loaders/evernote.py

@@ -74,12 +74,11 @@ class EverNoteLoader(BaseLoader):
 
             return html2text.html2text(content).strip()
         except ImportError as e:
-            logger.error(
+            raise ImportError(
                 "Could not import `html2text`. Although it is not a required package "
                 "to use Langchain, using the EverNote loader requires `html2text`. "
                 "Please install `html2text` via `pip install html2text` and try again."
-            )
-            raise e
+            ) from e
 
     @staticmethod
     def _parse_resource(resource: list) -> dict:

libs/langchain/langchain/document_loaders/geodataframe.py

@@ -20,7 +20,7 @@ class GeoDataFrameLoader(BaseLoader):
         try:
             import geopandas as gpd
         except ImportError:
-            raise ValueError(
+            raise ImportError(
                 "geopandas package not found, please install it with "
                 "`pip install geopandas`"
             )

libs/langchain/langchain/document_loaders/news.py

@@ -0,0 +1,124 @@
+"""Loader that uses unstructured to load HTML files."""
+import logging
+from typing import Any, Iterator, List
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+
+logger = logging.getLogger(__name__)
+
+
+class NewsURLLoader(BaseLoader):
+    """Loader that uses newspaper to load news articles from URLs.
+
+    Args:
+        urls: URLs to load. Each is loaded into its own document.
+        text_mode: If True, extract text from URL and use that for page content.
+            Otherwise, extract raw HTML.
+        nlp: If True, perform NLP on the extracted contents, like providing a summary
+            and extracting keywords.
+        continue_on_failure: If True, continue loading documents even if
+            loading fails for a particular URL.
+        show_progress_bar: If True, use tqdm to show a loading progress bar. Requires
+            tqdm to be installed, ``pip install tqdm``.
+        **newspaper_kwargs: Any additional named arguments to pass to
+            newspaper.Article().
+
+    Example:
+        .. code-block:: python
+
+            from langchain.document_loaders import NewsURLLoader
+
+            loader = NewsURLLoader(
+                urls=["<url-1>", "<url-2>"],
+            )
+            docs = loader.load()
+
+    Newspaper reference:
+        https://newspaper.readthedocs.io/en/latest/
+    """
+
+    def __init__(
+        self,
+        urls: List[str],
+        text_mode: bool = True,
+        nlp: bool = False,
+        continue_on_failure: bool = True,
+        show_progress_bar: bool = False,
+        **newspaper_kwargs: Any,
+    ) -> None:
+        """Initialize with file path."""
+        try:
+            import newspaper  # noqa:F401
+
+            self.__version = newspaper.__version__
+        except ImportError:
+            raise ImportError(
+                "newspaper package not found, please install it with "
+                "`pip install newspaper3k`"
+            )
+
+        self.urls = urls
+        self.text_mode = text_mode
+        self.nlp = nlp
+        self.continue_on_failure = continue_on_failure
+        self.newspaper_kwargs = newspaper_kwargs
+        self.show_progress_bar = show_progress_bar
+
+    def load(self) -> List[Document]:
+        iter = self.lazy_load()
+        if self.show_progress_bar:
+            try:
+                from tqdm import tqdm
+            except ImportError as e:
+                raise ImportError(
+                    "Package tqdm must be installed if show_progress_bar=True. "
+                    "Please install with 'pip install tqdm' or set "
+                    "show_progress_bar=False."
+                ) from e
+            iter = tqdm(iter)
+        return list(iter)
+
+    def lazy_load(self) -> Iterator[Document]:
+        try:
+            from newspaper import Article
+        except ImportError as e:
+            raise ImportError(
+                "Cannot import newspaper, please install with `pip install newspaper3k`"
+            ) from e
+
+        for url in self.urls:
+            try:
+                article = Article(url, **self.newspaper_kwargs)
+                article.download()
+                article.parse()
+
+                if self.nlp:
+                    article.nlp()
+
+            except Exception as e:
+                if self.continue_on_failure:
+                    logger.error(f"Error fetching or processing {url}, exception: {e}")
+                    continue
+                else:
+                    raise e
+
+            metadata = {
+                "title": getattr(article, "title", ""),
+                "link": getattr(article, "url", getattr(article, "canonical_link", "")),
+                "authors": getattr(article, "authors", []),
+                "language": getattr(article, "meta_lang", ""),
+                "description": getattr(article, "meta_description", ""),
+                "publish_date": getattr(article, "publish_date", ""),
+            }
+
+            if self.text_mode:
+                content = article.text
+            else:
+                content = article.html
+
+            if self.nlp:
+                metadata["keywords"] = getattr(article, "keywords", [])
+                metadata["summary"] = getattr(article, "summary", "")
+
+            yield Document(page_content=content, metadata=metadata)

libs/langchain/langchain/document_loaders/obs_directory.py

@@ -0,0 +1,82 @@
+# coding:utf-8
+from typing import List, Optional
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+from langchain.document_loaders.obs_file import OBSFileLoader
+
+
+class OBSDirectoryLoader(BaseLoader):
+    """Loading logic for loading documents from Huawei OBS."""
+
+    def __init__(
+        self,
+        bucket: str,
+        endpoint: str,
+        config: Optional[dict] = None,
+        prefix: str = "",
+    ):
+        """Initialize the OBSDirectoryLoader with the specified settings.
+
+        Args:
+            bucket (str): The name of the OBS bucket to be used.
+            endpoint (str): The endpoint URL of your OBS bucket.
+            config (dict): The parameters for connecting to OBS, provided as a dictionary. The dictionary could have the following keys:
+                - "ak" (str, optional): Your OBS access key (required if `get_token_from_ecs` is False and bucket policy is not public read).
+                - "sk" (str, optional): Your OBS secret key (required if `get_token_from_ecs` is False and bucket policy is not public read).
+                - "token" (str, optional): Your security token (required if using temporary credentials).
+                - "get_token_from_ecs" (bool, optional): Whether to retrieve the security token from ECS. Defaults to False if not provided. If set to True, `ak`, `sk`, and `token` will be ignored.
+            prefix (str, optional): The prefix to be added to the OBS key. Defaults to "".
+
+        Note:
+            Before using this class, make sure you have registered with OBS and have the necessary credentials. The `ak`, `sk`, and `endpoint` values are mandatory unless `get_token_from_ecs` is True or the bucket policy is public read. `token` is required when using temporary credentials.
+        Example:
+            To create a new OBSDirectoryLoader:
+            ```
+            config = {
+                "ak": "your-access-key",
+                "sk": "your-secret-key"
+            }
+            ```
+            directory_loader = OBSDirectoryLoader("your-bucket-name", "your-end-endpoint", config, "your-prefix")
+        """  # noqa: E501
+        try:
+            from obs import ObsClient
+        except ImportError:
+            raise ValueError(
+                "Could not import esdk-obs-python python package. "
+                "Please install it with `pip install esdk-obs-python`."
+            )
+        if not config:
+            config = dict()
+        if config.get("get_token_from_ecs"):
+            self.client = ObsClient(server=endpoint, security_provider_policy="ECS")
+        else:
+            self.client = ObsClient(
+                access_key_id=config.get("ak"),
+                secret_access_key=config.get("sk"),
+                security_token=config.get("token"),
+                server=endpoint,
+            )
+
+        self.bucket = bucket
+        self.prefix = prefix
+
+    def load(self) -> List[Document]:
+        """Load documents."""
+        max_num = 1000
+        mark = None
+        docs = []
+        while True:
+            resp = self.client.listObjects(
+                self.bucket, prefix=self.prefix, marker=mark, max_keys=max_num
+            )
+            if resp.status < 300:
+                for content in resp.body.contents:
+                    loader = OBSFileLoader(self.bucket, content.key, client=self.client)
+                    docs.extend(loader.load())
+                if resp.body.is_truncated is True:
+                    mark = resp.body.next_marker
+                else:
+                    break
+        return docs

libs/langchain/langchain/document_loaders/obs_file.py

@@ -0,0 +1,104 @@
+# coding:utf-8
+
+import os
+import tempfile
+from typing import Any, List, Optional
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+from langchain.document_loaders.unstructured import UnstructuredFileLoader
+
+
+class OBSFileLoader(BaseLoader):
+    """Loader for Huawei OBS file."""
+
+    def __init__(
+        self,
+        bucket: str,
+        key: str,
+        client: Any = None,
+        endpoint: str = "",
+        config: Optional[dict] = None,
+    ) -> None:
+        """Initialize the OBSFileLoader with the specified settings.
+
+        Args:
+            bucket (str): The name of the OBS bucket to be used.
+            key (str): The name of the object in the OBS bucket.
+            client (ObsClient, optional): An instance of the ObsClient to connect to OBS.
+            endpoint (str, optional): The endpoint URL of your OBS bucket. This parameter is mandatory if `client` is not provided.
+            config (dict, optional): The parameters for connecting to OBS, provided as a dictionary. This parameter is ignored if `client` is provided. The dictionary could have the following keys:
+                - "ak" (str, optional): Your OBS access key (required if `get_token_from_ecs` is False and bucket policy is not public read).
+                - "sk" (str, optional): Your OBS secret key (required if `get_token_from_ecs` is False and bucket policy is not public read).
+                - "token" (str, optional): Your security token (required if using temporary credentials).
+                - "get_token_from_ecs" (bool, optional): Whether to retrieve the security token from ECS. Defaults to False if not provided. If set to True, `ak`, `sk`, and `token` will be ignored.
+
+        Raises:
+            ValueError: If the `esdk-obs-python` package is not installed.
+            TypeError: If the provided `client` is not an instance of ObsClient.
+            ValueError: If `client` is not provided, but `endpoint` is missing.
+
+        Note:
+            Before using this class, make sure you have registered with OBS and have the necessary credentials. The `ak`, `sk`, and `endpoint` values are mandatory unless `get_token_from_ecs` is True or the bucket policy is public read. `token` is required when using temporary credentials.
+
+        Example:
+            To create a new OBSFileLoader with a new client:
+            ```
+            config = {
+                "ak": "your-access-key",
+                "sk": "your-secret-key"
+            }
+            obs_loader = OBSFileLoader("your-bucket-name", "your-object-key", config=config)
+            ```
+
+            To create a new OBSFileLoader with an existing client:
+            ```
+            from obs import ObsClient
+
+            # Assuming you have an existing ObsClient object 'obs_client'
+            obs_loader = OBSFileLoader("your-bucket-name", "your-object-key", client=obs_client)
+            ```
+
+            To create a new OBSFileLoader without an existing client:
+            ```
+            obs_loader = OBSFileLoader("your-bucket-name", "your-object-key", endpoint="your-endpoint-url")
+            ```
+        """  # noqa: E501
+        try:
+            from obs import ObsClient
+        except ImportError:
+            raise ValueError(
+                "Could not import esdk-obs-python python package. "
+                "Please install it with `pip install esdk-obs-python`."
+            )
+        if not client:
+            if not endpoint:
+                raise ValueError("Either OBSClient or endpoint must be provided.")
+            if not config:
+                config = dict()
+            if config.get("get_token_from_ecs"):
+                client = ObsClient(server=endpoint, security_provider_policy="ECS")
+            else:
+                client = ObsClient(
+                    access_key_id=config.get("ak"),
+                    secret_access_key=config.get("sk"),
+                    security_token=config.get("token"),
+                    server=endpoint,
+                )
+        if not isinstance(client, ObsClient):
+            raise TypeError("Client must be ObsClient type")
+        self.client = client
+        self.bucket = bucket
+        self.key = key
+
+    def load(self) -> List[Document]:
+        """Load documents."""
+        with tempfile.TemporaryDirectory() as temp_dir:
+            file_path = f"{temp_dir}/{self.bucket}/{self.key}"
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+            # Download the file to a destination
+            self.client.downloadFile(
+                bucketName=self.bucket, objectKey=self.key, downloadFile=file_path
+            )
+            loader = UnstructuredFileLoader(file_path)
+            return loader.load()

libs/langchain/langchain/document_loaders/parsers/audio.py

@@ -21,14 +21,14 @@ class OpenAIWhisperParser(BaseBlobParser):
         try:
             import openai
         except ImportError:
-            raise ValueError(
+            raise ImportError(
                 "openai package not found, please install it with "
                 "`pip install openai`"
             )
         try:
             from pydub import AudioSegment
         except ImportError:
-            raise ValueError(
+            raise ImportError(
                 "pydub package not found, please install it with " "`pip install pydub`"
             )
 
@@ -73,3 +73,107 @@ class OpenAIWhisperParser(BaseBlobParser):
                 page_content=transcript.text,
                 metadata={"source": blob.source, "chunk": split_number},
             )
+
+
+class OpenAIWhisperParserLocal(BaseBlobParser):
+    """Transcribe and parse audio files.
+    Audio transcription is with OpenAI Whisper model locally from transformers
+    NOTE: By default uses the gpu if available, if you want to use cpu,
+    please set device = "cpu"
+    """
+
+    def __init__(self, device: str = "0", lang_model: Optional[str] = None):
+        try:
+            from transformers import pipeline
+        except ImportError:
+            raise ImportError(
+                "transformers package not found, please install it with "
+                "`pip install transformers`"
+            )
+        try:
+            import torch
+        except ImportError:
+            raise ImportError(
+                "torch package not found, please install it with " "`pip install torch`"
+            )
+
+        # set device, cpu by default check if there is a GPU available
+        if device == "cpu":
+            self.device = "cpu"
+            if lang_model is not None:
+                self.lang_model = lang_model
+                print("WARNING! Model override. Using model: ", self.lang_model)
+            else:
+                # unless overridden, use the small base model on cpu
+                self.lang_model = "openai/whisper-base"
+        else:
+            if torch.cuda.is_available():
+                self.device = "cuda:0"
+                # check GPU memory and select automatically the model
+                mem = torch.cuda.get_device_properties(self.device).total_memory / (
+                    1024**2
+                )
+                if mem < 5000:
+                    rec_model = "openai/whisper-base"
+                elif mem < 7000:
+                    rec_model = "openai/whisper-small"
+                elif mem < 12000:
+                    rec_model = "openai/whisper-medium"
+                else:
+                    rec_model = "openai/whisper-large"
+
+                # check if model is overridden
+                if lang_model is not None:
+                    self.lang_model = lang_model
+                    print("WARNING! Model override. Might not fit in your GPU")
+                else:
+                    self.lang_model = rec_model
+            else:
+                "cpu"
+
+        print("Using the following model: ", self.lang_model)
+
+        # load model for inference
+        self.pipe = pipeline(
+            "automatic-speech-recognition",
+            model="openai/whisper-medium",
+            chunk_length_s=30,
+            device=self.device,
+        )
+
+    def lazy_parse(self, blob: Blob) -> Iterator[Document]:
+        """Lazily parse the blob."""
+
+        import io
+
+        try:
+            from pydub import AudioSegment
+        except ImportError:
+            raise ValueError(
+                "pydub package not found, please install it with " "`pip install pydub`"
+            )
+
+        try:
+            import librosa
+        except ImportError:
+            raise ValueError(
+                "librosa package not found, please install it with "
+                "`pip install librosa`"
+            )
+
+        # Audio file from disk
+        audio = AudioSegment.from_file(blob.path)
+
+        file_obj = io.BytesIO(audio.export(format="mp3").read())
+
+        # Transcribe
+        print(f"Transcribing part {blob.path}!")
+
+        y, sr = librosa.load(file_obj, sr=16000)
+
+        prediction = self.pipe(y.copy(), batch_size=8)["text"]
+
+        yield Document(
+            page_content=prediction,
+            metadata={"source": blob.source},
+        )