cr

.devcontainer/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 services:
   langchain:
     build:
-      dockerfile: dev.Dockerfile
+      dockerfile: libs/langchain/dev.Dockerfile
       context: ..
     volumes:
    # Update this to wherever you want VS Code to mount the folder of your project

.github/actions/poetry_setup/action.yml

@@ -52,11 +52,13 @@ runs:
 
     - name: Check Poetry File
       shell: bash
+      working-directory: ${{ inputs.working-directory }}
       run: |
         poetry check
 
     - name: Check lock file
       shell: bash
+      working-directory: ${{ inputs.working-directory }}
       run: |
         poetry lock --check
 

.github/workflows/_lint.yml

@@ -1,15 +1,21 @@
 name: lint
 
 on:
-  push:
-    branches: [master]
-  pull_request:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
 
 env:
   POETRY_VERSION: "1.4.2"
 
 jobs:
   build:
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -31,6 +37,10 @@ jobs:
       - name: Install dependencies
         run: |
           poetry install
+      - name: Install langchain editable
+        if: ${{ inputs.working-directory != 'langchain' }}
+        run: |
+          pip install -e ../langchain
       - name: Analysing the code with our lint
         run: |
           make lint

.github/workflows/_release.yml

@@ -1,13 +1,12 @@
 name: release
 
 on:
-  pull_request:
-    types:
-      - closed
-    branches:
-      - master
-    paths:
-      - 'pyproject.toml'
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
 
 env:
   POETRY_VERSION: "1.4.2"
@@ -18,6 +17,9 @@ jobs:
         ${{ github.event.pull_request.merged == true }}
         && ${{ contains(github.event.pull_request.labels.*.name, 'release') }}
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
     steps:
       - uses: actions/checkout@v3
       - name: Install poetry

.github/workflows/_test.yml

@@ -1,16 +1,25 @@
 name: test
 
 on:
-  push:
-    branches: [master]
-  pull_request:
-  workflow_dispatch:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+      test_type:
+        type: string
+        description: "Test types to run"
+        default: '["core", "extended"]'
 
 env:
   POETRY_VERSION: "1.4.2"
 
 jobs:
   build:
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -19,9 +28,7 @@ jobs:
           - "3.9"
           - "3.10"
           - "3.11"
-        test_type:
-          - "core"
-          - "extended"
+        test_type: ${{ fromJSON(inputs.test_type) }}
     name: Python ${{ matrix.python-version }} ${{ matrix.test_type }}
     steps:
       - uses: actions/checkout@v3
@@ -29,6 +36,7 @@ jobs:
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
+          working-directory: ${{ inputs.working-directory }}
           poetry-version: "1.4.2"
           cache-key: ${{ matrix.test_type }}
           install-command: |
@@ -39,6 +47,10 @@ jobs:
                 echo "Running extended tests, installing dependencies with poetry..."
                 poetry install -E extended_testing
               fi
+      - name: Install langchain editable
+        if: ${{ inputs.working-directory != 'langchain' }}
+        run: |
+          pip install -e ../langchain
       - name: Run ${{matrix.test_type}} tests
         run: |
           if [ "${{ matrix.test_type }}" == "core" ]; then

.github/workflows/langchain_ci.yml

@@ -0,0 +1,27 @@
+---
+name: libs/langchain CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    paths:
+      - '.github/workflows/_lint.yml'
+      - '.github/workflows/_test.yml'
+      - '.github/workflows/langchain_ci.yml'
+      - 'libs/langchain/**'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  lint:
+    uses:
+      ./.github/workflows/_lint.yml
+    with:
+      working-directory: libs/langchain
+    secrets: inherit
+  test:
+    uses:
+      ./.github/workflows/_test.yml
+    with:
+      working-directory: libs/langchain
+    secrets: inherit

.github/workflows/langchain_experimental_ci.yml

@@ -0,0 +1,28 @@
+---
+name: libs/langchain-experimental CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    paths:
+      - '.github/workflows/_lint.yml'
+      - '.github/workflows/_test.yml'
+      - '.github/workflows/langchain_experimental_ci.yml'
+      - 'libs/langchain-experimental/**'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  lint:
+    uses:
+      ./.github/workflows/_lint.yml
+    with:
+      working-directory: libs/langchain-experimental
+    secrets: inherit
+  test:
+    uses:
+      ./.github/workflows/_test.yml
+    with:
+      working-directory: libs/langchain-experimental
+      test_type: '["core"]'
+    secrets: inherit

.github/workflows/langchain_release.yml

@@ -0,0 +1,19 @@
+---
+name: libs/langchain Release
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - master
+    paths:
+      - 'libs/langchain/pyproject.toml'
+
+jobs:
+  release:
+    uses:
+      ./.github/workflows/_release.yml
+    with:
+      working-directory: libs/langchain
+    secrets: inherit

libs/langchain-experimental/Makefile

@@ -0,0 +1,105 @@
+.PHONY: all clean docs_build docs_clean docs_linkcheck api_docs_build api_docs_clean api_docs_linkcheck format lint test tests test_watch integration_tests docker_tests help extended_tests
+
+# Default target executed when no arguments are given to make.
+all: help
+
+######################
+# TESTING AND COVERAGE
+######################
+
+# Run unit tests and generate a coverage report.
+coverage:
+	poetry run pytest --cov \
+		--cov-config=.coveragerc \
+		--cov-report xml \
+		--cov-report term-missing:skip-covered
+
+######################
+# DOCUMENTATION
+######################
+
+clean: docs_clean api_docs_clean
+
+
+docs_build:
+	docs/.local_build.sh
+
+docs_clean:
+	rm -r docs/_dist
+
+docs_linkcheck:
+	poetry run linkchecker docs/_dist/docs_skeleton/ --ignore-url node_modules
+
+api_docs_build:
+	poetry run python docs/api_reference/create_api_rst.py
+	cd docs/api_reference && poetry run make html
+
+api_docs_clean:
+	rm -f docs/api_reference/api_reference.rst
+	cd docs/api_reference && poetry run make clean
+
+api_docs_linkcheck:
+	poetry run linkchecker docs/api_reference/_build/html/index.html
+
+# Define a variable for the test file path.
+TEST_FILE ?= tests/unit_tests/
+
+test:
+	poetry run pytest $(TEST_FILE)
+
+tests: 
+	poetry run pytest $(TEST_FILE)
+
+test_watch:
+	poetry run ptw --now . -- tests/unit_tests
+
+integration_tests:
+	poetry run pytest tests/integration_tests
+
+docker_tests:
+	docker build -t my-langchain-image:test .
+	docker run --rm my-langchain-image:test
+
+######################
+# LINTING AND FORMATTING
+######################
+
+# Define a variable for Python and notebook files.
+PYTHON_FILES=.
+lint format: PYTHON_FILES=.
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+
+lint lint_diff:
+	poetry run mypy $(PYTHON_FILES)
+	poetry run black $(PYTHON_FILES) --check
+	poetry run ruff .
+
+format format_diff:
+	poetry run black $(PYTHON_FILES)
+	poetry run ruff --select I --fix $(PYTHON_FILES)
+
+spell_check:
+	poetry run codespell --toml pyproject.toml
+
+spell_fix:
+	poetry run codespell --toml pyproject.toml -w
+
+######################
+# HELP
+######################
+
+help:
+	@echo '----'
+	@echo 'coverage                     - run unit tests and generate coverage report'
+	@echo 'docs_build                   - build the documentation'
+	@echo 'docs_clean                   - clean the documentation build artifacts'
+	@echo 'docs_linkcheck               - run linkchecker on the documentation'
+	@echo 'format                       - run code formatters'
+	@echo 'lint                         - run linters'
+	@echo 'test                         - run unit tests'
+	@echo 'tests                        - run unit tests'
+	@echo 'test TEST_FILE=<test_file>   - run all tests in file'
+	@echo 'extended_tests               - run only extended unit tests'
+	@echo 'test_watch                   - run unit tests in watch mode'
+	@echo 'integration_tests            - run integration tests'
+	@echo 'docker_tests                 - run unit tests in docker'

libs/langchain-experimental/README.md

@@ -0,0 +1,95 @@
+# 🦜️🔗 LangChain
+
+⚡ Building applications with LLMs through composability ⚡
+
+[![Release Notes](https://img.shields.io/github/release/hwchase17/langchain)](https://github.com/hwchase17/langchain/releases)
+[![lint](https://github.com/hwchase17/langchain/actions/workflows/lint.yml/badge.svg)](https://github.com/hwchase17/langchain/actions/workflows/lint.yml)
+[![test](https://github.com/hwchase17/langchain/actions/workflows/test.yml/badge.svg)](https://github.com/hwchase17/langchain/actions/workflows/test.yml)
+[![Downloads](https://static.pepy.tech/badge/langchain/month)](https://pepy.tech/project/langchain)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Twitter](https://img.shields.io/twitter/url/https/twitter.com/langchainai.svg?style=social&label=Follow%20%40LangChainAI)](https://twitter.com/langchainai)
+[![](https://dcbadge.vercel.app/api/server/6adMQxSpJS?compact=true&style=flat)](https://discord.gg/6adMQxSpJS)
+[![Open in Dev Containers](https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/hwchase17/langchain)
+[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/hwchase17/langchain)
+[![GitHub star chart](https://img.shields.io/github/stars/hwchase17/langchain?style=social)](https://star-history.com/#hwchase17/langchain)
+[![Dependency Status](https://img.shields.io/librariesio/github/hwchase17/langchain)](https://libraries.io/github/hwchase17/langchain)
+[![Open Issues](https://img.shields.io/github/issues-raw/hwchase17/langchain)](https://github.com/hwchase17/langchain/issues)
+
+
+Looking for the JS/TS version? Check out [LangChain.js](https://github.com/hwchase17/langchainjs).
+
+**Production Support:** As you move your LangChains into production, we'd love to offer more comprehensive support.
+Please fill out [this form](https://forms.gle/57d8AmXBYp8PP8tZA) and we'll set up a dedicated support Slack channel.
+
+## Quick Install
+
+`pip install langchain`
+or
+`pip install langsmith && conda install langchain -c conda-forge`
+
+## 🤔 What is this?
+
+Large language models (LLMs) are emerging as a transformative technology, enabling developers to build applications that they previously could not. However, using these LLMs in isolation is often insufficient for creating a truly powerful app - the real power comes when you can combine them with other sources of computation or knowledge.
+
+This library aims to assist in the development of those types of applications. Common examples of these applications include:
+
+**❓ Question Answering over specific documents**
+
+- [Documentation](https://python.langchain.com/docs/use_cases/question_answering/)
+- End-to-end Example: [Question Answering over Notion Database](https://github.com/hwchase17/notion-qa)
+
+**💬 Chatbots**
+
+- [Documentation](https://python.langchain.com/docs/use_cases/chatbots/)
+- End-to-end Example: [Chat-LangChain](https://github.com/hwchase17/chat-langchain)
+
+**🤖 Agents**
+
+- [Documentation](https://python.langchain.com/docs/modules/agents/)
+- End-to-end Example: [GPT+WolframAlpha](https://huggingface.co/spaces/JavaFXpert/Chat-GPT-LangChain)
+
+## 📖 Documentation
+
+Please see [here](https://python.langchain.com) for full documentation on:
+
+- Getting started (installation, setting up the environment, simple examples)
+- How-To examples (demos, integrations, helper functions)
+- Reference (full API docs)
+- Resources (high-level explanation of core concepts)
+
+## 🚀 What can this help with?
+
+There are six main areas that LangChain is designed to help with.
+These are, in increasing order of complexity:
+
+**📃 LLMs and Prompts:**
+
+This includes prompt management, prompt optimization, a generic interface for all LLMs, and common utilities for working with LLMs.
+
+**🔗 Chains:**
+
+Chains go beyond a single LLM call and involve sequences of calls (whether to an LLM or a different utility). LangChain provides a standard interface for chains, lots of integrations with other tools, and end-to-end chains for common applications.
+
+**📚 Data Augmented Generation:**
+
+Data Augmented Generation involves specific types of chains that first interact with an external data source to fetch data for use in the generation step. Examples include summarization of long pieces of text and question/answering over specific data sources.
+
+**🤖 Agents:**
+
+Agents involve an LLM making decisions about which Actions to take, taking that Action, seeing an Observation, and repeating that until done. LangChain provides a standard interface for agents, a selection of agents to choose from, and examples of end-to-end agents.
+
+**🧠 Memory:**
+
+Memory refers to persisting state between calls of a chain/agent. LangChain provides a standard interface for memory, a collection of memory implementations, and examples of chains/agents that use memory.
+
+**🧐 Evaluation:**
+
+[BETA] Generative models are notoriously hard to evaluate with traditional metrics. One new way of evaluating them is using language models themselves to do the evaluation. LangChain provides some prompts/chains for assisting in this.
+
+For more information on these concepts, please see our [full documentation](https://python.langchain.com).
+
+## 💁 Contributing
+
+As an open-source project in a rapidly developing field, we are extremely open to contributions, whether it be in the form of a new feature, improved infrastructure, or better documentation.
+
+For detailed information on how to contribute, see [here](.github/CONTRIBUTING.md).

libs/langchain-experimental/langchain_experimental/autonomous_agents/autogpt/agent.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+from typing import List, Optional
+
+from langchain.chains.llm import LLMChain
+from langchain.chat_models.base import BaseChatModel
+from langchain.experimental.autonomous_agents.autogpt.output_parser import (
+    AutoGPTOutputParser,
+    BaseAutoGPTOutputParser,
+)
+from langchain.experimental.autonomous_agents.autogpt.prompt import AutoGPTPrompt
+from langchain.experimental.autonomous_agents.autogpt.prompt_generator import (
+    FINISH_NAME,
+)
+from langchain.memory import ChatMessageHistory
+from langchain.schema import (
+    BaseChatMessageHistory,
+    Document,
+)
+from langchain.schema.messages import AIMessage, HumanMessage, SystemMessage
+from langchain.tools.base import BaseTool
+from langchain.tools.human.tool import HumanInputRun
+from langchain.vectorstores.base import VectorStoreRetriever
+from pydantic import ValidationError
+
+
+class AutoGPT:
+    """Agent class for interacting with Auto-GPT."""
+
+    def __init__(
+        self,
+        ai_name: str,
+        memory: VectorStoreRetriever,
+        chain: LLMChain,
+        output_parser: BaseAutoGPTOutputParser,
+        tools: List[BaseTool],
+        feedback_tool: Optional[HumanInputRun] = None,
+        chat_history_memory: Optional[BaseChatMessageHistory] = None,
+    ):
+        self.ai_name = ai_name
+        self.memory = memory
+        self.next_action_count = 0
+        self.chain = chain
+        self.output_parser = output_parser
+        self.tools = tools
+        self.feedback_tool = feedback_tool
+        self.chat_history_memory = chat_history_memory or ChatMessageHistory()
+
+    @classmethod
+    def from_llm_and_tools(
+        cls,
+        ai_name: str,
+        ai_role: str,
+        memory: VectorStoreRetriever,
+        tools: List[BaseTool],
+        llm: BaseChatModel,
+        human_in_the_loop: bool = False,
+        output_parser: Optional[BaseAutoGPTOutputParser] = None,
+        chat_history_memory: Optional[BaseChatMessageHistory] = None,
+    ) -> AutoGPT:
+        prompt = AutoGPTPrompt(
+            ai_name=ai_name,
+            ai_role=ai_role,
+            tools=tools,
+            input_variables=["memory", "messages", "goals", "user_input"],
+            token_counter=llm.get_num_tokens,
+        )
+        human_feedback_tool = HumanInputRun() if human_in_the_loop else None
+        chain = LLMChain(llm=llm, prompt=prompt)
+        return cls(
+            ai_name,
+            memory,
+            chain,
+            output_parser or AutoGPTOutputParser(),
+            tools,
+            feedback_tool=human_feedback_tool,
+            chat_history_memory=chat_history_memory,
+        )
+
+    def run(self, goals: List[str]) -> str:
+        user_input = (
+            "Determine which next command to use, "
+            "and respond using the format specified above:"
+        )
+        # Interaction Loop
+        loop_count = 0
+        while True:
+            # Discontinue if continuous limit is reached
+            loop_count += 1
+
+            # Send message to AI, get response
+            assistant_reply = self.chain.run(
+                goals=goals,
+                messages=self.chat_history_memory.messages,
+                memory=self.memory,
+                user_input=user_input,
+            )
+
+            # Print Assistant thoughts
+            print(assistant_reply)
+            self.chat_history_memory.add_message(HumanMessage(content=user_input))
+            self.chat_history_memory.add_message(AIMessage(content=assistant_reply))
+
+            # Get command name and arguments
+            action = self.output_parser.parse(assistant_reply)
+            tools = {t.name: t for t in self.tools}
+            if action.name == FINISH_NAME:
+                return action.args["response"]
+            if action.name in tools:
+                tool = tools[action.name]
+                try:
+                    observation = tool.run(action.args)
+                except ValidationError as e:
+                    observation = (
+                        f"Validation Error in args: {str(e)}, args: {action.args}"
+                    )
+                except Exception as e:
+                    observation = (
+                        f"Error: {str(e)}, {type(e).__name__}, args: {action.args}"
+                    )
+                result = f"Command {tool.name} returned: {observation}"
+            elif action.name == "ERROR":
+                result = f"Error: {action.args}. "
+            else:
+                result = (
+                    f"Unknown command '{action.name}'. "
+                    f"Please refer to the 'COMMANDS' list for available "
+                    f"commands and only respond in the specified JSON format."
+                )
+
+            memory_to_add = (
+                f"Assistant Reply: {assistant_reply} " f"\nResult: {result} "
+            )
+            if self.feedback_tool is not None:
+                feedback = f"\n{self.feedback_tool.run('Input: ')}"
+                if feedback in {"q", "stop"}:
+                    print("EXITING")
+                    return "EXITING"
+                memory_to_add += feedback
+
+            self.memory.add_documents([Document(page_content=memory_to_add)])
+            self.chat_history_memory.add_message(SystemMessage(content=result))

libs/langchain-experimental/langchain_experimental/autonomous_agents/autogpt/memory.py

@@ -0,0 +1,29 @@
+from typing import Any, Dict, List
+
+from langchain.memory.chat_memory import BaseChatMemory, get_prompt_input_key
+from langchain.vectorstores.base import VectorStoreRetriever
+from pydantic import Field
+
+
+class AutoGPTMemory(BaseChatMemory):
+    retriever: VectorStoreRetriever = Field(exclude=True)
+    """VectorStoreRetriever object to connect to."""
+
+    @property
+    def memory_variables(self) -> List[str]:
+        return ["chat_history", "relevant_context"]
+
+    def _get_prompt_input_key(self, inputs: Dict[str, Any]) -> str:
+        """Get the input key for the prompt."""
+        if self.input_key is None:
+            return get_prompt_input_key(inputs, self.memory_variables)
+        return self.input_key
+
+    def load_memory_variables(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        input_key = self._get_prompt_input_key(inputs)
+        query = inputs[input_key]
+        docs = self.retriever.get_relevant_documents(query)
+        return {
+            "chat_history": self.chat_memory.messages[-10:],
+            "relevant_context": docs,
+        }

libs/langchain-experimental/langchain_experimental/autonomous_agents/autogpt/prompt.py

@@ -0,0 +1,77 @@
+import time
+from typing import Any, Callable, List
+
+from langchain.experimental.autonomous_agents.autogpt.prompt_generator import get_prompt
+from langchain.prompts.chat import (
+    BaseChatPromptTemplate,
+)
+from langchain.schema.messages import BaseMessage, HumanMessage, SystemMessage
+from langchain.tools.base import BaseTool
+from langchain.vectorstores.base import VectorStoreRetriever
+from pydantic import BaseModel
+
+
+class AutoGPTPrompt(BaseChatPromptTemplate, BaseModel):
+    ai_name: str
+    ai_role: str
+    tools: List[BaseTool]
+    token_counter: Callable[[str], int]
+    send_token_limit: int = 4196
+
+    def construct_full_prompt(self, goals: List[str]) -> str:
+        prompt_start = (
+            "Your decisions must always be made independently "
+            "without seeking user assistance.\n"
+            "Play to your strengths as an LLM and pursue simple "
+            "strategies with no legal complications.\n"
+            "If you have completed all your tasks, make sure to "
+            'use the "finish" command.'
+        )
+        # Construct full prompt
+        full_prompt = (
+            f"You are {self.ai_name}, {self.ai_role}\n{prompt_start}\n\nGOALS:\n\n"
+        )
+        for i, goal in enumerate(goals):
+            full_prompt += f"{i+1}. {goal}\n"
+
+        full_prompt += f"\n\n{get_prompt(self.tools)}"
+        return full_prompt
+
+    def format_messages(self, **kwargs: Any) -> List[BaseMessage]:
+        base_prompt = SystemMessage(content=self.construct_full_prompt(kwargs["goals"]))
+        time_prompt = SystemMessage(
+            content=f"The current time and date is {time.strftime('%c')}"
+        )
+        used_tokens = self.token_counter(base_prompt.content) + self.token_counter(
+            time_prompt.content
+        )
+        memory: VectorStoreRetriever = kwargs["memory"]
+        previous_messages = kwargs["messages"]
+        relevant_docs = memory.get_relevant_documents(str(previous_messages[-10:]))
+        relevant_memory = [d.page_content for d in relevant_docs]
+        relevant_memory_tokens = sum(
+            [self.token_counter(doc) for doc in relevant_memory]
+        )
+        while used_tokens + relevant_memory_tokens > 2500:
+            relevant_memory = relevant_memory[:-1]
+            relevant_memory_tokens = sum(
+                [self.token_counter(doc) for doc in relevant_memory]
+            )
+        content_format = (
+            f"This reminds you of these events "
+            f"from your past:\n{relevant_memory}\n\n"
+        )
+        memory_message = SystemMessage(content=content_format)
+        used_tokens += self.token_counter(memory_message.content)
+        historical_messages: List[BaseMessage] = []
+        for message in previous_messages[-10:][::-1]:
+            message_tokens = self.token_counter(message.content)
+            if used_tokens + message_tokens > self.send_token_limit - 1000:
+                break
+            historical_messages = [message] + historical_messages
+            used_tokens += message_tokens
+        input_message = HumanMessage(content=kwargs["user_input"])
+        messages: List[BaseMessage] = [base_prompt, time_prompt, memory_message]
+        messages += historical_messages
+        messages.append(input_message)
+        return messages

libs/langchain-experimental/langchain_experimental/autonomous_agents/baby_agi/baby_agi.py

@@ -0,0 +1,186 @@
+"""BabyAGI agent."""
+from collections import deque
+from typing import Any, Dict, List, Optional
+
+from langchain.callbacks.manager import CallbackManagerForChainRun
+from langchain.chains.base import Chain
+from langchain.experimental.autonomous_agents.baby_agi.task_creation import (
+    TaskCreationChain,
+)
+from langchain.experimental.autonomous_agents.baby_agi.task_execution import (
+    TaskExecutionChain,
+)
+from langchain.experimental.autonomous_agents.baby_agi.task_prioritization import (
+    TaskPrioritizationChain,
+)
+from langchain.schema.language_model import BaseLanguageModel
+from langchain.vectorstores.base import VectorStore
+from pydantic import BaseModel, Field
+
+
+class BabyAGI(Chain, BaseModel):
+    """Controller model for the BabyAGI agent."""
+
+    task_list: deque = Field(default_factory=deque)
+    task_creation_chain: Chain = Field(...)
+    task_prioritization_chain: Chain = Field(...)
+    execution_chain: Chain = Field(...)
+    task_id_counter: int = Field(1)
+    vectorstore: VectorStore = Field(init=False)
+    max_iterations: Optional[int] = None
+
+    class Config:
+        """Configuration for this pydantic object."""
+
+        arbitrary_types_allowed = True
+
+    def add_task(self, task: Dict) -> None:
+        self.task_list.append(task)
+
+    def print_task_list(self) -> None:
+        print("\033[95m\033[1m" + "\n*****TASK LIST*****\n" + "\033[0m\033[0m")
+        for t in self.task_list:
+            print(str(t["task_id"]) + ": " + t["task_name"])
+
+    def print_next_task(self, task: Dict) -> None:
+        print("\033[92m\033[1m" + "\n*****NEXT TASK*****\n" + "\033[0m\033[0m")
+        print(str(task["task_id"]) + ": " + task["task_name"])
+
+    def print_task_result(self, result: str) -> None:
+        print("\033[93m\033[1m" + "\n*****TASK RESULT*****\n" + "\033[0m\033[0m")
+        print(result)
+
+    @property
+    def input_keys(self) -> List[str]:
+        return ["objective"]
+
+    @property
+    def output_keys(self) -> List[str]:
+        return []
+
+    def get_next_task(
+        self, result: str, task_description: str, objective: str
+    ) -> List[Dict]:
+        """Get the next task."""
+        task_names = [t["task_name"] for t in self.task_list]
+
+        incomplete_tasks = ", ".join(task_names)
+        response = self.task_creation_chain.run(
+            result=result,
+            task_description=task_description,
+            incomplete_tasks=incomplete_tasks,
+            objective=objective,
+        )
+        new_tasks = response.split("\n")
+        return [
+            {"task_name": task_name} for task_name in new_tasks if task_name.strip()
+        ]
+
+    def prioritize_tasks(self, this_task_id: int, objective: str) -> List[Dict]:
+        """Prioritize tasks."""
+        task_names = [t["task_name"] for t in list(self.task_list)]
+        next_task_id = int(this_task_id) + 1
+        response = self.task_prioritization_chain.run(
+            task_names=", ".join(task_names),
+            next_task_id=str(next_task_id),
+            objective=objective,
+        )
+        new_tasks = response.split("\n")
+        prioritized_task_list = []
+        for task_string in new_tasks:
+            if not task_string.strip():
+                continue
+            task_parts = task_string.strip().split(".", 1)
+            if len(task_parts) == 2:
+                task_id = task_parts[0].strip()
+                task_name = task_parts[1].strip()
+                prioritized_task_list.append(
+                    {"task_id": task_id, "task_name": task_name}
+                )
+        return prioritized_task_list
+
+    def _get_top_tasks(self, query: str, k: int) -> List[str]:
+        """Get the top k tasks based on the query."""
+        results = self.vectorstore.similarity_search(query, k=k)
+        if not results:
+            return []
+        return [str(item.metadata["task"]) for item in results]
+
+    def execute_task(self, objective: str, task: str, k: int = 5) -> str:
+        """Execute a task."""
+        context = self._get_top_tasks(query=objective, k=k)
+        return self.execution_chain.run(
+            objective=objective, context="\n".join(context), task=task
+        )
+
+    def _call(
+        self,
+        inputs: Dict[str, Any],
+        run_manager: Optional[CallbackManagerForChainRun] = None,
+    ) -> Dict[str, Any]:
+        """Run the agent."""
+        objective = inputs["objective"]
+        first_task = inputs.get("first_task", "Make a todo list")
+        self.add_task({"task_id": 1, "task_name": first_task})
+        num_iters = 0
+        while True:
+            if self.task_list:
+                self.print_task_list()
+
+                # Step 1: Pull the first task
+                task = self.task_list.popleft()
+                self.print_next_task(task)
+
+                # Step 2: Execute the task
+                result = self.execute_task(objective, task["task_name"])
+                this_task_id = int(task["task_id"])
+                self.print_task_result(result)
+
+                # Step 3: Store the result in Pinecone
+                result_id = f"result_{task['task_id']}"
+                self.vectorstore.add_texts(
+                    texts=[result],
+                    metadatas=[{"task": task["task_name"]}],
+                    ids=[result_id],
+                )
+
+                # Step 4: Create new tasks and reprioritize task list
+                new_tasks = self.get_next_task(result, task["task_name"], objective)
+                for new_task in new_tasks:
+                    self.task_id_counter += 1
+                    new_task.update({"task_id": self.task_id_counter})
+                    self.add_task(new_task)
+                self.task_list = deque(self.prioritize_tasks(this_task_id, objective))
+            num_iters += 1
+            if self.max_iterations is not None and num_iters == self.max_iterations:
+                print(
+                    "\033[91m\033[1m" + "\n*****TASK ENDING*****\n" + "\033[0m\033[0m"
+                )
+                break
+        return {}
+
+    @classmethod
+    def from_llm(
+        cls,
+        llm: BaseLanguageModel,
+        vectorstore: VectorStore,
+        verbose: bool = False,
+        task_execution_chain: Optional[Chain] = None,
+        **kwargs: Dict[str, Any],
+    ) -> "BabyAGI":
+        """Initialize the BabyAGI Controller."""
+        task_creation_chain = TaskCreationChain.from_llm(llm, verbose=verbose)
+        task_prioritization_chain = TaskPrioritizationChain.from_llm(
+            llm, verbose=verbose
+        )
+        if task_execution_chain is None:
+            execution_chain: Chain = TaskExecutionChain.from_llm(llm, verbose=verbose)
+        else:
+            execution_chain = task_execution_chain
+        return cls(
+            task_creation_chain=task_creation_chain,
+            task_prioritization_chain=task_prioritization_chain,
+            execution_chain=execution_chain,
+            vectorstore=vectorstore,
+            **kwargs,
+        )

libs/langchain-experimental/langchain_experimental/autonomous_agents/baby_agi/task_creation.py

@@ -1,30 +1,30 @@
-from langchain import LLMChain, PromptTemplate
-from langchain.schema.language_model import BaseLanguageModel
-
-
-class TaskCreationChain(LLMChain):
-    """Chain to generates tasks."""
-
-    @classmethod
-    def from_llm(cls, llm: BaseLanguageModel, verbose: bool = True) -> LLMChain:
-        """Get the response parser."""
-        task_creation_template = (
-            "You are an task creation AI that uses the result of an execution agent"
-            " to create new tasks with the following objective: {objective},"
-            " The last completed task has the result: {result}."
-            " This result was based on this task description: {task_description}."
-            " These are incomplete tasks: {incomplete_tasks}."
-            " Based on the result, create new tasks to be completed"
-            " by the AI system that do not overlap with incomplete tasks."
-            " Return the tasks as an array."
-        )
-        prompt = PromptTemplate(
-            template=task_creation_template,
-            input_variables=[
-                "result",
-                "task_description",
-                "incomplete_tasks",
-                "objective",
-            ],
-        )
-        return cls(prompt=prompt, llm=llm, verbose=verbose)
+from langchain import LLMChain, PromptTemplate
+from langchain.schema.language_model import BaseLanguageModel
+
+
+class TaskCreationChain(LLMChain):
+    """Chain to generates tasks."""
+
+    @classmethod
+    def from_llm(cls, llm: BaseLanguageModel, verbose: bool = True) -> LLMChain:
+        """Get the response parser."""
+        task_creation_template = (
+            "You are an task creation AI that uses the result of an execution agent"
+            " to create new tasks with the following objective: {objective},"
+            " The last completed task has the result: {result}."
+            " This result was based on this task description: {task_description}."
+            " These are incomplete tasks: {incomplete_tasks}."
+            " Based on the result, create new tasks to be completed"
+            " by the AI system that do not overlap with incomplete tasks."
+            " Return the tasks as an array."
+        )
+        prompt = PromptTemplate(
+            template=task_creation_template,
+            input_variables=[
+                "result",
+                "task_description",
+                "incomplete_tasks",
+                "objective",
+            ],
+        )
+        return cls(prompt=prompt, llm=llm, verbose=verbose)

libs/langchain-experimental/langchain_experimental/autonomous_agents/baby_agi/task_execution.py

@@ -1,21 +1,21 @@
-from langchain import LLMChain, PromptTemplate
-from langchain.schema.language_model import BaseLanguageModel
-
-
-class TaskExecutionChain(LLMChain):
-    """Chain to execute tasks."""
-
-    @classmethod
-    def from_llm(cls, llm: BaseLanguageModel, verbose: bool = True) -> LLMChain:
-        """Get the response parser."""
-        execution_template = (
-            "You are an AI who performs one task based on the following objective: "
-            "{objective}."
-            "Take into account these previously completed tasks: {context}."
-            " Your task: {task}. Response:"
-        )
-        prompt = PromptTemplate(
-            template=execution_template,
-            input_variables=["objective", "context", "task"],
-        )
-        return cls(prompt=prompt, llm=llm, verbose=verbose)
+from langchain import LLMChain, PromptTemplate
+from langchain.schema.language_model import BaseLanguageModel
+
+
+class TaskExecutionChain(LLMChain):
+    """Chain to execute tasks."""
+
+    @classmethod
+    def from_llm(cls, llm: BaseLanguageModel, verbose: bool = True) -> LLMChain:
+        """Get the response parser."""
+        execution_template = (
+            "You are an AI who performs one task based on the following objective: "
+            "{objective}."
+            "Take into account these previously completed tasks: {context}."
+            " Your task: {task}. Response:"
+        )
+        prompt = PromptTemplate(
+            template=execution_template,
+            input_variables=["objective", "context", "task"],
+        )
+        return cls(prompt=prompt, llm=llm, verbose=verbose)

libs/langchain-experimental/langchain_experimental/autonomous_agents/baby_agi/task_prioritization.py

@@ -1,24 +1,24 @@
-from langchain import LLMChain, PromptTemplate
-from langchain.schema.language_model import BaseLanguageModel
-
-
-class TaskPrioritizationChain(LLMChain):
-    """Chain to prioritize tasks."""
-
-    @classmethod
-    def from_llm(cls, llm: BaseLanguageModel, verbose: bool = True) -> LLMChain:
-        """Get the response parser."""
-        task_prioritization_template = (
-            "You are a task prioritization AI tasked with cleaning the formatting of "
-            "and reprioritizing the following tasks: {task_names}."
-            " Consider the ultimate objective of your team: {objective}."
-            " Do not remove any tasks. Return the result as a numbered list, like:"
-            " #. First task"
-            " #. Second task"
-            " Start the task list with number {next_task_id}."
-        )
-        prompt = PromptTemplate(
-            template=task_prioritization_template,
-            input_variables=["task_names", "next_task_id", "objective"],
-        )
-        return cls(prompt=prompt, llm=llm, verbose=verbose)
+from langchain import LLMChain, PromptTemplate
+from langchain.schema.language_model import BaseLanguageModel
+
+
+class TaskPrioritizationChain(LLMChain):
+    """Chain to prioritize tasks."""
+
+    @classmethod
+    def from_llm(cls, llm: BaseLanguageModel, verbose: bool = True) -> LLMChain:
+        """Get the response parser."""
+        task_prioritization_template = (
+            "You are a task prioritization AI tasked with cleaning the formatting of "
+            "and reprioritizing the following tasks: {task_names}."
+            " Consider the ultimate objective of your team: {objective}."
+            " Do not remove any tasks. Return the result as a numbered list, like:"
+            " #. First task"
+            " #. Second task"
+            " Start the task list with number {next_task_id}."
+        )
+        prompt = PromptTemplate(
+            template=task_prioritization_template,
+            input_variables=["task_names", "next_task_id", "objective"],
+        )
+        return cls(prompt=prompt, llm=llm, verbose=verbose)

libs/langchain-experimental/langchain_experimental/cpal/base.py

@@ -0,0 +1,270 @@
+"""
+CPAL Chain and its subchains
+"""
+from __future__ import annotations
+
+import json
+from typing import Any, ClassVar, Dict, List, Optional, Type
+
+import pydantic
+from langchain.base_language import BaseLanguageModel
+from langchain.callbacks.manager import CallbackManagerForChainRun
+from langchain.chains.base import Chain
+from langchain.chains.llm import LLMChain
+from langchain.experimental.cpal.constants import Constant
+from langchain.experimental.cpal.models import (
+    CausalModel,
+    InterventionModel,
+    NarrativeModel,
+    QueryModel,
+    StoryModel,
+)
+from langchain.experimental.cpal.templates.univariate.causal import (
+    template as causal_template,
+)
+from langchain.experimental.cpal.templates.univariate.intervention import (
+    template as intervention_template,
+)
+from langchain.experimental.cpal.templates.univariate.narrative import (
+    template as narrative_template,
+)
+from langchain.experimental.cpal.templates.univariate.query import (
+    template as query_template,
+)
+from langchain.output_parsers import PydanticOutputParser
+from langchain.prompts.prompt import PromptTemplate
+
+
+class _BaseStoryElementChain(Chain):
+    chain: LLMChain
+    input_key: str = Constant.narrative_input.value  #: :meta private:
+    output_key: str = Constant.chain_answer.value  #: :meta private:
+    pydantic_model: ClassVar[
+        Optional[Type[pydantic.BaseModel]]
+    ] = None  #: :meta private:
+    template: ClassVar[Optional[str]] = None  #: :meta private:
+
+    @classmethod
+    def parser(cls) -> PydanticOutputParser:
+        """Parse LLM output into a pydantic object."""
+        if cls.pydantic_model is None:
+            raise NotImplementedError(
+                f"pydantic_model not implemented for {cls.__name__}"
+            )
+        return PydanticOutputParser(pydantic_object=cls.pydantic_model)
+
+    @property
+    def input_keys(self) -> List[str]:
+        """Return the input keys.
+
+        :meta private:
+        """
+        return [self.input_key]
+
+    @property
+    def output_keys(self) -> List[str]:
+        """Return the output keys.
+
+        :meta private:
+        """
+        _output_keys = [self.output_key]
+        return _output_keys
+
+    @classmethod
+    def from_univariate_prompt(
+        cls,
+        llm: BaseLanguageModel,
+        **kwargs: Any,
+    ) -> Any:
+        return cls(
+            chain=LLMChain(
+                llm=llm,
+                prompt=PromptTemplate(
+                    input_variables=[Constant.narrative_input.value],
+                    template=kwargs.get("template", cls.template),
+                    partial_variables={
+                        "format_instructions": cls.parser().get_format_instructions()
+                    },
+                ),
+            ),
+            **kwargs,
+        )
+
+    def _call(
+        self,
+        inputs: Dict[str, Any],
+        run_manager: Optional[CallbackManagerForChainRun] = None,
+    ) -> Dict[str, Any]:
+        completion = self.chain.run(inputs[self.input_key])
+        pydantic_data = self.__class__.parser().parse(completion)
+        return {
+            Constant.chain_data.value: pydantic_data,
+            Constant.chain_answer.value: None,
+        }
+
+
+class NarrativeChain(_BaseStoryElementChain):
+    """Decompose the narrative into its story elements
+
+    - causal model
+    - query
+    - intervention
+    """
+
+    pydantic_model: ClassVar[Type[pydantic.BaseModel]] = NarrativeModel
+    template: ClassVar[str] = narrative_template
+
+
+class CausalChain(_BaseStoryElementChain):
+    """Translate the causal narrative into a stack of operations."""
+
+    pydantic_model: ClassVar[Type[pydantic.BaseModel]] = CausalModel
+    template: ClassVar[str] = causal_template
+
+
+class InterventionChain(_BaseStoryElementChain):
+    """Set the hypothetical conditions for the causal model."""
+
+    pydantic_model: ClassVar[Type[pydantic.BaseModel]] = InterventionModel
+    template: ClassVar[str] = intervention_template
+
+
+class QueryChain(_BaseStoryElementChain):
+    """Query the outcome table using SQL."""
+
+    pydantic_model: ClassVar[Type[pydantic.BaseModel]] = QueryModel
+    template: ClassVar[str] = query_template  # TODO: incl. table schema
+
+
+class CPALChain(_BaseStoryElementChain):
+    llm: BaseLanguageModel
+    narrative_chain: Optional[NarrativeChain] = None
+    causal_chain: Optional[CausalChain] = None
+    intervention_chain: Optional[InterventionChain] = None
+    query_chain: Optional[QueryChain] = None
+    _story: StoryModel = pydantic.PrivateAttr(default=None)  # TODO: change name ?
+
+    @classmethod
+    def from_univariate_prompt(
+        cls,
+        llm: BaseLanguageModel,
+        **kwargs: Any,
+    ) -> CPALChain:
+        """instantiation depends on component chains"""
+        return cls(
+            llm=llm,
+            chain=LLMChain(
+                llm=llm,
+                prompt=PromptTemplate(
+                    input_variables=["question", "query_result"],
+                    template=(
+                        "Summarize this answer '{query_result}' to this "
+                        "question '{question}'? "
+                    ),
+                ),
+            ),
+            narrative_chain=NarrativeChain.from_univariate_prompt(llm=llm),
+            causal_chain=CausalChain.from_univariate_prompt(llm=llm),
+            intervention_chain=InterventionChain.from_univariate_prompt(llm=llm),
+            query_chain=QueryChain.from_univariate_prompt(llm=llm),
+            **kwargs,
+        )
+
+    def _call(
+        self,
+        inputs: Dict[str, Any],
+        run_manager: Optional[CallbackManagerForChainRun] = None,
+        **kwargs: Any,
+    ) -> Dict[str, Any]:
+        # instantiate component chains
+        if self.narrative_chain is None:
+            self.narrative_chain = NarrativeChain.from_univariate_prompt(llm=self.llm)
+        if self.causal_chain is None:
+            self.causal_chain = CausalChain.from_univariate_prompt(llm=self.llm)
+        if self.intervention_chain is None:
+            self.intervention_chain = InterventionChain.from_univariate_prompt(
+                llm=self.llm
+            )
+        if self.query_chain is None:
+            self.query_chain = QueryChain.from_univariate_prompt(llm=self.llm)
+
+        # decompose narrative into three causal story elements
+        narrative = self.narrative_chain(inputs[Constant.narrative_input.value])[
+            Constant.chain_data.value
+        ]
+
+        story = StoryModel(
+            causal_operations=self.causal_chain(narrative.story_plot)[
+                Constant.chain_data.value
+            ],
+            intervention=self.intervention_chain(narrative.story_hypothetical)[
+                Constant.chain_data.value
+            ],
+            query=self.query_chain(narrative.story_outcome_question)[
+                Constant.chain_data.value
+            ],
+        )
+        self._story = story
+
+        def pretty_print_str(title: str, d: str) -> str:
+            return title + "\n" + d
+
+        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
+        _run_manager.on_text(
+            pretty_print_str("story outcome data", story._outcome_table.to_string()),
+            color="green",
+            end="\n\n",
+            verbose=self.verbose,
+        )
+
+        def pretty_print_dict(title: str, d: dict) -> str:
+            return title + "\n" + json.dumps(d, indent=4)
+
+        _run_manager.on_text(
+            pretty_print_dict("query data", story.query.dict()),
+            color="blue",
+            end="\n\n",
+            verbose=self.verbose,
+        )
+        if story.query._result_table.empty:
+            # prevent piping bad data into subsequent chains
+            raise ValueError(
+                (
+                    "unanswerable, query and outcome are incoherent\n"
+                    "\n"
+                    "outcome:\n"
+                    f"{story._outcome_table}\n"
+                    "query:\n"
+                    f"{story.query.dict()}"
+                )
+            )
+        else:
+            query_result = float(story.query._result_table.values[0][-1])
+            if False:
+                """TODO: add this back in when demanded by composable chains"""
+                reporting_chain = self.chain
+                human_report = reporting_chain.run(
+                    question=story.query.question, query_result=query_result
+                )
+                query_result = {
+                    "query_result": query_result,
+                    "human_report": human_report,
+                }
+        output = {
+            Constant.chain_data.value: story,
+            self.output_key: query_result,
+            **kwargs,
+        }
+        return output
+
+    def draw(self, **kwargs: Any) -> None:
+        """
+        CPAL chain can draw its resulting DAG.
+
+        Usage in a jupyter notebook:
+
+            >>> from IPython.display import SVG
+            >>> cpal_chain.draw(path="graph.svg")
+            >>> SVG('graph.svg')
+        """
+        self._story._networkx_wrapper.draw_graphviz(**kwargs)

libs/langchain-experimental/langchain_experimental/cpal/models.py

@@ -0,0 +1,244 @@
+from __future__ import annotations  # allows pydantic model to reference itself
+
+import re
+from typing import Any, Optional, Union
+
+import duckdb
+import pandas as pd
+from langchain.experimental.cpal.constants import Constant
+from langchain.graphs.networkx_graph import NetworkxEntityGraph
+from pydantic import BaseModel, Field, PrivateAttr, root_validator, validator
+
+
+class NarrativeModel(BaseModel):
+    """
+    Represent the narrative input as three story elements.
+    """
+
+    story_outcome_question: str
+    story_hypothetical: str
+    story_plot: str  # causal stack of operations
+
+    @validator("*", pre=True)
+    def empty_str_to_none(cls, v: str) -> Union[str, None]:
+        """Empty strings are not allowed"""
+        if v == "":
+            return None
+        return v
+
+
+class EntityModel(BaseModel):
+    name: str = Field(description="entity name")
+    code: str = Field(description="entity actions")
+    value: float = Field(description="entity initial value")
+    depends_on: list[str] = Field(default=[], description="ancestor entities")
+
+    # TODO: generalize to multivariate math
+    # TODO: acyclic graph
+
+    class Config:
+        validate_assignment = True
+
+    @validator("name")
+    def lower_case_name(cls, v: str) -> str:
+        v = v.lower()
+        return v
+
+
+class CausalModel(BaseModel):
+    attribute: str = Field(description="name of the attribute to be calculated")
+    entities: list[EntityModel] = Field(description="entities in the story")
+
+    # TODO: root validate each `entity.depends_on` using system's entity names
+
+
+class EntitySettingModel(BaseModel):
+    """
+    Initial conditions for an entity
+
+    {"name": "bud", "attribute": "pet_count", "value": 12}
+    """
+
+    name: str = Field(description="name of the entity")
+    attribute: str = Field(description="name of the attribute to be calculated")
+    value: float = Field(description="entity's attribute value (calculated)")
+
+    @validator("name")
+    def lower_case_transform(cls, v: str) -> str:
+        v = v.lower()
+        return v
+
+
+class SystemSettingModel(BaseModel):
+    """
+    Initial global conditions for the system.
+
+    {"parameter": "interest_rate", "value": .05}
+    """
+
+    parameter: str
+    value: float
+
+
+class InterventionModel(BaseModel):
+    """
+    aka initial conditions
+
+    >>> intervention.dict()
+    {
+        entity_settings: [
+            {"name": "bud", "attribute": "pet_count", "value": 12},
+            {"name": "pat", "attribute": "pet_count", "value": 0},
+        ],
+        system_settings: None,
+    }
+    """
+
+    entity_settings: list[EntitySettingModel]
+    system_settings: Optional[list[SystemSettingModel]] = None
+
+    @validator("system_settings")
+    def lower_case_name(cls, v: str) -> Union[str, None]:
+        if v is not None:
+            raise NotImplementedError("system_setting is not implemented yet")
+        return v
+
+
+class QueryModel(BaseModel):
+    """translate a question about the story outcome into a programmatic expression"""
+
+    question: str = Field(alias=Constant.narrative_input.value)  # input
+    expression: str  # output, part of llm completion
+    llm_error_msg: str  # output, part of llm completion
+    _result_table: str = PrivateAttr()  # result of the executed query
+
+
+class ResultModel(BaseModel):
+    question: str = Field(alias=Constant.narrative_input.value)  # input
+    _result_table: str = PrivateAttr()  # result of the executed query
+
+
+class StoryModel(BaseModel):
+    causal_operations: Any = Field(required=True)
+    intervention: Any = Field(required=True)
+    query: Any = Field(required=True)
+    _outcome_table: pd.DataFrame = PrivateAttr(default=None)
+    _networkx_wrapper: Any = PrivateAttr(default=None)
+
+    def __init__(self, **kwargs: Any):
+        super().__init__(**kwargs)
+        self._compute()
+
+        # TODO: when langchain adopts pydantic.v2 replace w/ `__post_init__`
+        # misses hints github.com/pydantic/pydantic/issues/1729#issuecomment-1300576214
+
+    @root_validator
+    def check_intervention_is_valid(cls, values: dict) -> dict:
+        valid_names = [e.name for e in values["causal_operations"].entities]
+        for setting in values["intervention"].entity_settings:
+            if setting.name not in valid_names:
+                error_msg = f"""
+                    Hypothetical question has an invalid entity name.
+                    `{setting.name}` not in `{valid_names}`
+                """
+                raise ValueError(error_msg)
+        return values
+
+    def _block_back_door_paths(self) -> None:
+        # stop intervention entities from depending on others
+        intervention_entities = [
+            entity_setting.name for entity_setting in self.intervention.entity_settings
+        ]
+        for entity in self.causal_operations.entities:
+            if entity.name in intervention_entities:
+                entity.depends_on = []
+                entity.code = "pass"
+
+    def _set_initial_conditions(self) -> None:
+        for entity_setting in self.intervention.entity_settings:
+            for entity in self.causal_operations.entities:
+                if entity.name == entity_setting.name:
+                    entity.value = entity_setting.value
+
+    def _make_graph(self) -> None:
+        self._networkx_wrapper = NetworkxEntityGraph()
+        for entity in self.causal_operations.entities:
+            for parent_name in entity.depends_on:
+                self._networkx_wrapper._graph.add_edge(
+                    parent_name, entity.name, relation=entity.code
+                )
+
+        # TODO: is it correct to drop entities with no impact on the outcome (?)
+        self.causal_operations.entities = [
+            entity
+            for entity in self.causal_operations.entities
+            if entity.name in self._networkx_wrapper.get_topological_sort()
+        ]
+
+    def _sort_entities(self) -> None:
+        # order the sequence of causal actions
+        sorted_nodes = self._networkx_wrapper.get_topological_sort()
+        self.causal_operations.entities.sort(key=lambda x: sorted_nodes.index(x.name))
+
+    def _forward_propagate(self) -> None:
+        entity_scope = {
+            entity.name: entity for entity in self.causal_operations.entities
+        }
+        for entity in self.causal_operations.entities:
+            if entity.code == "pass":
+                continue
+            else:
+                # gist.github.com/dean0x7d/df5ce97e4a1a05be4d56d1378726ff92
+                exec(entity.code, globals(), entity_scope)
+        row_values = [entity.dict() for entity in entity_scope.values()]
+        self._outcome_table = pd.DataFrame(row_values)
+
+    def _run_query(self) -> None:
+        def humanize_sql_error_msg(error: str) -> str:
+            pattern = r"column\s+(.*?)\s+not found"
+            col_match = re.search(pattern, error)
+            if col_match:
+                return (
+                    "SQL error: "
+                    + col_match.group(1)
+                    + " is not an attribute in your story!"
+                )
+            else:
+                return str(error)
+
+        if self.query.llm_error_msg == "":
+            try:
+                df = self._outcome_table  # noqa
+                query_result = duckdb.sql(self.query.expression).df()
+                self.query._result_table = query_result
+            except duckdb.BinderException as e:
+                self.query._result_table = humanize_sql_error_msg(str(e))
+            except Exception as e:
+                self.query._result_table = str(e)
+        else:
+            msg = "LLM maybe failed to translate question to SQL query."
+            raise ValueError(
+                {
+                    "question": self.query.question,
+                    "llm_error_msg": self.query.llm_error_msg,
+                    "msg": msg,
+                }
+            )
+
+    def _compute(self) -> Any:
+        self._block_back_door_paths()
+        self._set_initial_conditions()
+        self._make_graph()
+        self._sort_entities()
+        self._forward_propagate()
+        self._run_query()
+
+    def print_debug_report(self) -> None:
+        report = {
+            "outcome": self._outcome_table,
+            "query": self.query.dict(),
+            "result": self.query._result_table,
+        }
+        from pprint import pprint
+
+        pprint(report)

libs/langchain-experimental/langchain_experimental/generative_agents/generative_agent.py

@@ -0,0 +1,251 @@
+import re
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+from langchain import LLMChain
+from langchain.experimental.generative_agents.memory import GenerativeAgentMemory
+from langchain.prompts import PromptTemplate
+from langchain.schema.language_model import BaseLanguageModel
+from pydantic import BaseModel, Field
+
+
+class GenerativeAgent(BaseModel):
+    """A character with memory and innate characteristics."""
+
+    name: str
+    """The character's name."""
+
+    age: Optional[int] = None
+    """The optional age of the character."""
+    traits: str = "N/A"
+    """Permanent traits to ascribe to the character."""
+    status: str
+    """The traits of the character you wish not to change."""
+    memory: GenerativeAgentMemory
+    """The memory object that combines relevance, recency, and 'importance'."""
+    llm: BaseLanguageModel
+    """The underlying language model."""
+    verbose: bool = False
+    summary: str = ""  #: :meta private:
+    """Stateful self-summary generated via reflection on the character's memory."""
+
+    summary_refresh_seconds: int = 3600  #: :meta private:
+    """How frequently to re-generate the summary."""
+
+    last_refreshed: datetime = Field(default_factory=datetime.now)  # : :meta private:
+    """The last time the character's summary was regenerated."""
+
+    daily_summaries: List[str] = Field(default_factory=list)  # : :meta private:
+    """Summary of the events in the plan that the agent took."""
+
+    class Config:
+        """Configuration for this pydantic object."""
+
+        arbitrary_types_allowed = True
+
+    # LLM-related methods
+    @staticmethod
+    def _parse_list(text: str) -> List[str]:
+        """Parse a newline-separated string into a list of strings."""
+        lines = re.split(r"\n", text.strip())
+        return [re.sub(r"^\s*\d+\.\s*", "", line).strip() for line in lines]
+
+    def chain(self, prompt: PromptTemplate) -> LLMChain:
+        return LLMChain(
+            llm=self.llm, prompt=prompt, verbose=self.verbose, memory=self.memory
+        )
+
+    def _get_entity_from_observation(self, observation: str) -> str:
+        prompt = PromptTemplate.from_template(
+            "What is the observed entity in the following observation? {observation}"
+            + "\nEntity="
+        )
+        return self.chain(prompt).run(observation=observation).strip()
+
+    def _get_entity_action(self, observation: str, entity_name: str) -> str:
+        prompt = PromptTemplate.from_template(
+            "What is the {entity} doing in the following observation? {observation}"
+            + "\nThe {entity} is"
+        )
+        return (
+            self.chain(prompt).run(entity=entity_name, observation=observation).strip()
+        )
+
+    def summarize_related_memories(self, observation: str) -> str:
+        """Summarize memories that are most relevant to an observation."""
+        prompt = PromptTemplate.from_template(
+            """
+{q1}?
+Context from memory:
+{relevant_memories}
+Relevant context: 
+"""
+        )
+        entity_name = self._get_entity_from_observation(observation)
+        entity_action = self._get_entity_action(observation, entity_name)
+        q1 = f"What is the relationship between {self.name} and {entity_name}"
+        q2 = f"{entity_name} is {entity_action}"
+        return self.chain(prompt=prompt).run(q1=q1, queries=[q1, q2]).strip()
+
+    def _generate_reaction(
+        self, observation: str, suffix: str, now: Optional[datetime] = None
+    ) -> str:
+        """React to a given observation or dialogue act."""
+        prompt = PromptTemplate.from_template(
+            "{agent_summary_description}"
+            + "\nIt is {current_time}."
+            + "\n{agent_name}'s status: {agent_status}"
+            + "\nSummary of relevant context from {agent_name}'s memory:"
+            + "\n{relevant_memories}"
+            + "\nMost recent observations: {most_recent_memories}"
+            + "\nObservation: {observation}"
+            + "\n\n"
+            + suffix
+        )
+        agent_summary_description = self.get_summary(now=now)
+        relevant_memories_str = self.summarize_related_memories(observation)
+        current_time_str = (
+            datetime.now().strftime("%B %d, %Y, %I:%M %p")
+            if now is None
+            else now.strftime("%B %d, %Y, %I:%M %p")
+        )
+        kwargs: Dict[str, Any] = dict(
+            agent_summary_description=agent_summary_description,
+            current_time=current_time_str,
+            relevant_memories=relevant_memories_str,
+            agent_name=self.name,
+            observation=observation,
+            agent_status=self.status,
+        )
+        consumed_tokens = self.llm.get_num_tokens(
+            prompt.format(most_recent_memories="", **kwargs)
+        )
+        kwargs[self.memory.most_recent_memories_token_key] = consumed_tokens
+        return self.chain(prompt=prompt).run(**kwargs).strip()
+
+    def _clean_response(self, text: str) -> str:
+        return re.sub(f"^{self.name} ", "", text.strip()).strip()
+
+    def generate_reaction(
+        self, observation: str, now: Optional[datetime] = None
+    ) -> Tuple[bool, str]:
+        """React to a given observation."""
+        call_to_action_template = (
+            "Should {agent_name} react to the observation, and if so,"
+            + " what would be an appropriate reaction? Respond in one line."
+            + ' If the action is to engage in dialogue, write:\nSAY: "what to say"'
+            + "\notherwise, write:\nREACT: {agent_name}'s reaction (if anything)."
+            + "\nEither do nothing, react, or say something but not both.\n\n"
+        )
+        full_result = self._generate_reaction(
+            observation, call_to_action_template, now=now
+        )
+        result = full_result.strip().split("\n")[0]
+        # AAA
+        self.memory.save_context(
+            {},
+            {
+                self.memory.add_memory_key: f"{self.name} observed "
+                f"{observation} and reacted by {result}",
+                self.memory.now_key: now,
+            },
+        )
+        if "REACT:" in result:
+            reaction = self._clean_response(result.split("REACT:")[-1])
+            return False, f"{self.name} {reaction}"
+        if "SAY:" in result:
+            said_value = self._clean_response(result.split("SAY:")[-1])
+            return True, f"{self.name} said {said_value}"
+        else:
+            return False, result
+
+    def generate_dialogue_response(
+        self, observation: str, now: Optional[datetime] = None
+    ) -> Tuple[bool, str]:
+        """React to a given observation."""
+        call_to_action_template = (
+            "What would {agent_name} say? To end the conversation, write:"
+            ' GOODBYE: "what to say". Otherwise to continue the conversation,'
+            ' write: SAY: "what to say next"\n\n'
+        )
+        full_result = self._generate_reaction(
+            observation, call_to_action_template, now=now
+        )
+        result = full_result.strip().split("\n")[0]
+        if "GOODBYE:" in result:
+            farewell = self._clean_response(result.split("GOODBYE:")[-1])
+            self.memory.save_context(
+                {},
+                {
+                    self.memory.add_memory_key: f"{self.name} observed "
+                    f"{observation} and said {farewell}",
+                    self.memory.now_key: now,
+                },
+            )
+            return False, f"{self.name} said {farewell}"
+        if "SAY:" in result:
+            response_text = self._clean_response(result.split("SAY:")[-1])
+            self.memory.save_context(
+                {},
+                {
+                    self.memory.add_memory_key: f"{self.name} observed "
+                    f"{observation} and said {response_text}",
+                    self.memory.now_key: now,
+                },
+            )
+            return True, f"{self.name} said {response_text}"
+        else:
+            return False, result
+
+    ######################################################
+    # Agent stateful' summary methods.                   #
+    # Each dialog or response prompt includes a header   #
+    # summarizing the agent's self-description. This is  #
+    # updated periodically through probing its memories  #
+    ######################################################
+    def _compute_agent_summary(self) -> str:
+        """"""
+        prompt = PromptTemplate.from_template(
+            "How would you summarize {name}'s core characteristics given the"
+            + " following statements:\n"
+            + "{relevant_memories}"
+            + "Do not embellish."
+            + "\n\nSummary: "
+        )
+        # The agent seeks to think about their core characteristics.
+        return (
+            self.chain(prompt)
+            .run(name=self.name, queries=[f"{self.name}'s core characteristics"])
+            .strip()
+        )
+
+    def get_summary(
+        self, force_refresh: bool = False, now: Optional[datetime] = None
+    ) -> str:
+        """Return a descriptive summary of the agent."""
+        current_time = datetime.now() if now is None else now
+        since_refresh = (current_time - self.last_refreshed).seconds
+        if (
+            not self.summary
+            or since_refresh >= self.summary_refresh_seconds
+            or force_refresh
+        ):
+            self.summary = self._compute_agent_summary()
+            self.last_refreshed = current_time
+        age = self.age if self.age is not None else "N/A"
+        return (
+            f"Name: {self.name} (age: {age})"
+            + f"\nInnate traits: {self.traits}"
+            + f"\n{self.summary}"
+        )
+
+    def get_full_header(
+        self, force_refresh: bool = False, now: Optional[datetime] = None
+    ) -> str:
+        """Return a full header of the agent's status, summary, and current time."""
+        now = datetime.now() if now is None else now
+        summary = self.get_summary(force_refresh=force_refresh, now=now)
+        current_time_str = now.strftime("%B %d, %Y, %I:%M %p")
+        return (
+            f"{summary}\nIt is {current_time_str}.\n{self.name}'s status: {self.status}"
+        )

libs/langchain-experimental/langchain_experimental/llms/jsonformer_decoder.py

@@ -0,0 +1,60 @@
+"""Experimental implementation of jsonformer wrapped LLM."""
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any, List, Optional, cast
+
+from langchain.callbacks.manager import CallbackManagerForLLMRun
+from langchain.llms.huggingface_pipeline import HuggingFacePipeline
+from pydantic import Field, root_validator
+
+if TYPE_CHECKING:
+    import jsonformer
+
+
+def import_jsonformer() -> jsonformer:
+    """Lazily import jsonformer."""
+    try:
+        import jsonformer
+    except ImportError:
+        raise ValueError(
+            "Could not import jsonformer python package. "
+            "Please install it with `pip install jsonformer`."
+        )
+    return jsonformer
+
+
+class JsonFormer(HuggingFacePipeline):
+    json_schema: dict = Field(..., description="The JSON Schema to complete.")
+    max_new_tokens: int = Field(
+        default=200, description="Maximum number of new tokens to generate."
+    )
+    debug: bool = Field(default=False, description="Debug mode.")
+
+    @root_validator
+    def check_jsonformer_installation(cls, values: dict) -> dict:
+        import_jsonformer()
+        return values
+
+    def _call(
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> str:
+        jsonformer = import_jsonformer()
+        from transformers import Text2TextGenerationPipeline
+
+        pipeline = cast(Text2TextGenerationPipeline, self.pipeline)
+
+        model = jsonformer.Jsonformer(
+            model=pipeline.model,
+            tokenizer=pipeline.tokenizer,
+            json_schema=self.json_schema,
+            prompt=prompt,
+            max_number_tokens=self.max_new_tokens,
+            debug=self.debug,
+        )
+        text = model()
+        return json.dumps(text)

libs/langchain-experimental/langchain_experimental/llms/rellm_decoder.py

@@ -0,0 +1,67 @@
+"""Experimental implementation of RELLM wrapped LLM."""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, List, Optional, cast
+
+from langchain.callbacks.manager import CallbackManagerForLLMRun
+from langchain.llms.huggingface_pipeline import HuggingFacePipeline
+from langchain.llms.utils import enforce_stop_tokens
+from pydantic import Field, root_validator
+
+if TYPE_CHECKING:
+    import rellm
+    from regex import Pattern as RegexPattern
+else:
+    try:
+        from regex import Pattern as RegexPattern
+    except ImportError:
+        pass
+
+
+def import_rellm() -> rellm:
+    """Lazily import rellm."""
+    try:
+        import rellm
+    except ImportError:
+        raise ValueError(
+            "Could not import rellm python package. "
+            "Please install it with `pip install rellm`."
+        )
+    return rellm
+
+
+class RELLM(HuggingFacePipeline):
+    regex: RegexPattern = Field(..., description="The structured format to complete.")
+    max_new_tokens: int = Field(
+        default=200, description="Maximum number of new tokens to generate."
+    )
+
+    @root_validator
+    def check_rellm_installation(cls, values: dict) -> dict:
+        import_rellm()
+        return values
+
+    def _call(
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> str:
+        rellm = import_rellm()
+        from transformers import Text2TextGenerationPipeline
+
+        pipeline = cast(Text2TextGenerationPipeline, self.pipeline)
+
+        text = rellm.complete_re(
+            prompt,
+            self.regex,
+            tokenizer=pipeline.tokenizer,
+            model=pipeline.model,
+            max_new_tokens=self.max_new_tokens,
+        )
+        if stop is not None:
+            # This is a bit hacky, but I can't figure out a better way to enforce
+            # stop tokens when making calls to huggingface_hub.
+            text = enforce_stop_tokens(text, stop)
+        return text

libs/langchain-experimental/langchain_experimental/plan_and_execute/agent_executor.py

@@ -0,0 +1,59 @@
+from typing import Any, Dict, List, Optional
+
+from langchain.callbacks.manager import CallbackManagerForChainRun
+from langchain.chains.base import Chain
+from langchain.experimental.plan_and_execute.executors.base import BaseExecutor
+from langchain.experimental.plan_and_execute.planners.base import BasePlanner
+from langchain.experimental.plan_and_execute.schema import (
+    BaseStepContainer,
+    ListStepContainer,
+)
+from pydantic import Field
+
+
+class PlanAndExecute(Chain):
+    planner: BasePlanner
+    executor: BaseExecutor
+    step_container: BaseStepContainer = Field(default_factory=ListStepContainer)
+    input_key: str = "input"
+    output_key: str = "output"
+
+    @property
+    def input_keys(self) -> List[str]:
+        return [self.input_key]
+
+    @property
+    def output_keys(self) -> List[str]:
+        return [self.output_key]
+
+    def _call(
+        self,
+        inputs: Dict[str, Any],
+        run_manager: Optional[CallbackManagerForChainRun] = None,
+    ) -> Dict[str, Any]:
+        plan = self.planner.plan(
+            inputs,
+            callbacks=run_manager.get_child() if run_manager else None,
+        )
+        if run_manager:
+            run_manager.on_text(str(plan), verbose=self.verbose)
+        for step in plan.steps:
+            _new_inputs = {
+                "previous_steps": self.step_container,
+                "current_step": step,
+                "objective": inputs[self.input_key],
+            }
+            new_inputs = {**_new_inputs, **inputs}
+            response = self.executor.step(
+                new_inputs,
+                callbacks=run_manager.get_child() if run_manager else None,
+            )
+            if run_manager:
+                run_manager.on_text(
+                    f"*****\n\nStep: {step.value}", verbose=self.verbose
+                )
+                run_manager.on_text(
+                    f"\n\nResponse: {response.response}", verbose=self.verbose
+                )
+            self.step_container.add_step(step, response)
+        return {self.output_key: self.step_container.get_final_response()}

libs/langchain-experimental/langchain_experimental/plan_and_execute/executors/base.py

@@ -0,0 +1,39 @@
+from abc import abstractmethod
+from typing import Any
+
+from langchain.callbacks.manager import Callbacks
+from langchain.chains.base import Chain
+from langchain.experimental.plan_and_execute.schema import StepResponse
+from pydantic import BaseModel
+
+
+class BaseExecutor(BaseModel):
+    @abstractmethod
+    def step(
+        self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any
+    ) -> StepResponse:
+        """Take step."""
+
+    @abstractmethod
+    async def astep(
+        self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any
+    ) -> StepResponse:
+        """Take step."""
+
+
+class ChainExecutor(BaseExecutor):
+    chain: Chain
+
+    def step(
+        self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any
+    ) -> StepResponse:
+        """Take step."""
+        response = self.chain.run(**inputs, callbacks=callbacks)
+        return StepResponse(response=response)
+
+    async def astep(
+        self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any
+    ) -> StepResponse:
+        """Take step."""
+        response = await self.chain.arun(**inputs, callbacks=callbacks)
+        return StepResponse(response=response)

libs/langchain-experimental/langchain_experimental/plan_and_execute/planners/base.py

@@ -0,0 +1,39 @@
+from abc import abstractmethod
+from typing import Any, List, Optional
+
+from langchain.callbacks.manager import Callbacks
+from langchain.chains.llm import LLMChain
+from langchain.experimental.plan_and_execute.schema import Plan, PlanOutputParser
+from pydantic import BaseModel
+
+
+class BasePlanner(BaseModel):
+    @abstractmethod
+    def plan(self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any) -> Plan:
+        """Given input, decide what to do."""
+
+    @abstractmethod
+    async def aplan(
+        self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any
+    ) -> Plan:
+        """Given input, decide what to do."""
+
+
+class LLMPlanner(BasePlanner):
+    llm_chain: LLMChain
+    output_parser: PlanOutputParser
+    stop: Optional[List] = None
+
+    def plan(self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any) -> Plan:
+        """Given input, decide what to do."""
+        llm_response = self.llm_chain.run(**inputs, stop=self.stop, callbacks=callbacks)
+        return self.output_parser.parse(llm_response)
+
+    async def aplan(
+        self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any
+    ) -> Plan:
+        """Given input, decide what to do."""
+        llm_response = await self.llm_chain.arun(
+            **inputs, stop=self.stop, callbacks=callbacks
+        )
+        return self.output_parser.parse(llm_response)

libs/langchain-experimental/langchain_experimental/plan_and_execute/schema.py

@@ -0,0 +1,46 @@
+from abc import abstractmethod
+from typing import List, Tuple
+
+from langchain.schema import BaseOutputParser
+from pydantic import BaseModel, Field
+
+
+class Step(BaseModel):
+    value: str
+
+
+class Plan(BaseModel):
+    steps: List[Step]
+
+
+class StepResponse(BaseModel):
+    response: str
+
+
+class BaseStepContainer(BaseModel):
+    @abstractmethod
+    def add_step(self, step: Step, step_response: StepResponse) -> None:
+        """Add step and step response to the container."""
+
+    @abstractmethod
+    def get_final_response(self) -> str:
+        """Return the final response based on steps taken."""
+
+
+class ListStepContainer(BaseStepContainer):
+    steps: List[Tuple[Step, StepResponse]] = Field(default_factory=list)
+
+    def add_step(self, step: Step, step_response: StepResponse) -> None:
+        self.steps.append((step, step_response))
+
+    def get_steps(self) -> List[Tuple[Step, StepResponse]]:
+        return self.steps
+
+    def get_final_response(self) -> str:
+        return self.steps[-1][1].response
+
+
+class PlanOutputParser(BaseOutputParser):
+    @abstractmethod
+    def parse(self, text: str) -> Plan:
+        """Parse into a plan."""

libs/langchain-experimental/pyproject.toml

@@ -0,0 +1,77 @@
+[tool.poetry]
+name = "langchain_experimental"
+version = "0.0.1"
+description = "Experimental part of LangChain"
+authors = []
+license = "MIT"
+readme = "README.md"
+repository = "https://www.github.com/hwchase17/langchain"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+langchain = "*"
+
+[tool.poetry.group.lint.dependencies]
+ruff = "^0.0.249"
+black = "^23.1.0"
+
+[tool.poetry.group.typing.dependencies]
+mypy = "^0.991"
+
+[tool.poetry.group.dev.dependencies]
+jupyter = "^1.0.0"
+playwright = "^1.28.0"
+setuptools = "^67.6.1"
+
+[tool.poetry.group.test.dependencies]
+# The only dependencies that should be added are
+# dependencies used for running tests (e.g., pytest, freezegun, response).
+# Any dependencies that do not meet that criteria will be removed.
+pytest = "^7.3.0"
+
+[tool.ruff]
+select = [
+  "E",  # pycodestyle
+  "F",  # pyflakes
+  "I",  # isort
+]
+
+[tool.mypy]
+ignore_missing_imports = "True"
+disallow_untyped_defs = "True"
+exclude = ["notebooks", "examples", "example_data"]
+
+[tool.coverage.run]
+omit = [
+    "tests/*",
+]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+# --strict-markers will raise errors on unknown marks.
+# https://docs.pytest.org/en/7.1.x/how-to/mark.html#raising-errors-on-unknown-marks
+#
+# https://docs.pytest.org/en/7.1.x/reference/reference.html
+# --strict-config       any warnings encountered while parsing the `pytest`
+#                       section of the configuration file raise errors.
+#
+# https://github.com/tophat/syrupy
+# --snapshot-warn-unused    Prints a warning on unused snapshots rather than fail the test suite.
+addopts = "--strict-markers --strict-config --durations=5"
+# Registering custom markers.
+# https://docs.pytest.org/en/7.1.x/example/markers.html#registering-markers
+markers = [
+  "requires: mark tests as requiring a specific library"
+]
+
+[tool.codespell]
+skip = '.git,*.pdf,*.svg,*.pdf,*.yaml,*.ipynb,poetry.lock,*.min.js,*.css,package-lock.json,example_data,_dist,examples'
+# Ignore latin etc
+ignore-regex = '.*(Stati Uniti|Tense=Pres).*'
+# whats is a typo but used frequently in queries so kept as is
+# aapply - async apply
+# unsecure - typo but part of API, decided to not bother for now
+ignore-words-list = 'momento,collison,ned,foor,reworkd,parth,whats,aapply,mysogyny,unsecure,damon'

libs/langchain-experimental/tests/unit_tests/test_mock.py

@@ -0,0 +1,2 @@
+def test_mock() -> None:
+    assert True

libs/langchain/README.md

@@ -0,0 +1,95 @@
+# 🦜️🔗 LangChain
+
+⚡ Building applications with LLMs through composability ⚡
+
+[![Release Notes](https://img.shields.io/github/release/hwchase17/langchain)](https://github.com/hwchase17/langchain/releases)
+[![lint](https://github.com/hwchase17/langchain/actions/workflows/lint.yml/badge.svg)](https://github.com/hwchase17/langchain/actions/workflows/lint.yml)
+[![test](https://github.com/hwchase17/langchain/actions/workflows/test.yml/badge.svg)](https://github.com/hwchase17/langchain/actions/workflows/test.yml)
+[![Downloads](https://static.pepy.tech/badge/langchain/month)](https://pepy.tech/project/langchain)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Twitter](https://img.shields.io/twitter/url/https/twitter.com/langchainai.svg?style=social&label=Follow%20%40LangChainAI)](https://twitter.com/langchainai)
+[![](https://dcbadge.vercel.app/api/server/6adMQxSpJS?compact=true&style=flat)](https://discord.gg/6adMQxSpJS)
+[![Open in Dev Containers](https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/hwchase17/langchain)
+[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/hwchase17/langchain)
+[![GitHub star chart](https://img.shields.io/github/stars/hwchase17/langchain?style=social)](https://star-history.com/#hwchase17/langchain)
+[![Dependency Status](https://img.shields.io/librariesio/github/hwchase17/langchain)](https://libraries.io/github/hwchase17/langchain)
+[![Open Issues](https://img.shields.io/github/issues-raw/hwchase17/langchain)](https://github.com/hwchase17/langchain/issues)
+
+
+Looking for the JS/TS version? Check out [LangChain.js](https://github.com/hwchase17/langchainjs).
+
+**Production Support:** As you move your LangChains into production, we'd love to offer more comprehensive support.
+Please fill out [this form](https://forms.gle/57d8AmXBYp8PP8tZA) and we'll set up a dedicated support Slack channel.
+
+## Quick Install
+
+`pip install langchain`
+or
+`pip install langsmith && conda install langchain -c conda-forge`
+
+## 🤔 What is this?
+
+Large language models (LLMs) are emerging as a transformative technology, enabling developers to build applications that they previously could not. However, using these LLMs in isolation is often insufficient for creating a truly powerful app - the real power comes when you can combine them with other sources of computation or knowledge.
+
+This library aims to assist in the development of those types of applications. Common examples of these applications include:
+
+**❓ Question Answering over specific documents**
+
+- [Documentation](https://python.langchain.com/docs/use_cases/question_answering/)
+- End-to-end Example: [Question Answering over Notion Database](https://github.com/hwchase17/notion-qa)
+
+**💬 Chatbots**
+
+- [Documentation](https://python.langchain.com/docs/use_cases/chatbots/)
+- End-to-end Example: [Chat-LangChain](https://github.com/hwchase17/chat-langchain)
+
+**🤖 Agents**
+
+- [Documentation](https://python.langchain.com/docs/modules/agents/)
+- End-to-end Example: [GPT+WolframAlpha](https://huggingface.co/spaces/JavaFXpert/Chat-GPT-LangChain)
+
+## 📖 Documentation
+
+Please see [here](https://python.langchain.com) for full documentation on:
+
+- Getting started (installation, setting up the environment, simple examples)
+- How-To examples (demos, integrations, helper functions)
+- Reference (full API docs)
+- Resources (high-level explanation of core concepts)
+
+## 🚀 What can this help with?
+
+There are six main areas that LangChain is designed to help with.
+These are, in increasing order of complexity:
+
+**📃 LLMs and Prompts:**
+
+This includes prompt management, prompt optimization, a generic interface for all LLMs, and common utilities for working with LLMs.
+
+**🔗 Chains:**
+
+Chains go beyond a single LLM call and involve sequences of calls (whether to an LLM or a different utility). LangChain provides a standard interface for chains, lots of integrations with other tools, and end-to-end chains for common applications.
+
+**📚 Data Augmented Generation:**
+
+Data Augmented Generation involves specific types of chains that first interact with an external data source to fetch data for use in the generation step. Examples include summarization of long pieces of text and question/answering over specific data sources.
+
+**🤖 Agents:**
+
+Agents involve an LLM making decisions about which Actions to take, taking that Action, seeing an Observation, and repeating that until done. LangChain provides a standard interface for agents, a selection of agents to choose from, and examples of end-to-end agents.
+
+**🧠 Memory:**
+
+Memory refers to persisting state between calls of a chain/agent. LangChain provides a standard interface for memory, a collection of memory implementations, and examples of chains/agents that use memory.
+
+**🧐 Evaluation:**
+
+[BETA] Generative models are notoriously hard to evaluate with traditional metrics. One new way of evaluating them is using language models themselves to do the evaluation. LangChain provides some prompts/chains for assisting in this.
+
+For more information on these concepts, please see our [full documentation](https://python.langchain.com).
+
+## 💁 Contributing
+
+As an open-source project in a rapidly developing field, we are extremely open to contributions, whether it be in the form of a new feature, improved infrastructure, or better documentation.
+
+For detailed information on how to contribute, see [here](.github/CONTRIBUTING.md).