(WIP) set up experimental (#7959)

.devcontainer/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 services:
   langchain:
     build:
-      dockerfile: dev.Dockerfile
+      dockerfile: libs/langchain/dev.Dockerfile
       context: ..
     volumes:
    # Update this to wherever you want VS Code to mount the folder of your project

.github/actions/poetry_setup/action.yml

@@ -52,11 +52,13 @@ runs:
 
     - name: Check Poetry File
       shell: bash
+      working-directory: ${{ inputs.working-directory }}
       run: |
         poetry check
 
     - name: Check lock file
       shell: bash
+      working-directory: ${{ inputs.working-directory }}
       run: |
         poetry lock --check
 

.github/workflows/_lint.yml

@@ -1,15 +1,21 @@
 name: lint
 
 on:
-  push:
-    branches: [master]
-  pull_request:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
 
 env:
   POETRY_VERSION: "1.4.2"
 
 jobs:
   build:
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
     runs-on: ubuntu-latest
     strategy:
       matrix:

.github/workflows/_release.yml

@@ -1,13 +1,12 @@
 name: release
 
 on:
-  pull_request:
-    types:
-      - closed
-    branches:
-      - master
-    paths:
-      - 'pyproject.toml'
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
 
 env:
   POETRY_VERSION: "1.4.2"
@@ -18,6 +17,9 @@ jobs:
         ${{ github.event.pull_request.merged == true }}
         && ${{ contains(github.event.pull_request.labels.*.name, 'release') }}
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
     steps:
       - uses: actions/checkout@v3
       - name: Install poetry

.github/workflows/_test.yml

@@ -1,16 +1,21 @@
 name: test
 
 on:
-  push:
-    branches: [master]
-  pull_request:
-  workflow_dispatch:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
 
 env:
   POETRY_VERSION: "1.4.2"
 
 jobs:
   build:
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -29,6 +34,7 @@ jobs:
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
+          working-directory: ${{ inputs.working-directory }}
           poetry-version: "1.4.2"
           cache-key: ${{ matrix.test_type }}
           install-command: |

.github/workflows/langchain_ci.yml

@@ -0,0 +1,27 @@
+---
+name: libs/langchain CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    paths:
+      - '.github/workflows/_lint.yml'
+      - '.github/workflows/_test.yml'
+      - '.github/workflows/langchain_ci.yml'
+      - 'libs/langchain/**'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  lint:
+    uses:
+      ./.github/workflows/_lint.yml
+    with:
+      working-directory: libs/langchain
+    secrets: inherit
+  test:
+    uses:
+      ./.github/workflows/_test.yml
+    with:
+      working-directory: libs/langchain
+    secrets: inherit

.github/workflows/langchain_release.yml

@@ -0,0 +1,19 @@
+---
+name: libs/langchain Release
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - master
+    paths:
+      - 'libs/langchain/pyproject.toml'
+
+jobs:
+  release:
+    uses:
+      ./.github/workflows/_release.yml
+    with:
+      working-directory: libs/langchain
+    secrets: inherit

Makefile

@@ -1,18 +1,8 @@
-.PHONY: all clean docs_build docs_clean docs_linkcheck api_docs_build api_docs_clean api_docs_linkcheck format lint test tests test_watch integration_tests docker_tests help extended_tests
+.PHONY: all clean docs_build docs_clean docs_linkcheck api_docs_build api_docs_clean api_docs_linkcheck
 
 # Default target executed when no arguments are given to make.
 all: help
 
-######################
-# TESTING AND COVERAGE
-######################
-
-# Run unit tests and generate a coverage report.
-coverage:
-	poetry run pytest --cov \
-		--cov-config=.coveragerc \
-		--cov-report xml \
-		--cov-report term-missing:skip-covered
 
 ######################
 # DOCUMENTATION
@@ -41,45 +31,6 @@ api_docs_clean:
 api_docs_linkcheck:
 	poetry run linkchecker docs/api_reference/_build/html/index.html
 
-# Define a variable for the test file path.
-TEST_FILE ?= tests/unit_tests/
-
-test:
-	poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
-
-tests: 
-	poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
-
-extended_tests:
-	poetry run pytest --disable-socket --allow-unix-socket --only-extended tests/unit_tests
-
-test_watch:
-	poetry run ptw --now . -- tests/unit_tests
-
-integration_tests:
-	poetry run pytest tests/integration_tests
-
-docker_tests:
-	docker build -t my-langchain-image:test .
-	docker run --rm my-langchain-image:test
-
-######################
-# LINTING AND FORMATTING
-######################
-
-# Define a variable for Python and notebook files.
-PYTHON_FILES=.
-lint format: PYTHON_FILES=.
-lint_diff format_diff: PYTHON_FILES=$(shell git diff --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
-
-lint lint_diff:
-	poetry run mypy $(PYTHON_FILES)
-	poetry run black $(PYTHON_FILES) --check
-	poetry run ruff .
-
-format format_diff:
-	poetry run black $(PYTHON_FILES)
-	poetry run ruff --select I --fix $(PYTHON_FILES)
 
 spell_check:
 	poetry run codespell --toml pyproject.toml
@@ -97,12 +48,3 @@ help:
 	@echo 'docs_build                   - build the documentation'
 	@echo 'docs_clean                   - clean the documentation build artifacts'
 	@echo 'docs_linkcheck               - run linkchecker on the documentation'
-	@echo 'format                       - run code formatters'
-	@echo 'lint                         - run linters'
-	@echo 'test                         - run unit tests'
-	@echo 'tests                        - run unit tests'
-	@echo 'test TEST_FILE=<test_file>   - run all tests in file'
-	@echo 'extended_tests               - run only extended unit tests'
-	@echo 'test_watch                   - run unit tests in watch mode'
-	@echo 'integration_tests            - run integration tests'
-	@echo 'docker_tests                 - run unit tests in docker'

docs/api_reference/create_api_rst.py

@@ -4,7 +4,7 @@ import re
 from pathlib import Path
 
 ROOT_DIR = Path(__file__).parents[2].absolute()
-PKG_DIR = ROOT_DIR / "langchain"
+PKG_DIR = ROOT_DIR / "libs" / "langchain"
 WRITE_FILE = Path(__file__).parent / "api_reference.rst"
 
 

libs/langchain/Makefile

@@ -0,0 +1,108 @@
+.PHONY: all clean docs_build docs_clean docs_linkcheck api_docs_build api_docs_clean api_docs_linkcheck format lint test tests test_watch integration_tests docker_tests help extended_tests
+
+# Default target executed when no arguments are given to make.
+all: help
+
+######################
+# TESTING AND COVERAGE
+######################
+
+# Run unit tests and generate a coverage report.
+coverage:
+	poetry run pytest --cov \
+		--cov-config=.coveragerc \
+		--cov-report xml \
+		--cov-report term-missing:skip-covered
+
+######################
+# DOCUMENTATION
+######################
+
+clean: docs_clean api_docs_clean
+
+
+docs_build:
+	docs/.local_build.sh
+
+docs_clean:
+	rm -r docs/_dist
+
+docs_linkcheck:
+	poetry run linkchecker docs/_dist/docs_skeleton/ --ignore-url node_modules
+
+api_docs_build:
+	poetry run python docs/api_reference/create_api_rst.py
+	cd docs/api_reference && poetry run make html
+
+api_docs_clean:
+	rm -f docs/api_reference/api_reference.rst
+	cd docs/api_reference && poetry run make clean
+
+api_docs_linkcheck:
+	poetry run linkchecker docs/api_reference/_build/html/index.html
+
+# Define a variable for the test file path.
+TEST_FILE ?= tests/unit_tests/
+
+test:
+	poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
+
+tests: 
+	poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
+
+extended_tests:
+	poetry run pytest --disable-socket --allow-unix-socket --only-extended tests/unit_tests
+
+test_watch:
+	poetry run ptw --now . -- tests/unit_tests
+
+integration_tests:
+	poetry run pytest tests/integration_tests
+
+docker_tests:
+	docker build -t my-langchain-image:test .
+	docker run --rm my-langchain-image:test
+
+######################
+# LINTING AND FORMATTING
+######################
+
+# Define a variable for Python and notebook files.
+PYTHON_FILES=.
+lint format: PYTHON_FILES=.
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+
+lint lint_diff:
+	poetry run mypy $(PYTHON_FILES)
+	poetry run black $(PYTHON_FILES) --check
+	poetry run ruff .
+
+format format_diff:
+	poetry run black $(PYTHON_FILES)
+	poetry run ruff --select I --fix $(PYTHON_FILES)
+
+spell_check:
+	poetry run codespell --toml pyproject.toml
+
+spell_fix:
+	poetry run codespell --toml pyproject.toml -w
+
+######################
+# HELP
+######################
+
+help:
+	@echo '----'
+	@echo 'coverage                     - run unit tests and generate coverage report'
+	@echo 'docs_build                   - build the documentation'
+	@echo 'docs_clean                   - clean the documentation build artifacts'
+	@echo 'docs_linkcheck               - run linkchecker on the documentation'
+	@echo 'format                       - run code formatters'
+	@echo 'lint                         - run linters'
+	@echo 'test                         - run unit tests'
+	@echo 'tests                        - run unit tests'
+	@echo 'test TEST_FILE=<test_file>   - run all tests in file'
+	@echo 'extended_tests               - run only extended unit tests'
+	@echo 'test_watch                   - run unit tests in watch mode'
+	@echo 'integration_tests            - run integration tests'
+	@echo 'docker_tests                 - run unit tests in docker'

libs/langchain/README.md

@@ -0,0 +1,95 @@
+# 🦜️🔗 LangChain
+
+⚡ Building applications with LLMs through composability ⚡
+
+[![Release Notes](https://img.shields.io/github/release/hwchase17/langchain)](https://github.com/hwchase17/langchain/releases)
+[![lint](https://github.com/hwchase17/langchain/actions/workflows/lint.yml/badge.svg)](https://github.com/hwchase17/langchain/actions/workflows/lint.yml)
+[![test](https://github.com/hwchase17/langchain/actions/workflows/test.yml/badge.svg)](https://github.com/hwchase17/langchain/actions/workflows/test.yml)
+[![Downloads](https://static.pepy.tech/badge/langchain/month)](https://pepy.tech/project/langchain)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Twitter](https://img.shields.io/twitter/url/https/twitter.com/langchainai.svg?style=social&label=Follow%20%40LangChainAI)](https://twitter.com/langchainai)
+[![](https://dcbadge.vercel.app/api/server/6adMQxSpJS?compact=true&style=flat)](https://discord.gg/6adMQxSpJS)
+[![Open in Dev Containers](https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/hwchase17/langchain)
+[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/hwchase17/langchain)
+[![GitHub star chart](https://img.shields.io/github/stars/hwchase17/langchain?style=social)](https://star-history.com/#hwchase17/langchain)
+[![Dependency Status](https://img.shields.io/librariesio/github/hwchase17/langchain)](https://libraries.io/github/hwchase17/langchain)
+[![Open Issues](https://img.shields.io/github/issues-raw/hwchase17/langchain)](https://github.com/hwchase17/langchain/issues)
+
+
+Looking for the JS/TS version? Check out [LangChain.js](https://github.com/hwchase17/langchainjs).
+
+**Production Support:** As you move your LangChains into production, we'd love to offer more comprehensive support.
+Please fill out [this form](https://forms.gle/57d8AmXBYp8PP8tZA) and we'll set up a dedicated support Slack channel.
+
+## Quick Install
+
+`pip install langchain`
+or
+`pip install langsmith && conda install langchain -c conda-forge`
+
+## 🤔 What is this?
+
+Large language models (LLMs) are emerging as a transformative technology, enabling developers to build applications that they previously could not. However, using these LLMs in isolation is often insufficient for creating a truly powerful app - the real power comes when you can combine them with other sources of computation or knowledge.
+
+This library aims to assist in the development of those types of applications. Common examples of these applications include:
+
+**❓ Question Answering over specific documents**
+
+- [Documentation](https://python.langchain.com/docs/use_cases/question_answering/)
+- End-to-end Example: [Question Answering over Notion Database](https://github.com/hwchase17/notion-qa)
+
+**💬 Chatbots**
+
+- [Documentation](https://python.langchain.com/docs/use_cases/chatbots/)
+- End-to-end Example: [Chat-LangChain](https://github.com/hwchase17/chat-langchain)
+
+**🤖 Agents**
+
+- [Documentation](https://python.langchain.com/docs/modules/agents/)
+- End-to-end Example: [GPT+WolframAlpha](https://huggingface.co/spaces/JavaFXpert/Chat-GPT-LangChain)
+
+## 📖 Documentation
+
+Please see [here](https://python.langchain.com) for full documentation on:
+
+- Getting started (installation, setting up the environment, simple examples)
+- How-To examples (demos, integrations, helper functions)
+- Reference (full API docs)
+- Resources (high-level explanation of core concepts)
+
+## 🚀 What can this help with?
+
+There are six main areas that LangChain is designed to help with.
+These are, in increasing order of complexity:
+
+**📃 LLMs and Prompts:**
+
+This includes prompt management, prompt optimization, a generic interface for all LLMs, and common utilities for working with LLMs.
+
+**🔗 Chains:**
+
+Chains go beyond a single LLM call and involve sequences of calls (whether to an LLM or a different utility). LangChain provides a standard interface for chains, lots of integrations with other tools, and end-to-end chains for common applications.
+
+**📚 Data Augmented Generation:**
+
+Data Augmented Generation involves specific types of chains that first interact with an external data source to fetch data for use in the generation step. Examples include summarization of long pieces of text and question/answering over specific data sources.
+
+**🤖 Agents:**
+
+Agents involve an LLM making decisions about which Actions to take, taking that Action, seeing an Observation, and repeating that until done. LangChain provides a standard interface for agents, a selection of agents to choose from, and examples of end-to-end agents.
+
+**🧠 Memory:**
+
+Memory refers to persisting state between calls of a chain/agent. LangChain provides a standard interface for memory, a collection of memory implementations, and examples of chains/agents that use memory.
+
+**🧐 Evaluation:**
+
+[BETA] Generative models are notoriously hard to evaluate with traditional metrics. One new way of evaluating them is using language models themselves to do the evaluation. LangChain provides some prompts/chains for assisting in this.
+
+For more information on these concepts, please see our [full documentation](https://python.langchain.com).
+
+## 💁 Contributing
+
+As an open-source project in a rapidly developing field, we are extremely open to contributions, whether it be in the form of a new feature, improved infrastructure, or better documentation.
+
+For detailed information on how to contribute, see [here](.github/CONTRIBUTING.md).