Merge branch 'master' into harrison/improve-integration-docs

.github/DISCUSSION_TEMPLATE/q-a.yml

@@ -22,7 +22,7 @@ body:
         if there's another way to solve your problem:
         
         [LangChain documentation with the integrated search](https://python.langchain.com/docs/get_started/introduction),
-        [API Reference](https://api.python.langchain.com/en/stable/),
+        [API Reference](https://python.langchain.com/api_reference/),
         [GitHub search](https://github.com/langchain-ai/langchain),
         [LangChain Github Discussions](https://github.com/langchain-ai/langchain/discussions),
         [LangChain Github Issues](https://github.com/langchain-ai/langchain/issues?q=is%3Aissue),

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -16,7 +16,7 @@ body:
         if there's another way to solve your problem:
         
         [LangChain documentation with the integrated search](https://python.langchain.com/docs/get_started/introduction),
-        [API Reference](https://api.python.langchain.com/en/stable/),
+        [API Reference](https://python.langchain.com/api_reference/),
         [GitHub search](https://github.com/langchain-ai/langchain),
         [LangChain Github Discussions](https://github.com/langchain-ai/langchain/discussions),
         [LangChain Github Issues](https://github.com/langchain-ai/langchain/issues?q=is%3Aissue),

.github/ISSUE_TEMPLATE/documentation.yml

@@ -21,7 +21,7 @@ body:
       place to ask your question:
       
       [LangChain documentation with the integrated search](https://python.langchain.com/docs/get_started/introduction),
-      [API Reference](https://api.python.langchain.com/en/stable/),
+      [API Reference](https://python.langchain.com/api_reference/),
       [GitHub search](https://github.com/langchain-ai/langchain),
       [LangChain Github Discussions](https://github.com/langchain-ai/langchain/discussions),
       [LangChain Github Issues](https://github.com/langchain-ai/langchain/issues?q=is%3Aissue),

.github/scripts/check_diff.py

@@ -272,6 +272,7 @@ if __name__ == "__main__":
             # TODO: update to include all packages that rely on standard-tests (all partner packages)
             # note: won't run on external repo partners
             dirs_to_run["lint"].add("libs/standard-tests")
+            dirs_to_run["test"].add("libs/standard-tests")
             dirs_to_run["test"].add("libs/partners/mistralai")
             dirs_to_run["test"].add("libs/partners/openai")
             dirs_to_run["test"].add("libs/partners/anthropic")
@@ -279,8 +280,9 @@ if __name__ == "__main__":
             dirs_to_run["test"].add("libs/partners/groq")
 
         elif file.startswith("libs/cli"):
-            # todo: add cli makefile
-            pass
+            dirs_to_run["lint"].add("libs/cli")
+            dirs_to_run["test"].add("libs/cli")
+            
         elif file.startswith("libs/partners"):
             partner_dir = file.split("/")[2]
             if os.path.isdir(f"libs/partners/{partner_dir}") and [

.github/workflows/_release.yml

@@ -192,7 +192,12 @@ jobs:
           poetry-version: ${{ env.POETRY_VERSION }}
           working-directory: ${{ inputs.working-directory }}
 
-      - name: Import published package
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: ${{ inputs.working-directory }}/dist/
+
+      - name: Import dist package
         shell: bash
         working-directory: ${{ inputs.working-directory }}
         env:
@@ -208,15 +213,7 @@ jobs:
         # - attempt install again after 5 seconds if it fails because there is
         #   sometimes a delay in availability on test pypi
         run: |
-          poetry run pip install \
-            --extra-index-url https://test.pypi.org/simple/ \
-            "$PKG_NAME==$VERSION" || \
-          ( \
-            sleep 15 && \
-            poetry run pip install \
-              --extra-index-url https://test.pypi.org/simple/ \
-              "$PKG_NAME==$VERSION" \
-          )
+          poetry run pip install dist/*.whl
 
           # Replace all dashes in the package name with underscores,
           # since that's how Python imports packages with dashes in the name.
@@ -225,10 +222,10 @@ jobs:
           poetry run python -c "import $IMPORT_NAME; print(dir($IMPORT_NAME))"
 
       - name: Import test dependencies
-        run: poetry install --with test
+        run: poetry install --with test --no-root
         working-directory: ${{ inputs.working-directory }}
 
-      # Overwrite the local version of the package with the test PyPI version.
+      # Overwrite the local version of the package with the built version
       - name: Import published package (again)
         working-directory: ${{ inputs.working-directory }}
         shell: bash
@@ -236,9 +233,7 @@ jobs:
           PKG_NAME: ${{ needs.build.outputs.pkg-name }}
           VERSION: ${{ needs.build.outputs.version }}
         run: |
-          poetry run pip install \
-            --extra-index-url https://test.pypi.org/simple/ \
-            "$PKG_NAME==$VERSION"
+          poetry run pip install dist/*.whl
 
       - name: Run unit tests
         run: make tests

.github/workflows/scheduled_test.yml

@@ -14,17 +14,7 @@ on:
 
 env:
   POETRY_VERSION: "1.8.4"
-  DEFAULT_LIBS: >
-    [
-      "libs/partners/openai",
-      "libs/partners/anthropic",
-      "libs/partners/fireworks",
-      "libs/partners/groq",
-      "libs/partners/mistralai",
-      "libs/partners/google-vertexai",
-      "libs/partners/google-genai",
-      "libs/partners/aws"
-    ]
+  DEFAULT_LIBS: '["libs/partners/openai", "libs/partners/anthropic", "libs/partners/fireworks", "libs/partners/groq", "libs/partners/mistralai", "libs/partners/google-vertexai", "libs/partners/google-genai", "libs/partners/aws"]'
 
 jobs:
   compute-matrix:

Makefile

@@ -69,7 +69,11 @@ lint lint_package lint_tests:
 	poetry run ruff check docs cookbook
 	poetry run ruff format docs cookbook cookbook --diff
 	poetry run ruff check --select I docs cookbook
-	git grep 'from langchain import' docs/docs cookbook | grep -vE 'from langchain import (hub)' && exit 1 || exit 0
+	git --no-pager grep 'from langchain import' docs cookbook | grep -vE 'from langchain import (hub)' && echo "Error: no importing langchain from root in docs, except for hub" && exit 1 || exit 0
+	
+	git --no-pager grep 'api.python.langchain.com' -- docs/docs ':!docs/docs/additional_resources/arxiv_references.mdx' ':!docs/docs/integrations/document_loaders/sitemap.ipynb' || exit 0 && \
+	echo "Error: you should link python.langchain.com/api_reference, not api.python.langchain.com in the docs" && \
+	exit 1
 
 ## format: Format the project files.
 format format_diff:

README.md

@@ -123,7 +123,7 @@ Please see [here](https://python.langchain.com) for full documentation, which in
 - [Tutorials](https://python.langchain.com/docs/tutorials/): If you're looking to build something specific or are more of a hands-on learner, check out our tutorials. This is the best place to get started.
 - [How-to guides](https://python.langchain.com/docs/how_to/): Answers to “How do I….?” type questions. These guides are goal-oriented and concrete; they're meant to help you complete a specific task.
 - [Conceptual guide](https://python.langchain.com/docs/concepts/): Conceptual explanations of the key parts of the framework.
-- [API Reference](https://api.python.langchain.com): Thorough documentation of every class and method.
+- [API Reference](https://python.langchain.com/api_reference/): Thorough documentation of every class and method.
 
 ## 🌐 Ecosystem
 

SECURITY.md

@@ -1,5 +1,30 @@
 # Security Policy
 
+LangChain has a large ecosystem of integrations with various external resources like local and remote file systems, APIs and databases. These integrations allow developers to create versatile applications that combine the power of LLMs with the ability to access, interact with and manipulate external resources.
+
+## Best practices
+
+When building such applications developers should remember to follow good security practices:
+
+* [**Limit Permissions**](https://en.wikipedia.org/wiki/Principle_of_least_privilege): Scope permissions specifically to the application's need. Granting broad or excessive permissions can introduce significant security vulnerabilities. To avoid such vulnerabilities, consider using read-only credentials, disallowing access to sensitive resources, using sandboxing techniques (such as running inside a container), specifying proxy configurations to control external requests, etc. as appropriate for your application.
+* **Anticipate Potential Misuse**: Just as humans can err, so can Large Language Models (LLMs). Always assume that any system access or credentials may be used in any way allowed by the permissions they are assigned. For example, if a pair of database credentials allows deleting data, it’s safest to assume that any LLM able to use those credentials may in fact delete data.
+* [**Defense in Depth**](https://en.wikipedia.org/wiki/Defense_in_depth_(computing)): No security technique is perfect. Fine-tuning and good chain design can reduce, but not eliminate, the odds that a Large Language Model (LLM) may make a mistake. It’s best to combine multiple layered security approaches rather than relying on any single layer of defense to ensure security. For example: use both read-only permissions and sandboxing to ensure that LLMs are only able to access data that is explicitly meant for them to use.
+
+Risks of not doing so include, but are not limited to:
+* Data corruption or loss.
+* Unauthorized access to confidential information.
+* Compromised performance or availability of critical resources.
+
+Example scenarios with mitigation strategies:
+
+* A user may ask an agent with access to the file system to delete files that should not be deleted or read the content of files that contain sensitive information. To mitigate, limit the agent to only use a specific directory and only allow it to read or write files that are safe to read or write. Consider further sandboxing the agent by running it in a container.
+* A user may ask an agent with write access to an external API to write malicious data to the API, or delete data from that API. To mitigate, give the agent read-only API keys, or limit it to only use endpoints that are already resistant to such misuse.
+* A user may ask an agent with access to a database to drop a table or mutate the schema. To mitigate, scope the credentials to only the tables that the agent needs to access and consider issuing READ-ONLY credentials.
+
+If you're building applications that access external resources like file systems, APIs
+or databases, consider speaking with your company's security team to determine how to best
+design and secure your applications.
+
 ## Reporting OSS Vulnerabilities
 
 LangChain is partnered with [huntr by Protect AI](https://huntr.com/) to provide 
@@ -14,7 +39,7 @@ Before reporting a vulnerability, please review:
 
 1) In-Scope Targets and Out-of-Scope Targets below.
 2) The [langchain-ai/langchain](https://python.langchain.com/docs/contributing/repo_structure) monorepo structure.
-3) LangChain [security guidelines](https://python.langchain.com/docs/security) to
+3) The [Best practicies](#best-practices) above to
    understand what we consider to be a security vulnerability vs. developer
    responsibility.
 
@@ -33,13 +58,13 @@ The following packages and repositories are eligible for bug bounties:
 All out of scope targets defined by huntr as well as:
 
 - **langchain-experimental**: This repository is for experimental code and is not
-  eligible for bug bounties, bug reports to it will be marked as interesting or waste of
+  eligible for bug bounties (see [package warning](https://pypi.org/project/langchain-experimental/)), bug reports to it will be marked as interesting or waste of
   time and published with no bounty attached.
 - **tools**: Tools in either langchain or langchain-community are not eligible for bug
   bounties. This includes the following directories
-  - langchain/tools
-  - langchain-community/tools
-  - Please review our [security guidelines](https://python.langchain.com/docs/security)
+  - libs/langchain/langchain/tools
+  - libs/community/langchain_community/tools
+  - Please review the [best practices](#best-practices)
     for more details, but generally tools interact with the real world. Developers are
     expected to understand the security implications of their code and are responsible
     for the security of their tools.
@@ -47,7 +72,7 @@ All out of scope targets defined by huntr as well as:
   case basis, but likely will not be eligible for a bounty as the code is already
   documented with guidelines for developers that should be followed for making their
   application secure.
-- Any LangSmith related repositories or APIs see below.
+- Any LangSmith related repositories or APIs (see [Reporting LangSmith Vulnerabilities](#reporting-langsmith-vulnerabilities)).
 
 ## Reporting LangSmith Vulnerabilities
 

docs/Makefile

@@ -47,6 +47,7 @@ generate-files:
 	$(PYTHON) scripts/partner_pkg_table.py $(INTERMEDIATE_DIR)
 
 	curl https://raw.githubusercontent.com/langchain-ai/langserve/main/README.md | sed 's/<=/\&lt;=/g' > $(INTERMEDIATE_DIR)/langserve.md
+	cp ../SECURITY.md $(INTERMEDIATE_DIR)/security.md
 	$(PYTHON) scripts/resolve_local_links.py $(INTERMEDIATE_DIR)/langserve.md https://github.com/langchain-ai/langserve/tree/main/
 
 copy-infra:

docs/docs/additional_resources/arxiv_references.mdx

@@ -28,19 +28,19 @@ From the opposite direction, scientists use `LangChain` in research and referenc
 | `2307.09288v2` [Llama 2: Open Foundation and Fine-Tuned Chat Models](http://arxiv.org/abs/2307.09288v2) | Hugo Touvron, Louis Martin, Kevin Stone,  et al. | 2023‑07‑18 | `Cookbook:` [Semi Structured Rag](https://github.com/langchain-ai/langchain/blob/master/cookbook/Semi_Structured_RAG.ipynb)
 | `2307.03172v3` [Lost in the Middle: How Language Models Use Long Contexts](http://arxiv.org/abs/2307.03172v3) | Nelson F. Liu, Kevin Lin, John Hewitt,  et al. | 2023‑07‑06 | `Docs:` [docs/how_to/long_context_reorder](https://python.langchain.com/docs/how_to/long_context_reorder)
 | `2305.14283v3` [Query Rewriting for Retrieval-Augmented Large Language Models](http://arxiv.org/abs/2305.14283v3) | Xinbei Ma, Yeyun Gong, Pengcheng He,  et al. | 2023‑05‑23 | `Template:` [rewrite-retrieve-read](https://python.langchain.com/docs/templates/rewrite-retrieve-read), `Cookbook:` [Rewrite](https://github.com/langchain-ai/langchain/blob/master/cookbook/rewrite.ipynb)
-| `2305.08291v1` [Large Language Model Guided Tree-of-Thought](http://arxiv.org/abs/2305.08291v1) | Jieyi Long | 2023‑05‑15 | `API:` [langchain_experimental.tot](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.tot), `Cookbook:` [Tree Of Thought](https://github.com/langchain-ai/langchain/blob/master/cookbook/tree_of_thought.ipynb)
+| `2305.08291v1` [Large Language Model Guided Tree-of-Thought](http://arxiv.org/abs/2305.08291v1) | Jieyi Long | 2023‑05‑15 | `API:` [langchain_experimental.tot](https://python.langchain.com/api_reference/experimental/tot.html), `Cookbook:` [Tree Of Thought](https://github.com/langchain-ai/langchain/blob/master/cookbook/tree_of_thought.ipynb)
 | `2305.04091v3` [Plan-and-Solve Prompting: Improving Zero-Shot Chain-of-Thought Reasoning by Large Language Models](http://arxiv.org/abs/2305.04091v3) | Lei Wang, Wanyu Xu, Yihuai Lan,  et al. | 2023‑05‑06 | `Cookbook:` [Plan And Execute Agent](https://github.com/langchain-ai/langchain/blob/master/cookbook/plan_and_execute_agent.ipynb)
-| `2305.02156v1` [Zero-Shot Listwise Document Reranking with a Large Language Model](http://arxiv.org/abs/2305.02156v1) | Xueguang Ma, Xinyu Zhang, Ronak Pradeep,  et al. | 2023‑05‑03 | `Docs:` [docs/how_to/contextual_compression](https://python.langchain.com/docs/how_to/contextual_compression), `API:` [langchain...LLMListwiseRerank](https://api.python.langchain.com/en/latest/retrievers/langchain.retrievers.document_compressors.listwise_rerank.LLMListwiseRerank.html#langchain.retrievers.document_compressors.listwise_rerank.LLMListwiseRerank)
+| `2305.02156v1` [Zero-Shot Listwise Document Reranking with a Large Language Model](http://arxiv.org/abs/2305.02156v1) | Xueguang Ma, Xinyu Zhang, Ronak Pradeep,  et al. | 2023‑05‑03 | `Docs:` [docs/how_to/contextual_compression](https://python.langchain.com/docs/how_to/contextual_compression), `API:` [langchain...LLMListwiseRerank](https://python.langchain.com/api_reference/langchain/retrievers/langchain.retrievers.document_compressors.listwise_rerank.LLMListwiseRerank.html#)
 | `2304.08485v2` [Visual Instruction Tuning](http://arxiv.org/abs/2304.08485v2) | Haotian Liu, Chunyuan Li, Qingyang Wu,  et al. | 2023‑04‑17 | `Cookbook:` [Semi Structured Multi Modal Rag Llama2](https://github.com/langchain-ai/langchain/blob/master/cookbook/Semi_structured_multi_modal_RAG_LLaMA2.ipynb), [Semi Structured And Multi Modal Rag](https://github.com/langchain-ai/langchain/blob/master/cookbook/Semi_structured_and_multi_modal_RAG.ipynb)
 | `2304.03442v2` [Generative Agents: Interactive Simulacra of Human Behavior](http://arxiv.org/abs/2304.03442v2) | Joon Sung Park, Joseph C. O'Brien, Carrie J. Cai,  et al. | 2023‑04‑07 | `Cookbook:` [Generative Agents Interactive Simulacra Of Human Behavior](https://github.com/langchain-ai/langchain/blob/master/cookbook/generative_agents_interactive_simulacra_of_human_behavior.ipynb), [Multiagent Bidding](https://github.com/langchain-ai/langchain/blob/master/cookbook/multiagent_bidding.ipynb)
 | `2303.17760v2` [CAMEL: Communicative Agents for "Mind" Exploration of Large Language Model Society](http://arxiv.org/abs/2303.17760v2) | Guohao Li, Hasan Abed Al Kader Hammoud, Hani Itani,  et al. | 2023‑03‑31 | `Cookbook:` [Camel Role Playing](https://github.com/langchain-ai/langchain/blob/master/cookbook/camel_role_playing.ipynb)
-| `2303.17580v4` [HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in Hugging Face](http://arxiv.org/abs/2303.17580v4) | Yongliang Shen, Kaitao Song, Xu Tan,  et al. | 2023‑03‑30 | `API:` [langchain_experimental.autonomous_agents](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.autonomous_agents), `Cookbook:` [Hugginggpt](https://github.com/langchain-ai/langchain/blob/master/cookbook/hugginggpt.ipynb)
+| `2303.17580v4` [HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in Hugging Face](http://arxiv.org/abs/2303.17580v4) | Yongliang Shen, Kaitao Song, Xu Tan,  et al. | 2023‑03‑30 | `API:` [langchain_experimental.autonomous_agents](https://python.langchain.com/api_reference/experimental/autonomous_agents.html), `Cookbook:` [Hugginggpt](https://github.com/langchain-ai/langchain/blob/master/cookbook/hugginggpt.ipynb)
 | `2301.10226v4` [A Watermark for Large Language Models](http://arxiv.org/abs/2301.10226v4) | John Kirchenbauer, Jonas Geiping, Yuxin Wen,  et al. | 2023‑01‑24 | `API:` [langchain_community...OCIModelDeploymentTGI](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI.html#langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI), [langchain_huggingface...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint), [langchain_community...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_community...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint)
 | `2212.10496v1` [Precise Zero-Shot Dense Retrieval without Relevance Labels](http://arxiv.org/abs/2212.10496v1) | Luyu Gao, Xueguang Ma, Jimmy Lin,  et al. | 2022‑12‑20 | `Docs:` [docs/concepts](https://python.langchain.com/docs/concepts), `API:` [langchain...HypotheticalDocumentEmbedder](https://api.python.langchain.com/en/latest/chains/langchain.chains.hyde.base.HypotheticalDocumentEmbedder.html#langchain.chains.hyde.base.HypotheticalDocumentEmbedder), `Template:` [hyde](https://python.langchain.com/docs/templates/hyde), `Cookbook:` [Hypothetical Document Embeddings](https://github.com/langchain-ai/langchain/blob/master/cookbook/hypothetical_document_embeddings.ipynb)
 | `2212.08073v1` [Constitutional AI: Harmlessness from AI Feedback](http://arxiv.org/abs/2212.08073v1) | Yuntao Bai, Saurav Kadavath, Sandipan Kundu,  et al. | 2022‑12‑15 | `Docs:` [docs/versions/migrating_chains/constitutional_chain](https://python.langchain.com/docs/versions/migrating_chains/constitutional_chain)
-| `2212.07425v3` [Robust and Explainable Identification of Logical Fallacies in Natural Language Arguments](http://arxiv.org/abs/2212.07425v3) | Zhivar Sourati, Vishnu Priya Prasanna Venkatesh, Darshan Deshpande,  et al. | 2022‑12‑12 | `API:` [langchain_experimental.fallacy_removal](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.fallacy_removal)
+| `2212.07425v3` [Robust and Explainable Identification of Logical Fallacies in Natural Language Arguments](http://arxiv.org/abs/2212.07425v3) | Zhivar Sourati, Vishnu Priya Prasanna Venkatesh, Darshan Deshpande,  et al. | 2022‑12‑12 | `API:` [langchain_experimental.fallacy_removal](https://python.langchain.com/api_reference/experimental/fallacy_removal.html)
 | `2211.13892v2` [Complementary Explanations for Effective In-Context Learning](http://arxiv.org/abs/2211.13892v2) | Xi Ye, Srinivasan Iyer, Asli Celikyilmaz,  et al. | 2022‑11‑25 | `API:` [langchain_core...MaxMarginalRelevanceExampleSelector](https://api.python.langchain.com/en/latest/example_selectors/langchain_core.example_selectors.semantic_similarity.MaxMarginalRelevanceExampleSelector.html#langchain_core.example_selectors.semantic_similarity.MaxMarginalRelevanceExampleSelector)
-| `2211.10435v2` [PAL: Program-aided Language Models](http://arxiv.org/abs/2211.10435v2) | Luyu Gao, Aman Madaan, Shuyan Zhou,  et al. | 2022‑11‑18 | `API:` [langchain_experimental.pal_chain](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.pal_chain), [langchain_experimental...PALChain](https://api.python.langchain.com/en/latest/pal_chain/langchain_experimental.pal_chain.base.PALChain.html#langchain_experimental.pal_chain.base.PALChain), `Cookbook:` [Program Aided Language Model](https://github.com/langchain-ai/langchain/blob/master/cookbook/program_aided_language_model.ipynb)
+| `2211.10435v2` [PAL: Program-aided Language Models](http://arxiv.org/abs/2211.10435v2) | Luyu Gao, Aman Madaan, Shuyan Zhou,  et al. | 2022‑11‑18 | `API:` [langchain_experimental.pal_chain](https://python.langchain.com/api_reference/experimental/pal_chain.html), [langchain_experimental...PALChain](https://api.python.langchain.com/en/latest/pal_chain/langchain_experimental.pal_chain.base.PALChain.html#langchain_experimental.pal_chain.base.PALChain), `Cookbook:` [Program Aided Language Model](https://github.com/langchain-ai/langchain/blob/master/cookbook/program_aided_language_model.ipynb)
 | `2210.11934v2` [An Analysis of Fusion Functions for Hybrid Retrieval](http://arxiv.org/abs/2210.11934v2) | Sebastian Bruch, Siyu Gai, Amir Ingber | 2022‑10‑21 | `Docs:` [docs/concepts](https://python.langchain.com/docs/concepts)
 | `2210.03629v3` [ReAct: Synergizing Reasoning and Acting in Language Models](http://arxiv.org/abs/2210.03629v3) | Shunyu Yao, Jeffrey Zhao, Dian Yu,  et al. | 2022‑10‑06 | `Docs:` [docs/integrations/tools/ionic_shopping](https://python.langchain.com/docs/integrations/tools/ionic_shopping), [docs/integrations/providers/cohere](https://python.langchain.com/docs/integrations/providers/cohere), [docs/concepts](https://python.langchain.com/docs/concepts), `API:` [langchain...create_react_agent](https://api.python.langchain.com/en/latest/agents/langchain.agents.react.agent.create_react_agent.html#langchain.agents.react.agent.create_react_agent), [langchain...TrajectoryEvalChain](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain.html#langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain)
 | `2209.10785v2` [Deep Lake: a Lakehouse for Deep Learning](http://arxiv.org/abs/2209.10785v2) | Sasun Hambardzumyan, Abhinav Tuli, Levon Ghukasyan,  et al. | 2022‑09‑22 | `Docs:` [docs/integrations/providers/activeloop_deeplake](https://python.langchain.com/docs/integrations/providers/activeloop_deeplake)
@@ -49,7 +49,7 @@ From the opposite direction, scientists use `LangChain` in research and referenc
 | `2204.00498v1` [Evaluating the Text-to-SQL Capabilities of Large Language Models](http://arxiv.org/abs/2204.00498v1) | Nitarshan Rajkumar, Raymond Li, Dzmitry Bahdanau | 2022‑03‑15 | `Docs:` [docs/tutorials/sql_qa](https://python.langchain.com/docs/tutorials/sql_qa), `API:` [langchain_community...SQLDatabase](https://api.python.langchain.com/en/latest/utilities/langchain_community.utilities.sql_database.SQLDatabase.html#langchain_community.utilities.sql_database.SQLDatabase), [langchain_community...SparkSQL](https://api.python.langchain.com/en/latest/utilities/langchain_community.utilities.spark_sql.SparkSQL.html#langchain_community.utilities.spark_sql.SparkSQL)
 | `2202.00666v5` [Locally Typical Sampling](http://arxiv.org/abs/2202.00666v5) | Clara Meister, Tiago Pimentel, Gian Wiher,  et al. | 2022‑02‑01 | `API:` [langchain_huggingface...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint), [langchain_community...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_community...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint)
 | `2112.01488v3` [ColBERTv2: Effective and Efficient Retrieval via Lightweight Late Interaction](http://arxiv.org/abs/2112.01488v3) | Keshav Santhanam, Omar Khattab, Jon Saad-Falcon,  et al. | 2021‑12‑02 | `Docs:` [docs/integrations/retrievers/ragatouille](https://python.langchain.com/docs/integrations/retrievers/ragatouille), [docs/integrations/providers/ragatouille](https://python.langchain.com/docs/integrations/providers/ragatouille), [docs/concepts](https://python.langchain.com/docs/concepts), [docs/integrations/providers/dspy](https://python.langchain.com/docs/integrations/providers/dspy)
-| `2103.00020v1` [Learning Transferable Visual Models From Natural Language Supervision](http://arxiv.org/abs/2103.00020v1) | Alec Radford, Jong Wook Kim, Chris Hallacy,  et al. | 2021‑02‑26 | `API:` [langchain_experimental.open_clip](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.open_clip)
+| `2103.00020v1` [Learning Transferable Visual Models From Natural Language Supervision](http://arxiv.org/abs/2103.00020v1) | Alec Radford, Jong Wook Kim, Chris Hallacy,  et al. | 2021‑02‑26 | `API:` [langchain_experimental.open_clip](https://python.langchain.com/api_reference/experimental/open_clip.html)
 | `2005.14165v4` [Language Models are Few-Shot Learners](http://arxiv.org/abs/2005.14165v4) | Tom B. Brown, Benjamin Mann, Nick Ryder,  et al. | 2020‑05‑28 | `Docs:` [docs/concepts](https://python.langchain.com/docs/concepts)
 | `2005.11401v4` [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](http://arxiv.org/abs/2005.11401v4) | Patrick Lewis, Ethan Perez, Aleksandra Piktus,  et al. | 2020‑05‑22 | `Docs:` [docs/concepts](https://python.langchain.com/docs/concepts)
 | `1909.05858v2` [CTRL: A Conditional Transformer Language Model for Controllable Generation](http://arxiv.org/abs/1909.05858v2) | Nitish Shirish Keskar, Bryan McCann, Lav R. Varshney,  et al. | 2019‑09‑11 | `API:` [langchain_huggingface...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_huggingface.llms.huggingface_endpoint.HuggingFaceEndpoint), [langchain_community...HuggingFaceTextGenInference](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference.html#langchain_community.llms.huggingface_text_gen_inference.HuggingFaceTextGenInference), [langchain_community...HuggingFaceEndpoint](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html#langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint)
@@ -433,7 +433,7 @@ for retrieval-augmented LLM.
 - **arXiv id:** [2305.08291v1](http://arxiv.org/abs/2305.08291v1)  **Published Date:** 2023-05-15
 - **LangChain:**
 
-   - **API Reference:** [langchain_experimental.tot](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.tot)
+   - **API Reference:** [langchain_experimental.tot](https://python.langchain.com/api_reference/experimental/tot.html)
    - **Cookbook:** [tree_of_thought](https://github.com/langchain-ai/langchain/blob/master/cookbook/tree_of_thought.ipynb)
 
 **Abstract:** In this paper, we introduce the Tree-of-Thought (ToT) framework, a novel
@@ -490,7 +490,7 @@ https://github.com/AGI-Edgerunners/Plan-and-Solve-Prompting.
 - **LangChain:**
 
    - **Documentation:** [docs/how_to/contextual_compression](https://python.langchain.com/docs/how_to/contextual_compression)
-   - **API Reference:** [langchain...LLMListwiseRerank](https://api.python.langchain.com/en/latest/retrievers/langchain.retrievers.document_compressors.listwise_rerank.LLMListwiseRerank.html#langchain.retrievers.document_compressors.listwise_rerank.LLMListwiseRerank)
+   - **API Reference:** [langchain...LLMListwiseRerank](https://python.langchain.com/api_reference/langchain/retrievers/langchain.retrievers.document_compressors.listwise_rerank.LLMListwiseRerank.html#)
 
 **Abstract:** Supervised ranking methods based on bi-encoder or cross-encoder architectures
 have shown success in multi-stage text ranking tasks, but they require large
@@ -597,7 +597,7 @@ agents and beyond: https://github.com/camel-ai/camel.
 - **arXiv id:** [2303.17580v4](http://arxiv.org/abs/2303.17580v4)  **Published Date:** 2023-03-30
 - **LangChain:**
 
-   - **API Reference:** [langchain_experimental.autonomous_agents](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.autonomous_agents)
+   - **API Reference:** [langchain_experimental.autonomous_agents](https://python.langchain.com/api_reference/experimental/autonomous_agents.html)
    - **Cookbook:** [hugginggpt](https://github.com/langchain-ai/langchain/blob/master/cookbook/hugginggpt.ipynb)
 
 **Abstract:** Solving complicated AI tasks with different domains and modalities is a key
@@ -704,7 +704,7 @@ labels.
 - **arXiv id:** [2212.07425v3](http://arxiv.org/abs/2212.07425v3)  **Published Date:** 2022-12-12
 - **LangChain:**
 
-   - **API Reference:** [langchain_experimental.fallacy_removal](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.fallacy_removal)
+   - **API Reference:** [langchain_experimental.fallacy_removal](https://python.langchain.com/api_reference/experimental/fallacy_removal.html)
 
 **Abstract:** The spread of misinformation, propaganda, and flawed argumentation has been
 amplified in the Internet era. Given the volume of data and the subtlety of
@@ -759,7 +759,7 @@ performance across three real-world tasks on multiple LLMs.
 - **arXiv id:** [2211.10435v2](http://arxiv.org/abs/2211.10435v2)  **Published Date:** 2022-11-18
 - **LangChain:**
 
-   - **API Reference:** [langchain_experimental.pal_chain](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.pal_chain), [langchain_experimental...PALChain](https://api.python.langchain.com/en/latest/pal_chain/langchain_experimental.pal_chain.base.PALChain.html#langchain_experimental.pal_chain.base.PALChain)
+   - **API Reference:** [langchain_experimental.pal_chain](https://python.langchain.com/api_reference/experimental/pal_chain.html), [langchain_experimental...PALChain](https://api.python.langchain.com/en/latest/pal_chain/langchain_experimental.pal_chain.base.PALChain.html#langchain_experimental.pal_chain.base.PALChain)
    - **Cookbook:** [program_aided_language_model](https://github.com/langchain-ai/langchain/blob/master/cookbook/program_aided_language_model.ipynb)
 
 **Abstract:** Large language models (LLMs) have recently demonstrated an impressive ability
@@ -992,7 +992,7 @@ footprint of late interaction models by 6--10$\times$.
 - **arXiv id:** [2103.00020v1](http://arxiv.org/abs/2103.00020v1)  **Published Date:** 2021-02-26
 - **LangChain:**
 
-   - **API Reference:** [langchain_experimental.open_clip](https://api.python.langchain.com/en/latest/experimental_api_reference.html#module-langchain_experimental.open_clip)
+   - **API Reference:** [langchain_experimental.open_clip](https://python.langchain.com/api_reference/experimental/open_clip.html)
 
 **Abstract:** State-of-the-art computer vision systems are trained to predict a fixed set
 of predetermined object categories. This restricted form of supervision limits

docs/docs/concepts/embedding_models.mdx

@@ -3,7 +3,7 @@
 
 :::info[Prerequisites]
 
-* [Documents](https://api.python.langchain.com/en/latest/documents/langchain_core.documents.base.Document.html)
+* [Documents](https://python.langchain.com/api_reference/core/documents/langchain_core.documents.base.Document.html)
 
 :::
 

docs/docs/concepts/retrieval.mdx

@@ -221,7 +221,7 @@ They are particularly useful for storing and querying complex relationships betw
 LangChain provides a unified interface for interacting with various retrieval systems through the [retriever](/docs/concepts/retrievers/) concept. The interface is straightforward:
 
 1. Input: A query (string)
-2. Output: A list of documents (standardized LangChain [Document](https://api.python.langchain.com/en/latest/documents/langchain_core.documents.base.Document.html) objects)
+2. Output: A list of documents (standardized LangChain [Document](https://python.langchain.com/api_reference/core/documents/langchain_core.documents.base.Document.html) objects)
 
 You can create a retriever using any of the retrieval systems mentioned earlier. The query analysis techniques we discussed are particularly useful here, as they enable natural language interfaces for databases that typically require structured query languages.
 For example, you can build a retriever for a SQL database using text-to-SQL conversion. This allows a natural language query (string) to be transformed into a SQL query behind the scenes.

docs/docs/concepts/retrievers.mdx

@@ -18,7 +18,7 @@ Because of their importance and variability, LangChain provides a uniform interf
 The LangChain [retriever](/docs/concepts/retrievers/) interface is straightforward:
 
 1. Input: A query (string)
-2. Output: A list of documents (standardized LangChain [Document](https://api.python.langchain.com/en/latest/documents/langchain_core.documents.base.Document.html) objects)
+2. Output: A list of documents (standardized LangChain [Document](https://python.langchain.com/api_reference/core/documents/langchain_core.documents.base.Document.html) objects)
 
 ## Key concept
 
@@ -29,7 +29,7 @@ All retrievers implement a simple interface for retrieving documents using natur
 ## Interface 
 
 The only requirement for a retriever is the ability to accepts a query and return documents. 
-In particular, [LangChain's retriever class](https://api.python.langchain.com/en/latest/retrievers/langchain_core.retrievers.BaseRetriever.html) only requires that the `_get_relevant_documents` method is implemented, which takes a `query: str` and returns a list of [Document](https://api.python.langchain.com/en/latest/documents/langchain_core.documents.base.Document.html) objects that are most relevant to the query.
+In particular, [LangChain's retriever class](https://python.langchain.com/api_reference/core/retrievers/langchain_core.retrievers.BaseRetriever.html#) only requires that the `_get_relevant_documents` method is implemented, which takes a `query: str` and returns a list of [Document](https://python.langchain.com/api_reference/core/documents/langchain_core.documents.base.Document.html) objects that are most relevant to the query.
 The underlying logic used to get relevant documents is specified by the retriever and can be whatever is most useful for the application.
 
 A LangChain retriever is a [runnable](/docs/how_to/lcel_cheatsheet/), which is a standard interface is for LangChain components. 
@@ -39,7 +39,7 @@ This means that it has a few common methods, including `invoke`, that are used t
 docs = retriever.invoke(query)
 ```
 
-Retrievers return a list of [Document](https://api.python.langchain.com/en/latest/documents/langchain_core.documents.base.Document.html) objects, which have two attributes:
+Retrievers return a list of [Document](https://python.langchain.com/api_reference/core/documents/langchain_core.documents.base.Document.html) objects, which have two attributes:
 
 * `page_content`: The content of this document. Currently is a string.
 * `metadata`: Arbitrary metadata associated with this document (e.g., document id, file name, source, etc). 

docs/docs/concepts/runnables.mdx

@@ -125,7 +125,7 @@ Please see the [Configurable Runnables](#configurable-runnables) section for mor
 
 LangChain will automatically try to infer the input and output types of a Runnable based on available information.
 
-Currently, this inference does not work well for more complex Runnables that are built using [LCEL](/docs/concepts/lcel) composition, and the inferred input and / or output types may be incorrect. In these cases, we recommend that users override the inferred input and output types using the `with_types` method ([API Reference](https://api.python.langchain.com/en/latest/runnables/langchain_core.runnables.base.Runnable.html#langchain_core.runnables.base.Runnable.with_types
+Currently, this inference does not work well for more complex Runnables that are built using [LCEL](/docs/concepts/lcel) composition, and the inferred input and / or output types may be incorrect. In these cases, we recommend that users override the inferred input and output types using the `with_types` method ([API Reference](https://python.langchain.com/api_reference/core/runnables/langchain_core.runnables.base.Runnable.html#langchain_core.runnables.base.Runnable.with_types
 ).
 
 ## RunnableConfig

docs/docs/concepts/vectorstores.mdx

@@ -59,7 +59,7 @@ vector_store = InMemoryVectorStore(embedding=SomeEmbeddingModel())
 
 To add documents, use the `add_documents` method.
 
-This API works with a list of [Document](https://api.python.langchain.com/en/latest/documents/langchain_core.documents.base.Document.html) objects.
+This API works with a list of [Document](https://python.langchain.com/api_reference/core/documents/langchain_core.documents.base.Document.html) objects.
 `Document` objects all have `page_content` and `metadata` attributes, making them a universal way to store unstructured text and associated metadata.
 
 ```python
@@ -126,7 +126,7 @@ to the documentation of the specific vectorstore you are using to see what simil
 Given a similarity metric to measure the distance between the embedded query and any embedded document, we need an algorithm to efficiently search over *all* the embedded documents to find the most similar ones.
 There are various ways to do this. As an example, many vectorstores implement [HNSW (Hierarchical Navigable Small World)](https://www.pinecone.io/learn/series/faiss/hnsw/), a graph-based index structure that allows for efficient similarity search.
 Regardless of the search algorithm used under the hood, the LangChain vectorstore interface has a `similarity_search` method for all integrations. 
-This will take the search query, create an embedding, find similar documents, and return them as a list of [Documents](https://api.python.langchain.com/en/latest/documents/langchain_core.documents.base.Document.html).
+This will take the search query, create an embedding, find similar documents, and return them as a list of [Documents](https://python.langchain.com/api_reference/core/documents/langchain_core.documents.base.Document.html).
 
 ```python
 query = "my query"

docs/docs/how_to/custom_tools.ipynb

@@ -162,7 +162,7 @@
     "\n",
     "@tool\n",
     "def multiply_by_max(\n",
-    "    a: Annotated[str, \"scale factor\"],\n",
+    "    a: Annotated[int, \"scale factor\"],\n",
     "    b: Annotated[List[int], \"list of ints over which to take maximum\"],\n",
     ") -> int:\n",
     "    \"\"\"Multiply a by the maximum of b.\"\"\"\n",

docs/docs/how_to/graph_constructing.ipynb

@@ -245,7 +245,7 @@
     "    allowed_nodes=[\"Person\", \"Country\", \"Organization\"],\n",
     "    allowed_relationships=allowed_relationships,\n",
     ")\n",
-    "llm_transformer_tuple = llm_transformer_filtered.convert_to_graph_documents(documents)\n",
+    "graph_documents_filtered = llm_transformer_tuple.convert_to_graph_documents(documents)\n",
     "print(f\"Nodes:{graph_documents_filtered[0].nodes}\")\n",
     "print(f\"Relationships:{graph_documents_filtered[0].relationships}\")"
    ]

docs/docs/how_to/installation.mdx

@@ -51,7 +51,7 @@ pip install langchain-core
 
 Certain integrations like OpenAI and Anthropic have their own packages.
 Any integrations that require their own package will be documented as such in the [Integration docs](/docs/integrations/providers/).
-You can see a list of all integration packages in the [API reference](https://api.python.langchain.com) under the "Partner libs" dropdown.
+You can see a list of all integration packages in the [API reference](https://python.langchain.com/api_reference/) under the "Partner libs" dropdown.
 To install one of these run:
 
 ```bash

docs/docs/integrations/chat/cerebras.ipynb

@@ -17,7 +17,7 @@
    "source": [
     "# ChatCerebras\n",
     "\n",
-    "This notebook provides a quick overview for getting started with Cerebras [chat models](/docs/concepts/chat_models). For detailed documentation of all ChatCerebras features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/chat_models/langchain_cerebras.chat_models.ChatCerebras.html).\n",
+    "This notebook provides a quick overview for getting started with Cerebras [chat models](/docs/concepts/chat_models). For detailed documentation of all ChatCerebras features and configurations head to the [API reference](https://python.langchain.com/api_reference/cerebras/chat_models/langchain_cerebras.chat_models.ChatCerebras.html#).\n",
     "\n",
     "At Cerebras, we've developed the world's largest and fastest AI processor, the Wafer-Scale Engine-3 (WSE-3). The Cerebras CS-3 system, powered by the WSE-3, represents a new class of AI supercomputer that sets the standard for generative AI training and inference with unparalleled performance and scalability.\n",
     "\n",
@@ -37,7 +37,7 @@
     "\n",
     "| Class | Package | Local | Serializable | [JS support](https://js.langchain.com/docs/integrations/chat/cerebras) | Package downloads | Package latest |\n",
     "| :--- | :--- | :---: | :---: |  :---: | :---: | :---: |\n",
-    "| [ChatCerebras](https://api.python.langchain.com/en/latest/chat_models/langchain_cerebras.chat_models.ChatCerebras.html) | [langchain-cerebras](https://api.python.langchain.com/en/latest/cerebras_api_reference.html) | ❌ | beta | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain-cerebras?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain-cerebras?style=flat-square&label=%20) |\n",
+    "| [ChatCerebras](https://python.langchain.com/api_reference/cerebras/chat_models/langchain_cerebras.chat_models.ChatCerebras.html#) | [langchain-cerebras](https://python.langchain.com/api_reference/cerebras/index.html) | ❌ | beta | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain-cerebras?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain-cerebras?style=flat-square&label=%20) |\n",
     "\n",
     "### Model features\n",
     "| [Tool calling](/docs/how_to/tool_calling/) | [Structured output](/docs/how_to/structured_output/) | JSON mode | [Image input](/docs/how_to/multimodal_inputs/) | Audio input | Video input | [Token-level streaming](/docs/how_to/chat_streaming/) | Native async | [Token usage](/docs/how_to/chat_token_usage_tracking/) | [Logprobs](/docs/how_to/logprobs/) |\n",
@@ -396,7 +396,7 @@
    "source": [
     "## API reference\n",
     "\n",
-    "For detailed documentation of all ChatCerebras features and configurations head to the API reference: https://api.python.langchain.com/en/latest/chat_models/langchain_cerebras.chat_models.ChatCerebras.html"
+    "For detailed documentation of all ChatCerebras features and configurations head to the API reference: https://python.langchain.com/api_reference/cerebras/chat_models/langchain_cerebras.chat_models.ChatCerebras.html#"
    ]
   }
  ],

docs/docs/integrations/chat/oci_data_science.ipynb

@@ -19,7 +19,7 @@
    "source": [
     "# ChatOCIModelDeployment\n",
     "\n",
-    "This will help you getting started with OCIModelDeployment [chat models](/docs/concepts/chat_models). For detailed documentation of all ChatOCIModelDeployment features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.ChatOCIModelDeployment.html).\n",
+    "This will help you getting started with OCIModelDeployment [chat models](/docs/concepts/chat_models). For detailed documentation of all ChatOCIModelDeployment features and configurations head to the [API reference](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.oci_data_science.ChatOCIModelDeployment.html).\n",
     "\n",
     "[OCI Data Science](https://docs.oracle.com/en-us/iaas/data-science/using/home.htm) is a fully managed and serverless platform for data science teams to build, train, and manage machine learning models in the Oracle Cloud Infrastructure. You can use [AI Quick Actions](https://blogs.oracle.com/ai-and-datascience/post/ai-quick-actions-in-oci-data-science) to easily deploy LLMs on [OCI Data Science Model Deployment Service](https://docs.oracle.com/en-us/iaas/data-science/using/model-dep-about.htm). You may choose to deploy the model with popular inference frameworks such as vLLM or TGI. By default, the model deployment endpoint mimics the OpenAI API protocol.\n",
     "\n",
@@ -30,7 +30,7 @@
     "\n",
     "| Class | Package | Local | Serializable | JS support | Package downloads | Package latest |\n",
     "| :--- | :--- | :---: | :---: |  :---: | :---: | :---: |\n",
-    "| [ChatOCIModelDeployment](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.ChatOCIModelDeployment.html) | [langchain-community](https://api.python.langchain.com/en/latest/community_api_reference.html) | ❌ | beta | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain-community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain-community?style=flat-square&label=%20) |\n",
+    "| [ChatOCIModelDeployment](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.oci_data_science.ChatOCIModelDeployment.html) | [langchain-community](https://python.langchain.com/api_reference/community/index.html) | ❌ | beta | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain-community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain-community?style=flat-square&label=%20) |\n",
     "\n",
     "### Model features\n",
     "\n",
@@ -430,9 +430,9 @@
     "\n",
     "For comprehensive details on all features and configurations, please refer to the API reference documentation for each class:\n",
     "\n",
-    "* [ChatOCIModelDeployment](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.oci_data_science.ChatOCIModelDeployment.html)\n",
-    "* [ChatOCIModelDeploymentVLLM](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.oci_data_science.ChatOCIModelDeploymentVLLM.html)\n",
-    "* [ChatOCIModelDeploymentTGI](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.oci_data_science.ChatOCIModelDeploymentTGI.html)"
+    "* [ChatOCIModelDeployment](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.oci_data_science.ChatOCIModelDeployment.html)\n",
+    "* [ChatOCIModelDeploymentVLLM](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.oci_data_science.ChatOCIModelDeploymentVLLM.html)\n",
+    "* [ChatOCIModelDeploymentTGI](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.oci_data_science.ChatOCIModelDeploymentTGI.html)"
    ]
   }
  ],

docs/docs/integrations/chat/outlines.ipynb

@@ -17,7 +17,7 @@
             "source": [
                 "# ChatOutlines\n",
                 "\n",
-                "This will help you getting started with Outlines [chat models](/docs/concepts/chat_models/). For detailed documentation of all ChatOutlines features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/chat_models/outlines.chat_models.ChatOutlines.html).\n",
+                "This will help you getting started with Outlines [chat models](/docs/concepts/chat_models/). For detailed documentation of all ChatOutlines features and configurations head to the [API reference](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.outlines.ChatOutlines.html).\n",
                 "\n",
                 "[Outlines](https://github.com/outlines-dev/outlines) is a library for constrained language generation. It allows you to use large language models (LLMs) with various backends while applying constraints to the generated output.\n",
                 "\n",
@@ -26,7 +26,7 @@
                 "\n",
                 "| Class | Package | Local | Serializable | JS support | Package downloads | Package latest |\n",
                 "| :--- | :--- | :---: | :---: |  :---: | :---: | :---: |\n",
-                "| [ChatOutlines](https://api.python.langchain.com/en/latest/chat_models/outlines.chat_models.ChatOutlines.html) | [langchain-community](https://api.python.langchain.com/en/latest/community_api_reference.html) | ✅ | ❌ | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain-community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain-community?style=flat-square&label=%20) |\n",
+                "| [ChatOutlines](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.outlines.ChatOutlines.html) | [langchain-community](https://python.langchain.com/api_reference/community/index.html) | ✅ | ❌ | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain-community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain-community?style=flat-square&label=%20) |\n",
                 "\n",
                 "### Model features\n",
                 "| [Tool calling](/docs/how_to/tool_calling) | [Structured output](/docs/how_to/structured_output/) | JSON mode | [Image input](/docs/how_to/multimodal_inputs/) | Audio input | Video input | [Token-level streaming](/docs/how_to/chat_streaming/) | Native async | [Token usage](/docs/how_to/chat_token_usage_tracking/) | [Logprobs](/docs/how_to/logprobs/) |\n",
@@ -316,7 +316,7 @@
             "source": [
                 "## API reference\n",
                 "\n",
-                "For detailed documentation of all ChatOutlines features and configurations head to the API reference: https://api.python.langchain.com/en/latest/chat_models/outlines.chat_models.ChatOutlines.html\n",
+                "For detailed documentation of all ChatOutlines features and configurations head to the API reference: https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.outlines.ChatOutlines.html\n",
                 "\n",
                 "## Full Outlines Documentation: \n",
                 "\n",

docs/docs/integrations/chat/sambastudio.ipynb

@@ -19,7 +19,7 @@
    "source": [
     "# ChatSambaStudio\n",
     "\n",
-    "This will help you getting started with SambaStudio [chat models](/docs/concepts/chat_models). For detailed documentation of all ChatStudio features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html).\n",
+    "This will help you getting started with SambaStudio [chat models](/docs/concepts/chat_models). For detailed documentation of all ChatStudio features and configurations head to the [API reference](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html).\n",
     "\n",
     "**[SambaNova](https://sambanova.ai/)'s** [SambaStudio](https://docs.sambanova.ai/sambastudio/latest/sambastudio-intro.html) SambaStudio is a rich, GUI-based platform that provides the functionality to train, deploy, and manage models in SambaNova [DataScale](https://sambanova.ai/products/datascale) systems.\n",
     "\n",
@@ -28,7 +28,7 @@
     "\n",
     "| Class | Package | Local | Serializable | JS support | Package downloads | Package latest |\n",
     "| :--- | :--- | :---: | :---: |  :---: | :---: | :---: |\n",
-    "| [ChatSambaStudio](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html) | [langchain-community](https://python.langchain.com/api_reference/community/index.html) | ❌ | ❌ | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_community?style=flat-square&label=%20) |\n",
+    "| [ChatSambaStudio](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html) | [langchain-community](https://python.langchain.com/api_reference/community/index.html) | ❌ | ❌ | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_community?style=flat-square&label=%20) |\n",
     "\n",
     "### Model features\n",
     "\n",
@@ -355,7 +355,7 @@
    "source": [
     "## API reference\n",
     "\n",
-    "For detailed documentation of all ChatSambaStudio features and configurations head to the API reference: https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html"
+    "For detailed documentation of all ChatSambaStudio features and configurations head to the API reference: https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html"
    ]
   }
  ],

docs/docs/integrations/document_loaders/needle.ipynb

@@ -0,0 +1,253 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Needle Document Loader\n",
+    "[Needle](https://needle-ai.com) makes it easy to create your RAG pipelines with minimal effort. \n",
+    "\n",
+    "For more details, refer to our [API documentation](https://docs.needle-ai.com/docs/api-reference/needle-api)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Overview\n",
+    "The Needle Document Loader is a utility for integrating Needle collections with LangChain. It enables seamless storage, retrieval, and utilization of documents for Retrieval-Augmented Generation (RAG) workflows.\n",
+    "\n",
+    "This example demonstrates:\n",
+    "\n",
+    "* Storing documents into a Needle collection.\n",
+    "* Setting up a retriever to fetch documents.\n",
+    "* Building a Retrieval-Augmented Generation (RAG) pipeline."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Setup\n",
+    "Before starting, ensure you have the following environment variables set:\n",
+    "\n",
+    "* NEEDLE_API_KEY: Your API key for authenticating with Needle.\n",
+    "* OPENAI_API_KEY: Your OpenAI API key for language model operations."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"NEEDLE_API_KEY\"] = \"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"OPENAI_API_KEY\"] = \"\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Initialization\n",
+    "To initialize the NeedleLoader, you need the following parameters:\n",
+    "\n",
+    "* needle_api_key: Your Needle API key (or set it as an environment variable).\n",
+    "* collection_id: The ID of the Needle collection to work with."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Instantiation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders.needle import NeedleLoader\n",
+    "\n",
+    "collection_id = \"clt_01J87M9T6B71DHZTHNXYZQRG5H\"\n",
+    "\n",
+    "# Initialize NeedleLoader to store documents to the collection\n",
+    "document_loader = NeedleLoader(\n",
+    "    needle_api_key=os.getenv(\"NEEDLE_API_KEY\"),\n",
+    "    collection_id=collection_id,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load\n",
+    "To add files to the Needle collection:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "files = {\n",
+    "    \"tech-radar-30.pdf\": \"https://www.thoughtworks.com/content/dam/thoughtworks/documents/radar/2024/04/tr_technology_radar_vol_30_en.pdf\"\n",
+    "}\n",
+    "\n",
+    "document_loader.add_files(files=files)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Show the documents in the collection\n",
+    "# collections_documents = document_loader.load()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Lazy Load\n",
+    "The lazy_load method allows you to iteratively load documents from the Needle collection, yielding each document as it is fetched:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Show the documents in the collection\n",
+    "# collections_documents = document_loader.lazy_load()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Usage\n",
+    "### Use within a chain\n",
+    "Below is a complete example of setting up a RAG pipeline with Needle within a chain:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'input': 'Did RAG move to accepted?',\n",
+       " 'context': [Document(metadata={}, page_content='New Moved in/out No change\\n\\n© Thoughtworks, Inc. All Rights Reserved. 12\\n\\nTechniques\\n\\n1. Retrieval-augmented generation (RAG)\\nAdopt\\n\\nRetrieval-augmented generation (RAG) is the preferred pattern for our teams to improve the quality of \\nresponses generated by a large language model (LLM). We’ve successfully used it in several projects, \\nincluding the popular Jugalbandi AI Platform. With RAG, information about relevant and trustworthy \\ndocuments — in formats like HTML and PDF — are stored in databases that supports a vector data \\ntype or efficient document search, such as pgvector, Qdrant or Elasticsearch Relevance Engine. For \\na given prompt, the database is queried to retrieve relevant documents, which are then combined \\nwith the prompt to provide richer context to the LLM. This results in higher quality output and greatly \\nreduced hallucinations. The context window — which determines the maximum size of the LLM input \\n— is limited, which means that selecting the most relevant documents is crucial. We improve the \\nrelevancy of the content that is added to the prompt by reranking. Similarly, the documents are usually \\ntoo large to calculate an embedding, which means they must be split into smaller chunks. This is often \\na difficult problem, and one approach is to have the chunks overlap to a certain extent.'),\n",
+       "  Document(metadata={}, page_content='New Moved in/out No change\\n\\n© Thoughtworks, Inc. All Rights Reserved. 12\\n\\nTechniques\\n\\n1. Retrieval-augmented generation (RAG)\\nAdopt\\n\\nRetrieval-augmented generation (RAG) is the preferred pattern for our teams to improve the quality of \\nresponses generated by a large language model (LLM). We’ve successfully used it in several projects, \\nincluding the popular Jugalbandi AI Platform. With RAG, information about relevant and trustworthy \\ndocuments — in formats like HTML and PDF — are stored in databases that supports a vector data \\ntype or efficient document search, such as pgvector, Qdrant or Elasticsearch Relevance Engine. For \\na given prompt, the database is queried to retrieve relevant documents, which are then combined \\nwith the prompt to provide richer context to the LLM. This results in higher quality output and greatly \\nreduced hallucinations. The context window — which determines the maximum size of the LLM input \\n— is limited, which means that selecting the most relevant documents is crucial. We improve the \\nrelevancy of the content that is added to the prompt by reranking. Similarly, the documents are usually \\ntoo large to calculate an embedding, which means they must be split into smaller chunks. This is often \\na difficult problem, and one approach is to have the chunks overlap to a certain extent.'),\n",
+       "  Document(metadata={}, page_content='New Moved in/out No change\\n\\n© Thoughtworks, Inc. All Rights Reserved. 12\\n\\nTechniques\\n\\n1. Retrieval-augmented generation (RAG)\\nAdopt\\n\\nRetrieval-augmented generation (RAG) is the preferred pattern for our teams to improve the quality of \\nresponses generated by a large language model (LLM). We’ve successfully used it in several projects, \\nincluding the popular Jugalbandi AI Platform. With RAG, information about relevant and trustworthy \\ndocuments — in formats like HTML and PDF — are stored in databases that supports a vector data \\ntype or efficient document search, such as pgvector, Qdrant or Elasticsearch Relevance Engine. For \\na given prompt, the database is queried to retrieve relevant documents, which are then combined \\nwith the prompt to provide richer context to the LLM. This results in higher quality output and greatly \\nreduced hallucinations. The context window — which determines the maximum size of the LLM input \\n— is limited, which means that selecting the most relevant documents is crucial. We improve the \\nrelevancy of the content that is added to the prompt by reranking. Similarly, the documents are usually \\ntoo large to calculate an embedding, which means they must be split into smaller chunks. This is often \\na difficult problem, and one approach is to have the chunks overlap to a certain extent.'),\n",
+       "  Document(metadata={}, page_content='New Moved in/out No change\\n\\n© Thoughtworks, Inc. All Rights Reserved. 12\\n\\nTechniques\\n\\n1. Retrieval-augmented generation (RAG)\\nAdopt\\n\\nRetrieval-augmented generation (RAG) is the preferred pattern for our teams to improve the quality of \\nresponses generated by a large language model (LLM). We’ve successfully used it in several projects, \\nincluding the popular Jugalbandi AI Platform. With RAG, information about relevant and trustworthy \\ndocuments — in formats like HTML and PDF — are stored in databases that supports a vector data \\ntype or efficient document search, such as pgvector, Qdrant or Elasticsearch Relevance Engine. For \\na given prompt, the database is queried to retrieve relevant documents, which are then combined \\nwith the prompt to provide richer context to the LLM. This results in higher quality output and greatly \\nreduced hallucinations. The context window — which determines the maximum size of the LLM input \\n— is limited, which means that selecting the most relevant documents is crucial. We improve the \\nrelevancy of the content that is added to the prompt by reranking. Similarly, the documents are usually \\ntoo large to calculate an embedding, which means they must be split into smaller chunks. This is often \\na difficult problem, and one approach is to have the chunks overlap to a certain extent.'),\n",
+       "  Document(metadata={}, page_content='New Moved in/out No change\\n\\n© Thoughtworks, Inc. All Rights Reserved. 12\\n\\nTechniques\\n\\n1. Retrieval-augmented generation (RAG)\\nAdopt\\n\\nRetrieval-augmented generation (RAG) is the preferred pattern for our teams to improve the quality of \\nresponses generated by a large language model (LLM). We’ve successfully used it in several projects, \\nincluding the popular Jugalbandi AI Platform. With RAG, information about relevant and trustworthy \\ndocuments — in formats like HTML and PDF — are stored in databases that supports a vector data \\ntype or efficient document search, such as pgvector, Qdrant or Elasticsearch Relevance Engine. For \\na given prompt, the database is queried to retrieve relevant documents, which are then combined \\nwith the prompt to provide richer context to the LLM. This results in higher quality output and greatly \\nreduced hallucinations. The context window — which determines the maximum size of the LLM input \\n— is limited, which means that selecting the most relevant documents is crucial. We improve the \\nrelevancy of the content that is added to the prompt by reranking. Similarly, the documents are usually \\ntoo large to calculate an embedding, which means they must be split into smaller chunks. This is often \\na difficult problem, and one approach is to have the chunks overlap to a certain extent.')],\n",
+       " 'answer': 'Yes, RAG has been adopted as the preferred pattern for improving the quality of responses generated by a large language model.'}"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "\n",
+    "from langchain.chains import create_retrieval_chain\n",
+    "from langchain.chains.combine_documents import create_stuff_documents_chain\n",
+    "from langchain_community.retrievers.needle import NeedleRetriever\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "llm = ChatOpenAI(temperature=0)\n",
+    "\n",
+    "# Initialize the Needle retriever (make sure your Needle API key is set as an environment variable)\n",
+    "retriever = NeedleRetriever(\n",
+    "    needle_api_key=os.getenv(\"NEEDLE_API_KEY\"),\n",
+    "    collection_id=\"clt_01J87M9T6B71DHZTHNXYZQRG5H\",\n",
+    ")\n",
+    "\n",
+    "# Define system prompt for the assistant\n",
+    "system_prompt = \"\"\"\n",
+    "    You are an assistant for question-answering tasks. \n",
+    "    Use the following pieces of retrieved context to answer the question.\n",
+    "    If you don't know, say so concisely.\\n\\n{context}\n",
+    "    \"\"\"\n",
+    "\n",
+    "prompt = ChatPromptTemplate.from_messages(\n",
+    "    [(\"system\", system_prompt), (\"human\", \"{input}\")]\n",
+    ")\n",
+    "\n",
+    "# Define the question-answering chain using a document chain (stuff chain) and the retriever\n",
+    "question_answer_chain = create_stuff_documents_chain(llm, prompt)\n",
+    "\n",
+    "# Create the RAG (Retrieval-Augmented Generation) chain by combining the retriever and the question-answering chain\n",
+    "rag_chain = create_retrieval_chain(retriever, question_answer_chain)\n",
+    "\n",
+    "# Define the input query\n",
+    "query = {\"input\": \"Did RAG move to accepted?\"}\n",
+    "\n",
+    "response = rag_chain.invoke(query)\n",
+    "\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## API reference\n",
+    "\n",
+    "For detailed documentation of all `Needle` features and configurations head to the API reference: https://docs.needle-ai.com"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

docs/docs/integrations/document_loaders/sitemap.ipynb

@@ -103,7 +103,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [

docs/docs/integrations/document_loaders/tomarkdown.ipynb

@@ -153,7 +153,7 @@
       "\n",
       "Best practices for developing with LangChain.\n",
       "\n",
-      "### [API reference](https://api.python.langchain.com) [​](\\#api-reference \"Direct link to api-reference\")\n",
+      "### [API reference](https://python.langchain.com/api_reference/) [​](\\#api-reference \"Direct link to api-reference\")\n",
       "\n",
       "Head to the reference section for full documentation of all classes and methods in the LangChain and LangChain Experimental Python packages.\n",
       "\n",

docs/docs/integrations/llms/ai21.ipynb

@@ -17,6 +17,9 @@
    "source": [
     "# AI21LLM\n",
     "\n",
+    ":::caution This service is deprecated.\n",
+    "See [this page](https://python.langchain.com/docs/integrations/chat/ai21/) for the updated ChatAI21 object. :::\n",
+    "\n",
     "This example goes over how to use LangChain to interact with `AI21` Jurassic models. To use the Jamba model, use the [ChatAI21 object](https://python.langchain.com/docs/integrations/chat/ai21/) instead.\n",
     "\n",
     "[See a full list of AI21 models and tools on LangChain.](https://pypi.org/project/langchain-ai21/)\n",

docs/docs/integrations/llms/oci_model_deployment_endpoint.ipynb

@@ -180,9 +180,9 @@
     "\n",
     "For comprehensive details on all features and configurations, please refer to the API reference documentation for each class:\n",
     "\n",
-    "* [OCIModelDeploymentLLM](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentLLM.html)\n",
-    "* [OCIModelDeploymentVLLM](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentVLLM.html)\n",
-    "* [OCIModelDeploymentTGI](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI.html)"
+    "* [OCIModelDeploymentLLM](https://python.langchain.com/api_reference/community/llms/langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentLLM.html)\n",
+    "* [OCIModelDeploymentVLLM](https://python.langchain.com/api_reference/community/llms/langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentVLLM.html)\n",
+    "* [OCIModelDeploymentTGI](https://python.langchain.com/api_reference/community/llms/langchain_community.llms.oci_data_science_model_deployment_endpoint.OCIModelDeploymentTGI.html)"
    ]
   }
  ],

docs/docs/integrations/llms/outlines.ipynb

@@ -6,7 +6,7 @@
    "source": [
     "# Outlines\n",
     "\n",
-    "This will help you getting started with Outlines LLM. For detailed documentation of all Outlines features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/llms/outlines.llms.Outlines.html).\n",
+    "This will help you getting started with Outlines LLM. For detailed documentation of all Outlines features and configurations head to the [API reference](https://python.langchain.com/api_reference/community/llms/langchain_community.llms.outlines.Outlines.html).\n",
     "\n",
     "[Outlines](https://github.com/outlines-dev/outlines) is a library for constrained language generation. It allows you to use large language models (LLMs) with various backends while applying constraints to the generated output.\n",
     "\n",
@@ -236,7 +236,7 @@
    "source": [
     "## API reference\n",
     "\n",
-    "For detailed documentation of all ChatOutlines features and configurations head to the API reference: https://api.python.langchain.com/en/latest/chat_models/outlines.chat_models.ChatOutlines.html\n",
+    "For detailed documentation of all ChatOutlines features and configurations head to the API reference: https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.outlines.ChatOutlines.html\n",
     "\n",
     "## Outlines Documentation: \n",
     "\n",

docs/docs/integrations/providers/aerospike.mdx

@@ -0,0 +1,24 @@
+# Aerospike
+
+>[Aerospike](https://aerospike.com/docs/vector) is a high-performance, distributed database known for its speed and scalability, now with support for vector storage and search, enabling retrieval and search of embedding vectors for machine learning and AI applications.
+> See the documentation for Aerospike Vector Search (AVS) [here](https://aerospike.com/docs/vector).
+
+## Installation and Setup
+
+Install the AVS Python SDK and AVS langchain vector store:
+
+```bash
+pip install aerospike-vector-search langchain-community
+
+See the documentation for the Ptyhon SDK [here](https://aerospike-vector-search-python-client.readthedocs.io/en/latest/index.html).
+The documentation for the AVS langchain vector store is [here](https://python.langchain.com/api_reference/community/vectorstores/langchain_community.vectorstores.aerospike.Aerospike.html).
+
+## Vector Store
+
+To import this vectorstore:
+
+```python
+from langchain_community.vectorstores import Aerospike
+
+See a usage example [here](https://python.langchain.com/docs/integrations/vectorstores/aerospike/).
+

docs/docs/integrations/providers/ai21.mdx

@@ -15,26 +15,6 @@ This page covers how to use the `AI21` ecosystem within `LangChain`.
 pip install langchain-ai21
 ```
 
-## LLMs
-
-See a [usage example](/docs/integrations/llms/ai21).
-
-### AI21 LLM
-
-```python
-from langchain_ai21 import AI21LLM
-```
-
-### AI21 Contextual Answer
-
-You can use AI21’s contextual answers model to receive text or document, 
-serving as a context, and a question and return an answer based entirely on this context.
-
-```python
-from langchain_ai21 import AI21ContextualAnswers
-```
-
-
 ## Chat models
 
 ### AI21 Chat 
@@ -45,23 +25,32 @@ See a [usage example](/docs/integrations/chat/ai21).
 from langchain_ai21 import ChatAI21
 ```
 
-## Embedding models
+## Deprecated features
+
+:::caution The following features are deprecated. 
+:::
+
+### AI21 LLM
+
+```python
+from langchain_ai21 import AI21LLM
+```
+
+### AI21 Contextual Answer
+
+```python
+from langchain_ai21 import AI21ContextualAnswers
+```
 
 ### AI21 Embeddings
 
-See a [usage example](/docs/integrations/text_embedding/ai21).
-
 ```python
 from langchain_ai21 import AI21Embeddings
 ```
-
 ## Text splitters
 
 ### AI21 Semantic Text Splitter
 
-See a [usage example](/docs/integrations/document_transformers/ai21_semantic_text_splitter).
-
 ```python
 from langchain_ai21 import AI21SemanticTextSplitter
-```
-
+```

docs/docs/integrations/providers/unstructured.mdx

@@ -22,7 +22,7 @@ dependencies running.
 
 - To run everything locally, install the open-source python package with `pip install unstructured`
   along with `pip install langchain-community` and use the same `UnstructuredLoader` as mentioned above.
-    - You can install document specific dependencies with extras, e.g. `pip install "unstructured[docx]"`.
+    - You can install document specific dependencies with extras, e.g. `pip install "unstructured[docx]"`. Learn more about extras [here](https://docs.unstructured.io/open-source/installation/full-installation).
     - To install the dependencies for all document types, use `pip install "unstructured[all-docs]"`.
 - Install the following system dependencies if they are not already available on your system with e.g. `brew install` for Mac.
   Depending on what document types you're parsing, you may not need all of these.

docs/docs/integrations/retrievers/fleet_context.ipynb

@@ -9,7 +9,7 @@
     "\n",
     ">[Fleet AI Context](https://www.fleet.so/context) is a dataset of high-quality embeddings of the top 1200 most popular & permissive Python Libraries & their documentation.\n",
     ">\n",
-    ">The `Fleet AI` team is on a mission to embed the world's most important data. They've started by embedding the top 1200 Python libraries to enable code generation with up-to-date knowledge. They've been kind enough to share their embeddings of the [LangChain docs](/docs/introduction) and [API reference](https://api.python.langchain.com/en/latest/api_reference.html).\n",
+    ">The `Fleet AI` team is on a mission to embed the world's most important data. They've started by embedding the top 1200 Python libraries to enable code generation with up-to-date knowledge. They've been kind enough to share their embeddings of the [LangChain docs](/docs/introduction) and [API reference](https://python.langchain.com/api_reference/).\n",
     "\n",
     "Let's take a look at how we can use these embeddings to power a docs retrieval system and ultimately a simple code-generating chain!"
    ]

docs/docs/integrations/retrievers/needle.ipynb

@@ -0,0 +1,235 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Needle Retriever\n",
+    "[Needle](https://needle-ai.com) makes it easy to create your RAG pipelines with minimal effort. \n",
+    "\n",
+    "For more details, refer to our [API documentation](https://docs.needle-ai.com/docs/api-reference/needle-api)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Overview\n",
+    "The Needle Document Loader is a utility for integrating Needle collections with LangChain. It enables seamless storage, retrieval, and utilization of documents for Retrieval-Augmented Generation (RAG) workflows.\n",
+    "\n",
+    "This example demonstrates:\n",
+    "\n",
+    "* Storing documents into a Needle collection.\n",
+    "* Setting up a retriever to fetch documents.\n",
+    "* Building a Retrieval-Augmented Generation (RAG) pipeline."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Setup\n",
+    "Before starting, ensure you have the following environment variables set:\n",
+    "\n",
+    "* NEEDLE_API_KEY: Your API key for authenticating with Needle.\n",
+    "* OPENAI_API_KEY: Your OpenAI API key for language model operations."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Initialization\n",
+    "To initialize the NeedleLoader, you need the following parameters:\n",
+    "\n",
+    "* needle_api_key: Your Needle API key (or set it as an environment variable).\n",
+    "* collection_id: The ID of the Needle collection to work with."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"NEEDLE_API_KEY\"] = \"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"OPENAI_API_KEY\"] = \"\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Instantiation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders.needle import NeedleLoader\n",
+    "\n",
+    "collection_id = \"clt_01J87M9T6B71DHZTHNXYZQRG5H\"\n",
+    "\n",
+    "# Initialize NeedleLoader to store documents to the collection\n",
+    "document_loader = NeedleLoader(\n",
+    "    needle_api_key=os.getenv(\"NEEDLE_API_KEY\"),\n",
+    "    collection_id=collection_id,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load\n",
+    "To add files to the Needle collection:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "files = {\n",
+    "    \"tech-radar-30.pdf\": \"https://www.thoughtworks.com/content/dam/thoughtworks/documents/radar/2024/04/tr_technology_radar_vol_30_en.pdf\"\n",
+    "}\n",
+    "\n",
+    "document_loader.add_files(files=files)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Show the documents in the collection\n",
+    "# collections_documents = document_loader.load()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Usage\n",
+    "### Use within a chain\n",
+    "Below is a complete example of setting up a RAG pipeline with Needle within a chain:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'input': 'Did RAG move to accepted?',\n",
+       " 'context': [Document(metadata={}, page_content='New Moved in/out No change\\n\\n© Thoughtworks, Inc. All Rights Reserved. 12\\n\\nTechniques\\n\\n1. Retrieval-augmented generation (RAG)\\nAdopt\\n\\nRetrieval-augmented generation (RAG) is the preferred pattern for our teams to improve the quality of \\nresponses generated by a large language model (LLM). We’ve successfully used it in several projects, \\nincluding the popular Jugalbandi AI Platform. With RAG, information about relevant and trustworthy \\ndocuments — in formats like HTML and PDF — are stored in databases that supports a vector data \\ntype or efficient document search, such as pgvector, Qdrant or Elasticsearch Relevance Engine. For \\na given prompt, the database is queried to retrieve relevant documents, which are then combined \\nwith the prompt to provide richer context to the LLM. This results in higher quality output and greatly \\nreduced hallucinations. The context window — which determines the maximum size of the LLM input \\n— is limited, which means that selecting the most relevant documents is crucial. We improve the \\nrelevancy of the content that is added to the prompt by reranking. Similarly, the documents are usually \\ntoo large to calculate an embedding, which means they must be split into smaller chunks. This is often \\na difficult problem, and one approach is to have the chunks overlap to a certain extent.'),\n",
+       "  Document(metadata={}, page_content='New Moved in/out No change\\n\\n© Thoughtworks, Inc. All Rights Reserved. 12\\n\\nTechniques\\n\\n1. Retrieval-augmented generation (RAG)\\nAdopt\\n\\nRetrieval-augmented generation (RAG) is the preferred pattern for our teams to improve the quality of \\nresponses generated by a large language model (LLM). We’ve successfully used it in several projects, \\nincluding the popular Jugalbandi AI Platform. With RAG, information about relevant and trustworthy \\ndocuments — in formats like HTML and PDF — are stored in databases that supports a vector data \\ntype or efficient document search, such as pgvector, Qdrant or Elasticsearch Relevance Engine. For \\na given prompt, the database is queried to retrieve relevant documents, which are then combined \\nwith the prompt to provide richer context to the LLM. This results in higher quality output and greatly \\nreduced hallucinations. The context window — which determines the maximum size of the LLM input \\n— is limited, which means that selecting the most relevant documents is crucial. We improve the \\nrelevancy of the content that is added to the prompt by reranking. Similarly, the documents are usually \\ntoo large to calculate an embedding, which means they must be split into smaller chunks. This is often \\na difficult problem, and one approach is to have the chunks overlap to a certain extent.'),\n",
+       "  Document(metadata={}, page_content='New Moved in/out No change\\n\\n© Thoughtworks, Inc. All Rights Reserved. 12\\n\\nTechniques\\n\\n1. Retrieval-augmented generation (RAG)\\nAdopt\\n\\nRetrieval-augmented generation (RAG) is the preferred pattern for our teams to improve the quality of \\nresponses generated by a large language model (LLM). We’ve successfully used it in several projects, \\nincluding the popular Jugalbandi AI Platform. With RAG, information about relevant and trustworthy \\ndocuments — in formats like HTML and PDF — are stored in databases that supports a vector data \\ntype or efficient document search, such as pgvector, Qdrant or Elasticsearch Relevance Engine. For \\na given prompt, the database is queried to retrieve relevant documents, which are then combined \\nwith the prompt to provide richer context to the LLM. This results in higher quality output and greatly \\nreduced hallucinations. The context window — which determines the maximum size of the LLM input \\n— is limited, which means that selecting the most relevant documents is crucial. We improve the \\nrelevancy of the content that is added to the prompt by reranking. Similarly, the documents are usually \\ntoo large to calculate an embedding, which means they must be split into smaller chunks. This is often \\na difficult problem, and one approach is to have the chunks overlap to a certain extent.'),\n",
+       "  Document(metadata={}, page_content='New Moved in/out No change\\n\\n© Thoughtworks, Inc. All Rights Reserved. 12\\n\\nTechniques\\n\\n1. Retrieval-augmented generation (RAG)\\nAdopt\\n\\nRetrieval-augmented generation (RAG) is the preferred pattern for our teams to improve the quality of \\nresponses generated by a large language model (LLM). We’ve successfully used it in several projects, \\nincluding the popular Jugalbandi AI Platform. With RAG, information about relevant and trustworthy \\ndocuments — in formats like HTML and PDF — are stored in databases that supports a vector data \\ntype or efficient document search, such as pgvector, Qdrant or Elasticsearch Relevance Engine. For \\na given prompt, the database is queried to retrieve relevant documents, which are then combined \\nwith the prompt to provide richer context to the LLM. This results in higher quality output and greatly \\nreduced hallucinations. The context window — which determines the maximum size of the LLM input \\n— is limited, which means that selecting the most relevant documents is crucial. We improve the \\nrelevancy of the content that is added to the prompt by reranking. Similarly, the documents are usually \\ntoo large to calculate an embedding, which means they must be split into smaller chunks. This is often \\na difficult problem, and one approach is to have the chunks overlap to a certain extent.'),\n",
+       "  Document(metadata={}, page_content='https://www.thoughtworks.com/radar/tools/nemo-guardrails\\nhttps://www.thoughtworks.com/radar/platforms/langfuse\\nhttps://www.thoughtworks.com/radar/techniques/retrieval-augmented-generation-rag\\nhttps://cruisecontrol.sourceforge.net/\\nhttps://martinfowler.com/articles/continuousIntegration.html\\nhttps://www.thoughtworks.com/radar/techniques/peer-review-equals-pull-request\\nhttps://martinfowler.com/bliki/ContinuousIntegrationCertification.html\\nhttps://linearb.io/platform/gitstream\\nhttps://www.thoughtworks.com/radar/tools/github-merge-queue\\nhttps://stacking.dev/\\n\\n© Thoughtworks, Inc. All Rights Reserved. 8\\n\\nHold HoldAssess AssessTrial TrialAdopt Adopt\\n\\n18\\n\\n8\\n\\n24\\n\\n29\\n\\n30\\n31\\n\\n32\\n33\\n\\n34 35\\n\\n36\\n37\\n\\n38 39\\n\\n40\\n41\\n\\n42\\n43\\n\\n26\\n\\n2\\n\\n3\\n\\n4\\n\\n5\\n\\n6 7\\n\\n9\\n\\n15\\n\\n16\\n\\n17\\n\\n10\\n\\n11\\n\\n12\\n\\n13 14\\n\\n44\\n\\n47\\n49\\n\\n50\\n\\n65\\n66\\n\\n67 68\\n69\\n\\n70\\n71\\n\\n72\\n\\n73 74\\n\\n75\\n\\n76 77\\n\\n78\\n79\\n\\n80\\n81\\n\\n82\\n\\n83\\n\\n51\\n\\n52 54\\n\\n59\\n\\n53\\n56\\n\\n58\\n\\n61\\n\\n62\\n63\\n\\n64\\n\\n85\\n\\n88 89\\n\\n90 91\\n\\n92\\n93\\n\\n94\\n95 96\\n\\n97\\n\\n98 99\\n\\n100\\n\\n101\\n102\\n\\n103\\n\\n104\\n\\n86\\n\\n87\\n1921\\n\\n22\\n\\n20\\n28\\n\\n25\\n\\n27\\n\\n23\\n\\n84\\n\\n105\\n\\n1\\n45\\n\\n46\\n\\n48\\n\\n55\\n57')],\n",
+       " 'answer': 'Yes, RAG has moved to the \"Adopt\" status.'}"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "\n",
+    "from langchain.chains import create_retrieval_chain\n",
+    "from langchain.chains.combine_documents import create_stuff_documents_chain\n",
+    "from langchain_community.retrievers.needle import NeedleRetriever\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "llm = ChatOpenAI(temperature=0)\n",
+    "\n",
+    "# Initialize the Needle retriever (make sure your Needle API key is set as an environment variable)\n",
+    "retriever = NeedleRetriever(\n",
+    "    needle_api_key=os.getenv(\"NEEDLE_API_KEY\"),\n",
+    "    collection_id=\"clt_01J87M9T6B71DHZTHNXYZQRG5H\",\n",
+    ")\n",
+    "\n",
+    "# Define system prompt for the assistant\n",
+    "system_prompt = \"\"\"\n",
+    "    You are an assistant for question-answering tasks. \n",
+    "    Use the following pieces of retrieved context to answer the question.\n",
+    "    If you don't know, say so concisely.\\n\\n{context}\n",
+    "    \"\"\"\n",
+    "\n",
+    "prompt = ChatPromptTemplate.from_messages(\n",
+    "    [(\"system\", system_prompt), (\"human\", \"{input}\")]\n",
+    ")\n",
+    "\n",
+    "# Define the question-answering chain using a document chain (stuff chain) and the retriever\n",
+    "question_answer_chain = create_stuff_documents_chain(llm, prompt)\n",
+    "\n",
+    "# Create the RAG (Retrieval-Augmented Generation) chain by combining the retriever and the question-answering chain\n",
+    "rag_chain = create_retrieval_chain(retriever, question_answer_chain)\n",
+    "\n",
+    "# Define the input query\n",
+    "query = {\"input\": \"Did RAG move to accepted?\"}\n",
+    "\n",
+    "response = rag_chain.invoke(query)\n",
+    "\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## API reference\n",
+    "\n",
+    "For detailed documentation of all `Needle` features and configurations head to the API reference: https://docs.needle-ai.com"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

docs/docs/integrations/text_embedding/ai21.ipynb

@@ -17,6 +17,8 @@
    "source": [
     "# AI21Embeddings\n",
     "\n",
+    ":::caution This service is deprecated. :::\n",
+    "\n",
     "This will help you get started with AI21 embedding models using LangChain. For detailed documentation on `AI21Embeddings` features and configuration options, please refer to the [API reference](https://python.langchain.com/api_reference/ai21/embeddings/langchain_ai21.embeddings.AI21Embeddings.html).\n",
     "\n",
     "## Overview\n",

docs/docs/integrations/text_embedding/databricks.ipynb

@@ -28,7 +28,7 @@
     "\n",
     "| Class | Package |\n",
     "| :--- | :--- |\n",
-    "| [DatabricksEmbeddings](https://api.python.langchain.com/en/latest/embeddings/langchain_databricks.embeddings.DatabricksEmbeddings.html) | [databricks-langchain](https://python.langchain.com/docs/integrations/providers/databricks/) |\n",
+    "| [DatabricksEmbeddings](https://python.langchain.com/api_reference/community/embeddings/langchain_community.embeddings.databricks.DatabricksEmbeddings.html) | [databricks-langchain](https://python.langchain.com/docs/integrations/providers/databricks/) |\n",
     "\n",
     "### Supported Methods\n",
     "\n",

docs/docs/integrations/text_embedding/zhipuai.ipynb

@@ -251,7 +251,7 @@
    "source": [
     "## API Reference\n",
     "\n",
-    "For detailed documentation on `ZhipuAIEmbeddings` features and configuration options, please refer to the [API reference](https://api.python.langchain.com/en/latest/embeddings/langchain_community.embeddings.zhipuai.ZhipuAIEmbeddings.html).\n"
+    "For detailed documentation on `ZhipuAIEmbeddings` features and configuration options, please refer to the [API reference](https://python.langchain.com/api_reference/community/embeddings/langchain_community.embeddings.zhipuai.ZhipuAIEmbeddings.html).\n"
    ]
   }
  ],

docs/docs/integrations/vectorstores/databricks_vector_search.ipynb

@@ -506,7 +506,7 @@
    "source": [
     "## API reference\n",
     "\n",
-    "For detailed documentation of all DatabricksVectorSearch features and configurations head to the API reference: https://api.python.langchain.com/en/latest/vectorstores/langchain_databricks.vectorstores.DatabricksVectorSearch.html"
+    "For detailed documentation of all DatabricksVectorSearch features and configurations head to the API reference: https://python.langchain.com/api_reference/databricks/vectorstores/langchain_databricks.vectorstores.DatabricksVectorSearch.html"
    ]
   }
  ],

docs/docs/integrations/vectorstores/mongodb_atlas.ipynb

@@ -439,7 +439,7 @@
    "source": [
     "#### Other search methods\n",
     "\n",
-    "There are a variety of other search methods that are not covered in this notebook, such as MMR search or searching by vector. For a full list of the search abilities available for `MongoDBAtlasVectorStore` check out the [API reference](https://api.python.langchain.com/en/latest/vectorstores/langchain_mongodb.vectorstores.MongoDBAtlasVectorSearch.html)."
+    "There are a variety of other search methods that are not covered in this notebook, such as MMR search or searching by vector. For a full list of the search abilities available for `MongoDBAtlasVectorStore` check out the [API reference](https://python.langchain.com/api_reference/mongodb/vectorstores/langchain_mongodb.vectorstores.MongoDBAtlasVectorSearch.html)."
    ]
   },
   {

docs/docs/integrations/vectorstores/sqlitevec.ipynb

@@ -155,7 +155,7 @@
    "cell_type": "markdown",
    "source": [
     "## API reference\n",
-    "For detailed documentation of all SQLiteVec features and configurations head to the API reference:https://api.python.langchain.com/en/latest/vectorstores/langchain_community.vectorstores.sqlitevec.SQLiteVec.html"
+    "For detailed documentation of all SQLiteVec features and configurations head to the API reference: https://python.langchain.com/api_reference/community/vectorstores/langchain_community.vectorstores.sqlitevec.SQLiteVec.html"
    ]
   },
   {

docs/docs/security.md

@@ -1,30 +0,0 @@
-# Security
-
-LangChain has a large ecosystem of integrations with various external resources like local and remote file systems, APIs and databases. These integrations allow developers to create versatile applications that combine the power of LLMs with the ability to access, interact with and manipulate external resources.
-
-## Best practices
-
-When building such applications developers should remember to follow good security practices:
-
-* [**Limit Permissions**](https://en.wikipedia.org/wiki/Principle_of_least_privilege): Scope permissions specifically to the application's need. Granting broad or excessive permissions can introduce significant security vulnerabilities. To avoid such vulnerabilities, consider using read-only credentials, disallowing access to sensitive resources, using sandboxing techniques (such as running inside a container), specifying proxy configurations to control external requests, etc. as appropriate for your application.
-* **Anticipate Potential Misuse**: Just as humans can err, so can Large Language Models (LLMs). Always assume that any system access or credentials may be used in any way allowed by the permissions they are assigned. For example, if a pair of database credentials allows deleting data, it’s safest to assume that any LLM able to use those credentials may in fact delete data.
-* [**Defense in Depth**](https://en.wikipedia.org/wiki/Defense_in_depth_(computing)): No security technique is perfect. Fine-tuning and good chain design can reduce, but not eliminate, the odds that a Large Language Model (LLM) may make a mistake. It’s best to combine multiple layered security approaches rather than relying on any single layer of defense to ensure security. For example: use both read-only permissions and sandboxing to ensure that LLMs are only able to access data that is explicitly meant for them to use.
-
-Risks of not doing so include, but are not limited to:
-* Data corruption or loss.
-* Unauthorized access to confidential information.
-* Compromised performance or availability of critical resources.
-
-Example scenarios with mitigation strategies:
-
-* A user may ask an agent with access to the file system to delete files that should not be deleted or read the content of files that contain sensitive information. To mitigate, limit the agent to only use a specific directory and only allow it to read or write files that are safe to read or write. Consider further sandboxing the agent by running it in a container.
-* A user may ask an agent with write access to an external API to write malicious data to the API, or delete data from that API. To mitigate, give the agent read-only API keys, or limit it to only use endpoints that are already resistant to such misuse.
-* A user may ask an agent with access to a database to drop a table or mutate the schema. To mitigate, scope the credentials to only the tables that the agent needs to access and consider issuing READ-ONLY credentials.
-
-If you're building applications that access external resources like file systems, APIs
-or databases, consider speaking with your company's security team to determine how to best
-design and secure your applications.
-
-## Reporting a vulnerability
-
-Please report security vulnerabilities by email to security@langchain.dev. This will ensure the issue is promptly triaged and acted upon as needed.

docs/docs/versions/migrating_memory/conversation_buffer_window_memory.ipynb

@@ -426,7 +426,7 @@
     "\n",
     "## Usage with a pre-built langgraph agent\n",
     "\n",
-    "This example shows usage of an Agent Executor with a pre-built agent constructed using the [create_tool_calling_agent](https://api.python.langchain.com/en/latest/agents/langchain.agents.tool_calling_agent.base.create_tool_calling_agent.html) function.\n",
+    "This example shows usage of an Agent Executor with a pre-built agent constructed using the [create_tool_calling_agent](https://python.langchain.com/api_reference/langchain/agents/langchain.agents.tool_calling_agent.base.create_tool_calling_agent.html) function.\n",
     "\n",
     "If you are using one of the [old LangChain pre-built agents](https://python.langchain.com/v0.1/docs/modules/agents/agent_types/), you should be able\n",
     "to replace that code with the new [langgraph pre-built agent](https://langchain-ai.github.io/langgraph/how-tos/create-react-agent/) which leverages\n",
@@ -673,7 +673,7 @@
     "</details>\n",
     "\n",
     "If you need to implement more efficient logic and want to use `RunnableWithMessageHistory` for now the way to achieve this\n",
-    "is to subclass from [BaseChatMessageHistory](https://api.python.langchain.com/en/latest/chat_history/langchain_core.chat_history.BaseChatMessageHistory.html) and\n",
+    "is to subclass from [BaseChatMessageHistory](https://python.langchain.com/api_reference/core/chat_history/langchain_core.chat_history.BaseChatMessageHistory.html) and\n",
     "define appropriate logic for `add_messages` (that doesn't simply append the history, but instead re-writes it).\n",
     "\n",
     "Unless you have a good reason to implement this solution, you should instead use LangGraph."

libs/cli/.gitignore

@@ -158,3 +158,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+.integration_test

libs/cli/Makefile

@@ -1,8 +1,47 @@
-lint lint_diff:
-	poetry run poe lint
 
-test:
-	poetry run poe test
+######################
+# LINTING AND FORMATTING
+######################
 
-format:
-	poetry run poe format
+# Define a variable for Python and notebook files.
+PYTHON_FILES=.
+MYPY_CACHE=.mypy_cache
+lint format: PYTHON_FILES=.
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=libs/cli --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+lint_package: PYTHON_FILES=langchain_cli
+lint_tests: PYTHON_FILES=tests
+lint_tests: MYPY_CACHE=.mypy_cache_test
+
+lint lint_diff lint_package lint_tests:
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff check $(PYTHON_FILES)
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES) --diff
+	[ "$(PYTHON_FILES)" = "" ] || mkdir -p $(MYPY_CACHE) && poetry run mypy $(PYTHON_FILES) --cache-dir $(MYPY_CACHE)
+
+format format_diff:
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES)
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff check --select I --fix $(PYTHON_FILES)
+
+test tests: _test _e2e_test
+
+PYTHON = .venv/bin/python
+
+_test:
+	poetry run pytest tests
+
+# custom integration testing for cli integration flow
+# currently ignores vectorstores test because lacks implementation
+_e2e_test:
+	rm -rf .integration_test
+	mkdir .integration_test
+	cd .integration_test && \
+		python3 -m venv .venv && \
+		$(PYTHON) -m pip install --upgrade poetry && \
+		$(PYTHON) -m pip install -e .. && \
+		$(PYTHON) -m langchain_cli.cli integration new --name parrot-link --name-class ParrotLink && \
+		cd langchain-parrot-link && \
+			poetry install --with lint,typing,test && \
+			poetry run pip install -e ../../../standard-tests && \
+			make format lint tests && \
+			poetry install --with test_integration && \
+			rm tests/integration_tests/test_vectorstores.py && \
+			make integration_test

libs/cli/langchain_cli/dev_scripts.py

@@ -1,3 +1,4 @@
+# type: ignore
 """
 Development Scripts for template packages
 """

libs/cli/langchain_cli/integration_template/Makefile

@@ -33,13 +33,13 @@ lint_tests: PYTHON_FILES=tests
 lint_tests: MYPY_CACHE=.mypy_cache_test
 
 lint lint_diff lint_package lint_tests:
-	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff $(PYTHON_FILES)
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff check $(PYTHON_FILES)
 	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES) --diff
 	[ "$(PYTHON_FILES)" = "" ] || mkdir -p $(MYPY_CACHE) && poetry run mypy $(PYTHON_FILES) --cache-dir $(MYPY_CACHE)
 
 format format_diff:
 	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES)
-	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff --select I --fix $(PYTHON_FILES)
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff check --select I --fix $(PYTHON_FILES)
 
 spell_check:
 	poetry run codespell --toml pyproject.toml

libs/cli/langchain_cli/integration_template/integration_template/__init__.py

@@ -1,8 +1,11 @@
 from importlib import metadata
 
 from __module_name__.chat_models import Chat__ModuleName__
+from __module_name__.document_loaders import __ModuleName__Loader
 from __module_name__.embeddings import __ModuleName__Embeddings
-from __module_name__.llms import __ModuleName__LLM
+from __module_name__.retrievers import __ModuleName__Retriever
+from __module_name__.toolkits import __ModuleName__Toolkit
+from __module_name__.tools import __ModuleName__Tool
 from __module_name__.vectorstores import __ModuleName__VectorStore
 
 try:
@@ -14,8 +17,11 @@ del metadata  # optional, avoids polluting the results of dir(__package__)
 
 __all__ = [
     "Chat__ModuleName__",
-    "__ModuleName__LLM",
     "__ModuleName__VectorStore",
     "__ModuleName__Embeddings",
+    "__ModuleName__Loader",
+    "__ModuleName__Retriever",
+    "__ModuleName__Toolkit",
+    "__ModuleName__Tool",
     "__version__",
 ]

libs/cli/langchain_cli/integration_template/integration_template/chat_models.py

@@ -1,13 +1,19 @@
 """__ModuleName__ chat models."""
 
-from typing import Any, List, Optional
+from typing import Any, Dict, Iterator, List, Optional
 
 from langchain_core.callbacks import (
     CallbackManagerForLLMRun,
 )
-from langchain_core.language_models.chat_models import BaseChatModel
-from langchain_core.messages import BaseMessage
-from langchain_core.outputs import ChatResult
+from langchain_core.language_models import BaseChatModel
+from langchain_core.messages import (
+    AIMessage,
+    AIMessageChunk,
+    BaseMessage,
+)
+from langchain_core.messages.ai import UsageMetadata
+from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
+from pydantic import Field
 
 
 class Chat__ModuleName__(BaseChatModel):
@@ -15,6 +21,8 @@ class Chat__ModuleName__(BaseChatModel):
     # https://github.com/langchain-ai/langchain/blob/7ff05357bac6eaedf5058a2af88f23a1817d40fe/libs/partners/openai/langchain_openai/chat_models/base.py#L1120
     """__ModuleName__ chat model integration.
 
+    The default implementation echoes the first `parrot_buffer_length` characters of the input.
+
     # TODO: Replace with relevant packages, env vars.
     Setup:
         Install ``__package_name__`` and set environment variable ``__MODULE_NAME___API_KEY``.
@@ -258,7 +266,36 @@ class Chat__ModuleName__(BaseChatModel):
 
     """  # noqa: E501
 
-    # TODO: This method must be implemented to generate chat responses.
+    model_name: str = Field(alias="model")
+    """The name of the model"""
+    parrot_buffer_length: int
+    """The number of characters from the last message of the prompt to be echoed."""
+    temperature: Optional[float] = None
+    max_tokens: Optional[int] = None
+    timeout: Optional[int] = None
+    stop: Optional[List[str]] = None
+    max_retries: int = 2
+
+    @property
+    def _llm_type(self) -> str:
+        """Return type of chat model."""
+        return "chat-__package_name_short__"
+
+    @property
+    def _identifying_params(self) -> Dict[str, Any]:
+        """Return a dictionary of identifying parameters.
+
+        This information is used by the LangChain callback system, which
+        is used for tracing purposes make it possible to monitor LLMs.
+        """
+        return {
+            # The model name allows users to specify custom token counting
+            # rules in LLM monitoring applications (e.g., in LangSmith users
+            # can provide per token pricing for their model and monitor
+            # costs for the given LLM.)
+            "model_name": self.model_name,
+        }
+
     def _generate(
         self,
         messages: List[BaseMessage],
@@ -266,16 +303,101 @@ class Chat__ModuleName__(BaseChatModel):
         run_manager: Optional[CallbackManagerForLLMRun] = None,
         **kwargs: Any,
     ) -> ChatResult:
-        raise NotImplementedError()
+        """Override the _generate method to implement the chat model logic.
 
-    # TODO: Implement if Chat__ModuleName__ supports streaming. Otherwise delete method.
-    # def _stream(
-    #     self,
-    #     messages: List[BaseMessage],
-    #     stop: Optional[List[str]] = None,
-    #     run_manager: Optional[CallbackManagerForLLMRun] = None,
-    #     **kwargs: Any,
-    # ) -> Iterator[ChatGenerationChunk]:
+        This can be a call to an API, a call to a local model, or any other
+        implementation that generates a response to the input prompt.
+
+        Args:
+            messages: the prompt composed of a list of messages.
+            stop: a list of strings on which the model should stop generating.
+                  If generation stops due to a stop token, the stop token itself
+                  SHOULD BE INCLUDED as part of the output. This is not enforced
+                  across models right now, but it's a good practice to follow since
+                  it makes it much easier to parse the output of the model
+                  downstream and understand why generation stopped.
+            run_manager: A run manager with callbacks for the LLM.
+        """
+        # Replace this with actual logic to generate a response from a list
+        # of messages.
+        last_message = messages[-1]
+        tokens = last_message.content[: self.parrot_buffer_length]
+        ct_input_tokens = sum(len(message.content) for message in messages)
+        ct_output_tokens = len(tokens)
+        message = AIMessage(
+            content=tokens,
+            additional_kwargs={},  # Used to add additional payload to the message
+            response_metadata={  # Use for response metadata
+                "time_in_seconds": 3,
+            },
+            usage_metadata={
+                "input_tokens": ct_input_tokens,
+                "output_tokens": ct_output_tokens,
+                "total_tokens": ct_input_tokens + ct_output_tokens,
+            },
+        )
+        ##
+
+        generation = ChatGeneration(message=message)
+        return ChatResult(generations=[generation])
+
+    def _stream(
+        self,
+        messages: List[BaseMessage],
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> Iterator[ChatGenerationChunk]:
+        """Stream the output of the model.
+
+        This method should be implemented if the model can generate output
+        in a streaming fashion. If the model does not support streaming,
+        do not implement it. In that case streaming requests will be automatically
+        handled by the _generate method.
+
+        Args:
+            messages: the prompt composed of a list of messages.
+            stop: a list of strings on which the model should stop generating.
+                  If generation stops due to a stop token, the stop token itself
+                  SHOULD BE INCLUDED as part of the output. This is not enforced
+                  across models right now, but it's a good practice to follow since
+                  it makes it much easier to parse the output of the model
+                  downstream and understand why generation stopped.
+            run_manager: A run manager with callbacks for the LLM.
+        """
+        last_message = messages[-1]
+        tokens = str(last_message.content[: self.parrot_buffer_length])
+        ct_input_tokens = sum(len(message.content) for message in messages)
+
+        for token in tokens:
+            usage_metadata = UsageMetadata(
+                {
+                    "input_tokens": ct_input_tokens,
+                    "output_tokens": 1,
+                    "total_tokens": ct_input_tokens + 1,
+                }
+            )
+            ct_input_tokens = 0
+            chunk = ChatGenerationChunk(
+                message=AIMessageChunk(content=token, usage_metadata=usage_metadata)
+            )
+
+            if run_manager:
+                # This is optional in newer versions of LangChain
+                # The on_llm_new_token will be called automatically
+                run_manager.on_llm_new_token(token, chunk=chunk)
+
+            yield chunk
+
+        # Let's add some other information (e.g., response metadata)
+        chunk = ChatGenerationChunk(
+            message=AIMessageChunk(content="", response_metadata={"time_in_sec": 3})
+        )
+        if run_manager:
+            # This is optional in newer versions of LangChain
+            # The on_llm_new_token will be called automatically
+            run_manager.on_llm_new_token(token, chunk=chunk)
+        yield chunk
 
     # TODO: Implement if Chat__ModuleName__ supports async streaming. Otherwise delete.
     # async def _astream(
@@ -294,8 +416,3 @@ class Chat__ModuleName__(BaseChatModel):
     #     run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
     #     **kwargs: Any,
     # ) -> ChatResult:
-
-    @property
-    def _llm_type(self) -> str:
-        """Return type of chat model."""
-        return "chat-__package_name_short__"

libs/cli/langchain_cli/integration_template/integration_template/embeddings.py

@@ -8,7 +8,8 @@ class __ModuleName__Embeddings(Embeddings):
 
     # TODO: Replace with relevant packages, env vars.
     Setup:
-        Install ``__package_name__`` and set environment variable ``__MODULE_NAME___API_KEY``.
+        Install ``__package_name__`` and set environment variable
+        ``__MODULE_NAME___API_KEY``.
 
         .. code-block:: bash
 
@@ -70,21 +71,26 @@ class __ModuleName__Embeddings(Embeddings):
 
     """
 
+    def __init__(self, model: str):
+        self.model = model
+
     def embed_documents(self, texts: List[str]) -> List[List[float]]:
         """Embed search docs."""
-        raise NotImplementedError
+        return [[0.5, 0.6, 0.7] for _ in texts]
 
     def embed_query(self, text: str) -> List[float]:
         """Embed query text."""
-        raise NotImplementedError
+        return self.embed_documents([text])[0]
 
-    # only keep aembed_documents and aembed_query if they're implemented!
-    # delete them otherwise to use the base class' default
-    # implementation, which calls the sync version in an executor
-    async def aembed_documents(self, texts: List[str]) -> List[List[float]]:
-        """Asynchronous Embed search docs."""
-        raise NotImplementedError
+    # optional: add custom async implementations here
+    # you can also delete these, and the base class will
+    # use the default implementation, which calls the sync
+    # version in an async executor:
 
-    async def aembed_query(self, text: str) -> List[float]:
-        """Asynchronous Embed query text."""
-        raise NotImplementedError
+    # async def aembed_documents(self, texts: List[str]) -> List[List[float]]:
+    #     """Asynchronous Embed search docs."""
+    #     ...
+
+    # async def aembed_query(self, text: str) -> List[float]:
+    #     """Asynchronous Embed query text."""
+    #     ...

libs/cli/langchain_cli/integration_template/integration_template/llms.py

@@ -1,155 +0,0 @@
-"""__ModuleName__ large language models."""
-
-from typing import (
-    Any,
-    List,
-    Optional,
-)
-
-from langchain_core.callbacks import (
-    CallbackManagerForLLMRun,
-)
-from langchain_core.language_models import BaseLLM
-from langchain_core.outputs import LLMResult
-
-
-class __ModuleName__LLM(BaseLLM):
-    """__ModuleName__ completion model integration.
-
-    # TODO: Replace with relevant packages, env vars.
-    Setup:
-        Install ``__package_name__`` and set environment variable ``__MODULE_NAME___API_KEY``.
-
-        .. code-block:: bash
-
-            pip install -U __package_name__
-            export __MODULE_NAME___API_KEY="your-api-key"
-
-    # TODO: Populate with relevant params.
-    Key init args — completion params:
-        model: str
-            Name of __ModuleName__ model to use.
-        temperature: float
-            Sampling temperature.
-        max_tokens: Optional[int]
-            Max number of tokens to generate.
-
-    # TODO: Populate with relevant params.
-    Key init args — client params:
-        timeout: Optional[float]
-            Timeout for requests.
-        max_retries: int
-            Max number of retries.
-        api_key: Optional[str]
-            __ModuleName__ API key. If not passed in will be read from env var __MODULE_NAME___API_KEY.
-
-    See full list of supported init args and their descriptions in the params section.
-
-    # TODO: Replace with relevant init params.
-    Instantiate:
-        .. code-block:: python
-
-            from __module_name__ import __ModuleName__LLM
-
-            llm = __ModuleName__LLM(
-                model="...",
-                temperature=0,
-                max_tokens=None,
-                timeout=None,
-                max_retries=2,
-                # api_key="...",
-                # other params...
-            )
-
-    Invoke:
-        .. code-block:: python
-
-            input_text = "The meaning of life is "
-            llm.invoke(input_text)
-
-        .. code-block:: python
-
-            # TODO: Example output.
-
-    # TODO: Delete if token-level streaming isn't supported.
-    Stream:
-        .. code-block:: python
-
-            for chunk in llm.stream(input_text):
-                print(chunk)
-
-        .. code-block:: python
-
-            # TODO: Example output.
-
-        .. code-block:: python
-
-            ''.join(llm.stream(input_text))
-
-        .. code-block:: python
-
-            # TODO: Example output.
-
-    # TODO: Delete if native async isn't supported.
-    Async:
-        .. code-block:: python
-
-            await llm.ainvoke(input_text)
-
-            # stream:
-            # async for chunk in (await llm.astream(input_text))
-
-            # batch:
-            # await llm.abatch([input_text])
-
-        .. code-block:: python
-
-            # TODO: Example output.
-    """
-
-    # TODO: This method must be implemented to generate text completions.
-    def _generate(
-        self,
-        prompts: List[str],
-        stop: Optional[List[str]] = None,
-        run_manager: Optional[CallbackManagerForLLMRun] = None,
-        **kwargs: Any,
-    ) -> LLMResult:
-        raise NotImplementedError
-
-    # TODO: Implement if __ModuleName__LLM supports async generation. Otherwise
-    # delete method.
-    # async def _agenerate(
-    #     self,
-    #     prompts: List[str],
-    #     stop: Optional[List[str]] = None,
-    #     run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
-    #     **kwargs: Any,
-    # ) -> LLMResult:
-    #     raise NotImplementedError
-
-    # TODO: Implement if __ModuleName__LLM supports streaming. Otherwise delete method.
-    # def _stream(
-    #     self,
-    #     prompt: str,
-    #     stop: Optional[List[str]] = None,
-    #     run_manager: Optional[CallbackManagerForLLMRun] = None,
-    #     **kwargs: Any,
-    # ) -> Iterator[GenerationChunk]:
-    #     raise NotImplementedError
-
-    # TODO: Implement if __ModuleName__LLM supports async streaming. Otherwise delete
-    # method.
-    # async def _astream(
-    #     self,
-    #     prompt: str,
-    #     stop: Optional[List[str]] = None,
-    #     run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
-    #     **kwargs: Any,
-    # ) -> AsyncIterator[GenerationChunk]:
-    #     raise NotImplementedError
-
-    @property
-    def _llm_type(self) -> str:
-        """Return type of LLM."""
-        return "__package_name_short__-llm"

libs/cli/langchain_cli/integration_template/integration_template/retrievers.py

@@ -1,7 +1,8 @@
 """__ModuleName__ retrievers."""
 
-from typing import List
+from typing import Any, List
 
+from langchain_core.callbacks import CallbackManagerForRetrieverRun
 from langchain_core.documents import Document
 from langchain_core.retrievers import BaseRetriever
 
@@ -13,7 +14,8 @@ class __ModuleName__Retriever(BaseRetriever):
 
     # TODO: Replace with relevant packages, env vars, etc.
     Setup:
-        Install ``__package_name__`` and set environment variable ``__MODULE_NAME___API_KEY``.
+        Install ``__package_name__`` and set environment variable
+        ``__MODULE_NAME___API_KEY``.
 
         .. code-block:: bash
 
@@ -82,8 +84,24 @@ class __ModuleName__Retriever(BaseRetriever):
 
              # TODO: Example output.
 
-    """  # noqa: E501
+    """
+
+    k: int = 3
 
     # TODO: This method must be implemented to retrieve documents.
-    def _get_relevant_documents(self, query: str) -> List[Document]:
-        raise NotImplementedError()
+    def _get_relevant_documents(
+        self, query: str, *, run_manager: CallbackManagerForRetrieverRun, **kwargs: Any
+    ) -> List[Document]:
+        k = kwargs.get("k", self.k)
+        return [
+            Document(page_content=f"Result {i} for query: {query}") for i in range(k)
+        ]
+
+    # optional: add custom async implementations here
+    # async def _aget_relevant_documents(
+    #     self,
+    #     query: str,
+    #     *,
+    #     run_manager: AsyncCallbackManagerForRetrieverRun,
+    #     **kwargs: Any,
+    # ) -> List[Document]: ...

libs/cli/langchain_cli/integration_template/integration_template/toolkits.py

@@ -2,10 +2,10 @@
 
 from typing import List
 
-from langchain_core.tools import BaseTool, BaseToolKit
+from langchain_core.tools import BaseTool, BaseToolkit
 
 
-class __ModuleName__Toolkit(BaseToolKit):
+class __ModuleName__Toolkit(BaseToolkit):
     # TODO: Replace all TODOs in docstring. See example docstring:
     # https://github.com/langchain-ai/langchain/blob/c123cb2b304f52ab65db4714eeec46af69a861ec/libs/community/langchain_community/agent_toolkits/sql/toolkit.py#L19
     """__ModuleName__ toolkit.

libs/cli/langchain_cli/integration_template/integration_template/tools.py

@@ -6,10 +6,10 @@ from langchain_core.callbacks import (
     CallbackManagerForToolRun,
 )
 from langchain_core.tools import BaseTool
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 
-class __ModuleName__Input(BaseModel):
+class __ModuleName__ToolInput(BaseModel):
     """Input schema for __ModuleName__ tool.
 
     This docstring is **not** part of what is sent to the model when performing tool
@@ -18,12 +18,11 @@ class __ModuleName__Input(BaseModel):
     """
 
     # TODO: Add input args and descriptions.
-    # a: int = Field(..., description="first number")
-    # b: int = Field(0, description="second number")
-    ...
+    a: int = Field(..., description="first number to add")
+    b: int = Field(..., description="second number to add")
 
 
-class __ModuleName__Tool(BaseTool):
+class __ModuleName__Tool(BaseTool):  # type: ignore[override]
     """__ModuleName__ tool.
 
     Setup:
@@ -69,24 +68,26 @@ class __ModuleName__Tool(BaseTool):
     """The name that is passed to the model when performing tool calling."""
     description: str = "TODO: Tool description."
     """The description that is passed to the model when performing tool calling."""
-    args_schema: Type[BaseModel] = __ModuleName__Input
+    args_schema: Type[BaseModel] = __ModuleName__ToolInput
     """The schema that is passed to the model when performing tool calling."""
 
     # TODO: Add any other init params for the tool.
     # param1: Optional[str]
     # """param1 determines foobar"""
 
-    # TODO: Replaced *args with real tool arguments.
+    # TODO: Replaced (a, b) with real tool arguments.
     def _run(
-        self, *args, run_manager: Optional[CallbackManagerForToolRun] = None
+        self, a: int, b: int, *, run_manager: Optional[CallbackManagerForToolRun] = None
     ) -> str:
-        raise NotImplementedError
+        return str(a + b + 80)
 
     # TODO: Implement if tool has native async functionality, otherwise delete.
 
     # async def _arun(
     #     self,
-    #     *args,
+    #     a: int,
+    #     b: int,
+    #     *,
     #     run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
     # ) -> str:
     #     ...

libs/cli/langchain_cli/integration_template/integration_template/vectorstores.py

@@ -2,8 +2,6 @@
 
 from __future__ import annotations
 
-import asyncio
-from functools import partial
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -160,6 +158,8 @@ class __ModuleName__VectorStore(VectorStore):
 
     """  # noqa: E501
 
+    _database: dict[str, tuple[Document, list[float]]] = {}
+
     def add_texts(
         self,
         texts: Iterable[str],
@@ -168,65 +168,70 @@ class __ModuleName__VectorStore(VectorStore):
     ) -> List[str]:
         raise NotImplementedError
 
-    async def aadd_texts(
-        self,
-        texts: Iterable[str],
-        metadatas: Optional[List[dict]] = None,
-        **kwargs: Any,
-    ) -> List[str]:
-        return await asyncio.get_running_loop().run_in_executor(
-            None, partial(self.add_texts, **kwargs), texts, metadatas
-        )
+    # optional: add custom async implementations
+    # async def aadd_texts(
+    #     self,
+    #     texts: Iterable[str],
+    #     metadatas: Optional[List[dict]] = None,
+    #     **kwargs: Any,
+    # ) -> List[str]:
+    #     return await asyncio.get_running_loop().run_in_executor(
+    #         None, partial(self.add_texts, **kwargs), texts, metadatas
+    #     )
 
     def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> Optional[bool]:
         raise NotImplementedError
 
-    async def adelete(
-        self, ids: Optional[List[str]] = None, **kwargs: Any
-    ) -> Optional[bool]:
-        raise NotImplementedError
+    # optional: add custom async implementations
+    # async def adelete(
+    #     self, ids: Optional[List[str]] = None, **kwargs: Any
+    # ) -> Optional[bool]:
+    #     raise NotImplementedError
 
     def similarity_search(
         self, query: str, k: int = 4, **kwargs: Any
     ) -> List[Document]:
         raise NotImplementedError
 
-    async def asimilarity_search(
-        self, query: str, k: int = 4, **kwargs: Any
-    ) -> List[Document]:
-        # This is a temporary workaround to make the similarity search
-        # asynchronous. The proper solution is to make the similarity search
-        # asynchronous in the vector store implementations.
-        func = partial(self.similarity_search, query, k=k, **kwargs)
-        return await asyncio.get_event_loop().run_in_executor(None, func)
+    # optional: add custom async implementations
+    # async def asimilarity_search(
+    #     self, query: str, k: int = 4, **kwargs: Any
+    # ) -> List[Document]:
+    #     # This is a temporary workaround to make the similarity search
+    #     # asynchronous. The proper solution is to make the similarity search
+    #     # asynchronous in the vector store implementations.
+    #     func = partial(self.similarity_search, query, k=k, **kwargs)
+    #     return await asyncio.get_event_loop().run_in_executor(None, func)
 
     def similarity_search_with_score(
         self, *args: Any, **kwargs: Any
     ) -> List[Tuple[Document, float]]:
         raise NotImplementedError
 
-    async def asimilarity_search_with_score(
-        self, *args: Any, **kwargs: Any
-    ) -> List[Tuple[Document, float]]:
-        # This is a temporary workaround to make the similarity search
-        # asynchronous. The proper solution is to make the similarity search
-        # asynchronous in the vector store implementations.
-        func = partial(self.similarity_search_with_score, *args, **kwargs)
-        return await asyncio.get_event_loop().run_in_executor(None, func)
+    # optional: add custom async implementations
+    # async def asimilarity_search_with_score(
+    #     self, *args: Any, **kwargs: Any
+    # ) -> List[Tuple[Document, float]]:
+    #     # This is a temporary workaround to make the similarity search
+    #     # asynchronous. The proper solution is to make the similarity search
+    #     # asynchronous in the vector store implementations.
+    #     func = partial(self.similarity_search_with_score, *args, **kwargs)
+    #     return await asyncio.get_event_loop().run_in_executor(None, func)
 
     def similarity_search_by_vector(
         self, embedding: List[float], k: int = 4, **kwargs: Any
     ) -> List[Document]:
         raise NotImplementedError
 
-    async def asimilarity_search_by_vector(
-        self, embedding: List[float], k: int = 4, **kwargs: Any
-    ) -> List[Document]:
-        # This is a temporary workaround to make the similarity search
-        # asynchronous. The proper solution is to make the similarity search
-        # asynchronous in the vector store implementations.
-        func = partial(self.similarity_search_by_vector, embedding, k=k, **kwargs)
-        return await asyncio.get_event_loop().run_in_executor(None, func)
+    # optional: add custom async implementations
+    # async def asimilarity_search_by_vector(
+    #     self, embedding: List[float], k: int = 4, **kwargs: Any
+    # ) -> List[Document]:
+    #     # This is a temporary workaround to make the similarity search
+    #     # asynchronous. The proper solution is to make the similarity search
+    #     # asynchronous in the vector store implementations.
+    #     func = partial(self.similarity_search_by_vector, embedding, k=k, **kwargs)
+    #     return await asyncio.get_event_loop().run_in_executor(None, func)
 
     def max_marginal_relevance_search(
         self,
@@ -238,26 +243,27 @@ class __ModuleName__VectorStore(VectorStore):
     ) -> List[Document]:
         raise NotImplementedError
 
-    async def amax_marginal_relevance_search(
-        self,
-        query: str,
-        k: int = 4,
-        fetch_k: int = 20,
-        lambda_mult: float = 0.5,
-        **kwargs: Any,
-    ) -> List[Document]:
-        # This is a temporary workaround to make the similarity search
-        # asynchronous. The proper solution is to make the similarity search
-        # asynchronous in the vector store implementations.
-        func = partial(
-            self.max_marginal_relevance_search,
-            query,
-            k=k,
-            fetch_k=fetch_k,
-            lambda_mult=lambda_mult,
-            **kwargs,
-        )
-        return await asyncio.get_event_loop().run_in_executor(None, func)
+    # optional: add custom async implementations
+    # async def amax_marginal_relevance_search(
+    #     self,
+    #     query: str,
+    #     k: int = 4,
+    #     fetch_k: int = 20,
+    #     lambda_mult: float = 0.5,
+    #     **kwargs: Any,
+    # ) -> List[Document]:
+    #     # This is a temporary workaround to make the similarity search
+    #     # asynchronous. The proper solution is to make the similarity search
+    #     # asynchronous in the vector store implementations.
+    #     func = partial(
+    #         self.max_marginal_relevance_search,
+    #         query,
+    #         k=k,
+    #         fetch_k=fetch_k,
+    #         lambda_mult=lambda_mult,
+    #         **kwargs,
+    #     )
+    #     return await asyncio.get_event_loop().run_in_executor(None, func)
 
     def max_marginal_relevance_search_by_vector(
         self,
@@ -269,15 +275,16 @@ class __ModuleName__VectorStore(VectorStore):
     ) -> List[Document]:
         raise NotImplementedError
 
-    async def amax_marginal_relevance_search_by_vector(
-        self,
-        embedding: List[float],
-        k: int = 4,
-        fetch_k: int = 20,
-        lambda_mult: float = 0.5,
-        **kwargs: Any,
-    ) -> List[Document]:
-        raise NotImplementedError
+    # optional: add custom async implementations
+    # async def amax_marginal_relevance_search_by_vector(
+    #     self,
+    #     embedding: List[float],
+    #     k: int = 4,
+    #     fetch_k: int = 20,
+    #     lambda_mult: float = 0.5,
+    #     **kwargs: Any,
+    # ) -> List[Document]:
+    #     raise NotImplementedError
 
     @classmethod
     def from_texts(
@@ -289,17 +296,18 @@ class __ModuleName__VectorStore(VectorStore):
     ) -> VST:
         raise NotImplementedError
 
-    @classmethod
-    async def afrom_texts(
-        cls: Type[VST],
-        texts: List[str],
-        embedding: Embeddings,
-        metadatas: Optional[List[dict]] = None,
-        **kwargs: Any,
-    ) -> VST:
-        return await asyncio.get_running_loop().run_in_executor(
-            None, partial(cls.from_texts, **kwargs), texts, embedding, metadatas
-        )
+    # optional: add custom async implementations
+    # @classmethod
+    # async def afrom_texts(
+    #     cls: Type[VST],
+    #     texts: List[str],
+    #     embedding: Embeddings,
+    #     metadatas: Optional[List[dict]] = None,
+    #     **kwargs: Any,
+    # ) -> VST:
+    #     return await asyncio.get_running_loop().run_in_executor(
+    #         None, partial(cls.from_texts, **kwargs), texts, embedding, metadatas
+    #     )
 
     def _select_relevance_score_fn(self) -> Callable[[float], float]:
         raise NotImplementedError

libs/cli/langchain_cli/integration_template/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = [ "poetry-core>=1.0.0",]
+requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
@@ -23,14 +23,16 @@ python = ">=3.9,<4.0"
 langchain-core = "^0.3.15"
 
 [tool.ruff.lint]
-select = [ "E", "F", "I", "T201",]
+select = ["E", "F", "I", "T201"]
 
 [tool.coverage.run]
-omit = [ "tests/*",]
+omit = ["tests/*"]
 
 [tool.pytest.ini_options]
 addopts = "--strict-markers --strict-config --durations=5"
-markers = [ "compile: mark placeholder test used to compile integration tests without running them",]
+markers = [
+    "compile: mark placeholder test used to compile integration tests without running them",
+]
 asyncio_mode = "auto"
 
 [tool.poetry.group.test]
@@ -48,11 +50,14 @@ optional = true
 [tool.poetry.group.dev]
 optional = true
 
+[tool.poetry.group.dev.dependencies]
+
 [tool.poetry.group.test.dependencies]
 pytest = "^7.4.3"
 pytest-asyncio = "^0.23.2"
 pytest-socket = "^0.7.0"
 pytest-watcher = "^0.3.4"
+langchain-tests = "^0.3.5"
 
 [tool.poetry.group.codespell.dependencies]
 codespell = "^2.2.6"
@@ -64,15 +69,3 @@ ruff = "^0.5"
 
 [tool.poetry.group.typing.dependencies]
 mypy = "^1.10"
-
-[tool.poetry.group.test.dependencies.langchain-core]
-path = "../../core"
-develop = true
-
-[tool.poetry.group.dev.dependencies.langchain-core]
-path = "../../core"
-develop = true
-
-[tool.poetry.group.typing.dependencies.langchain-core]
-path = "../../core"
-develop = true

libs/cli/langchain_cli/integration_template/tests/integration_tests/test_chat_models.py

@@ -1,64 +1,21 @@
 """Test Chat__ModuleName__ chat model."""
 
+from typing import Type
+
 from __module_name__.chat_models import Chat__ModuleName__
+from langchain_tests.integration_tests import ChatModelIntegrationTests
 
 
-def test_stream() -> None:
-    """Test streaming tokens from OpenAI."""
-    llm = Chat__ModuleName__()
+class TestChatParrotLinkIntegration(ChatModelIntegrationTests):
+    @property
+    def chat_model_class(self) -> Type[Chat__ModuleName__]:
+        return Chat__ModuleName__
 
-    for token in llm.stream("I'm Pickle Rick"):
-        assert isinstance(token.content, str)
-
-
-async def test_astream() -> None:
-    """Test streaming tokens from OpenAI."""
-    llm = Chat__ModuleName__()
-
-    async for token in llm.astream("I'm Pickle Rick"):
-        assert isinstance(token.content, str)
-
-
-async def test_abatch() -> None:
-    """Test streaming tokens from Chat__ModuleName__."""
-    llm = Chat__ModuleName__()
-
-    result = await llm.abatch(["I'm Pickle Rick", "I'm not Pickle Rick"])
-    for token in result:
-        assert isinstance(token.content, str)
-
-
-async def test_abatch_tags() -> None:
-    """Test batch tokens from Chat__ModuleName__."""
-    llm = Chat__ModuleName__()
-
-    result = await llm.abatch(
-        ["I'm Pickle Rick", "I'm not Pickle Rick"], config={"tags": ["foo"]}
-    )
-    for token in result:
-        assert isinstance(token.content, str)
-
-
-def test_batch() -> None:
-    """Test batch tokens from Chat__ModuleName__."""
-    llm = Chat__ModuleName__()
-
-    result = llm.batch(["I'm Pickle Rick", "I'm not Pickle Rick"])
-    for token in result:
-        assert isinstance(token.content, str)
-
-
-async def test_ainvoke() -> None:
-    """Test invoke tokens from Chat__ModuleName__."""
-    llm = Chat__ModuleName__()
-
-    result = await llm.ainvoke("I'm Pickle Rick", config={"tags": ["foo"]})
-    assert isinstance(result.content, str)
-
-
-def test_invoke() -> None:
-    """Test invoke tokens from Chat__ModuleName__."""
-    llm = Chat__ModuleName__()
-
-    result = llm.invoke("I'm Pickle Rick", config=dict(tags=["foo"]))
-    assert isinstance(result.content, str)
+    @property
+    def chat_model_params(self) -> dict:
+        # These should be parameters used to initialize your integration for testing
+        return {
+            "model": "bird-brain-001",
+            "temperature": 0,
+            "parrot_buffer_length": 50,
+        }

libs/cli/langchain_cli/integration_template/tests/integration_tests/test_embeddings.py

@@ -1,20 +1,16 @@
 """Test __ModuleName__ embeddings."""
 
+from typing import Type
+
 from __module_name__.embeddings import __ModuleName__Embeddings
+from langchain_tests.integration_tests import EmbeddingsIntegrationTests
 
 
-def test___module_name___embedding_documents() -> None:
-    """Test cohere embeddings."""
-    documents = ["foo bar"]
-    embedding = __ModuleName__Embeddings()
-    output = embedding.embed_documents(documents)
-    assert len(output) == 1
-    assert len(output[0]) > 0
+class TestParrotLinkEmbeddingsIntegration(EmbeddingsIntegrationTests):
+    @property
+    def embeddings_class(self) -> Type[__ModuleName__Embeddings]:
+        return __ModuleName__Embeddings
 
-
-def test___module_name___embedding_query() -> None:
-    """Test cohere embeddings."""
-    document = "foo bar"
-    embedding = __ModuleName__Embeddings()
-    output = embedding.embed_query(document)
-    assert len(output) > 0
+    @property
+    def embedding_model_params(self) -> dict:
+        return {"model": "nest-embed-001"}

libs/cli/langchain_cli/integration_template/tests/integration_tests/test_llms.py

@@ -1,64 +0,0 @@
-"""Test __ModuleName__LLM llm."""
-
-from __module_name__.llms import __ModuleName__LLM
-
-
-def test_stream() -> None:
-    """Test streaming tokens from OpenAI."""
-    llm = __ModuleName__LLM()
-
-    for token in llm.stream("I'm Pickle Rick"):
-        assert isinstance(token, str)
-
-
-async def test_astream() -> None:
-    """Test streaming tokens from OpenAI."""
-    llm = __ModuleName__LLM()
-
-    async for token in llm.astream("I'm Pickle Rick"):
-        assert isinstance(token, str)
-
-
-async def test_abatch() -> None:
-    """Test streaming tokens from __ModuleName__LLM."""
-    llm = __ModuleName__LLM()
-
-    result = await llm.abatch(["I'm Pickle Rick", "I'm not Pickle Rick"])
-    for token in result:
-        assert isinstance(token, str)
-
-
-async def test_abatch_tags() -> None:
-    """Test batch tokens from __ModuleName__LLM."""
-    llm = __ModuleName__LLM()
-
-    result = await llm.abatch(
-        ["I'm Pickle Rick", "I'm not Pickle Rick"], config={"tags": ["foo"]}
-    )
-    for token in result:
-        assert isinstance(token, str)
-
-
-def test_batch() -> None:
-    """Test batch tokens from __ModuleName__LLM."""
-    llm = __ModuleName__LLM()
-
-    result = llm.batch(["I'm Pickle Rick", "I'm not Pickle Rick"])
-    for token in result:
-        assert isinstance(token, str)
-
-
-async def test_ainvoke() -> None:
-    """Test invoke tokens from __ModuleName__LLM."""
-    llm = __ModuleName__LLM()
-
-    result = await llm.ainvoke("I'm Pickle Rick", config={"tags": ["foo"]})
-    assert isinstance(result, str)
-
-
-def test_invoke() -> None:
-    """Test invoke tokens from __ModuleName__LLM."""
-    llm = __ModuleName__LLM()
-
-    result = llm.invoke("I'm Pickle Rick", config=dict(tags=["foo"]))
-    assert isinstance(result, str)

libs/cli/langchain_cli/integration_template/tests/integration_tests/test_retrievers.py

@@ -0,0 +1,24 @@
+from typing import Type
+
+from __module_name__.retrievers import __ModuleName__Retriever
+from langchain_tests.integration_tests import (
+    RetrieversIntegrationTests,
+)
+
+
+class Test__ModuleName__Retriever(RetrieversIntegrationTests):
+    @property
+    def retriever_constructor(self) -> Type[__ModuleName__Retriever]:
+        """Get an empty vectorstore for unit tests."""
+        return __ModuleName__Retriever
+
+    @property
+    def retriever_constructor_params(self) -> dict:
+        return {"k": 2}
+
+    @property
+    def retriever_query_example(self) -> str:
+        """
+        Returns a dictionary representing the "args" of an example retriever call.
+        """
+        return "example query"

libs/cli/langchain_cli/integration_template/tests/integration_tests/test_tools.py

@@ -0,0 +1,27 @@
+from typing import Type
+
+from __module_name__.tools import __ModuleName__Tool
+from langchain_tests.integration_tests import ToolsIntegrationTests
+
+
+class TestParrotMultiplyToolIntegration(ToolsIntegrationTests):
+    @property
+    def tool_constructor(self) -> Type[__ModuleName__Tool]:
+        return __ModuleName__Tool
+
+    @property
+    def tool_constructor_params(self) -> dict:
+        # if your tool constructor instead required initialization arguments like
+        # `def __init__(self, some_arg: int):`, you would return those here
+        # as a dictionary, e.g.: `return {'some_arg': 42}`
+        return {}
+
+    @property
+    def tool_invoke_params_example(self) -> dict:
+        """
+        Returns a dictionary representing the "args" of an example tool call.
+
+        This should NOT be a ToolCall dict - i.e. it should not
+        have {"name", "id", "args"} keys.
+        """
+        return {"a": 2, "b": 3}

libs/cli/langchain_cli/integration_template/tests/integration_tests/test_vectorstores.py

@@ -0,0 +1,37 @@
+from typing import AsyncGenerator, Generator
+
+import pytest
+from __module_name__.vectorstores import __ModuleName__VectorStore
+from langchain_core.vectorstores import VectorStore
+from langchain_tests.integration_tests import (
+    AsyncReadWriteTestSuite,
+    ReadWriteTestSuite,
+)
+
+
+class Test__ModuleName__VectorStoreSync(ReadWriteTestSuite):
+    @pytest.fixture()
+    def vectorstore(self) -> Generator[VectorStore, None, None]:  # type: ignore
+        """Get an empty vectorstore for unit tests."""
+        store = __ModuleName__VectorStore()
+        # note: store should be EMPTY at this point
+        # if you need to delete data, you may do so here
+        try:
+            yield store
+        finally:
+            # cleanup operations, or deleting data
+            pass
+
+
+class Test__ModuleName__VectorStoreAsync(AsyncReadWriteTestSuite):
+    @pytest.fixture()
+    async def vectorstore(self) -> AsyncGenerator[VectorStore, None]:  # type: ignore
+        """Get an empty vectorstore for unit tests."""
+        store = __ModuleName__VectorStore()
+        # note: store should be EMPTY at this point
+        # if you need to delete data, you may do so here
+        try:
+            yield store
+        finally:
+            # cleanup operations, or deleting data
+            pass

libs/cli/langchain_cli/integration_template/tests/unit_tests/test_chat_models.py

@@ -1,8 +1,21 @@
 """Test chat model integration."""
 
+from typing import Type
+
 from __module_name__.chat_models import Chat__ModuleName__
+from langchain_tests.unit_tests import ChatModelUnitTests
 
 
-def test_initialization() -> None:
-    """Test chat model initialization."""
-    Chat__ModuleName__()
+class TestChat__ModuleName__Unit(ChatModelUnitTests):
+    @property
+    def chat_model_class(self) -> Type[Chat__ModuleName__]:
+        return Chat__ModuleName__
+
+    @property
+    def chat_model_params(self) -> dict:
+        # These should be parameters used to initialize your integration for testing
+        return {
+            "model": "bird-brain-001",
+            "temperature": 0,
+            "parrot_buffer_length": 50,
+        }

libs/cli/langchain_cli/integration_template/tests/unit_tests/test_embeddings.py

@@ -1,8 +1,16 @@
 """Test embedding model integration."""
 
+from typing import Type
+
 from __module_name__.embeddings import __ModuleName__Embeddings
+from langchain_tests.unit_tests import EmbeddingsUnitTests
 
 
-def test_initialization() -> None:
-    """Test embedding model initialization."""
-    __ModuleName__Embeddings()
+class TestParrotLinkEmbeddingsUnit(EmbeddingsUnitTests):
+    @property
+    def embeddings_class(self) -> Type[__ModuleName__Embeddings]:
+        return __ModuleName__Embeddings
+
+    @property
+    def embedding_model_params(self) -> dict:
+        return {"model": "nest-embed-001"}

libs/cli/langchain_cli/integration_template/tests/unit_tests/test_imports.py

@@ -1,12 +0,0 @@
-from __module_name__ import __all__
-
-EXPECTED_ALL = [
-    "__ModuleName__LLM",
-    "Chat__ModuleName__",
-    "__ModuleName__VectorStore",
-    "__ModuleName__Embeddings",
-]
-
-
-def test_all_imports() -> None:
-    assert sorted(EXPECTED_ALL) == sorted(__all__)

libs/cli/langchain_cli/integration_template/tests/unit_tests/test_llms.py

@@ -1,8 +0,0 @@
-"""Test __ModuleName__ Chat API wrapper."""
-
-from __module_name__ import __ModuleName__LLM
-
-
-def test_initialization() -> None:
-    """Test integration initialization."""
-    __ModuleName__LLM()

libs/cli/langchain_cli/integration_template/tests/unit_tests/test_tools.py

@@ -0,0 +1,27 @@
+from typing import Type
+
+from __module_name__.tools import __ModuleName__Tool
+from langchain_tests.unit_tests import ToolsUnitTests
+
+
+class TestParrotMultiplyToolUnit(ToolsUnitTests):
+    @property
+    def tool_constructor(self) -> Type[__ModuleName__Tool]:
+        return __ModuleName__Tool
+
+    @property
+    def tool_constructor_params(self) -> dict:
+        # if your tool constructor instead required initialization arguments like
+        # `def __init__(self, some_arg: int):`, you would return those here
+        # as a dictionary, e.g.: `return {'some_arg': 42}`
+        return {}
+
+    @property
+    def tool_invoke_params_example(self) -> dict:
+        """
+        Returns a dictionary representing the "args" of an example tool call.
+
+        This should NOT be a ToolCall dict - i.e. it should not
+        have {"name", "id", "args"} keys.
+        """
+        return {"a": 2, "b": 3}

libs/cli/langchain_cli/integration_template/tests/unit_tests/test_vectorstores.py

@@ -1,6 +0,0 @@
-from __module_name__.vectorstores import __ModuleName__VectorStore
-
-
-def test_initialization() -> None:
-    """Test integration vectorstore initialization."""
-    __ModuleName__VectorStore()

libs/cli/langchain_cli/namespaces/integration.py

@@ -6,7 +6,7 @@ import re
 import shutil
 import subprocess
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional, cast
 
 import typer
 from typing_extensions import Annotated, TypedDict
@@ -15,19 +15,17 @@ from langchain_cli.utils.find_replace import replace_file, replace_glob
 
 integration_cli = typer.Typer(no_args_is_help=True, add_completion=False)
 
-Replacements = TypedDict(
-    "Replacements",
-    {
-        "__package_name__": str,
-        "__module_name__": str,
-        "__ModuleName__": str,
-        "__MODULE_NAME__": str,
-        "__package_name_short__": str,
-    },
-)
+
+class Replacements(TypedDict):
+    __package_name__: str
+    __module_name__: str
+    __ModuleName__: str
+    __MODULE_NAME__: str
+    __package_name_short__: str
+    __package_name_short_snake__: str
 
 
-def _process_name(name: str, *, community: bool = False):
+def _process_name(name: str, *, community: bool = False) -> Replacements:
     preprocessed = name.replace("_", "-").lower()
 
     if preprocessed.startswith("langchain-"):
@@ -42,7 +40,7 @@ def _process_name(name: str, *, community: bool = False):
         raise ValueError("Name should not end with `-`.")
     if preprocessed.find("--") != -1:
         raise ValueError("Name should not contain consecutive hyphens.")
-    replacements = {
+    replacements: Replacements = {
         "__package_name__": f"langchain-{preprocessed}",
         "__module_name__": "langchain_" + preprocessed.replace("-", "_"),
         "__ModuleName__": preprocessed.title().replace("-", ""),
@@ -52,7 +50,7 @@ def _process_name(name: str, *, community: bool = False):
     }
     if community:
         replacements["__module_name__"] = preprocessed.replace("-", "_")
-    return Replacements(replacements)
+    return replacements
 
 
 @integration_cli.command()
@@ -74,16 +72,7 @@ def new(
 ):
     """
     Creates a new integration package.
-
-    Should be run from libs/partners
     """
-    # confirm that we are in the right directory
-    if not Path.cwd().name == "partners" or not Path.cwd().parent.name == "libs":
-        typer.echo(
-            "This command should be run from the `libs/partners` directory in the "
-            "langchain-ai/langchain monorepo. Continuing is NOT recommended."
-        )
-        typer.confirm("Are you sure you want to continue?", abort=True)
 
     try:
         replacements = _process_name(name)
@@ -104,7 +93,7 @@ def new(
             "Name of integration in PascalCase", default=replacements["__ModuleName__"]
         )
 
-    destination_dir = Path.cwd() / replacements["__package_name_short__"]
+    destination_dir = Path.cwd() / replacements["__package_name__"]
     if destination_dir.exists():
         typer.echo(f"Folder {destination_dir} exists.")
         raise typer.Exit(code=1)
@@ -118,7 +107,7 @@ def new(
     shutil.move(destination_dir / "integration_template", package_dir)
 
     # replacements in files
-    replace_glob(destination_dir, "**/*", replacements)
+    replace_glob(destination_dir, "**/*", cast(Dict[str, str], replacements))
 
     # poetry install
     subprocess.run(
@@ -226,4 +215,4 @@ def create_doc(
     shutil.copy(docs_template, destination_path)
 
     # replacements in file
-    replace_file(destination_path, replacements)
+    replace_file(destination_path, cast(Dict[str, str], replacements))

libs/cli/langchain_cli/namespaces/migrate/generate/utils.py

@@ -17,7 +17,7 @@ PARTNER_PKGS = PKGS_ROOT / "partners"
 class ImportExtractor(ast.NodeVisitor):
     def __init__(self, *, from_package: Optional[str] = None) -> None:
         """Extract all imports from the given code, optionally filtering by package."""
-        self.imports = []
+        self.imports: list = []
         self.package = from_package
 
     def visit_ImportFrom(self, node):
@@ -68,7 +68,7 @@ def find_subclasses_in_module(module, classes_: List[Type]) -> List[str]:
     return subclasses
 
 
-def _get_all_classnames_from_file(file: str, pkg: str) -> List[Tuple[str, str]]:
+def _get_all_classnames_from_file(file: Path, pkg: str) -> List[Tuple[str, str]]:
     """Extract all class names from a file."""
     with open(file, encoding="utf-8") as f:
         code = f.read()
@@ -145,7 +145,7 @@ def find_imports_from_package(
     return extractor.imports
 
 
-def _get_current_module(path: str, pkg_root: str) -> str:
+def _get_current_module(path: Path, pkg_root: str) -> str:
     """Convert a path to a module name."""
     path_as_pathlib = pathlib.Path(os.path.abspath(path))
     relative_path = path_as_pathlib.relative_to(pkg_root).with_suffix("")

libs/cli/langchain_cli/namespaces/migrate/main.py

@@ -4,7 +4,7 @@ from pathlib import Path
 
 import rich
 import typer
-from gritql import run
+from gritql import run  # type: ignore
 from typer import Option
 
 

libs/cli/langchain_cli/utils/find_replace.py

@@ -13,7 +13,7 @@ def find_and_replace(source: str, replacements: Dict[str, str]) -> str:
     return rtn
 
 
-def replace_file(source: Path, replacements: Dict[str, str]) -> None:
+def replace_file(source: Path, replacements: dict[str, str]) -> None:
     try:
         content = source.read_text()
     except UnicodeDecodeError:
@@ -24,7 +24,7 @@ def replace_file(source: Path, replacements: Dict[str, str]) -> None:
         source.write_text(new_content)
 
 
-def replace_glob(parent: Path, glob: str, replacements: Dict[str, str]) -> None:
+def replace_glob(parent: Path, glob: str, replacements: dict[str, str]) -> None:
     for file in parent.glob(glob):
         if not file.is_file():
             continue

libs/cli/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langchain-cli"
-version = "0.0.33"
+version = "0.0.34"
 description = "CLI for interacting with LangChain"
 authors = ["Erick Friis <erick@langchain.dev>"]
 readme = "README.md"
@@ -25,16 +25,18 @@ langchain = "langchain_cli.cli:app"
 langchain-cli = "langchain_cli.cli:app"
 
 [tool.poetry.group.dev.dependencies]
-poethepoet = "^0.24.1"
 pytest = "^7.4.2"
 pytest-watch = "^4.2.0"
 
 [tool.poetry.group.lint.dependencies]
 ruff = "^0.5"
+mypy = "^1.13.0"
 
 [tool.poetry.group.test.dependencies]
+langchain = {path = "../langchain", develop = true}
 
 [tool.poetry.group.typing.dependencies]
+langchain = {path = "../langchain", develop = true}
 
 [tool.poetry.group.test_integration.dependencies]
 
@@ -50,22 +52,11 @@ select = [
   "T201", # print
 ]
 
-[tool.poe.tasks]
-test = "poetry run pytest tests"
-watch = "poetry run ptw"
-version = "poetry version --short"
-bump = ["_bump_1", "_bump_2"]
-lint = ["_lint", "_check_formatting"]
-format = ["_format", "_lint_fix"]
-
-_bump_2.shell = """sed -i "" "/^__version__ =/c\\ \n__version__ = \\"$version\\"\n" langchain_cli/cli.py"""
-_bump_2.uses = { version = "version" }
-
-_bump_1 = "poetry version patch"
-_check_formatting = "poetry run ruff format . --diff"
-_lint = "poetry run ruff check ."
-_format = "poetry run ruff format ."
-_lint_fix = "poetry run ruff check . --fix"
+[tool.mypy]
+exclude = [
+  "langchain_cli/integration_template",
+  "langchain_cli/package_template",
+]
 
 [build-system]
 requires = ["poetry-core"]

libs/cli/scripts/generate_migrations.py

@@ -1,3 +1,4 @@
+# type: ignore
 """Script to generate migrations for the migration script."""
 
 import json

libs/cli/tests/integration_tests/test_compile.py

@@ -0,0 +1,7 @@
+import pytest
+
+
+@pytest.mark.compile
+def test_placeholder() -> None:
+    """Used for compiling integration tests without running any real tests."""
+    pass

libs/cli/tests/unit_tests/migrate/cli_runner/test_cli.py

@@ -41,6 +41,7 @@ def find_issue(current: Folder, expected: Folder) -> str:
     return "Unknown"
 
 
+@pytest.mark.xfail(reason="grit may not be installed in env")
 def test_command_line(tmp_path: Path) -> None:
     runner = CliRunner()
 

libs/community/README.md

@@ -13,7 +13,7 @@ pip install langchain-community
 
 LangChain Community contains third-party integrations that implement the base interfaces defined in LangChain Core, making them ready-to-use in any LangChain application.
 
-For full documentation see the [API reference](https://api.python.langchain.com/en/stable/community_api_reference.html).
+For full documentation see the [API reference](https://python.langchain.com/api_reference/community/index.html).
 
 ![Diagram outlining the hierarchical organization of the LangChain framework, displaying the interconnected parts across multiple layers.](https://raw.githubusercontent.com/langchain-ai/langchain/e1d113ea84a2edcf4a7709fc5be0e972ea74a5d9/docs/static/svg/langchain_stack_112024.svg "LangChain Framework Overview")
 

libs/community/extended_testing_deps.txt

@@ -46,6 +46,7 @@ motor>=3.3.1,<4
 msal>=1.25.0,<2
 mwparserfromhell>=0.6.4,<0.7
 mwxml>=0.3.3,<0.4
+needle-python>=0.4
 networkx>=3.2.1,<4
 newspaper3k>=0.2.8,<0.3
 numexpr>=2.8.6,<3

libs/community/langchain_community/document_loaders/__init__.py

@@ -299,6 +299,9 @@ if TYPE_CHECKING:
     from langchain_community.document_loaders.mongodb import (
         MongodbLoader,
     )
+    from langchain_community.document_loaders.needle import (
+        NeedleLoader,
+    )
     from langchain_community.document_loaders.news import (
         NewsURLLoader,
     )
@@ -631,6 +634,7 @@ _module_lookup = {
     "MergedDataLoader": "langchain_community.document_loaders.merge",
     "ModernTreasuryLoader": "langchain_community.document_loaders.modern_treasury",
     "MongodbLoader": "langchain_community.document_loaders.mongodb",
+    "NeedleLoader": "langchain_community.document_loaders.needle",
     "NewsURLLoader": "langchain_community.document_loaders.news",
     "NotebookLoader": "langchain_community.document_loaders.notebook",
     "NotionDBLoader": "langchain_community.document_loaders.notiondb",
@@ -837,6 +841,7 @@ __all__ = [
     "MergedDataLoader",
     "ModernTreasuryLoader",
     "MongodbLoader",
+    "NeedleLoader",
     "NewsURLLoader",
     "NotebookLoader",
     "NotionDBLoader",

libs/community/langchain_community/document_loaders/needle.py

@@ -0,0 +1,164 @@
+from typing import Dict, Iterator, List, Optional
+
+from langchain_core.document_loaders.base import BaseLoader
+from langchain_core.documents import Document
+
+
+class NeedleLoader(BaseLoader):
+    """
+    NeedleLoader is a document loader for managing documents stored in a collection.
+
+    Setup:
+        Install the `needle-python` library and set your Needle API key.
+
+        .. code-block:: bash
+
+            pip install needle-python
+            export NEEDLE_API_KEY="your-api-key"
+
+    Key init args:
+        - `needle_api_key` (Optional[str]): API key for authenticating with Needle.
+        - `collection_id` (str): Needle collection to load documents from.
+
+    Usage:
+        .. code-block:: python
+
+            from langchain_community.document_loaders.needle import NeedleLoader
+
+            loader = NeedleLoader(
+                needle_api_key="your-api-key",
+                collection_id="your-collection-id"
+            )
+
+            # Load documents
+            documents = loader.load()
+            for doc in documents:
+                print(doc.metadata)
+
+            # Lazy load documents
+            for doc in loader.lazy_load():
+                print(doc.metadata)
+    """
+
+    def __init__(
+        self,
+        needle_api_key: Optional[str] = None,
+        collection_id: Optional[str] = None,
+    ) -> None:
+        """
+        Initializes the NeedleLoader with API key and collection ID.
+
+        Args:
+            needle_api_key (Optional[str]): API key for authenticating with Needle.
+            collection_id (Optional[str]): Identifier for the Needle collection.
+
+        Raises:
+            ImportError: If the `needle-python` library is not installed.
+            ValueError: If the collection ID is not provided.
+        """
+        try:
+            from needle.v1 import NeedleClient
+        except ImportError:
+            raise ImportError(
+                "Please install with `pip install needle-python` to use NeedleLoader."
+            )
+
+        super().__init__()
+        self.needle_api_key = needle_api_key
+        self.collection_id = collection_id
+        self.client: Optional[NeedleClient] = None
+
+        if self.needle_api_key:
+            self.client = NeedleClient(api_key=self.needle_api_key)
+
+        if not self.collection_id:
+            raise ValueError("Collection ID must be provided.")
+
+    def _get_collection(self) -> None:
+        """
+        Ensures the Needle collection is set and the client is initialized.
+
+        Raises:
+            ValueError: If the Needle client is not initialized or
+                        if the collection ID is missing.
+        """
+        if self.client is None:
+            raise ValueError(
+                "NeedleClient is not initialized. Provide a valid API key."
+            )
+        if not self.collection_id:
+            raise ValueError("Collection ID must be provided.")
+
+    def add_files(self, files: Dict[str, str]) -> None:
+        """
+        Adds files to the Needle collection.
+
+        Args:
+            files (Dict[str, str]): Dictionary where keys are file names and values
+                                    are file URLs.
+
+        Raises:
+            ImportError: If the `needle-python` library is not installed.
+            ValueError: If the collection is not properly initialized.
+        """
+        try:
+            from needle.v1.models import FileToAdd
+        except ImportError:
+            raise ImportError(
+                "Please install with `pip install needle-python` to add files."
+            )
+
+        self._get_collection()
+        assert self.client is not None, "NeedleClient must be initialized."
+
+        files_to_add = [FileToAdd(name=name, url=url) for name, url in files.items()]
+
+        self.client.collections.files.add(
+            collection_id=self.collection_id, files=files_to_add
+        )
+
+    def _fetch_documents(self) -> List[Document]:
+        """
+        Fetches metadata for documents from the Needle collection.
+
+        Returns:
+            List[Document]: A list of documents with metadata. Content is excluded.
+
+        Raises:
+            ValueError: If the collection is not properly initialized.
+        """
+        self._get_collection()
+        assert self.client is not None, "NeedleClient must be initialized."
+
+        files = self.client.collections.files.list(self.collection_id)
+        docs = [
+            Document(
+                page_content="",  # Needle doesn't provide file content fetching
+                metadata={
+                    "source": file.url,
+                    "title": file.name,
+                    "size": getattr(file, "size", None),
+                },
+            )
+            for file in files
+            if file.status == "indexed"
+        ]
+        return docs
+
+    def load(self) -> List[Document]:
+        """
+        Loads all documents from the Needle collection.
+
+        Returns:
+            List[Document]: A list of documents from the collection.
+        """
+        return self._fetch_documents()
+
+    def lazy_load(self) -> Iterator[Document]:
+        """
+        Lazily loads documents from the Needle collection.
+
+        Yields:
+            Iterator[Document]: An iterator over the documents.
+        """
+        yield from self._fetch_documents()

libs/community/langchain_community/query_constructors/databricks_vector_search.py

@@ -90,5 +90,5 @@ class DatabricksVectorSearchTranslator(Visitor):
         if structured_query.filter is None:
             kwargs = {}
         else:
-            kwargs = {"filters": structured_query.filter.accept(self)}
+            kwargs = {"filter": structured_query.filter.accept(self)}
         return structured_query.query, kwargs

libs/community/langchain_community/retrievers/__init__.py

@@ -93,6 +93,7 @@ if TYPE_CHECKING:
         MilvusRetriever,
     )
     from langchain_community.retrievers.nanopq import NanoPQRetriever
+    from langchain_community.retrievers.needle import NeedleRetriever
     from langchain_community.retrievers.outline import (
         OutlineRetriever,
     )
@@ -173,6 +174,7 @@ _module_lookup = {
     "MetalRetriever": "langchain_community.retrievers.metal",
     "MilvusRetriever": "langchain_community.retrievers.milvus",
     "NanoPQRetriever": "langchain_community.retrievers.nanopq",
+    "NeedleRetriever": "langchain_community.retrievers.needle",
     "OutlineRetriever": "langchain_community.retrievers.outline",
     "PineconeHybridSearchRetriever": "langchain_community.retrievers.pinecone_hybrid_search",  # noqa: E501
     "PubMedRetriever": "langchain_community.retrievers.pubmed",
@@ -229,6 +231,7 @@ __all__ = [
     "MetalRetriever",
     "MilvusRetriever",
     "NanoPQRetriever",
+    "NeedleRetriever",
     "NeuralDBRetriever",
     "OutlineRetriever",
     "PineconeHybridSearchRetriever",

libs/community/langchain_community/retrievers/bm25.py

@@ -33,6 +33,7 @@ class BM25Retriever(BaseRetriever):
         cls,
         texts: Iterable[str],
         metadatas: Optional[Iterable[dict]] = None,
+        ids: Optional[Iterable[str]] = None,
         bm25_params: Optional[Dict[str, Any]] = None,
         preprocess_func: Callable[[str], List[str]] = default_preprocessing_func,
         **kwargs: Any,
@@ -42,6 +43,7 @@ class BM25Retriever(BaseRetriever):
         Args:
             texts: A list of texts to vectorize.
             metadatas: A list of metadata dicts to associate with each text.
+            ids: A list of ids to associate with each text.
             bm25_params: Parameters to pass to the BM25 vectorizer.
             preprocess_func: A function to preprocess each text before vectorization.
             **kwargs: Any other arguments to pass to the retriever.
@@ -61,7 +63,15 @@ class BM25Retriever(BaseRetriever):
         bm25_params = bm25_params or {}
         vectorizer = BM25Okapi(texts_processed, **bm25_params)
         metadatas = metadatas or ({} for _ in texts)
-        docs = [Document(page_content=t, metadata=m) for t, m in zip(texts, metadatas)]
+        if ids:
+            docs = [
+                Document(page_content=t, metadata=m, id=i)
+                for t, m, i in zip(texts, metadatas, ids)
+            ]
+        else:
+            docs = [
+                Document(page_content=t, metadata=m) for t, m in zip(texts, metadatas)
+            ]
         return cls(
             vectorizer=vectorizer, docs=docs, preprocess_func=preprocess_func, **kwargs
         )
@@ -86,11 +96,14 @@ class BM25Retriever(BaseRetriever):
         Returns:
             A BM25Retriever instance.
         """
-        texts, metadatas = zip(*((d.page_content, d.metadata) for d in documents))
+        texts, metadatas, ids = zip(
+            *((d.page_content, d.metadata, d.id) for d in documents)
+        )
         return cls.from_texts(
             texts=texts,
             bm25_params=bm25_params,
             metadatas=metadatas,
+            ids=ids,
             preprocess_func=preprocess_func,
             **kwargs,
         )

libs/community/langchain_community/retrievers/needle.py

@@ -0,0 +1,96 @@
+from typing import Any, List, Optional  # noqa: I001
+
+from langchain_core.callbacks import CallbackManagerForRetrieverRun
+from langchain_core.documents import Document
+from langchain_core.retrievers import BaseRetriever
+from pydantic import BaseModel, Field
+
+
+class NeedleRetriever(BaseRetriever, BaseModel):
+    """
+    NeedleRetriever retrieves relevant documents or context from a Needle collection
+    based on a search query.
+
+    Setup:
+        Install the `needle-python` library and set your Needle API key.
+
+        .. code-block:: bash
+
+            pip install needle-python
+            export NEEDLE_API_KEY="your-api-key"
+
+    Key init args:
+        - `needle_api_key` (Optional[str]): The API key for authenticating with Needle.
+        - `collection_id` (str): The ID of the Needle collection to search in.
+        - `client` (Optional[NeedleClient]): An optional instance of the NeedleClient.
+
+    Usage:
+        .. code-block:: python
+
+            from langchain_community.retrievers.needle import NeedleRetriever
+
+            retriever = NeedleRetriever(
+                needle_api_key="your-api-key",
+                collection_id="your-collection-id"
+            )
+
+            results = retriever.retrieve("example query")
+            for doc in results:
+                print(doc.page_content)
+    """
+
+    client: Optional[Any] = None
+    """Optional instance of NeedleClient."""
+    needle_api_key: Optional[str] = Field(None, description="Needle API Key")
+    collection_id: Optional[str] = Field(
+        ..., description="The ID of the Needle collection to search in"
+    )
+
+    def _initialize_client(self) -> None:
+        """
+        Initialize the NeedleClient with the provided API key.
+
+        If a client instance is already provided, this method does nothing.
+        """
+        try:
+            from needle.v1 import NeedleClient
+        except ImportError:
+            raise ImportError("Please install with `pip install needle-python`.")
+
+        if not self.client:
+            self.client = NeedleClient(api_key=self.needle_api_key)
+
+    def _search_collection(self, query: str) -> List[Document]:
+        """
+        Search the Needle collection for relevant documents.
+
+        Args:
+            query (str): The search query used to find relevant documents.
+
+        Returns:
+            List[Document]: A list of documents matching the search query.
+        """
+        self._initialize_client()
+        if self.client is None:
+            raise ValueError("NeedleClient is not initialized. Provide an API key.")
+
+        results = self.client.collections.search(
+            collection_id=self.collection_id, text=query
+        )
+        docs = [Document(page_content=result.content) for result in results]
+        return docs
+
+    def _get_relevant_documents(
+        self, query: str, *, run_manager: CallbackManagerForRetrieverRun
+    ) -> List[Document]:
+        """
+        Retrieve relevant documents based on the query.
+
+        Args:
+            query (str): The query string used to search the collection.
+        Returns:
+            List[Document]: A list of documents relevant to the query.
+        """
+        # The `run_manager` parameter is included to match the superclass signature,
+        # but it is not used in this implementation.
+        return self._search_collection(query)

libs/community/langchain_community/utilities/pubmed.py

@@ -31,6 +31,7 @@ class PubMedAPIWrapper(BaseModel):
         sleep_time: time to wait between retries.
           Default is 0.2 seconds.
         email: email address to be used for the PubMed API.
+        api_key: API key to be used for the PubMed API.
     """
 
     parse: Any  #: :meta private:
@@ -47,6 +48,7 @@ class PubMedAPIWrapper(BaseModel):
     MAX_QUERY_LENGTH: int = 300
     doc_content_chars_max: int = 2000
     email: str = "your_email@example.com"
+    api_key: str = ""
 
     @model_validator(mode="before")
     @classmethod
@@ -101,6 +103,8 @@ class PubMedAPIWrapper(BaseModel):
             + str({urllib.parse.quote(query)})
             + f"&retmode=json&retmax={self.top_k_results}&usehistory=y"
         )
+        if self.api_key != "":
+            url += f"&api_key={self.api_key}"
         result = urllib.request.urlopen(url)
         text = result.read().decode("utf-8")
         json_text = json.loads(text)
@@ -135,6 +139,8 @@ class PubMedAPIWrapper(BaseModel):
             + "&webenv="
             + webenv
         )
+        if self.api_key != "":
+            url += f"&api_key={self.api_key}"
 
         retry = 0
         while True:

libs/community/langchain_community/vectorstores/tencentvectordb.py

@@ -6,7 +6,18 @@ import json
 import logging
 import time
 from enum import Enum
-from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union, cast
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    cast,
+)
 
 import numpy as np
 from langchain_core.documents import Document
@@ -168,8 +179,8 @@ class TencentVectorDB(VectorStore):
         tcvectordb = guard_import("tcvectordb")
         tcollection = guard_import("tcvectordb.model.collection")
         enum = guard_import("tcvectordb.model.enum")
-
-        if t_vdb_embedding:
+        self.embedding_model = None
+        if embedding is None and t_vdb_embedding:
             embedding_model = [
                 model
                 for model in enum.EmbeddingModel
@@ -566,3 +577,17 @@ class TencentVectorDB(VectorStore):
         )
         # Reorder the values and return.
         return [documents[x] for x in new_ordering if x != -1]
+
+    def _select_relevance_score_fn(self) -> Callable[[float], float]:
+        metric_type = self.index_params.metric_type
+        if metric_type == "COSINE":
+            return self._cosine_relevance_score_fn
+        elif metric_type == "L2":
+            return self._euclidean_relevance_score_fn
+        elif metric_type == "IP":
+            return self._max_inner_product_relevance_score_fn
+        else:
+            raise ValueError(
+                "No supported normalization function"
+                f" for distance metric of type: {metric_type}."
+            )

libs/community/tests/unit_tests/document_loaders/test_imports.py

@@ -105,6 +105,7 @@ EXPECTED_ALL = [
     "MergedDataLoader",
     "ModernTreasuryLoader",
     "MongodbLoader",
+    "NeedleLoader",
     "NewsURLLoader",
     "NotebookLoader",
     "NotionDBLoader",

libs/community/tests/unit_tests/document_loaders/test_needle.py

@@ -0,0 +1,75 @@
+import pytest
+from pytest_mock import MockerFixture
+
+
+@pytest.mark.requires("needle")
+def test_add_and_fetch_files(mocker: MockerFixture) -> None:
+    """
+    Test adding and fetching files using the NeedleLoader with a mock.
+    """
+    from langchain_community.document_loaders.needle import NeedleLoader  # noqa: I001
+    from needle.v1.models import CollectionFile  # noqa: I001
+
+    # Create mock instances using mocker
+    # Create mock instances using mocker
+    mock_files = mocker.Mock()
+    mock_files.add.return_value = [
+        CollectionFile(
+            id="mock_id",
+            name="tech-radar-30.pdf",
+            url="https://example.com/",
+            status="indexed",
+            type="mock_type",
+            user_id="mock_user_id",
+            connector_id="mock_connector_id",
+            size=1234,
+            md5_hash="mock_md5_hash",
+            created_at="2024-01-01T00:00:00Z",
+            updated_at="2024-01-01T00:00:00Z",
+        )
+    ]
+    mock_files.list.return_value = [
+        CollectionFile(
+            id="mock_id",
+            name="tech-radar-30.pdf",
+            url="https://example.com/",
+            status="indexed",
+            type="mock_type",
+            user_id="mock_user_id",
+            connector_id="mock_connector_id",
+            size=1234,
+            md5_hash="mock_md5_hash",
+            created_at="2024-01-01T00:00:00Z",
+            updated_at="2024-01-01T00:00:00Z",
+        )
+    ]
+
+    mock_collections = mocker.Mock()
+    mock_collections.files = mock_files
+
+    mock_needle_client = mocker.Mock()
+    mock_needle_client.collections = mock_collections
+
+    # Patch the NeedleClient to return the mock client
+    mocker.patch("needle.v1.NeedleClient", return_value=mock_needle_client)
+
+    # Initialize NeedleLoader with mock API key and collection ID
+    document_store = NeedleLoader(
+        needle_api_key="fake_api_key",
+        collection_id="fake_collection_id",
+    )
+
+    # Define files to add
+    files = {
+        "tech-radar-30.pdf": "https://www.thoughtworks.com/content/dam/thoughtworks/documents/radar/2024/04/tr_technology_radar_vol_30_en.pdf"
+    }
+
+    # Add files to the collection using the mock client
+    document_store.add_files(files=files)
+
+    # Fetch the added files using the mock client
+    added_files = document_store._fetch_documents()
+
+    # Assertions to verify that the file was added and fetched correctly
+    assert isinstance(added_files[0].metadata["title"], str)
+    assert isinstance(added_files[0].metadata["source"], str)

libs/community/tests/unit_tests/query_constructors/test_databricks_vector_search.py

@@ -109,7 +109,7 @@ def test_visit_structured_query_with_one_arg_filter() -> None:
         filter=comp,
     )
 
-    expected = (query, {"filters": {"country": "France"}})
+    expected = (query, {"filter": {"country": "France"}})
 
     actual = DEFAULT_TRANSLATOR.visit_structured_query(structured_query)
     assert expected == actual
@@ -134,7 +134,7 @@ def test_visit_structured_query_with_multiple_arg_filter_and_operator() -> None:
 
     expected = (
         query,
-        {"filters": {"country": "France", "year >=": 1888, "year <=": 1900}},
+        {"filter": {"country": "France", "year >=": 1888, "year <=": 1900}},
     )
 
     actual = DEFAULT_TRANSLATOR.visit_structured_query(structured_query)

libs/community/tests/unit_tests/retrievers/test_bm25.py

@@ -43,3 +43,42 @@ def test_repr() -> None:
     ]
     bm25_retriever = BM25Retriever.from_documents(documents=input_docs)
     assert "I have a pen" not in repr(bm25_retriever)
+
+
+@pytest.mark.requires("rank_bm25")
+def test_doc_id() -> None:
+    docs_with_ids = [
+        Document(page_content="I have a pen.", id="1"),
+        Document(page_content="Do you have a pen?", id="2"),
+        Document(page_content="I have a bag.", id="3"),
+    ]
+    docs_without_ids = [
+        Document(page_content="I have a pen."),
+        Document(page_content="Do you have a pen?"),
+        Document(page_content="I have a bag."),
+    ]
+    docs_with_some_ids = [
+        Document(page_content="I have a pen.", id="1"),
+        Document(page_content="Do you have a pen?"),
+        Document(page_content="I have a bag.", id="3"),
+    ]
+    bm25_retriever_with_ids = BM25Retriever.from_documents(documents=docs_with_ids)
+    bm25_retriever_without_ids = BM25Retriever.from_documents(
+        documents=docs_without_ids
+    )
+    bm25_retriever_with_some_ids = BM25Retriever.from_documents(
+        documents=docs_with_some_ids
+    )
+    for doc in bm25_retriever_with_ids.docs:
+        assert doc.id is not None
+    for doc in bm25_retriever_without_ids.docs:
+        assert doc.id is None
+    for doc in bm25_retriever_with_some_ids.docs:
+        if doc.page_content == "I have a pen.":
+            assert doc.id == "1"
+        elif doc.page_content == "Do you have a pen?":
+            assert doc.id is None
+        elif doc.page_content == "I have a bag.":
+            assert doc.id == "3"
+        else:
+            raise ValueError("Unexpected document")

libs/community/tests/unit_tests/retrievers/test_imports.py

@@ -26,6 +26,7 @@ EXPECTED_ALL = [
     "MetalRetriever",
     "MilvusRetriever",
     "NanoPQRetriever",
+    "NeedleRetriever",
     "OutlineRetriever",
     "PineconeHybridSearchRetriever",
     "PubMedRetriever",

libs/community/tests/unit_tests/retrievers/test_needle.py

@@ -0,0 +1,72 @@
+from typing import Any
+
+import pytest
+from pytest_mock import MockerFixture
+
+
+# Mock class to simulate search results from Needle API
+class MockSearchResult:
+    def __init__(self, content: str) -> None:
+        self.content = content
+
+
+# Mock class to simulate NeedleClient and its collections behavior
+class MockNeedleClient:
+    def __init__(self, api_key: str) -> None:
+        self.api_key = api_key
+        self.collections = self.MockCollections()
+
+    class MockCollections:
+        def search(self, collection_id: str, text: str) -> list[MockSearchResult]:
+            return [
+                MockSearchResult(content=f"Result for query: {text}"),
+                MockSearchResult(content=f"Another result for query: {text}"),
+            ]
+
+
+@pytest.mark.requires("needle")
+def test_needle_retriever_initialization() -> None:
+    """
+    Test that the NeedleRetriever is initialized correctly.
+    """
+    from langchain_community.retrievers.needle import NeedleRetriever  # noqa: I001
+
+    retriever = NeedleRetriever(
+        needle_api_key="mock_api_key",
+        collection_id="mock_collection_id",
+    )
+
+    assert retriever.needle_api_key == "mock_api_key"
+    assert retriever.collection_id == "mock_collection_id"
+
+
+@pytest.mark.requires("needle")
+def test_get_relevant_documents(mocker: MockerFixture) -> None:
+    """
+    Test that the retriever correctly fetches documents.
+    """
+    from langchain_community.retrievers.needle import NeedleRetriever  # noqa: I001
+
+    # Patch the actual NeedleClient import path used in the NeedleRetriever
+    mocker.patch("needle.v1.NeedleClient", new=MockNeedleClient)
+
+    # Initialize the retriever with mocked API key and collection ID
+    retriever = NeedleRetriever(
+        needle_api_key="mock_api_key",
+        collection_id="mock_collection_id",
+    )
+
+    mock_run_manager: Any = None
+
+    # Perform the search
+    query = "What is RAG?"
+    retrieved_documents = retriever._get_relevant_documents(
+        query, run_manager=mock_run_manager
+    )
+
+    # Validate the results
+    assert len(retrieved_documents) == 2
+    assert retrieved_documents[0].page_content == "Result for query: What is RAG?"
+    assert (
+        retrieved_documents[1].page_content == "Another result for query: What is RAG?"
+    )

libs/core/README.md

@@ -17,7 +17,7 @@ These abstractions are designed to be as modular and simple as possible. Example
 
 The benefit of having these abstractions is that any provider can implement the required interface and then easily be used in the rest of the LangChain ecosystem.
 
-For full documentation see the [API reference](https://api.python.langchain.com/en/stable/core_api_reference.html).
+For full documentation see the [API reference](https://python.langchain.com/api_reference/core/index.html).
 
 ## 1️⃣ Core Interface: Runnables
 

libs/langchain/langchain/chains/router/multi_prompt.py

@@ -20,9 +20,8 @@ from langchain.chains.router.multi_prompt_prompt import MULTI_PROMPT_ROUTER_TEMP
     since="0.2.12",
     removal="1.0",
     message=(
-        "Use RunnableLambda to select from multiple prompt templates. See example "
-        "in API reference: "
-        "https://api.python.langchain.com/en/latest/chains/langchain.chains.router.multi_prompt.MultiPromptChain.html"  # noqa: E501
+        "Please see migration guide here for recommended implementation: "
+        "https://python.langchain.com/docs/versions/migrating_chains/multi_prompt_chain/"  # noqa: E501
     ),
 )
 class MultiPromptChain(MultiRouteChain):
@@ -37,60 +36,109 @@ class MultiPromptChain(MultiRouteChain):
 
             from operator import itemgetter
             from typing import Literal
-            from typing_extensions import TypedDict
 
             from langchain_core.output_parsers import StrOutputParser
             from langchain_core.prompts import ChatPromptTemplate
-            from langchain_core.runnables import RunnableLambda, RunnablePassthrough
+            from langchain_core.runnables import RunnableConfig
             from langchain_openai import ChatOpenAI
+            from langgraph.graph import END, START, StateGraph
+            from typing_extensions import TypedDict
 
             llm = ChatOpenAI(model="gpt-4o-mini")
 
+            # Define the prompts we will route to
             prompt_1 = ChatPromptTemplate.from_messages(
                 [
                     ("system", "You are an expert on animals."),
-                    ("human", "{query}"),
+                    ("human", "{input}"),
                 ]
             )
             prompt_2 = ChatPromptTemplate.from_messages(
                 [
                     ("system", "You are an expert on vegetables."),
-                    ("human", "{query}"),
+                    ("human", "{input}"),
                 ]
             )
 
+            # Construct the chains we will route to. These format the input query
+            # into the respective prompt, run it through a chat model, and cast
+            # the result to a string.
             chain_1 = prompt_1 | llm | StrOutputParser()
             chain_2 = prompt_2 | llm | StrOutputParser()
 
+
+            # Next: define the chain that selects which branch to route to.
+            # Here we will take advantage of tool-calling features to force
+            # the output to select one of two desired branches.
             route_system = "Route the user's query to either the animal or vegetable expert."
             route_prompt = ChatPromptTemplate.from_messages(
                 [
                     ("system", route_system),
-                    ("human", "{query}"),
+                    ("human", "{input}"),
                 ]
             )
 
 
+            # Define schema for output:
             class RouteQuery(TypedDict):
-                \"\"\"Route query to destination.\"\"\"
+                \"\"\"Route query to destination expert.\"\"\"
+
                 destination: Literal["animal", "vegetable"]
 
 
-            route_chain = (
-                route_prompt
-                | llm.with_structured_output(RouteQuery)
-                | itemgetter("destination")
-            )
+            route_chain = route_prompt | llm.with_structured_output(RouteQuery)
 
-            chain = {
-                "destination": route_chain,  # "animal" or "vegetable"
-                "query": lambda x: x["query"],  # pass through input query
-            } | RunnableLambda(
-                # if animal, chain_1. otherwise, chain_2.
-                lambda x: chain_1 if x["destination"] == "animal" else chain_2,
-            )
 
-            chain.invoke({"query": "what color are carrots"})
+            # For LangGraph, we will define the state of the graph to hold the query,
+            # destination, and final answer.
+            class State(TypedDict):
+                query: str
+                destination: RouteQuery
+                answer: str
+
+
+            # We define functions for each node, including routing the query:
+            async def route_query(state: State, config: RunnableConfig):
+                destination = await route_chain.ainvoke(state["query"], config)
+                return {"destination": destination}
+
+
+            # And one node for each prompt
+            async def prompt_1(state: State, config: RunnableConfig):
+                return {"answer": await chain_1.ainvoke(state["query"], config)}
+
+
+            async def prompt_2(state: State, config: RunnableConfig):
+                return {"answer": await chain_2.ainvoke(state["query"], config)}
+
+
+            # We then define logic that selects the prompt based on the classification
+            def select_node(state: State) -> Literal["prompt_1", "prompt_2"]:
+                if state["destination"] == "animal":
+                    return "prompt_1"
+                else:
+                    return "prompt_2"
+
+
+            # Finally, assemble the multi-prompt chain. This is a sequence of two steps:
+            # 1) Select "animal" or "vegetable" via the route_chain, and collect the answer
+            # alongside the input query.
+            # 2) Route the input query to chain_1 or chain_2, based on the
+            # selection.
+            graph = StateGraph(State)
+            graph.add_node("route_query", route_query)
+            graph.add_node("prompt_1", prompt_1)
+            graph.add_node("prompt_2", prompt_2)
+
+            graph.add_edge(START, "route_query")
+            graph.add_conditional_edges("route_query", select_node)
+            graph.add_edge("prompt_1", END)
+            graph.add_edge("prompt_2", END)
+            app = graph.compile()
+
+            result = await app.ainvoke({"query": "what color are carrots"})
+            print(result["destination"])
+            print(result["answer"])
     """  # noqa: E501
 
     @property

libs/partners/anthropic/Makefile

@@ -8,7 +8,7 @@ TEST_FILE ?= tests/unit_tests/
 integration_test integration_tests: TEST_FILE=tests/integration_tests/
 
 test tests integration_test integration_tests:
-	poetry run pytest $(TEST_FILE)
+	poetry run pytest -vvv --timeout 10 $(TEST_FILE)
 
 test_watch:
 	poetry run ptw --snapshot-update --now . -- -vv $(TEST_FILE)

libs/partners/anthropic/poetry.lock

@@ -852,6 +852,20 @@ pytest = ">=6.2.5"
 [package.extras]
 dev = ["pre-commit", "pytest-asyncio", "tox"]
 
+[[package]]
+name = "pytest-timeout"
+version = "2.3.1"
+description = "pytest plugin to abort hanging tests"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pytest-timeout-2.3.1.tar.gz", hash = "sha256:12397729125c6ecbdaca01035b9e5239d4db97352320af155b3f5de1ba5165d9"},
+    {file = "pytest_timeout-2.3.1-py3-none-any.whl", hash = "sha256:68188cb703edfc6a18fad98dc25a3c61e9f24d644b0b70f33af545219fc7813e"},
+]
+
+[package.dependencies]
+pytest = ">=7.0.0"
+
 [[package]]
 name = "pytest-watcher"
 version = "0.3.5"
@@ -1140,4 +1154,4 @@ watchmedo = ["PyYAML (>=3.10)"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<4.0"
-content-hash = "ee4aaa06307b4dc7f7913147bf58f3f36245193c9d4a79c43aba07641f7b6ab9"
+content-hash = "7d24a5eb5b867fa9ad80cbc9fb80d630e7cb00a490b62502cdb57c2fe95cd125"

libs/partners/anthropic/pyproject.toml

@@ -65,6 +65,7 @@ syrupy = "^4.0.2"
 pytest-watcher = "^0.3.4"
 pytest-asyncio = "^0.21.1"
 defusedxml = "^0.7.1"
+pytest-timeout = "^2.3.1"
 
 [tool.poetry.group.codespell.dependencies]
 codespell = "^2.2.0"

libs/partners/anthropic/tests/integration_tests/test_chat_models.py

@@ -22,7 +22,7 @@ from pydantic import BaseModel, Field
 from langchain_anthropic import ChatAnthropic, ChatAnthropicMessages
 from tests.unit_tests._utils import FakeCallbackHandler
 
-MODEL_NAME = "claude-3-sonnet-20240229"
+MODEL_NAME = "claude-3-5-sonnet-20240620"
 
 
 def test_stream() -> None:

libs/partners/chroma/langchain_chroma/vectorstores.py

@@ -16,6 +16,7 @@ from typing import (
     Iterable,
     List,
     Optional,
+    Sequence,
     Tuple,
     Type,
     Union,
@@ -44,10 +45,14 @@ def _results_to_docs_and_scores(results: Any) -> List[Tuple[Document, float]]:
     return [
         # TODO: Chroma can do batch querying,
         # we shouldn't hard code to the 1st result
-        (Document(page_content=result[0], metadata=result[1] or {}), result[2])
+        (
+            Document(page_content=result[0], metadata=result[1] or {}, id=result[2]),
+            result[3],
+        )
         for result in zip(
             results["documents"][0],
             results["metadatas"][0],
+            results["ids"][0],
             results["distances"][0],
         )
     ]
@@ -513,6 +518,11 @@ class Chroma(VectorStore):
         """
         if ids is None:
             ids = [str(uuid.uuid4()) for _ in texts]
+        else:
+            # Assign strings to any null IDs
+            for idx, _id in enumerate(ids):
+                if _id is None:
+                    ids[idx] = str(uuid.uuid4())
         embeddings = None
         texts = list(texts)
         if self._embedding_function is not None:
@@ -748,7 +758,7 @@ class Chroma(VectorStore):
 
         The most similar documents will have the lowest relevance score. Default
         relevance score function is euclidean distance. Distance metric must be
-        provided in `collection_metadata` during initizalition of Chroma object.
+        provided in `collection_metadata` during initialization of Chroma object.
         Example: collection_metadata={"hnsw:space": "cosine"}. Available distance
         metrics are: 'cosine', 'l2' and 'ip'.
 
@@ -1024,6 +1034,38 @@ class Chroma(VectorStore):
 
         return self._collection.get(**kwargs)  # type: ignore
 
+    def get_by_ids(self, ids: Sequence[str], /) -> list[Document]:
+        """Get documents by their IDs.
+
+        The returned documents are expected to have the ID field set to the ID of the
+        document in the vector store.
+
+        Fewer documents may be returned than requested if some IDs are not found or
+        if there are duplicated IDs.
+
+        Users should not assume that the order of the returned documents matches
+        the order of the input IDs. Instead, users should rely on the ID field of the
+        returned documents.
+
+        This method should **NOT** raise exceptions if no documents are found for
+        some IDs.
+
+        Args:
+            ids: List of ids to retrieve.
+
+        Returns:
+            List of Documents.
+
+        .. versionadded:: 0.2.1
+        """
+        results = self.get(ids=list(ids))
+        return [
+            Document(page_content=doc, metadata=meta, id=doc_id)
+            for doc, meta, doc_id in zip(
+                results["documents"], results["metadatas"], results["ids"]
+            )
+        ]
+
     def update_document(self, document_id: str, document: Document) -> None:
         """Update a document in the collection.
 
@@ -1185,6 +1227,8 @@ class Chroma(VectorStore):
         """
         texts = [doc.page_content for doc in documents]
         metadatas = [doc.metadata for doc in documents]
+        if ids is None:
+            ids = [doc.id if doc.id else str(uuid.uuid4()) for doc in documents]
         return cls.from_texts(
             texts=texts,
             embedding=embedding,