infra: support third-party integration packages in API ref build (#31021)

.github/scripts/prep_api_docs_build.py

@@ -69,7 +69,7 @@ def main():
         clean_target_directories([
             p
             for p in package_yaml["packages"]
-            if p["repo"].startswith("langchain-ai/")
+            if (p["repo"].startswith("langchain-ai/") or p.get("include_in_api_ref"))
             and p["repo"] != "langchain-ai/langchain"
         ])
 
@@ -78,7 +78,7 @@ def main():
             p
             for p in package_yaml["packages"]
             if not p.get("disabled", False)
-            and p["repo"].startswith("langchain-ai/")
+            and (p["repo"].startswith("langchain-ai/") or p.get("include_in_api_ref"))
             and p["repo"] != "langchain-ai/langchain"
         ])
 

.github/workflows/api_doc_build.yml

@@ -26,7 +26,20 @@ jobs:
         id: get-unsorted-repos
         uses: mikefarah/yq@master
         with:
-          cmd: yq '.packages[].repo' langchain/libs/packages.yml
+          cmd: |
+            yq '
+              .packages[]
+              | select(
+                  (
+                    (.repo | test("^langchain-ai/"))
+                    and
+                    (.repo != "langchain-ai/langchain")
+                  )
+                  or
+                  (.include_in_api_ref // false)
+                )
+              | .repo
+            ' langchain/libs/packages.yml
 
       - name: Parse YAML and checkout repos
         env:
@@ -38,11 +51,9 @@ jobs:
           
           # Checkout each unique repository that is in langchain-ai org
           for repo in $REPOS; do
-            if [[ "$repo" != "langchain-ai/langchain" && "$repo" == langchain-ai/* ]]; then
             REPO_NAME=$(echo $repo | cut -d'/' -f2)
             echo "Checking out $repo to $REPO_NAME"
             git clone --depth 1 https://github.com/$repo.git $REPO_NAME
-            fi
           done
 
       - name: Setup python ${{ env.PYTHON_VERSION }}

docs/vercel_overrides.txt

@@ -1,2 +1,4 @@
 httpx
 grpcio
+aiohttp<3.11
+protobuf<3.21

libs/packages.yml

@@ -549,6 +549,7 @@ packages:
 - name: langchain-tavily
   path: .
   repo: tavily-ai/langchain-tavily
+  include_in_api_ref: true
   downloads: 13796
   downloads_updated_at: '2025-04-22T15:25:24.644345+00:00'
 - name: langchain-zotero-retriever