From 48ab91b520eec910f0592f1ae7fb8fdfb8d3b93e Mon Sep 17 00:00:00 2001
From: Erick Friis <erick@langchain.dev>
Date: Thu, 12 Dec 2024 19:07:24 -0800
Subject: [PATCH] docs: more useful vercel warnings (#28699)
---
docs/Makefile | 19 +++----
docs/scripts/generate_api_reference_links.py | 53 ++++++++++++++++----
docs/scripts/partner_deps_list.py | 26 ++++++++++
docs/vercel_requirements.txt | 5 --
libs/packages.yml | 4 +-
5 files changed, 76 insertions(+), 31 deletions(-)
create mode 100644 docs/scripts/partner_deps_list.py
diff --git a/docs/Makefile b/docs/Makefile
index 107bb1feb4d..5df104397aa 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -13,28 +13,21 @@ OUTPUT_NEW_DOCS_DIR = $(OUTPUT_NEW_DIR)/docs
PYTHON = .venv/bin/python
-PARTNER_DEPS_LIST := $(shell find ../libs/partners -mindepth 1 -maxdepth 1 -type d -exec sh -c ' \
-for dir; do \
- if find "$$dir" -maxdepth 1 -type f \( -name "pyproject.toml" -o -name "setup.py" \) | grep -q .; then \
- echo "$$dir"; \
- fi \
-done' sh {} + | grep -vE "airbyte|ibm|databricks" | tr '\n' ' ')
-
PORT ?= 3001
clean:
rm -rf build
install-vercel-deps:
- yum -y update
- yum install gcc bzip2-devel libffi-devel zlib-devel wget tar gzip rsync -y
+ yum -y -q update
+ yum -y -q install gcc bzip2-devel libffi-devel zlib-devel wget tar gzip rsync -y
install-py-deps:
python3 -m venv .venv
- $(PYTHON) -m pip install --upgrade pip
- $(PYTHON) -m pip install --upgrade uv
- $(PYTHON) -m uv pip install --pre -r vercel_requirements.txt
- $(PYTHON) -m uv pip install --pre --editable $(PARTNER_DEPS_LIST)
+ $(PYTHON) -m pip install -q --upgrade pip
+ $(PYTHON) -m pip install -q --upgrade uv
+ $(PYTHON) -m uv pip install -q --pre -r vercel_requirements.txt
+ $(PYTHON) -m uv pip install -q --pre $$($(PYTHON) scripts/partner_deps_list.py)
generate-files:
mkdir -p $(INTERMEDIATE_DIR)
diff --git a/docs/scripts/generate_api_reference_links.py b/docs/scripts/generate_api_reference_links.py
index 662d1dd12f7..dd9ba388c31 100644
--- a/docs/scripts/generate_api_reference_links.py
+++ b/docs/scripts/generate_api_reference_links.py
@@ -5,6 +5,7 @@ import json
import logging
import os
import re
+import warnings
from pathlib import Path
from typing import List, Literal, Optional
@@ -110,16 +111,36 @@ def find_files(path):
def get_full_module_name(module_path, class_name) -> Optional[str]:
"""Get full module name using inspect"""
- try:
- module = importlib.import_module(module_path)
- class_ = getattr(module, class_name)
- return inspect.getmodule(class_).__name__
- except AttributeError as e:
- logger.warning(f"Could not find module for {class_name}, {e}")
- return None
- except ImportError as e:
- logger.warning(f"Failed to load for class {class_name}, {e}")
- return None
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ try:
+ module = importlib.import_module(module_path)
+ except ImportError:
+ # check if it's a submodule
+ try:
+ module = importlib.import_module(module_path + "." + class_name)
+ except ImportError:
+ raise ValueError(f"Failed to import module {module_path}")
+ return module.__name__
+
+ try:
+ class_ = getattr(module, class_name)
+ except AttributeError:
+ # check if it's a submodule
+ try:
+ module = importlib.import_module(module_path + "." + class_name)
+ except ImportError:
+ raise ValueError(
+ f"Failed to import class {class_name} from module {module_path}"
+ )
+ return module.__name__
+
+ inspectmodule = inspect.getmodule(class_)
+ if inspectmodule is None:
+ # it wasn't a class, it's a primitive (e.g. END="__end__")
+ # so no documentation link is necessary
+ return None
+ return inspectmodule.__name__
def get_args() -> argparse.Namespace:
@@ -228,7 +249,17 @@ def _get_imports(
if imp.strip()
]
for class_name in imported_classes:
- module_path = get_full_module_name(module, class_name)
+ try:
+ module_path = get_full_module_name(module, class_name)
+ except ValueError as e:
+ logger.warning(e)
+ continue
+ except Exception as e:
+ logger.error(
+ f"Failed to get full module name for {module}.{class_name}"
+ )
+ logger.error(e)
+ continue
if not module_path:
continue
if len(module_path.split(".")) < 2:
diff --git a/docs/scripts/partner_deps_list.py b/docs/scripts/partner_deps_list.py
new file mode 100644
index 00000000000..2103619f5b0
--- /dev/null
+++ b/docs/scripts/partner_deps_list.py
@@ -0,0 +1,26 @@
+import yaml
+
+if __name__ == "__main__":
+ with open("../libs/packages.yml", "r") as f:
+ packages_yaml = yaml.safe_load(f)
+
+ packages = packages_yaml["packages"]
+
+ comaintain_packages = [
+ p
+ for p in packages
+ if not p.get("disabled", False)
+ and p["repo"].startswith("langchain-ai/")
+ and p["repo"] != "langchain-ai/langchain"
+ ]
+ monorepo_packages = [
+ p
+ for p in packages
+ if not p.get("disabled", False) and p["repo"] == "langchain-ai/langchain"
+ ]
+
+ for p in monorepo_packages:
+ print("--editable ../" + p["path"])
+
+ for p in comaintain_packages:
+ print(p["name"])
diff --git a/docs/vercel_requirements.txt b/docs/vercel_requirements.txt
index 66cd83bf0c3..b4a7d5d070b 100644
--- a/docs/vercel_requirements.txt
+++ b/docs/vercel_requirements.txt
@@ -1,10 +1,5 @@
--e ../libs/core
--e ../libs/langchain
--e ../libs/community
--e ../libs/text-splitters
langgraph
pyyaml
-langchain-cohere
urllib3==1.26.19
nbconvert==7.16.4
diff --git a/libs/packages.yml b/libs/packages.yml
index 7d995ec5d63..77e6b15a7c5 100644
--- a/libs/packages.yml
+++ b/libs/packages.yml
@@ -42,8 +42,8 @@ packages:
repo: langchain-ai/langchain
path: libs/partners/huggingface
- name: langchain-ibm
- repo: langchain-ai/langchain
- path: libs/partners/ibm
+ repo: langchain-ai/langchain-ibm
+ path: libs/ibm
- name: langchain-milvus
repo: langchain-ai/langchain-milvus
path: libs/milvus