From 82d4afcac0c2268c3cd181e26a96f1bf3787cf9c Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 29 Apr 2024 15:34:03 -0400 Subject: [PATCH] langchain[minor]: Code to handle dynamic imports (#20893) Proposing to centralize code for handling dynamic imports. This allows treating langchain-community as an optional dependency. --- The proposal is to scan the code base and to replace all existing imports with dynamic imports using this functionality. --- libs/langchain/langchain/_api/__init__.py | 2 + .../langchain/langchain/_api/module_import.py | 128 ++++++++++++++++++ libs/langchain/langchain/chains/__init__.py | 10 +- .../chat_loaders/facebook_messenger.py | 32 ++++- libs/langchain/langchain/graphs/__init__.py | 20 +-- .../langchain/retrievers/__init__.py | 25 +--- .../tests/unit_tests/_api/test_importing.py | 13 ++ 7 files changed, 187 insertions(+), 43 deletions(-) create mode 100644 libs/langchain/langchain/_api/module_import.py create mode 100644 libs/langchain/tests/unit_tests/_api/test_importing.py diff --git a/libs/langchain/langchain/_api/__init__.py b/libs/langchain/langchain/_api/__init__.py index e013a72129f..1f746db82aa 100644 --- a/libs/langchain/langchain/_api/__init__.py +++ b/libs/langchain/langchain/_api/__init__.py @@ -16,6 +16,7 @@ from .deprecation import ( surface_langchain_deprecation_warnings, warn_deprecated, ) +from .module_import import create_importer __all__ = [ "deprecated", @@ -23,4 +24,5 @@ __all__ = [ "suppress_langchain_deprecation_warning", "surface_langchain_deprecation_warnings", "warn_deprecated", + "create_importer", ] diff --git a/libs/langchain/langchain/_api/module_import.py b/libs/langchain/langchain/_api/module_import.py new file mode 100644 index 00000000000..b352d257b9d --- /dev/null +++ b/libs/langchain/langchain/_api/module_import.py @@ -0,0 +1,128 @@ +import importlib +import os +import pathlib +import warnings +from typing import Any, Callable, Dict, Optional + +from langchain_core._api import LangChainDeprecationWarning + +from langchain.utils.interactive_env import is_interactive_env + +ALLOWED_TOP_LEVEL_PKGS = { + "langchain_community", + "langchain_core", + "langchain", +} + + +HERE = pathlib.Path(__file__).parent +ROOT = HERE.parent.parent + + +def _get_current_module(path: str) -> str: + """Convert a path to a module name.""" + path_as_pathlib = pathlib.Path(os.path.abspath(path)) + relative_path = path_as_pathlib.relative_to(ROOT).with_suffix("") + posix_path = relative_path.as_posix() + norm_path = os.path.normpath(str(posix_path)) + fully_qualified_module = norm_path.replace("/", ".") + # Strip off __init__ if present + if fully_qualified_module.endswith(".__init__"): + return fully_qualified_module[:-9] + return fully_qualified_module + + +def create_importer( + here: str, + *, + module_lookup: Optional[Dict[str, str]] = None, + fallback_module: Optional[str] = None, +) -> Callable[[str], Any]: + """Create a function that helps retrieve objects from their new locations. + + The goal of this function is to help users transition from deprecated + imports to new imports. + + This function will raise warnings when the old imports are used and + suggest the new imports. + + This function should ideally only be used with deprecated imports not with + existing imports that are valid, as in addition to raising deprecation warnings + the dynamic imports can create other issues for developers (e.g., + loss of type information, IDE support for going to definition etc). + + Args: + here: path of the current file. Use __file__ + module_lookup: maps name of object to the module where it is defined. + e.g., + { + "MyDocumentLoader": ( + "langchain_community.document_loaders.my_document_loader" + ) + } + fallback_module: module to import from if the object is not found in + module_lookup or if module_lookup is not provided. + + Returns: + A function that imports objects from the specified modules. + """ + current_module = _get_current_module(here) + + def import_by_name(name: str) -> Any: + """Import stores from langchain_community.""" + # If not in interactive env, raise warning. + if module_lookup and name in module_lookup: + new_module = module_lookup[name] + if new_module.split(".")[0] not in ALLOWED_TOP_LEVEL_PKGS: + raise AssertionError( + f"Importing from {new_module} is not allowed. " + f"Allowed top-level packages are: {ALLOWED_TOP_LEVEL_PKGS}" + ) + + try: + module = importlib.import_module(new_module) + except ModuleNotFoundError as e: + if new_module.startswith("langchain_community"): + raise ModuleNotFoundError( + f"Module {new_module} not found. " + "Please install langchain-community to access this module. " + "You can install it using `pip install -U langchain-community`" + ) from e + raise + + try: + result = getattr(module, name) + if not is_interactive_env(): + warnings.warn( + f"Importing {name} from {current_module} is deprecated. " + "Please replace the import with the following:\n" + f"from {new_module} import {name}", + category=LangChainDeprecationWarning, + ) + return result + except Exception as e: + raise AttributeError( + f"module {new_module} has no attribute {name}" + ) from e + + if fallback_module: + try: + module = importlib.import_module(fallback_module) + result = getattr(module, name) + if not is_interactive_env(): + warnings.warn( + f"Importing {name} from {current_module} is deprecated. " + "Please replace the import with the following:\n" + f"from {fallback_module} import {name}", + category=LangChainDeprecationWarning, + ) + return result + + except Exception as e: + raise AttributeError( + f"module {fallback_module} has no attribute {name}" + ) from e + + raise AttributeError(f"module {current_module} has no attribute {name}") + + return import_by_name diff --git a/libs/langchain/langchain/chains/__init__.py b/libs/langchain/langchain/chains/__init__.py index d4dc8a40f94..f1263ae7e38 100644 --- a/libs/langchain/langchain/chains/__init__.py +++ b/libs/langchain/langchain/chains/__init__.py @@ -17,9 +17,10 @@ The Chain interface makes it easy to create apps that are: Chain --> Chain # Examples: LLMChain, MapReduceChain, RouterChain """ -import importlib from typing import Any +from langchain._api import create_importer + _module_lookup = { "APIChain": "langchain.chains.api.base", "OpenAPIEndpointChain": "langchain.chains.api.openapi.chain", @@ -84,12 +85,11 @@ _module_lookup = { "TransformChain": "langchain.chains.transform", } +importer = create_importer(__file__, module_lookup=_module_lookup) + def __getattr__(name: str) -> Any: - if name in _module_lookup: - module = importlib.import_module(_module_lookup[name]) - return getattr(module, name) - raise AttributeError(f"module {__name__} has no attribute {name}") + return importer(name) __all__ = list(_module_lookup.keys()) diff --git a/libs/langchain/langchain/chat_loaders/facebook_messenger.py b/libs/langchain/langchain/chat_loaders/facebook_messenger.py index 389ae17b265..cbb9929dd68 100644 --- a/libs/langchain/langchain/chat_loaders/facebook_messenger.py +++ b/libs/langchain/langchain/chat_loaders/facebook_messenger.py @@ -1,6 +1,32 @@ -from langchain_community.chat_loaders.facebook_messenger import ( - FolderFacebookMessengerChatLoader, - SingleFileFacebookMessengerChatLoader, +from typing import TYPE_CHECKING, Any + +from langchain._api.module_import import create_importer + +if TYPE_CHECKING: + from langchain_community.chat_loaders.facebook_messenger import ( + FolderFacebookMessengerChatLoader, + SingleFileFacebookMessengerChatLoader, + ) + +module_lookup = { + "SingleFileFacebookMessengerChatLoader": ( + "langchain_community.chat_loaders.facebook_messenger" + ), + "FolderFacebookMessengerChatLoader": ( + "langchain_community.chat_loaders.facebook_messenger" + ), +} + +# Temporary code for backwards compatibility for deprecated imports. +# This will eventually be removed. +import_lookup = create_importer( + __file__, + module_lookup=module_lookup, ) + +def __getattr__(name: str) -> Any: + return import_lookup(name) + + __all__ = ["SingleFileFacebookMessengerChatLoader", "FolderFacebookMessengerChatLoader"] diff --git a/libs/langchain/langchain/graphs/__init__.py b/libs/langchain/langchain/graphs/__init__.py index 3189534975e..7ada923a6a9 100644 --- a/libs/langchain/langchain/graphs/__init__.py +++ b/libs/langchain/langchain/graphs/__init__.py @@ -1,27 +1,13 @@ """**Graphs** provide a natural language interface to graph databases.""" -import warnings from typing import Any -from langchain_core._api import LangChainDeprecationWarning +from langchain._api import create_importer -from langchain.utils.interactive_env import is_interactive_env +importer = create_importer(__file__, fallback_module="langchain_community.graphs") def __getattr__(name: str) -> Any: - from langchain_community import graphs - - # If not in interactive env, raise warning. - if not is_interactive_env(): - warnings.warn( - "Importing graphs from langchain is deprecated. Importing from " - "langchain will no longer be supported as of langchain==0.2.0. " - "Please import from langchain-community instead:\n\n" - f"`from langchain_community.graphs import {name}`.\n\n" - "To install langchain-community run `pip install -U langchain-community`.", - category=LangChainDeprecationWarning, - ) - - return getattr(graphs, name) + return importer(name) __all__ = [ diff --git a/libs/langchain/langchain/retrievers/__init__.py b/libs/langchain/langchain/retrievers/__init__.py index e2a44de0d51..29fce7a61a8 100644 --- a/libs/langchain/langchain/retrievers/__init__.py +++ b/libs/langchain/langchain/retrievers/__init__.py @@ -17,11 +17,9 @@ the backbone of a retriever, but there are other types of retrievers as well. Document, Serializable, Callbacks, CallbackManagerForRetrieverRun, AsyncCallbackManagerForRetrieverRun """ -import warnings from typing import Any -from langchain_core._api import LangChainDeprecationWarning - +from langchain._api.module_import import create_importer from langchain.retrievers.contextual_compression import ContextualCompressionRetriever from langchain.retrievers.ensemble import EnsembleRetriever from langchain.retrievers.merger_retriever import MergerRetriever @@ -35,24 +33,15 @@ from langchain.retrievers.time_weighted_retriever import ( TimeWeightedVectorStoreRetriever, ) from langchain.retrievers.web_research import WebResearchRetriever -from langchain.utils.interactive_env import is_interactive_env + +import_lookup = create_importer( + __file__, fallback_module="langchain_community.retrievers" +) def __getattr__(name: str) -> Any: - from langchain_community import retrievers - - # If not in interactive env, raise warning. - if not is_interactive_env(): - warnings.warn( - "Importing this retriever from langchain is deprecated. Importing it from " - "langchain will no longer be supported as of langchain==0.2.0. " - "Please import from langchain-community instead:\n\n" - f"`from langchain_community.retrievers import {name}`.\n\n" - "To install langchain-community run `pip install -U langchain-community`.", - category=LangChainDeprecationWarning, - ) - - return getattr(retrievers, name) + """Import retrievers from langchain_community.""" + return import_lookup(name) __all__ = [ diff --git a/libs/langchain/tests/unit_tests/_api/test_importing.py b/libs/langchain/tests/unit_tests/_api/test_importing.py new file mode 100644 index 00000000000..9768cfc240a --- /dev/null +++ b/libs/langchain/tests/unit_tests/_api/test_importing.py @@ -0,0 +1,13 @@ +from langchain._api.module_import import create_importer + + +def test_importing_module() -> None: + """Test importing all modules in langchain.""" + module_lookup = { + "Document": "langchain_core.documents", + } + lookup = create_importer(__file__, module_lookup=module_lookup) + imported_doc = lookup("Document") + from langchain_core.documents import Document + + assert imported_doc is Document