more

.scripts/community_split/libs/community/langchain_community/callbacks/__init__.py

@@ -0,0 +1,83 @@
+"""**Callback handlers** allow listening to events in LangChain.
+
+**Class hierarchy:**
+
+.. code-block::
+
+    BaseCallbackHandler --> <name>CallbackHandler  # Example: AimCallbackHandler
+"""
+
+from langchain_core.callbacks import (
+    StdOutCallbackHandler,
+    StreamingStdOutCallbackHandler,
+)
+from langchain_core.tracers.langchain import LangChainTracer
+
+from langchain_community.callbacks.aim_callback import AimCallbackHandler
+from langchain_community.callbacks.argilla_callback import ArgillaCallbackHandler
+from langchain_community.callbacks.arize_callback import ArizeCallbackHandler
+from langchain_community.callbacks.arthur_callback import ArthurCallbackHandler
+from langchain_community.callbacks.clearml_callback import ClearMLCallbackHandler
+from langchain_community.callbacks.comet_ml_callback import CometCallbackHandler
+from langchain_community.callbacks.context_callback import ContextCallbackHandler
+from langchain_community.callbacks.file import FileCallbackHandler
+from langchain_community.callbacks.flyte_callback import FlyteCallbackHandler
+from langchain_community.callbacks.human import HumanApprovalCallbackHandler
+from langchain_community.callbacks.infino_callback import InfinoCallbackHandler
+from langchain_community.callbacks.labelstudio_callback import (
+    LabelStudioCallbackHandler,
+)
+from langchain_community.callbacks.llmonitor_callback import LLMonitorCallbackHandler
+from langchain_community.callbacks.manager import (
+    get_openai_callback,
+    wandb_tracing_enabled,
+)
+from langchain_community.callbacks.mlflow_callback import MlflowCallbackHandler
+from langchain_community.callbacks.openai_info import OpenAICallbackHandler
+from langchain_community.callbacks.promptlayer_callback import (
+    PromptLayerCallbackHandler,
+)
+from langchain_community.callbacks.sagemaker_callback import SageMakerCallbackHandler
+from langchain_community.callbacks.streaming_aiter import AsyncIteratorCallbackHandler
+from langchain_community.callbacks.streaming_stdout_final_only import (
+    FinalStreamingStdOutCallbackHandler,
+)
+from langchain_community.callbacks.streamlit import (
+    LLMThoughtLabeler,
+    StreamlitCallbackHandler,
+)
+from langchain_community.callbacks.trubrics_callback import TrubricsCallbackHandler
+from langchain_community.callbacks.wandb_callback import WandbCallbackHandler
+from langchain_community.callbacks.whylabs_callback import WhyLabsCallbackHandler
+
+__all__ = [
+    "AimCallbackHandler",
+    "ArgillaCallbackHandler",
+    "ArizeCallbackHandler",
+    "PromptLayerCallbackHandler",
+    "ArthurCallbackHandler",
+    "ClearMLCallbackHandler",
+    "CometCallbackHandler",
+    "ContextCallbackHandler",
+    "FileCallbackHandler",
+    "HumanApprovalCallbackHandler",
+    "InfinoCallbackHandler",
+    "MlflowCallbackHandler",
+    "LLMonitorCallbackHandler",
+    "OpenAICallbackHandler",
+    "StdOutCallbackHandler",
+    "AsyncIteratorCallbackHandler",
+    "StreamingStdOutCallbackHandler",
+    "FinalStreamingStdOutCallbackHandler",
+    "LLMThoughtLabeler",
+    "LangChainTracer",
+    "StreamlitCallbackHandler",
+    "WandbCallbackHandler",
+    "WhyLabsCallbackHandler",
+    "get_openai_callback",
+    "wandb_tracing_enabled",
+    "FlyteCallbackHandler",
+    "SageMakerCallbackHandler",
+    "LabelStudioCallbackHandler",
+    "TrubricsCallbackHandler",
+]

.scripts/community_split/libs/community/langchain_community/callbacks/manager.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import logging
+from contextlib import contextmanager
+from contextvars import ContextVar
+from typing import (
+    Generator,
+    Optional,
+)
+
+from langchain_core.tracers.context import register_configure_hook
+
+from langchain_community.callbacks.openai_info import OpenAICallbackHandler
+from langchain_community.callbacks.tracers.wandb import WandbTracer
+
+logger = logging.getLogger(__name__)
+
+openai_callback_var: ContextVar[Optional[OpenAICallbackHandler]] = ContextVar(
+    "openai_callback", default=None
+)
+wandb_tracing_callback_var: ContextVar[Optional[WandbTracer]] = ContextVar(  # noqa: E501
+    "tracing_wandb_callback", default=None
+)
+
+register_configure_hook(openai_callback_var, True)
+register_configure_hook(
+    wandb_tracing_callback_var, True, WandbTracer, "LANGCHAIN_WANDB_TRACING"
+)
+
+
+@contextmanager
+def get_openai_callback() -> Generator[OpenAICallbackHandler, None, None]:
+    """Get the OpenAI callback handler in a context manager.
+    which conveniently exposes token and cost information.
+
+    Returns:
+        OpenAICallbackHandler: The OpenAI callback handler.
+
+    Example:
+        >>> with get_openai_callback() as cb:
+        ...     # Use the OpenAI callback handler
+    """
+    cb = OpenAICallbackHandler()
+    openai_callback_var.set(cb)
+    yield cb
+    openai_callback_var.set(None)
+
+
+@contextmanager
+def wandb_tracing_enabled(
+    session_name: str = "default",
+) -> Generator[None, None, None]:
+    """Get the WandbTracer in a context manager.
+
+    Args:
+        session_name (str, optional): The name of the session.
+            Defaults to "default".
+
+    Returns:
+        None
+
+    Example:
+        >>> with wandb_tracing_enabled() as session:
+        ...     # Use the WandbTracer session
+    """
+    cb = WandbTracer()
+    wandb_tracing_callback_var.set(cb)
+    yield None
+    wandb_tracing_callback_var.set(None)

.scripts/community_split/libs/community/langchain_community/callbacks/tracers/__init__.py

@@ -0,0 +1,20 @@
+"""Tracers that record execution of LangChain runs."""
+
+from langchain_core.tracers.langchain import LangChainTracer
+from langchain_core.tracers.langchain_v1 import LangChainTracerV1
+from langchain_core.tracers.stdout import (
+    ConsoleCallbackHandler,
+    FunctionCallbackHandler,
+)
+
+from langchain_community.callbacks.tracers.logging import LoggingCallbackHandler
+from langchain_community.callbacks.tracers.wandb import WandbTracer
+
+__all__ = [
+    "ConsoleCallbackHandler",
+    "FunctionCallbackHandler",
+    "LoggingCallbackHandler",
+    "LangChainTracer",
+    "LangChainTracerV1",
+    "WandbTracer",
+]

.scripts/community_split/libs/core/langchain_core/utils/__init__.py

@@ -0,0 +1,49 @@
+"""
+**Utility functions** for LangChain.
+
+These functions do not depend on any other LangChain module.
+"""
+
+from langchain_core.utils.env import get_from_dict_or_env, get_from_env
+from langchain_core.utils.formatting import StrictFormatter, formatter
+from langchain_core.utils.input import (
+    get_bolded_text,
+    get_color_mapping,
+    get_colored_text,
+    print_text,
+)
+from langchain_core.utils.loading import try_load_from_hub
+from langchain_core.utils.strings import comma_list, stringify_dict, stringify_value
+from langchain_core.utils.utils import (
+    build_extra_kwargs,
+    check_package_version,
+    convert_to_secret_str,
+    get_pydantic_field_names,
+    guard_import,
+    mock_now,
+    raise_for_status_with_text,
+    xor_args,
+)
+
+__all__ = [
+    "StrictFormatter",
+    "check_package_version",
+    "convert_to_secret_str",
+    "formatter",
+    "get_bolded_text",
+    "get_color_mapping",
+    "get_colored_text",
+    "get_pydantic_field_names",
+    "guard_import",
+    "mock_now",
+    "print_text",
+    "raise_for_status_with_text",
+    "xor_args",
+    "try_load_from_hub",
+    "build_extra_kwargs",
+    "get_from_env",
+    "get_from_dict_or_env",
+    "stringify_dict",
+    "comma_list",
+    "stringify_value",
+]

.scripts/community_split/libs/langchain/langchain/callbacks/__init__.py

@@ -0,0 +1,83 @@
+"""**Callback handlers** allow listening to events in LangChain.
+
+**Class hierarchy:**
+
+.. code-block::
+
+    BaseCallbackHandler --> <name>CallbackHandler  # Example: AimCallbackHandler
+"""
+
+from langchain_core.callbacks import StdOutCallbackHandler, StreamingStdOutCallbackHandler
+from langchain_core.tracers.langchain import LangChainTracer
+from langchain_core.tracers.context import (
+    collect_runs,
+    tracing_enabled,
+    tracing_v2_enabled,
+)
+
+from langchain_community.callbacks.aim_callback import AimCallbackHandler
+from langchain_community.callbacks.argilla_callback import ArgillaCallbackHandler
+from langchain_community.callbacks.arize_callback import ArizeCallbackHandler
+from langchain_community.callbacks.arthur_callback import ArthurCallbackHandler
+from langchain_community.callbacks.clearml_callback import ClearMLCallbackHandler
+from langchain_community.callbacks.comet_ml_callback import CometCallbackHandler
+from langchain_community.callbacks.context_callback import ContextCallbackHandler
+from langchain_community.callbacks.file import FileCallbackHandler
+from langchain_community.callbacks.flyte_callback import FlyteCallbackHandler
+from langchain_community.callbacks.human import HumanApprovalCallbackHandler
+from langchain_community.callbacks.infino_callback import InfinoCallbackHandler
+from langchain_community.callbacks.labelstudio_callback import LabelStudioCallbackHandler
+from langchain_community.callbacks.llmonitor_callback import LLMonitorCallbackHandler
+from langchain_community.callbacks.mlflow_callback import MlflowCallbackHandler
+from langchain_community.callbacks.openai_info import OpenAICallbackHandler
+from langchain_community.callbacks.promptlayer_callback import PromptLayerCallbackHandler
+from langchain_community.callbacks.sagemaker_callback import SageMakerCallbackHandler
+from langchain_community.callbacks.streaming_aiter import AsyncIteratorCallbackHandler
+from langchain_community.callbacks.streaming_stdout_final_only import (
+    FinalStreamingStdOutCallbackHandler,
+)
+from langchain_community.callbacks.streamlit import LLMThoughtLabeler, StreamlitCallbackHandler
+from langchain_community.callbacks.trubrics_callback import TrubricsCallbackHandler
+from langchain_community.callbacks.wandb_callback import WandbCallbackHandler
+from langchain_community.callbacks.whylabs_callback import WhyLabsCallbackHandler
+
+from langchain_community.callbacks.manager import (
+    get_openai_callback,
+    wandb_tracing_enabled,
+)
+
+
+__all__ = [
+    "AimCallbackHandler",
+    "ArgillaCallbackHandler",
+    "ArizeCallbackHandler",
+    "PromptLayerCallbackHandler",
+    "ArthurCallbackHandler",
+    "ClearMLCallbackHandler",
+    "CometCallbackHandler",
+    "ContextCallbackHandler",
+    "FileCallbackHandler",
+    "HumanApprovalCallbackHandler",
+    "InfinoCallbackHandler",
+    "MlflowCallbackHandler",
+    "LLMonitorCallbackHandler",
+    "OpenAICallbackHandler",
+    "StdOutCallbackHandler",
+    "AsyncIteratorCallbackHandler",
+    "StreamingStdOutCallbackHandler",
+    "FinalStreamingStdOutCallbackHandler",
+    "LLMThoughtLabeler",
+    "LangChainTracer",
+    "StreamlitCallbackHandler",
+    "WandbCallbackHandler",
+    "WhyLabsCallbackHandler",
+    "get_openai_callback",
+    "tracing_enabled",
+    "tracing_v2_enabled",
+    "collect_runs",
+    "wandb_tracing_enabled",
+    "FlyteCallbackHandler",
+    "SageMakerCallbackHandler",
+    "LabelStudioCallbackHandler",
+    "TrubricsCallbackHandler",
+]

.scripts/community_split/libs/langchain/langchain/callbacks/manager.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from langchain_core.callbacks.manager import (
+    AsyncCallbackManager,
+    AsyncCallbackManagerForChainGroup,
+    AsyncCallbackManagerForChainRun,
+    AsyncCallbackManagerForLLMRun,
+    AsyncCallbackManagerForRetrieverRun,
+    AsyncCallbackManagerForToolRun,
+    AsyncParentRunManager,
+    AsyncRunManager,
+    BaseRunManager,
+    CallbackManager,
+    CallbackManagerForChainGroup,
+    CallbackManagerForChainRun,
+    CallbackManagerForLLMRun,
+    CallbackManagerForRetrieverRun,
+    CallbackManagerForToolRun,
+    Callbacks,
+    ParentRunManager,
+    RunManager,
+    ahandle_event,
+    atrace_as_chain_group,
+    handle_event,
+    trace_as_chain_group,
+)
+from langchain_core.tracers.context import (
+    collect_runs,
+    tracing_enabled,
+    tracing_v2_enabled,
+)
+from langchain_core.utils.env import env_var_is_set
+from langchain_community.callbacks.manager import (
+    get_openai_callback,
+    wandb_tracing_enabled,
+)
+
+
+__all__ = [
+    "BaseRunManager",
+    "RunManager",
+    "ParentRunManager",
+    "AsyncRunManager",
+    "AsyncParentRunManager",
+    "CallbackManagerForLLMRun",
+    "AsyncCallbackManagerForLLMRun",
+    "CallbackManagerForChainRun",
+    "AsyncCallbackManagerForChainRun",
+    "CallbackManagerForToolRun",
+    "AsyncCallbackManagerForToolRun",
+    "CallbackManagerForRetrieverRun",
+    "AsyncCallbackManagerForRetrieverRun",
+    "CallbackManager",
+    "CallbackManagerForChainGroup",
+    "AsyncCallbackManager",
+    "AsyncCallbackManagerForChainGroup",
+    "tracing_enabled",
+    "tracing_v2_enabled",
+    "collect_runs",
+    "atrace_as_chain_group",
+    "trace_as_chain_group",
+    "handle_event",
+    "ahandle_event",
+    "Callbacks",
+    "env_var_is_set",
+    "get_openai_callback",
+    "wandb_tracing_enabled",
+]

.scripts/community_split/libs/partners/openai/langchain_openai/__init__.py

@@ -0,0 +1,20 @@
+from langchain_openai.chat_models import AzureChatOpenAI, ChatOpenAI, _import_tiktoken
+from langchain_openai.embeddings import AzureOpenAIEmbeddings, OpenAIEmbeddings
+from langchain_openai.functions import (
+    convert_pydantic_to_openai_function,
+    convert_pydantic_to_openai_tool,
+)
+from langchain_openai.llms import AzureOpenAI, BaseOpenAI, OpenAI
+
+__all__ = [
+    "_import_tiktoken",
+    "OpenAI",
+    "AzureOpenAI",
+    "ChatOpenAI",
+    "AzureChatOpenAI",
+    "OpenAIEmbeddings",
+    "AzureOpenAIEmbeddings",
+    "convert_pydantic_to_openai_function",
+    "convert_pydantic_to_openai_tool",
+    "BaseOpenAI",
+]

.scripts/community_split/libs/partners/openai/langchain_openai/chat_models/__init__.py

@@ -0,0 +1,17 @@
+from langchain_openai.chat_models.azure import AzureChatOpenAI
+from langchain_openai.chat_models.base import (
+    ChatOpenAI,
+    _convert_delta_to_message_chunk,
+    _create_retry_decorator,
+    _import_tiktoken,
+    acompletion_with_retry,
+)
+
+__all__ = [
+    "_create_retry_decorator",
+    "acompletion_with_retry",
+    "_convert_delta_to_message_chunk",
+    "_import_tiktoken",
+    "ChatOpenAI",
+    "AzureChatOpenAI",
+]

.scripts/community_split/libs/partners/openai/langchain_openai/embeddings/__init__.py

@@ -0,0 +1,21 @@
+from langchain_openai.embeddings.azure import AzureOpenAIEmbeddings
+from langchain_openai.embeddings.base import (
+    OpenAIEmbeddings,
+    _async_retry_decorator,
+    _check_response,
+    _create_retry_decorator,
+    _is_openai_v1,
+    async_embed_with_retry,
+    embed_with_retry,
+)
+
+__all__ = [
+    "_create_retry_decorator",
+    "_async_retry_decorator",
+    "_check_response",
+    "embed_with_retry",
+    "async_embed_with_retry",
+    "_is_openai_v1",
+    "OpenAIEmbeddings",
+    "AzureOpenAIEmbeddings",
+]

.scripts/community_split/libs/partners/openai/langchain_openai/llms/__init__.py

@@ -0,0 +1,27 @@
+from langchain_openai.llms.base import (
+    AzureOpenAI,
+    BaseOpenAI,
+    OpenAI,
+    OpenAIChat,
+    _create_retry_decorator,
+    _stream_response_to_generation_chunk,
+    _streaming_response_template,
+    _update_response,
+    acompletion_with_retry,
+    completion_with_retry,
+    update_token_usage,
+)
+
+__all__ = [
+    "update_token_usage",
+    "_stream_response_to_generation_chunk",
+    "_update_response",
+    "_streaming_response_template",
+    "_create_retry_decorator",
+    "completion_with_retry",
+    "acompletion_with_rety",
+    "OpenAIChat",
+    "OpenAI",
+    "AzureOpenAI",
+    "BaseOpenAI",
+]

.scripts/community_split/script_integrations.sh

@@ -221,7 +221,8 @@ git checkout master -- langchain/tests/unit_tests/document_loaders/blob_loaders/
 git checkout master -- langchain/tests/unit_tests/document_loaders/parsers/test_public_api.py
 git checkout master -- langchain/tests/unit_tests/vectorstores/test_public_api.py
 git checkout master -- langchain/tests/unit_tests/schema
-touch langchain/tests/unit_tests/{llms,chat_models,tools,callbacks,runnables}/__init__.py
+touch langchain/tests/unit_tests/{llms,chat_models,tools,callbacks,runnables,document_loaders,docstore,document_transformers,embeddings,graphs,storage,utilities,vectorstores}/__init__.py
+touch langchain/tests/unit_tests/document_loaders/{blo_loaders,parsers}/__init__.py
 
 cp core/Makefile community
 cp core/Makefile partners/openai
@@ -230,65 +231,11 @@ sed -i '' 's/libs\/core/libs\/partners\/openai/g' partners/openai/Makefile
 cp -r core/scripts community
 cp -r core/scripts partners/openai
 
-printf 'from langchain_openai.llms import BaseOpenAI, OpenAI, AzureOpenAI\nfrom langchain_openai.chat_models import _import_tiktoken, ChatOpenAI, AzureChatOpenAI\nfrom langchain_openai.embeddings import OpenAIEmbeddings, AzureOpenAIEmbeddings\nfrom langchain_openai.functions import convert_pydantic_to_openai_function, convert_pydantic_to_openai_tool\n\n__all__ = ["_import_tiktoken", "OpenAI", "AzureOpenAI", "ChatOpenAI", "AzureChatOpenAI", "OpenAIEmbeddings", "AzureOpenAIEmbeddings", "convert_pydantic_to_openai_function", "convert_pydantic_to_openai_tool", "BaseOpenAI"]' >> partners/openai/langchain_openai/__init__.py
-printf 'from langchain_openai.llms.base import update_token_usage, _stream_response_to_generation_chunk, _update_response, _streaming_response_template, _create_retry_decorator, completion_with_retry, acompletion_with_retry, BaseOpenAI, OpenAIChat, OpenAI, AzureOpenAI\n\n__all__ = ["update_token_usage", "_stream_response_to_generation_chunk", "_update_response", "_streaming_response_template", "_create_retry_decorator", "completion_with_retry", "acompletion_with_rety", "OpenAIChat", "OpenAI", "AzureOpenAI", "BaseOpenAI"]' >> partners/openai/langchain_openai/llms/__init__.py
-printf 'from langchain_openai.chat_models.base import _create_retry_decorator, acompletion_with_retry, _convert_delta_to_message_chunk, _import_tiktoken, ChatOpenAI\nfrom langchain_openai.chat_models.azure import AzureChatOpenAI\n\n__all__ = ["_create_retry_decorator", "acompletion_with_retry", "_convert_delta_to_message_chunk", "_import_tiktoken", "ChatOpenAI", "AzureChatOpenAI"]' >> partners/openai/langchain_openai/chat_models/__init__.py
-printf 'from langchain_openai.embeddings.base import _create_retry_decorator, _async_retry_decorator, _check_response, embed_with_retry, async_embed_with_retry, _is_openai_v1, OpenAIEmbeddings\nfrom langchain_openai.embeddings.azure import AzureOpenAIEmbeddings\n\n\n__all__ = ["_create_retry_decorator", "_async_retry_decorator", "_check_response", "embed_with_retry", "async_embed_with_retry", "_is_openai_v1", "OpenAIEmbeddings", "AzureOpenAIEmbeddings"]' >> partners/openai/langchain_openai/embeddings/__init__.py
-
 
 sed -i '' 's/from\ langchain_openai.chat_models\ /from\ langchain_openai.chat_models.base\ /g' partners/openai/langchain_openai/chat_models/azure.py
 sed -i '' 's/from\ langchain_openai.embeddings\ /from\ langchain_openai.embeddings.base\ /g' partners/openai/langchain_openai/embeddings/azure.py
 
-echo '"""
-**Utility functions** for LangChain.
-
-These functions do not depend on any other LangChain module.
-"""
-
-from langchain_core.utils.formatting import StrictFormatter, formatter
-from langchain_core.utils.input import (
-    get_bolded_text,
-    get_color_mapping,
-    get_colored_text,
-    print_text,
-)
-from langchain_core.utils.loading import try_load_from_hub
-from langchain_core.utils.utils import (
-    build_extra_kwargs,
-    check_package_version,
-    convert_to_secret_str,
-    get_pydantic_field_names,
-    guard_import,
-    mock_now,
-    raise_for_status_with_text,
-    xor_args,
-)
-from langchain_core.utils.env import get_from_env, get_from_dict_or_env
-from langchain_core.utils.strings import stringify_dict, comma_list, stringify_value
-
-__all__ = [
-    "StrictFormatter",
-    "check_package_version",
-    "convert_to_secret_str",
-    "formatter",
-    "get_bolded_text",
-    "get_color_mapping",
-    "get_colored_text",
-    "get_pydantic_field_names",
-    "guard_import",
-    "mock_now",
-    "print_text",
-    "raise_for_status_with_text",
-    "xor_args",
-    "try_load_from_hub",
-    "build_extra_kwargs",
-    "get_from_env",
-    "get_from_dict_or_env",
-    "stringify_dict",
-    "comma_list",
-    "stringify_value",
-]
-' > core/langchain_core/utils/__init__.py
+cp -r ../.scripts/community_split/libs/* .
 
 cd core
 make format