community[major], core[patch], langchain[patch], experimental[patch]: Create langchain-community (#14463)

Moved the following modules to new package langchain-community in a backwards compatible fashion: ``` mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community ``` Moved the following to core ``` mv langchain/langchain/utils/json_schema.py core/langchain_core/utils mv langchain/langchain/utils/html.py core/langchain_core/utils mv langchain/langchain/utils/strings.py core/langchain_core/utils cat langchain/langchain/utils/env.py >> core/langchain_core/utils/env.py rm langchain/langchain/utils/env.py ``` See .scripts/community_split/script_integrations.sh for all changes
2025-09-29 07:19:59 +00:00 · 2023-12-11 13:53:30 -08:00
parent c0f4b95aa9
commit ed58eeb9c5
2446 changed files with 171805 additions and 137118 deletions
--- a/libs/community/langchain_community/document_loaders/tensorflow_datasets.py
+++ b/libs/community/langchain_community/document_loaders/tensorflow_datasets.py
@@ -0,0 +1,80 @@
+from typing import Callable, Dict, Iterator, List, Optional
+
+from langchain_core.documents import Document
+
+from langchain_community.document_loaders.base import BaseLoader
+from langchain_community.utilities.tensorflow_datasets import TensorflowDatasets
+
+
+class TensorflowDatasetLoader(BaseLoader):
+    """Load from `TensorFlow Dataset`.
+
+    Attributes:
+        dataset_name: the name of the dataset to load
+        split_name: the name of the split to load.
+        load_max_docs: a limit to the number of loaded documents. Defaults to 100.
+        sample_to_document_function: a function that converts a dataset sample
+          into a Document
+
+    Example:
+        .. code-block:: python
+
+            from langchain_community.document_loaders import TensorflowDatasetLoader
+
+            def mlqaen_example_to_document(example: dict) -> Document:
+                return Document(
+                    page_content=decode_to_str(example["context"]),
+                    metadata={
+                        "id": decode_to_str(example["id"]),
+                        "title": decode_to_str(example["title"]),
+                        "question": decode_to_str(example["question"]),
+                        "answer": decode_to_str(example["answers"]["text"][0]),
+                    },
+                )
+
+            tsds_client = TensorflowDatasetLoader(
+                    dataset_name="mlqa/en",
+                    split_name="test",
+                    load_max_docs=100,
+                    sample_to_document_function=mlqaen_example_to_document,
+                )
+
+    """
+
+    def __init__(
+        self,
+        dataset_name: str,
+        split_name: str,
+        load_max_docs: Optional[int] = 100,
+        sample_to_document_function: Optional[Callable[[Dict], Document]] = None,
+    ):
+        """Initialize the TensorflowDatasetLoader.
+
+        Args:
+            dataset_name: the name of the dataset to load
+            split_name: the name of the split to load.
+            load_max_docs: a limit to the number of loaded documents. Defaults to 100.
+            sample_to_document_function: a function that converts a dataset sample
+                into a Document.
+        """
+        self.dataset_name: str = dataset_name
+        self.split_name: str = split_name
+        self.load_max_docs = load_max_docs
+        """The maximum number of documents to load."""
+        self.sample_to_document_function: Optional[
+            Callable[[Dict], Document]
+        ] = sample_to_document_function
+        """Custom function that transform a dataset sample into a Document."""
+
+        self._tfds_client = TensorflowDatasets(
+            dataset_name=self.dataset_name,
+            split_name=self.split_name,
+            load_max_docs=self.load_max_docs,
+            sample_to_document_function=self.sample_to_document_function,
+        )
+
+    def lazy_load(self) -> Iterator[Document]:
+        yield from self._tfds_client.lazy_load()
+
+    def load(self) -> List[Document]:
+        return list(self.lazy_load())