mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-07 14:03:26 +00:00
community[major], core[patch], langchain[patch], experimental[patch]: Create langchain-community (#14463)
Moved the following modules to new package langchain-community in a backwards compatible fashion: ``` mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community ``` Moved the following to core ``` mv langchain/langchain/utils/json_schema.py core/langchain_core/utils mv langchain/langchain/utils/html.py core/langchain_core/utils mv langchain/langchain/utils/strings.py core/langchain_core/utils cat langchain/langchain/utils/env.py >> core/langchain_core/utils/env.py rm langchain/langchain/utils/env.py ``` See .scripts/community_split/script_integrations.sh for all changes
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Callable, List, Optional
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
from langchain_community.document_loaders.base import BaseLoader
|
||||
from langchain_community.document_loaders.unstructured import UnstructuredFileLoader
|
||||
from langchain_community.utilities.vertexai import get_client_info
|
||||
|
||||
|
||||
class GCSFileLoader(BaseLoader):
|
||||
"""Load from GCS file."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
project_name: str,
|
||||
bucket: str,
|
||||
blob: str,
|
||||
loader_func: Optional[Callable[[str], BaseLoader]] = None,
|
||||
):
|
||||
"""Initialize with bucket and key name.
|
||||
|
||||
Args:
|
||||
project_name: The name of the project to load
|
||||
bucket: The name of the GCS bucket.
|
||||
blob: The name of the GCS blob to load.
|
||||
loader_func: A loader function that instantiates a loader based on a
|
||||
file_path argument. If nothing is provided, the
|
||||
UnstructuredFileLoader is used.
|
||||
|
||||
Examples:
|
||||
To use an alternative PDF loader:
|
||||
>> from from langchain_community.document_loaders import PyPDFLoader
|
||||
>> loader = GCSFileLoader(..., loader_func=PyPDFLoader)
|
||||
|
||||
To use UnstructuredFileLoader with additional arguments:
|
||||
>> loader = GCSFileLoader(...,
|
||||
>> loader_func=lambda x: UnstructuredFileLoader(x, mode="elements"))
|
||||
|
||||
"""
|
||||
self.bucket = bucket
|
||||
self.blob = blob
|
||||
self.project_name = project_name
|
||||
|
||||
def default_loader_func(file_path: str) -> BaseLoader:
|
||||
return UnstructuredFileLoader(file_path)
|
||||
|
||||
self._loader_func = loader_func if loader_func else default_loader_func
|
||||
|
||||
def load(self) -> List[Document]:
|
||||
"""Load documents."""
|
||||
try:
|
||||
from google.cloud import storage
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"Could not import google-cloud-storage python package. "
|
||||
"Please install it with `pip install google-cloud-storage`."
|
||||
)
|
||||
|
||||
# Initialise a client
|
||||
storage_client = storage.Client(
|
||||
self.project_name, client_info=get_client_info("google-cloud-storage")
|
||||
)
|
||||
# Create a bucket object for our bucket
|
||||
bucket = storage_client.get_bucket(self.bucket)
|
||||
# Create a blob object from the filepath
|
||||
blob = bucket.blob(self.blob)
|
||||
# retrieve custom metadata associated with the blob
|
||||
metadata = bucket.get_blob(self.blob).metadata
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
file_path = f"{temp_dir}/{self.blob}"
|
||||
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
||||
# Download the file to a destination
|
||||
blob.download_to_filename(file_path)
|
||||
loader = self._loader_func(file_path)
|
||||
docs = loader.load()
|
||||
for doc in docs:
|
||||
if "source" in doc.metadata:
|
||||
doc.metadata["source"] = f"gs://{self.bucket}/{self.blob}"
|
||||
if metadata:
|
||||
doc.metadata.update(metadata)
|
||||
return docs
|
Reference in New Issue
Block a user