mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-29 15:28:54 +00:00
8516a03a02
This PR upgrades langchain-community to pydantic 2.
* Most of this PR was auto-generated using code mods with gritql
(https://github.com/eyurtsev/migrate-pydantic/tree/main)
* Subsequently, some code was fixed manually due to accommodate
differences between pydantic 1 and 2
Breaking Changes:
- Use TEXTEMBED_API_KEY and TEXTEMBEB_API_URL for env variables for text
embed integrations:
cbea780492
Other changes:
- Added pydantic_settings as a required dependency for community. This
may be removed if we have enough time to convert the dependency into an
optional one.
---------
Co-authored-by: Eugene Yurtsev <eyurtsev@gmail.com>
Co-authored-by: Bagatur <baskaryan@gmail.com>
83 lines
2.9 KiB
Python
83 lines
2.9 KiB
Python
from typing import Any, Callable, Dict, List
|
|
|
|
from langchain_core.documents import Document
|
|
from pydantic import BaseModel, model_validator
|
|
|
|
from langchain_community.document_loaders.base import BaseLoader
|
|
|
|
|
|
class ApifyDatasetLoader(BaseLoader, BaseModel):
|
|
"""Load datasets from `Apify` web scraping, crawling, and data extraction platform.
|
|
|
|
For details, see https://docs.apify.com/platform/integrations/langchain
|
|
|
|
Example:
|
|
.. code-block:: python
|
|
|
|
from langchain_community.document_loaders import ApifyDatasetLoader
|
|
from langchain_core.documents import Document
|
|
|
|
loader = ApifyDatasetLoader(
|
|
dataset_id="YOUR-DATASET-ID",
|
|
dataset_mapping_function=lambda dataset_item: Document(
|
|
page_content=dataset_item["text"], metadata={"source": dataset_item["url"]}
|
|
),
|
|
)
|
|
documents = loader.load()
|
|
""" # noqa: E501
|
|
|
|
apify_client: Any
|
|
"""An instance of the ApifyClient class from the apify-client Python package."""
|
|
dataset_id: str
|
|
"""The ID of the dataset on the Apify platform."""
|
|
dataset_mapping_function: Callable[[Dict], Document]
|
|
"""A custom function that takes a single dictionary (an Apify dataset item)
|
|
and converts it to an instance of the Document class."""
|
|
|
|
def __init__(
|
|
self, dataset_id: str, dataset_mapping_function: Callable[[Dict], Document]
|
|
):
|
|
"""Initialize the loader with an Apify dataset ID and a mapping function.
|
|
|
|
Args:
|
|
dataset_id (str): The ID of the dataset on the Apify platform.
|
|
dataset_mapping_function (Callable): A function that takes a single
|
|
dictionary (an Apify dataset item) and converts it to an instance
|
|
of the Document class.
|
|
"""
|
|
super().__init__(
|
|
dataset_id=dataset_id, dataset_mapping_function=dataset_mapping_function
|
|
)
|
|
|
|
@model_validator(mode="before")
|
|
@classmethod
|
|
def validate_environment(cls, values: Dict) -> Any:
|
|
"""Validate environment.
|
|
|
|
Args:
|
|
values: The values to validate.
|
|
"""
|
|
|
|
try:
|
|
from apify_client import ApifyClient
|
|
|
|
client = ApifyClient()
|
|
if httpx_client := getattr(client.http_client, "httpx_client"):
|
|
httpx_client.headers["user-agent"] += "; Origin/langchain"
|
|
|
|
values["apify_client"] = client
|
|
except ImportError:
|
|
raise ImportError(
|
|
"Could not import apify-client Python package. "
|
|
"Please install it with `pip install apify-client`."
|
|
)
|
|
|
|
return values
|
|
|
|
def load(self) -> List[Document]:
|
|
"""Load documents."""
|
|
dataset_items = (
|
|
self.apify_client.dataset(self.dataset_id).list_items(clean=True).items
|
|
)
|
|
return list(map(self.dataset_mapping_function, dataset_items))
|