mirror of
https://github.com/hwchase17/langchain.git
synced 2025-06-20 13:54:48 +00:00
community: add **request_kwargs and expect TimeError AsyncHtmlLoader (#23068)
- **Description:** add `**request_kwargs` and expect `TimeError` in `_fetch` function for AsyncHtmlLoader. This allows you to fill in the kwargs parameter when using the `load()` method of the `AsyncHtmlLoader` class. Co-authored-by: Yucolu <yucolu@tencent.com>
This commit is contained in:
parent
109a70fc64
commit
6b46b5e9ce
@ -137,6 +137,7 @@ class AsyncHtmlLoader(BaseLoader):
|
||||
url,
|
||||
headers=self.session.headers,
|
||||
ssl=None if self.session.verify else False,
|
||||
**self.requests_kwargs,
|
||||
) as response:
|
||||
try:
|
||||
text = await response.text()
|
||||
@ -144,7 +145,7 @@ class AsyncHtmlLoader(BaseLoader):
|
||||
logger.error(f"Failed to decode content from {url}")
|
||||
text = ""
|
||||
return text
|
||||
except aiohttp.ClientConnectionError as e:
|
||||
except (aiohttp.ClientConnectionError, TimeoutError) as e:
|
||||
if i == retries - 1 and self.ignore_load_errors:
|
||||
logger.warning(f"Error fetching {url} after {retries} retries.")
|
||||
return ""
|
||||
|
Loading…
Reference in New Issue
Block a user