github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-06-21 14:18:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

community: add **request_kwargs and expect TimeError AsyncHtmlLoader (#23068)

Browse Source 
- **Description:** add `**request_kwargs` and expect `TimeError` in
`_fetch` function for AsyncHtmlLoader. This allows you to fill in the
kwargs parameter when using the `load()` method of the `AsyncHtmlLoader`
class.

Co-authored-by: Yucolu <yucolu@tencent.com>
This commit is contained in:
鹿鹿鹿鲨 2024-06-19 11:02:46 +08:00 committed by GitHub
parent 109a70fc64
commit 6b46b5e9ce
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
libs/community/langchain_community/document_loaders/async_html.py
View File

@ -137,6 +137,7 @@ class AsyncHtmlLoader(BaseLoader):
url, url,
headers=self.session.headers, headers=self.session.headers,
ssl=None if self.session.verify else False, ssl=None if self.session.verify else False,
**self.requests_kwargs,
) as response: ) as response:
try: try:
text = await response.text() text = await response.text()
@ -144,7 +145,7 @@ class AsyncHtmlLoader(BaseLoader):
logger.error(f"Failed to decode content from {url}") logger.error(f"Failed to decode content from {url}")
text = "" text = ""
return text return text
except aiohttp.ClientConnectionError as e: except (aiohttp.ClientConnectionError, TimeoutError) as e:
if i == retries - 1 and self.ignore_load_errors: if i == retries - 1 and self.ignore_load_errors:
logger.warning(f"Error fetching {url} after {retries} retries.") logger.warning(f"Error fetching {url} after {retries} retries.")
return "" return ""