From fa155a422f54a0bf07c00e5fc05cbdf17940433d Mon Sep 17 00:00:00 2001 From: Mohammad Mohtashim <45242107+keenborder786@users.noreply.github.com> Date: Thu, 12 Dec 2024 04:46:54 +0500 Subject: [PATCH] [Community]: `requests_kwargs` not being used in _fetch (#28646) - **Description:** `requests_kwargs` is not being passed to `_fetch` which is fetching pages asynchronously. In this PR, making sure that we are passing `requests_kwargs` to `_fetch` just like `_scrape`. - **Issue:** #28634 --------- Co-authored-by: Erick Friis --- .../langchain_community/document_loaders/web_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/community/langchain_community/document_loaders/web_base.py b/libs/community/langchain_community/document_loaders/web_base.py index 3eab7f351a1..26bc5504d58 100644 --- a/libs/community/langchain_community/document_loaders/web_base.py +++ b/libs/community/langchain_community/document_loaders/web_base.py @@ -208,7 +208,10 @@ class WebBaseLoader(BaseLoader): ) if not self.session.verify: kwargs["ssl"] = False - async with session.get(url, **kwargs) as response: + + async with session.get( + url, **(self.requests_kwargs | kwargs) + ) as response: if self.raise_for_status: response.raise_for_status() return await response.text()