github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-05-28 10:39:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

cleanup recursive url session (#10863)

Browse Source
This commit is contained in:
Bagatur 2023-09-21 07:22:13 -07:00 committed by GitHub
parent 777b33b873
commit 5097007407
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
libs/langchain/langchain/document_loaders/recursive_url_loader.py
View File

@ -179,10 +179,14 @@ class RecursiveUrlLoader(BaseLoader):
# Disable SSL verification because websites may have invalid SSL certificates,
# but won't cause any security issues for us.
close_session = session is None
session = session or aiohttp.ClientSession(
connector=aiohttp.TCPConnector(ssl=False),
timeout=aiohttp.ClientTimeout(total=self.timeout),
headers=self.headers,
session = (
session
if session is not None
else aiohttp.ClientSession(
connector=aiohttp.TCPConnector(ssl=False),
timeout=aiohttp.ClientTimeout(total=self.timeout),
headers=self.headers,
)
)
async with self._lock: # type: ignore
visited.add(url)
@ -194,6 +198,8 @@ class RecursiveUrlLoader(BaseLoader):
f"Unable to load {url}. Received error {e} of type "
f"{e.__class__.__name__}"
)
if close_session:
await session.close()
return []
results = []
content = self.extractor(text)