github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-06-19 05:13:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

cleanup recursive url session (#10863)

Browse Source
This commit is contained in:
Bagatur 2023-09-21 07:22:13 -07:00 committed by GitHub
parent 777b33b873
commit 5097007407
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libs/langchain/langchain/document_loaders/recursive_url_loader.py
View File

@ -179,11 +179,15 @@ class RecursiveUrlLoader(BaseLoader):
# Disable SSL verification because websites may have invalid SSL certificates, # Disable SSL verification because websites may have invalid SSL certificates,
# but won't cause any security issues for us. # but won't cause any security issues for us.
close_session = session is None close_session = session is None
session = session or aiohttp.ClientSession( session = (
session
if session is not None
else aiohttp.ClientSession(
connector=aiohttp.TCPConnector(ssl=False), connector=aiohttp.TCPConnector(ssl=False),
timeout=aiohttp.ClientTimeout(total=self.timeout), timeout=aiohttp.ClientTimeout(total=self.timeout),
headers=self.headers, headers=self.headers,
) )
)
async with self._lock: # type: ignore async with self._lock: # type: ignore
visited.add(url) visited.add(url)
try: try:
@ -194,6 +198,8 @@ class RecursiveUrlLoader(BaseLoader):
f"Unable to load {url}. Received error {e} of type " f"Unable to load {url}. Received error {e} of type "
f"{e.__class__.__name__}" f"{e.__class__.__name__}"
) )
if close_session:
await session.close()
return [] return []
results = [] results = []
content = self.extractor(text) content = self.extractor(text)