github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-07-06 05:08:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

add continue to fix 'continue_on_failure' parameter for URL doc loader (#2735)

Browse Source 
Currently, the function still fails if `continue_on_failure` is set to
True, because `elements` is not set.

---------

Co-authored-by: leecjohnny <johnny-lee1255@users.noreply.github.com>
This commit is contained in:
Johnny Lee 2023-04-12 00:12:39 -04:00 committed by GitHub
parent 4bdcedab54
commit 0ab364404e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
langchain/document_loaders/url.py
View File

@ -63,6 +63,7 @@ class UnstructuredURLLoader(BaseLoader):
except Exception as e: except Exception as e:
if self.continue_on_failure: if self.continue_on_failure:
logger.error(f"Error fetching or processing {url}, exeption: {e}") logger.error(f"Error fetching or processing {url}, exeption: {e}")
continue
else: else:
raise e raise e
text = "\n\n".join([str(el) for el in elements]) text = "\n\n".join([str(el) for el in elements])

16
tests/integration_tests/document_loaders/test_url.py Normal file
View File

@ -0,0 +1,16 @@
import pytest
from langchain.document_loaders import UnstructuredURLLoader
def test_continue_on_failure_true() -> None:
"""Test exception is not raised when continue_on_failure=True."""
loader = UnstructuredURLLoader(["badurl.foobar"])
loader.load()
def test_continue_on_failure_false() -> None:
"""Test exception is raised when continue_on_failure=False."""
loader = UnstructuredURLLoader(["badurl.foobar"], continue_on_failure=False)
with pytest.raises(Exception):
loader.load()