mirror of
https://github.com/hwchase17/langchain.git
synced 2025-05-28 10:39:23 +00:00
fix (Html2TextTransformer): allow configuration of html2text (#9914)
Hi, this PR enables configuring the html2text package, instead of being bound to use the hardcoded values. While simply passing `ignore_links` and `ignore_images` to the `transform_documents` method was possible, I preferred passing them to the `__init__` method for 2 reasons: 1. It is more efficient in case of subsequent calls to `transform_documents`. 2. It allows to move the "complexity" to the instantiation, keeping the actual execution simple and general enough. IMO the transformers should all follow this pattern, allowing something like this: ```python # Instantiate transformers transformers = [ TransformerA(foo='bar'), TransformerB(bar='foo'), # others ] # During execution, call them sequentially documents = ... for tr in transformers: documents = tr.transform_documents(documents) ``` Thanks for the reviews! --------- Co-authored-by: taamedag <Davide.Menini@swisscom.com>
This commit is contained in:
parent
abd8681341
commit
b8baead70c
@ -5,13 +5,22 @@ from langchain.schema import BaseDocumentTransformer, Document
|
||||
|
||||
class Html2TextTransformer(BaseDocumentTransformer):
|
||||
"""Replace occurrences of a particular search pattern with a replacement string
|
||||
|
||||
Arguments:
|
||||
ignore_links: Whether links should be ignored; defaults to True.
|
||||
ignore_images: Whether images should be ignored; defaults to True.
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
from langchain.document_transformers import Html2TextTransformer
|
||||
html2text=Html2TextTransformer()
|
||||
docs_transform=html2text.transform_documents(docs)
|
||||
html2text = Html2TextTransformer()
|
||||
docs_transform = html2text.transform_documents(docs)
|
||||
"""
|
||||
|
||||
def __init__(self, ignore_links: bool = True, ignore_images: bool = True) -> None:
|
||||
self.ignore_links = ignore_links
|
||||
self.ignore_images = ignore_images
|
||||
|
||||
def transform_documents(
|
||||
self,
|
||||
documents: Sequence[Document],
|
||||
@ -25,10 +34,11 @@ class Html2TextTransformer(BaseDocumentTransformer):
|
||||
install it with `pip install html2text`"""
|
||||
)
|
||||
|
||||
# Create an html2text.HTML2Text object and override some properties
|
||||
# Create a html2text.HTML2Text object and override some properties
|
||||
h = html2text.HTML2Text()
|
||||
h.ignore_links = True
|
||||
h.ignore_images = True
|
||||
h.ignore_links = self.ignore_links
|
||||
h.ignore_images = self.ignore_images
|
||||
|
||||
for d in documents:
|
||||
d.page_content = h.handle(d.page_content)
|
||||
return documents
|
||||
|
Loading…
Reference in New Issue
Block a user