mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-02 11:39:18 +00:00
community[major], core[patch], langchain[patch], experimental[patch]: Create langchain-community (#14463)
Moved the following modules to new package langchain-community in a backwards compatible fashion: ``` mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community ``` Moved the following to core ``` mv langchain/langchain/utils/json_schema.py core/langchain_core/utils mv langchain/langchain/utils/html.py core/langchain_core/utils mv langchain/langchain/utils/strings.py core/langchain_core/utils cat langchain/langchain/utils/env.py >> core/langchain_core/utils/env.py rm langchain/langchain/utils/env.py ``` See .scripts/community_split/script_integrations.sh for all changes
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
from typing import Any, Sequence
|
||||
|
||||
from langchain_core.documents import BaseDocumentTransformer, Document
|
||||
|
||||
|
||||
class Html2TextTransformer(BaseDocumentTransformer):
|
||||
"""Replace occurrences of a particular search pattern with a replacement string
|
||||
|
||||
Arguments:
|
||||
ignore_links: Whether links should be ignored; defaults to True.
|
||||
ignore_images: Whether images should be ignored; defaults to True.
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
from langchain_community.document_transformers import Html2TextTransformer
|
||||
html2text = Html2TextTransformer()
|
||||
docs_transform = html2text.transform_documents(docs)
|
||||
"""
|
||||
|
||||
def __init__(self, ignore_links: bool = True, ignore_images: bool = True) -> None:
|
||||
self.ignore_links = ignore_links
|
||||
self.ignore_images = ignore_images
|
||||
|
||||
def transform_documents(
|
||||
self,
|
||||
documents: Sequence[Document],
|
||||
**kwargs: Any,
|
||||
) -> Sequence[Document]:
|
||||
try:
|
||||
import html2text
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"""html2text package not found, please
|
||||
install it with `pip install html2text`"""
|
||||
)
|
||||
|
||||
# Create a html2text.HTML2Text object and override some properties
|
||||
h = html2text.HTML2Text()
|
||||
h.ignore_links = self.ignore_links
|
||||
h.ignore_images = self.ignore_images
|
||||
|
||||
new_documents = []
|
||||
|
||||
for d in documents:
|
||||
new_document = Document(
|
||||
page_content=h.handle(d.page_content), metadata={**d.metadata}
|
||||
)
|
||||
new_documents.append(new_document)
|
||||
return new_documents
|
||||
|
||||
async def atransform_documents(
|
||||
self,
|
||||
documents: Sequence[Document],
|
||||
**kwargs: Any,
|
||||
) -> Sequence[Document]:
|
||||
raise NotImplementedError
|
Reference in New Issue
Block a user