cr

Typing, change var, etc
Revert pyproject.toml changes
2026-01-30 05:47:54 +00:00 · 2023-10-27 15:39:50 -07:00 · 2023-10-27 16:42:20 -04:00 · 2023-10-26 19:56:10 -04:00 · 2023-10-26 19:55:33 -04:00 · 2023-10-26 18:59:46 -04:00
4 changed files with 166 additions and 15 deletions
--- a/libs/langchain/langchain/document_loaders/readthedocs.py
+++ b/libs/langchain/langchain/document_loaders/readthedocs.py
@@ -1,9 +1,19 @@
+from __future__ import annotations
+
+import logging
 from pathlib import Path
-from typing import Any, List, Optional, Sequence, Tuple, Union
+from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Sequence, Tuple, Union

 from langchain.docstore.document import Document
 from langchain.document_loaders.base import BaseLoader

+if TYPE_CHECKING:
+    from bs4 import NavigableString
+    from bs4.element import Comment, Tag
+
+
+logger = logging.getLogger(__name__)
+

 class ReadTheDocsLoader(BaseLoader):
    """Load `ReadTheDocs` documentation directory."""
@@ -15,7 +25,8 @@ class ReadTheDocsLoader(BaseLoader):
        errors: Optional[str] = None,
        custom_html_tag: Optional[Tuple[str, dict]] = None,
        patterns: Sequence[str] = ("*.htm", "*.html"),
-        **kwargs: Optional[Any]
+        exclude_links_ratio: float = 1.0,
+        **kwargs: Optional[Any],
    ):
        """
        Initialize ReadTheDocsLoader
@@ -36,6 +47,9 @@ class ReadTheDocsLoader(BaseLoader):
            custom_html_tag: Optional custom html tag to retrieve the content from
                files.
            patterns: The file patterns to load, passed to `glob.rglob`.
+            exclude_links_ratio: The ratio of links:content to exclude pages from.
+                This is to reduce the frequency at which index pages make their
+                way into retrieved results. Recommended: 0.5
            kwargs: named arguments passed to `bs4.BeautifulSoup`.
        """
        try:
@@ -48,7 +62,9 @@ class ReadTheDocsLoader(BaseLoader):

        try:
            _ = BeautifulSoup(
-                "<html><body>Parser builder library test.</body></html>", **kwargs
+                "<html><body>Parser builder library test.</body></html>",
+                "html.parser",
+                **kwargs,
            )
        except Exception as e:
            raise ValueError("Parsing kwargs do not appear valid") from e
@@ -59,24 +75,26 @@ class ReadTheDocsLoader(BaseLoader):
        self.custom_html_tag = custom_html_tag
        self.patterns = patterns
        self.bs_kwargs = kwargs
+        self.exclude_links_ratio = exclude_links_ratio

-    def load(self) -> List[Document]:
-        """Load documents."""
-        docs = []
+    def lazy_load(self) -> Iterator[Document]:
+        """A lazy loader for Documents."""
        for file_pattern in self.patterns:
            for p in self.file_path.rglob(file_pattern):
                if p.is_dir():
                    continue
                with open(p, encoding=self.encoding, errors=self.errors) as f:
                    text = self._clean_data(f.read())
-                metadata = {"source": str(p)}
-                docs.append(Document(page_content=text, metadata=metadata))
-        return docs
+                yield Document(page_content=text, metadata={"source": str(p)})
+
+    def load(self) -> List[Document]:
+        """Load documents."""
+        return list(self.lazy_load())

    def _clean_data(self, data: str) -> str:
        from bs4 import BeautifulSoup

-        soup = BeautifulSoup(data, **self.bs_kwargs)
+        soup = BeautifulSoup(data, "html.parser", **self.bs_kwargs)

        # default tags
        html_tags = [
@@ -87,18 +105,122 @@ class ReadTheDocsLoader(BaseLoader):
        if self.custom_html_tag is not None:
            html_tags.append(self.custom_html_tag)

-        text = None
+        element = None

        # reversed order. check the custom one first
        for tag, attrs in html_tags[::-1]:
-            text = soup.find(tag, attrs)
+            element = soup.find(tag, attrs)
            # if found, break
-            if text is not None:
+            if element is not None:
                break

-        if text is not None:
-            text = text.get_text()
+        if element is not None and _get_link_ratio(element) <= self.exclude_links_ratio:
+            logger.info("=" * 100, "\n", element, type(element), "\n", "=" * 100)
+            text = _get_clean_text(element)
        else:
            text = ""
        # trim empty lines
        return "\n".join([t for t in text.split("\n") if t])
+
+
+def _get_clean_text(element: Tag) -> str:
+    """Returns cleaned text with newlines preserved and irrelevant elements removed."""
+    elements_to_skip = [
+        "script",
+        "noscript",
+        "canvas",
+        "meta",
+        "svg",
+        "map",
+        "area",
+        "audio",
+        "source",
+        "track",
+        "video",
+        "embed",
+        "object",
+        "param",
+        "picture",
+        "iframe",
+        "frame",
+        "frameset",
+        "noframes",
+        "applet",
+        "form",
+        "button",
+        "select",
+        "base",
+        "style",
+        "img",
+    ]
+
+    newline_elements = [
+        "p",
+        "div",
+        "ul",
+        "ol",
+        "li",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "pre",
+        "table",
+        "tr",
+    ]
+
+    text = _process_element(element, elements_to_skip, newline_elements)
+    return text.strip()
+
+
+def _get_link_ratio(section: Tag) -> float:
+    links = section.find_all("a")
+    total_text = "".join(str(s) for s in section.stripped_strings)
+    if len(total_text) == 0:
+        return 0
+
+    link_text = "".join(
+        str(string.string.strip())
+        for link in links
+        for string in link.strings
+        if string
+    )
+    return len(link_text) / len(total_text)
+
+
+def _process_element(
+    element: Union[Tag, NavigableString, Comment],
+    elements_to_skip: List[str],
+    newline_elements: List[str],
+) -> str:
+    """
+    Traverse through HTML tree recursively to preserve newline and skip
+    unwanted (code/binary) elements
+    """
+    from bs4 import NavigableString
+    from bs4.element import Comment, Tag
+
+    tag_name = getattr(element, "name", None)
+    if isinstance(element, Comment) or tag_name in elements_to_skip:
+        return ""
+    elif isinstance(element, NavigableString):
+        return element
+    elif tag_name == "br":
+        return "\n"
+    elif tag_name in newline_elements:
+        return (
+            "".join(
+                _process_element(child, elements_to_skip, newline_elements)
+                for child in element.children
+                if isinstance(child, (Tag, NavigableString, Comment))
+            )
+            + "\n"
+        )
+    else:
+        return "".join(
+            _process_element(child, elements_to_skip, newline_elements)
+            for child in element.children
+            if isinstance(child, (Tag, NavigableString, Comment))
+        )
--- a/libs/langchain/tests/unit_tests/document_loaders/test_docs/readthedocs/index_page/test.html
+++ b/libs/langchain/tests/unit_tests/document_loaders/test_docs/readthedocs/index_page/test.html
@@ -0,0 +1,10 @@
+<html>
+  <main id="main-content">
+    Websites:
+    <a href="https://langchain.com">Langchain</a>
+    <a href="https://docs.langchain.com">Langchain Docs</a>
+    <a href="https://api.python.langchain.com/en/latest/api_reference.html"
+      >Langchain API Reference</a
+    >
+  </main>
+</html>
--- a/libs/langchain/tests/unit_tests/document_loaders/test_docs/readthedocs/nested_html_structure/test.html
+++ b/libs/langchain/tests/unit_tests/document_loaders/test_docs/readthedocs/nested_html_structure/test.html
@@ -0,0 +1,5 @@
+<html>
+  <main id="main-content">
+    Hello <span><em>World</em>!</span>
+  </main>
+</html>
--- a/libs/langchain/tests/unit_tests/document_loaders/test_readthedoc.py
+++ b/libs/langchain/tests/unit_tests/document_loaders/test_readthedoc.py
@@ -31,6 +31,20 @@ def test_custom() -> None:
    assert len(documents[0].page_content) != 0


+@pytest.mark.requires("bs4")
+def test_nested_html_structure() -> None:
+    loader = ReadTheDocsLoader(PARENT_DIR / "nested_html_structure")
+    documents = loader.load()
+    assert documents[0].page_content == "Hello World!"
+
+
+@pytest.mark.requires("bs4")
+def test_index_page() -> None:
+    loader = ReadTheDocsLoader(PARENT_DIR / "index_page", exclude_links_ratio=0.5)
+    documents = loader.load()
+    assert len(documents[0].page_content) == 0
+
+
@pytest.mark.requires("bs4")
 def test_empty() -> None:
    loader = ReadTheDocsLoader(
Author	SHA1	Message	Date
Bagatur	2b2912b020	cr	2023-10-27 15:39:50 -07:00
Andrew Zhou	2c334d4d4f	Typing, change var, etc	2023-10-27 16:42:20 -04:00
Andrew Zhou	f19dc245a7	Revert pyproject.toml changes	2023-10-26 19:56:10 -04:00
Andrew Zhou	51cee6ddaf	Revert poetry.lock changes	2023-10-26 19:55:33 -04:00
Andrew Zhou	4f2a6ae8dd	Write tests	2023-10-26 18:59:46 -04:00
Andrew Zhou	b4fa3bd9f6	Clean code	2023-10-26 18:35:19 -04:00
Andrew Zhou	a904b8423b	Add exclude index pages option; preserve newline data dnd exclude binary/code information from readthedocs	2023-10-26 18:04:09 -04:00