diff --git a/langchain/chains/research/api.py b/langchain/chains/research/api.py index f65b6a58719..f5a771ef48d 100644 --- a/langchain/chains/research/api.py +++ b/langchain/chains/research/api.py @@ -85,10 +85,12 @@ class Research(Chain): urls = search_results["urls"] blobs = self.downloader.download(urls) parser = MarkdownifyHTMLParser() - docs = itertools.chain.from_iterable(parser.lazy_parse(blob) for blob in blobs) - inputs = [{"doc": doc, "question": question} for doc in docs] + docs = itertools.chain.from_iterable( + parser.lazy_parse(blob) for blob in blobs if blob is not None + ) + _inputs = [{"doc": doc, "question": question} for doc in docs] results = self.reader( - inputs, callbacks=run_manager.get_child() if run_manager else None + _inputs, callbacks=run_manager.get_child() if run_manager else None ) return { "docs": [result["answer"] for result in results["inputs"]], @@ -111,9 +113,9 @@ class Research(Chain): docs = itertools.chain.from_iterable( parser.lazy_parse(blob) for blob in blobs if blob is not None ) - inputs = [{"doc": doc, "question": question} for doc in docs] + _inputs = [{"doc": doc, "question": question} for doc in docs] results = await self.reader.acall( - inputs, + _inputs, callbacks=run_manager.get_child() if run_manager else None, ) return { @@ -157,7 +159,9 @@ class Research(Chain): if isinstance(text_splitter, str): if text_splitter == "recursive": _text_splitter_kwargs = text_splitter_kwargs or {} - _text_splitter = RecursiveCharacterTextSplitter(**_text_splitter_kwargs) + _text_splitter: TextSplitter = RecursiveCharacterTextSplitter( + **_text_splitter_kwargs + ) else: raise ValueError(f"Invalid text splitter: {text_splitter}") elif isinstance(text_splitter, TextSplitter): @@ -167,7 +171,7 @@ class Research(Chain): if isinstance(download_handler, str): if download_handler == "auto": - _download_handler = AutoDownloadHandler() + _download_handler: DownloadHandler = AutoDownloadHandler() else: raise ValueError(f"Invalid download handler: {download_handler}") elif isinstance(download_handler, DownloadHandler): diff --git a/langchain/chains/research/crawling.py b/langchain/chains/research/crawling.py index 00fef1c1400..13255e1ca5d 100644 --- a/langchain/chains/research/crawling.py +++ b/langchain/chains/research/crawling.py @@ -79,6 +79,8 @@ class ChainCrawler(BlobCrawler): def crawl(self, blob: Blob, question: str) -> List[str]: """Explore the blob and suggest additional content to explore.""" + if not blob.source: + raise NotImplementedError() records, columns = _extract_records(blob) result = self.chain( diff --git a/langchain/chains/research/search.py b/langchain/chains/research/search.py index 63dbf29d21b..9bf50e23316 100644 --- a/langchain/chains/research/search.py +++ b/langchain/chains/research/search.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio +import typing from typing import Any, Dict, List, Mapping, Optional, Sequence from bs4 import BeautifulSoup @@ -257,8 +258,12 @@ class GenericSearcher(Chain): run_manager: Optional[CallbackManagerForChainRun] = None, ) -> Dict[str, Any]: question = inputs["question"] - queries = self.query_generator.predict_and_parse( - callbacks=run_manager.get_child(), question=question + queries = typing.cast( + List[str], + self.query_generator.predict_and_parse( + callbacks=run_manager.get_child() if run_manager else None, + question=question, + ), ) results = _run_searches(queries, top_k=self.top_k_per_search) deuped_results = _deduplicate_objects(results, "link") @@ -271,7 +276,7 @@ class GenericSearcher(Chain): "question": question, "choices": records, }, - callbacks=run_manager.get_child(), + callbacks=run_manager.get_child() if run_manager else None, ) return {"urls": [result["link"] for result in response_["selected"]]} @@ -281,8 +286,12 @@ class GenericSearcher(Chain): run_manager: Optional[AsyncCallbackManagerForChainRun] = None, ) -> Dict[str, Any]: question = inputs["question"] - queries = await self.query_generator.apredict_and_parse( - callbacks=run_manager.get_child(), question=question + queries = typing.cast( + List[str], + await self.query_generator.apredict_and_parse( + callbacks=run_manager.get_child() if run_manager else None, + question=question, + ), ) results = _run_searches(queries, top_k=self.top_k_per_search) deuped_results = _deduplicate_objects(results, "link") @@ -295,7 +304,7 @@ class GenericSearcher(Chain): "question": question, "choices": records, }, - callbacks=run_manager.get_child(), + callbacks=run_manager.get_child() if run_manager else None, ) return {"urls": [result["link"] for result in response_["selected"]]} diff --git a/tests/unit_tests/chains/research/test_downloader.py b/tests/unit_tests/chains/research/test_downloader.py index 50ef2d15e71..fa270fb88cf 100644 --- a/tests/unit_tests/chains/research/test_downloader.py +++ b/tests/unit_tests/chains/research/test_downloader.py @@ -4,7 +4,7 @@ from langchain.chains.research.download import ( ) -def test_is_javascript_required(): +def test_is_javascript_required() -> None: """Check whether a given page should be re-downloaded with javascript executed.""" assert not _is_javascript_required( """