From 926c64da606ffc825d834acf3fb40b20f82dfbf9 Mon Sep 17 00:00:00 2001 From: Jacques Arnoux Date: Fri, 11 Aug 2023 10:50:37 -0700 Subject: [PATCH] Fix web research retriever for unknown links in results (#9115) Fixes an issue with web research retriever for unknown links in results. This is currently making the retrieve crash sometimes. @rlancemartin --- libs/langchain/langchain/retrievers/web_research.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libs/langchain/langchain/retrievers/web_research.py b/libs/langchain/langchain/retrievers/web_research.py index e8ebfd7b435..478005cb358 100644 --- a/libs/langchain/langchain/retrievers/web_research.py +++ b/libs/langchain/langchain/retrievers/web_research.py @@ -179,15 +179,16 @@ class WebResearchRetriever(BaseRetriever): logger.info(f"Questions for Google Search: {questions}") # Get urls - logger.info("Searching for relevat urls ...") + logger.info("Searching for relevant urls...") urls_to_look = [] for query in questions: # Google search search_results = self.search_tool(query, self.num_search_results) - logger.info("Searching for relevat urls ...") + logger.info("Searching for relevant urls...") logger.info(f"Search results: {search_results}") for res in search_results: - urls_to_look.append(res["link"]) + if res.get("link", None): + urls_to_look.append(res["link"]) # Relevant urls urls = set(urls_to_look)