langchain[minor]: Add serpapi tools (#13934)

- **Description:** Added some of the more endpoints supported by serpapi
that are not suported on langchain at the moment, like google trends,
google finance, google jobs, and google lens
- **Issue:** [Add support for many of the querying endpoints with
serpapi #11811](https://github.com/langchain-ai/langchain/issues/11811)

---------

Co-authored-by: zushenglu <58179949+zushenglu@users.noreply.github.com>
Co-authored-by: Erick Friis <erick@langchain.dev>
Co-authored-by: Ian Xu <ian.xu@mail.utoronto.ca>
Co-authored-by: zushenglu <zushenglu1809@gmail.com>
Co-authored-by: KevinT928 <96837880+KevinT928@users.noreply.github.com>
Co-authored-by: Bagatur <baskaryan@gmail.com>
This commit is contained in:
Jawad Arshad
2023-11-29 17:02:57 -05:00
committed by GitHub
parent dbaeb163aa
commit 00a6e8962c
19 changed files with 1662 additions and 1 deletions

View File

@@ -34,9 +34,13 @@ from langchain.tools.base import BaseTool
from langchain.tools.bing_search.tool import BingSearchRun
from langchain.tools.ddg_search.tool import DuckDuckGoSearchRun
from langchain.tools.google_cloud.texttospeech import GoogleCloudTextToSpeechTool
from langchain.tools.google_lens.tool import GoogleLensQueryRun
from langchain.tools.google_search.tool import GoogleSearchResults, GoogleSearchRun
from langchain.tools.google_scholar.tool import GoogleScholarQueryRun
from langchain.tools.google_finance.tool import GoogleFinanceQueryRun
from langchain.tools.google_trends.tool import GoogleTrendsQueryRun
from langchain.tools.metaphor_search.tool import MetaphorSearchResults
from langchain.tools.google_jobs.tool import GoogleJobsQueryRun
from langchain.tools.google_serper.tool import GoogleSerperResults, GoogleSerperRun
from langchain.tools.searchapi.tool import SearchAPIResults, SearchAPIRun
from langchain.tools.graphql.tool import BaseGraphQLTool
@@ -65,9 +69,13 @@ from langchain.utilities.golden_query import GoldenQueryAPIWrapper
from langchain.utilities.pubmed import PubMedAPIWrapper
from langchain.utilities.bing_search import BingSearchAPIWrapper
from langchain.utilities.duckduckgo_search import DuckDuckGoSearchAPIWrapper
from langchain.utilities.google_lens import GoogleLensAPIWrapper
from langchain.utilities.google_jobs import GoogleJobsAPIWrapper
from langchain.utilities.google_search import GoogleSearchAPIWrapper
from langchain.utilities.google_serper import GoogleSerperAPIWrapper
from langchain.utilities.google_scholar import GoogleScholarAPIWrapper
from langchain.utilities.google_finance import GoogleFinanceAPIWrapper
from langchain.utilities.google_trends import GoogleTrendsAPIWrapper
from langchain.utilities.metaphor_search import MetaphorSearchAPIWrapper
from langchain.utilities.awslambda import LambdaWrapper
from langchain.utilities.graphql import GraphQLAPIWrapper
@@ -238,6 +246,14 @@ def _get_pubmed(**kwargs: Any) -> BaseTool:
return PubmedQueryRun(api_wrapper=PubMedAPIWrapper(**kwargs))
def _get_google_jobs(**kwargs: Any) -> BaseTool:
return GoogleJobsQueryRun(api_wrapper=GoogleJobsAPIWrapper(**kwargs))
def _get_google_lens(**kwargs: Any) -> BaseTool:
return GoogleLensQueryRun(api_wrapper=GoogleLensAPIWrapper(**kwargs))
def _get_google_serper(**kwargs: Any) -> BaseTool:
return GoogleSerperRun(api_wrapper=GoogleSerperAPIWrapper(**kwargs))
@@ -246,6 +262,14 @@ def _get_google_scholar(**kwargs: Any) -> BaseTool:
return GoogleScholarQueryRun(api_wrapper=GoogleScholarAPIWrapper(**kwargs))
def _get_google_finance(**kwargs: Any) -> BaseTool:
return GoogleFinanceQueryRun(api_wrapper=GoogleFinanceAPIWrapper(**kwargs))
def _get_google_trends(**kwargs: Any) -> BaseTool:
return GoogleTrendsQueryRun(api_wrapper=GoogleTrendsAPIWrapper(**kwargs))
def _get_google_serper_results_json(**kwargs: Any) -> BaseTool:
return GoogleSerperResults(api_wrapper=GoogleSerperAPIWrapper(**kwargs))
@@ -373,11 +397,24 @@ _EXTRA_OPTIONAL_TOOLS: Dict[str, Tuple[Callable[[KwArg(Any)], BaseTool], List[st
"bing-search": (_get_bing_search, ["bing_subscription_key", "bing_search_url"]),
"metaphor-search": (_get_metaphor_search, ["metaphor_api_key"]),
"ddg-search": (_get_ddg_search, []),
"google-lens": (_get_google_lens, ["serp_api_key"]),
"google-serper": (_get_google_serper, ["serper_api_key", "aiosession"]),
"google-scholar": (
_get_google_scholar,
["top_k_results", "hl", "lr", "serp_api_key"],
),
"google-finance": (
_get_google_finance,
["serp_api_key"],
),
"google-trends": (
_get_google_trends,
["serp_api_key"],
),
"google-jobs": (
_get_google_jobs,
["serp_api_key"],
),
"google-serper-results-json": (
_get_google_serper_results_json,
["serper_api_key", "aiosession"],
@@ -519,13 +556,14 @@ def load_tools(
callbacks = _handle_callbacks(
callback_manager=kwargs.get("callback_manager"), callbacks=callbacks
)
# print(_BASE_TOOLS)
# print(1)
for name in tool_names:
if name == "requests":
warnings.warn(
"tool name `requests` is deprecated - "
"please use `requests_all` or specify the requests method"
)
if name == "requests_all":
# expand requests into various methods
requests_method_tools = [

View File

@@ -0,0 +1,5 @@
"""Google Finance API Toolkit."""
from langchain.tools.google_finance.tool import GoogleFinanceQueryRun
__all__ = ["GoogleFinanceQueryRun"]

View File

@@ -0,0 +1,28 @@
"""Tool for the Google Finance"""
from typing import Optional
from langchain.callbacks.manager import CallbackManagerForToolRun
from langchain.tools.base import BaseTool
from langchain.utilities.google_finance import GoogleFinanceAPIWrapper
class GoogleFinanceQueryRun(BaseTool):
"""Tool that queries the Google Finance API."""
name: str = "google_finance"
description: str = (
"A wrapper around Google Finance Search. "
"Useful for when you need to get information about"
"google search Finance from Google Finance"
"Input should be a search query."
)
api_wrapper: GoogleFinanceAPIWrapper
def _run(
self,
query: str,
run_manager: Optional[CallbackManagerForToolRun] = None,
) -> str:
"""Use the tool."""
return self.api_wrapper.run(query)

View File

@@ -0,0 +1,5 @@
"""Google Jobs API Toolkit."""
from langchain.tools.google_jobs.tool import GoogleJobsQueryRun
__all__ = ["GoogleJobsQueryRun"]

View File

@@ -0,0 +1,28 @@
"""Tool for the Google Trends"""
from typing import Optional
from langchain.callbacks.manager import CallbackManagerForToolRun
from langchain.tools.base import BaseTool
from langchain.utilities.google_jobs import GoogleJobsAPIWrapper
class GoogleJobsQueryRun(BaseTool):
"""Tool that queries the Google Jobs API."""
name: str = "google_jobs"
description: str = (
"A wrapper around Google Jobs Search. "
"Useful for when you need to get information about"
"google search Jobs from Google Jobs"
"Input should be a search query."
)
api_wrapper: GoogleJobsAPIWrapper
def _run(
self,
query: str,
run_manager: Optional[CallbackManagerForToolRun] = None,
) -> str:
"""Use the tool."""
return self.api_wrapper.run(query)

View File

@@ -0,0 +1,5 @@
"""Google Lens API Toolkit."""
from langchain.tools.google_lens.tool import GoogleLensQueryRun
__all__ = ["GoogleLensQueryRun"]

View File

@@ -0,0 +1,28 @@
"""Tool for the Google Lens"""
from typing import Optional
from langchain.callbacks.manager import CallbackManagerForToolRun
from langchain.tools.base import BaseTool
from langchain.utilities.google_lens import GoogleLensAPIWrapper
class GoogleLensQueryRun(BaseTool):
"""Tool that queries the Google Lens API."""
name: str = "google_Lens"
description: str = (
"A wrapper around Google Lens Search. "
"Useful for when you need to get information related"
"to an image from Google Lens"
"Input should be a url to an image."
)
api_wrapper: GoogleLensAPIWrapper
def _run(
self,
query: str,
run_manager: Optional[CallbackManagerForToolRun] = None,
) -> str:
"""Use the tool."""
return self.api_wrapper.run(query)

View File

@@ -0,0 +1,5 @@
"""Google Trends API Toolkit."""
from langchain.tools.google_trends.tool import GoogleTrendsQueryRun
__all__ = ["GoogleTrendsQueryRun"]

View File

@@ -0,0 +1,28 @@
"""Tool for the Google Trends"""
from typing import Optional
from langchain.callbacks.manager import CallbackManagerForToolRun
from langchain.tools.base import BaseTool
from langchain.utilities.google_trends import GoogleTrendsAPIWrapper
class GoogleTrendsQueryRun(BaseTool):
"""Tool that queries the Google trends API."""
name: str = "google_trends"
description: str = (
"A wrapper around Google Trends Search. "
"Useful for when you need to get information about"
"google search trends from Google Trends"
"Input should be a search query."
)
api_wrapper: GoogleTrendsAPIWrapper
def _run(
self,
query: str,
run_manager: Optional[CallbackManagerForToolRun] = None,
) -> str:
"""Use the tool."""
return self.api_wrapper.run(query)

View File

@@ -68,18 +68,42 @@ def _import_golden_query() -> Any:
return GoldenQueryAPIWrapper
def _import_google_lens() -> Any:
from langchain.utilities.google_lens import GoogleLensAPIWrapper
return GoogleLensAPIWrapper
def _import_google_places_api() -> Any:
from langchain.utilities.google_places_api import GooglePlacesAPIWrapper
return GooglePlacesAPIWrapper
def _import_google_jobs() -> Any:
from langchain.utilities.google_jobs import GoogleJobsAPIWrapper
return GoogleJobsAPIWrapper
def _import_google_scholar() -> Any:
from langchain.utilities.google_scholar import GoogleScholarAPIWrapper
return GoogleScholarAPIWrapper
def _import_google_trends() -> Any:
from langchain.utilities.google_trends import GoogleTrendsAPIWrapper
return GoogleTrendsAPIWrapper
def _import_google_finance() -> Any:
from langchain.utilities.google_finance import GoogleFinanceAPIWrapper
return GoogleFinanceAPIWrapper
def _import_google_search() -> Any:
from langchain.utilities.google_search import GoogleSearchAPIWrapper
@@ -243,10 +267,18 @@ def __getattr__(name: str) -> Any:
return _import_brave_search()
elif name == "DuckDuckGoSearchAPIWrapper":
return _import_duckduckgo_search()
elif name == "GoogleLensAPIWrapper":
return _import_google_lens()
elif name == "GoldenQueryAPIWrapper":
return _import_golden_query()
elif name == "GoogleJobsAPIWrapper":
return _import_google_jobs()
elif name == "GoogleScholarAPIWrapper":
return _import_google_scholar()
elif name == "GoogleFinanceAPIWrapper":
return _import_google_finance()
elif name == "GoogleTrendsAPIWrapper":
return _import_google_trends()
elif name == "GooglePlacesAPIWrapper":
return _import_google_places_api()
elif name == "GoogleSearchAPIWrapper":
@@ -311,8 +343,12 @@ __all__ = [
"BraveSearchWrapper",
"DuckDuckGoSearchAPIWrapper",
"GoldenQueryAPIWrapper",
"GoogleFinanceAPIWrapper",
"GoogleLensAPIWrapper",
"GoogleJobsAPIWrapper",
"GooglePlacesAPIWrapper",
"GoogleScholarAPIWrapper",
"GoogleTrendsAPIWrapper",
"GoogleSearchAPIWrapper",
"GoogleSerperAPIWrapper",
"GraphQLAPIWrapper",

View File

@@ -0,0 +1,97 @@
"""Util that calls Google Finance Search."""
from typing import Any, Dict, Optional, cast
from langchain.pydantic_v1 import BaseModel, Extra, SecretStr, root_validator
from langchain.utils import convert_to_secret_str, get_from_dict_or_env
class GoogleFinanceAPIWrapper(BaseModel):
"""Wrapper for SerpApi's Google Finance API
You can create SerpApi.com key by signing up at: https://serpapi.com/users/sign_up.
The wrapper uses the SerpApi.com python package:
https://serpapi.com/integrations/python
To use, you should have the environment variable ``SERPAPI_API_KEY``
set with your API key, or pass `serp_api_key` as a named parameter
to the constructor.
Example:
.. code-block:: python
from langchain.utilities import GoogleFinanceAPIWrapper
google_Finance = GoogleFinanceAPIWrapper()
google_Finance.run('langchain')
"""
serp_search_engine: Any
serp_api_key: Optional[SecretStr] = None
class Config:
"""Configuration for this pydantic object."""
extra = Extra.forbid
@root_validator()
def validate_environment(cls, values: Dict) -> Dict:
"""Validate that api key and python package exists in environment."""
values["serp_api_key"] = convert_to_secret_str(
get_from_dict_or_env(values, "serp_api_key", "SERPAPI_API_KEY")
)
try:
from serpapi import SerpApiClient
except ImportError:
raise ImportError(
"google-search-results is not installed. "
"Please install it with `pip install google-search-results"
">=2.4.2`"
)
serp_search_engine = SerpApiClient
values["serp_search_engine"] = serp_search_engine
return values
def run(self, query: str) -> str:
"""Run query through Google Finance with Serpapi"""
serpapi_api_key = cast(SecretStr, self.serp_api_key)
params = {
"engine": "google_finance",
"api_key": serpapi_api_key.get_secret_value(),
"q": query,
}
total_results = {}
client = self.serp_search_engine(params)
total_results = client.get_dict()
if not total_results:
return "Nothing was found from the query: " + query
markets = total_results.get("markets", {})
res = "\nQuery: " + query + "\n"
if "futures_chain" in total_results:
futures_chain = total_results.get("futures_chain", [])[0]
stock = futures_chain["stock"]
price = futures_chain["price"]
temp = futures_chain["price_movement"]
percentage = temp["percentage"]
movement = temp["movement"]
res += (
f"stock: {stock}\n"
+ f"price: {price}\n"
+ f"percentage: {percentage}\n"
+ f"movement: {movement}\n"
)
else:
res += "No summary information\n"
for key in markets:
if (key == "us") or (key == "asia") or (key == "europe"):
res += key
res += ": price = "
res += str(markets[key][0]["price"])
res += ", movement = "
res += markets[key][0]["price_movement"]["movement"]
res += "\n"
return res

View File

@@ -0,0 +1,80 @@
"""Util that calls Google Scholar Search."""
from typing import Any, Dict, Optional, cast
from langchain.pydantic_v1 import BaseModel, Extra, SecretStr, root_validator
from langchain.utils import convert_to_secret_str, get_from_dict_or_env
class GoogleJobsAPIWrapper(BaseModel):
"""Wrapper for SerpApi's Google Scholar API
You can create SerpApi.com key by signing up at: https://serpapi.com/users/sign_up.
The wrapper uses the SerpApi.com python package:
https://serpapi.com/integrations/python
To use, you should have the environment variable ``SERPAPI_API_KEY``
set with your API key, or pass `serp_api_key` as a named parameter
to the constructor.
Example:
.. code-block:: python
from langchain.utilities import GoogleJobsAPIWrapper
google_Jobs = GoogleJobsAPIWrapper()
google_Jobs.run('langchain')
"""
serp_search_engine: Any
serp_api_key: Optional[SecretStr] = None
class Config:
"""Configuration for this pydantic object."""
extra = Extra.forbid
@root_validator()
def validate_environment(cls, values: Dict) -> Dict:
"""Validate that api key and python package exists in environment."""
values["serp_api_key"] = convert_to_secret_str(
get_from_dict_or_env(values, "serp_api_key", "SERPAPI_API_KEY")
)
try:
from serpapi import SerpApiClient
except ImportError:
raise ImportError(
"google-search-results is not installed. "
"Please install it with `pip install google-search-results"
">=2.4.2`"
)
serp_search_engine = SerpApiClient
values["serp_search_engine"] = serp_search_engine
return values
def run(self, query: str) -> str:
"""Run query through Google Trends with Serpapi"""
# set up query
serpapi_api_key = cast(SecretStr, self.serp_api_key)
params = {
"engine": "google_jobs",
"api_key": serpapi_api_key.get_secret_value(),
"q": query,
}
total_results = []
client = self.serp_search_engine(params)
total_results = client.get_dict()["jobs_results"]
# extract 1 job info:
res_str = ""
for i in range(1):
job = total_results[i]
res_str += (
"\n_______________________________________________"
+ f"\nJob Title: {job['title']}\n"
+ f"Company Name: {job['company_name']}\n"
+ f"Location: {job['location']}\n"
+ f"Description: {job['description']}"
+ "\n_______________________________________________\n"
)
return res_str + "\n"

View File

@@ -0,0 +1,85 @@
"""Util that calls Google Lens Search."""
from typing import Any, Dict, Optional, cast
import requests
from langchain.pydantic_v1 import BaseModel, Extra, SecretStr, root_validator
from langchain.utils import convert_to_secret_str, get_from_dict_or_env
class GoogleLensAPIWrapper(BaseModel):
"""Wrapper for SerpApi's Google Lens API
You can create SerpApi.com key by signing up at: https://serpapi.com/users/sign_up.
The wrapper uses the SerpApi.com python package:
https://serpapi.com/integrations/python
To use, you should have the environment variable ``SERPAPI_API_KEY``
set with your API key, or pass `serp_api_key` as a named parameter
to the constructor.
Example:
.. code-block:: python
from langchain.utilities import GoogleLensAPIWrapper
google_lens = GoogleLensAPIWrapper()
google_lens.run('langchain')
"""
serp_search_engine: Any
serp_api_key: Optional[SecretStr] = None
class Config:
"""Configuration for this pydantic object."""
extra = Extra.forbid
@root_validator()
def validate_environment(cls, values: Dict) -> Dict:
"""Validate that api key and python package exists in environment."""
values["serp_api_key"] = convert_to_secret_str(
get_from_dict_or_env(values, "serp_api_key", "SERPAPI_API_KEY")
)
return values
def run(self, query: str) -> str:
"""Run query through Google Trends with Serpapi"""
serpapi_api_key = cast(SecretStr, self.serp_api_key)
params = {
"engine": "google_lens",
"api_key": serpapi_api_key.get_secret_value(),
"url": query,
}
queryURL = f"https://serpapi.com/search?engine={params['engine']}&api_key={params['api_key']}&url={params['url']}"
response = requests.get(queryURL)
if response.status_code != 200:
return "Google Lens search failed"
responseValue = response.json()
if responseValue["search_metadata"]["status"] != "Success":
return "Google Lens search failed"
xs = ""
if len(responseValue["knowledge_graph"]) > 0:
subject = responseValue["knowledge_graph"][0]
xs += f"Subject:{subject['title']}({subject['subtitle']})\n"
xs += f"Link to subject:{subject['link']}\n\n"
xs += "Related Images:\n\n"
for image in responseValue["visual_matches"]:
xs += f"Title: {image['title']}\n"
xs += f"Source({image['source']}): {image['link']}\n"
xs += f"Image: {image['thumbnail']}\n\n"
xs += (
"Reverse Image Search"
+ f"Link: {responseValue['reverse_image_search']['link']}\n"
)
print(xs)
docs = [xs]
return "\n\n".join(docs)

View File

@@ -0,0 +1,116 @@
"""Util that calls Google Scholar Search."""
from typing import Any, Dict, Optional, cast
from langchain.pydantic_v1 import BaseModel, Extra, SecretStr, root_validator
from langchain.utils import convert_to_secret_str, get_from_dict_or_env
class GoogleTrendsAPIWrapper(BaseModel):
"""Wrapper for SerpApi's Google Scholar API
You can create SerpApi.com key by signing up at: https://serpapi.com/users/sign_up.
The wrapper uses the SerpApi.com python package:
https://serpapi.com/integrations/python
To use, you should have the environment variable ``SERPAPI_API_KEY``
set with your API key, or pass `serp_api_key` as a named parameter
to the constructor.
Example:
.. code-block:: python
from langchain.utilities import GoogleTrendsAPIWrapper
google_trends = GoogleTrendsAPIWrapper()
google_trends.run('langchain')
"""
serp_search_engine: Any
serp_api_key: Optional[SecretStr] = None
class Config:
"""Configuration for this pydantic object."""
extra = Extra.forbid
@root_validator()
def validate_environment(cls, values: Dict) -> Dict:
"""Validate that api key and python package exists in environment."""
values["serp_api_key"] = convert_to_secret_str(
get_from_dict_or_env(values, "serp_api_key", "SERPAPI_API_KEY")
)
try:
from serpapi import SerpApiClient
except ImportError:
raise ImportError(
"google-search-results is not installed. "
"Please install it with `pip install google-search-results"
">=2.4.2`"
)
serp_search_engine = SerpApiClient
values["serp_search_engine"] = serp_search_engine
return values
def run(self, query: str) -> str:
"""Run query through Google Trends with Serpapi"""
serpapi_api_key = cast(SecretStr, self.serp_api_key)
params = {
"engine": "google_trends",
"api_key": serpapi_api_key.get_secret_value(),
"q": query,
}
total_results = []
client = self.serp_search_engine(params)
total_results = client.get_dict()["interest_over_time"]["timeline_data"]
if not total_results:
return "No good Trend Result was found"
start_date = total_results[0]["date"].split()
end_date = total_results[-1]["date"].split()
values = [
results.get("values")[0].get("extracted_value") for results in total_results
]
min_value = min(values)
max_value = max(values)
avg_value = sum(values) / len(values)
percentage_change = (
(values[-1] - values[0])
/ (values[0] if values[0] != 0 else 1)
* (100 if values[0] != 0 else 1)
)
params = {
"engine": "google_trends",
"api_key": serpapi_api_key.get_secret_value(),
"data_type": "RELATED_QUERIES",
"q": query,
}
total_results2 = {}
client = self.serp_search_engine(params)
total_results2 = client.get_dict().get("related_queries", {})
rising = []
top = []
rising = [results.get("query") for results in total_results2.get("rising", [])]
top = [results.get("query") for results in total_results2.get("top", [])]
doc = [
f"Query: {query}\n"
f"Date From: {start_date[0]} {start_date[1]}, {start_date[-1]}\n"
f"Date To: {end_date[0]} {end_date[3]} {end_date[-1]}\n"
f"Min Value: {min_value}\n"
f"Max Value: {max_value}\n"
f"Average Value: {avg_value}\n"
f"Percent Change: {str(percentage_change) + '%'}\n"
f"Trend values: {', '.join([str(x) for x in values])}\n"
f"Rising Related Queries: {', '.join(rising)}\n"
f"Top Related Queries: {', '.join(top)}"
]
return "\n\n".join(doc)

View File

@@ -10,10 +10,14 @@ EXPECTED_ALL = [
"BraveSearchWrapper",
"DuckDuckGoSearchAPIWrapper",
"GoldenQueryAPIWrapper",
"GoogleFinanceAPIWrapper",
"GoogleJobsAPIWrapper",
"GoogleLensAPIWrapper",
"GooglePlacesAPIWrapper",
"GoogleScholarAPIWrapper",
"GoogleSearchAPIWrapper",
"GoogleSerperAPIWrapper",
"GoogleTrendsAPIWrapper",
"GraphQLAPIWrapper",
"JiraAPIWrapper",
"LambdaWrapper",