Harrison/nasa tool (#14245)

Co-authored-by: Jacob Matias <88005863+matiasjacob25@users.noreply.github.com>
Co-authored-by: Karam Daid <karam.daid@mail.utoronto.ca>
Co-authored-by: Jumana <jumana.fanous@mail.utoronto.ca>
Co-authored-by: KaramDaid <38271127+KaramDaid@users.noreply.github.com>
Co-authored-by: Anna Chester <74325334+CodeMakesMeSmile@users.noreply.github.com>
Co-authored-by: Jumana <144748640+jfanous@users.noreply.github.com>

libs/langchain/langchain/agents/agent_toolkits/__init__.py

@@ -34,6 +34,7 @@ from langchain.agents.agent_toolkits.jira.toolkit import JiraToolkit
 from langchain.agents.agent_toolkits.json.base import create_json_agent
 from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit
 from langchain.agents.agent_toolkits.multion.toolkit import MultionToolkit
+from langchain.agents.agent_toolkits.nasa.toolkit import NasaToolkit
 from langchain.agents.agent_toolkits.nla.toolkit import NLAToolkit
 from langchain.agents.agent_toolkits.office365.toolkit import O365Toolkit
 from langchain.agents.agent_toolkits.openapi.base import create_openapi_agent
@@ -93,6 +94,7 @@ __all__ = [
     "JiraToolkit",
     "JsonToolkit",
     "MultionToolkit",
+    "NasaToolkit",
     "NLAToolkit",
     "O365Toolkit",
     "OpenAPIToolkit",

libs/langchain/langchain/agents/agent_toolkits/nasa/__init__.py

@@ -0,0 +1 @@
+"""NASA Toolkit"""

libs/langchain/langchain/agents/agent_toolkits/nasa/toolkit.py

@@ -0,0 +1,57 @@
+from typing import Dict, List
+
+from langchain.agents.agent_toolkits.base import BaseToolkit
+from langchain.tools import BaseTool
+from langchain.tools.nasa.prompt import (
+    NASA_CAPTIONS_PROMPT,
+    NASA_MANIFEST_PROMPT,
+    NASA_METADATA_PROMPT,
+    NASA_SEARCH_PROMPT,
+)
+from langchain.tools.nasa.tool import NasaAction
+from langchain.utilities.nasa import NasaAPIWrapper
+
+
+class NasaToolkit(BaseToolkit):
+    """Nasa Toolkit."""
+
+    tools: List[BaseTool] = []
+
+    @classmethod
+    def from_nasa_api_wrapper(cls, nasa_api_wrapper: NasaAPIWrapper) -> "NasaToolkit":
+        operations: List[Dict] = [
+            {
+                "mode": "search_media",
+                "name": "Search NASA Image and Video Library media",
+                "description": NASA_SEARCH_PROMPT,
+            },
+            {
+                "mode": "get_media_metadata_manifest",
+                "name": "Get NASA Image and Video Library media metadata manifest",
+                "description": NASA_MANIFEST_PROMPT,
+            },
+            {
+                "mode": "get_media_metadata_location",
+                "name": "Get NASA Image and Video Library media metadata location",
+                "description": NASA_METADATA_PROMPT,
+            },
+            {
+                "mode": "get_video_captions_location",
+                "name": "Get NASA Image and Video Library video captions location",
+                "description": NASA_CAPTIONS_PROMPT,
+            },
+        ]
+        tools = [
+            NasaAction(
+                name=action["name"],
+                description=action["description"],
+                mode=action["mode"],
+                api_wrapper=nasa_api_wrapper,
+            )
+            for action in operations
+        ]
+        return cls(tools=tools)
+
+    def get_tools(self) -> List[BaseTool]:
+        """Get the tools in the toolkit."""
+        return self.tools

libs/langchain/langchain/tools/__init__.py

@@ -338,6 +338,12 @@ def _import_metaphor_search() -> Any:
     return MetaphorSearchResults
 
 
+def _import_nasa_tool() -> Any:
+    from langchain.tools.nasa.tool import NasaAction
+
+    return NasaAction
+
+
 def _import_office365_create_draft_message() -> Any:
     from langchain.tools.office365.create_draft_message import O365CreateDraftMessage
 
@@ -831,6 +837,8 @@ def __getattr__(name: str) -> Any:
         return _import_merriam_webster_tool()
     elif name == "MetaphorSearchResults":
         return _import_metaphor_search()
+    elif name == "NasaAction":
+        return _import_nasa_tool()
     elif name == "O365CreateDraftMessage":
         return _import_office365_create_draft_message()
     elif name == "O365SearchEvents":
@@ -1030,6 +1038,7 @@ __all__ = [
     "MerriamWebsterQueryRun",
     "MetaphorSearchResults",
     "MoveFileTool",
+    "NasaAction",
     "NavigateBackTool",
     "NavigateTool",
     "O365CreateDraftMessage",

libs/langchain/langchain/tools/nasa/prompt.py

@@ -0,0 +1,82 @@
+# flake8: noqa
+NASA_SEARCH_PROMPT = """
+    This tool is a wrapper around NASA's search API, useful when you need to search through NASA's Image and Video Library. 
+    The input to this tool is a query specified by the user, and will be passed into NASA's `search` function.
+    
+    At least one parameter must be provided.
+
+    There are optional parameters that can be passed by the user based on their query
+    specifications. Each item in this list contains pound sign (#) separated values, the first value is the parameter name, 
+    the second value is the datatype and the third value is the description: {{
+
+        - q#string#Free text search terms to compare to all indexed metadata.
+        - center#string#NASA center which published the media.
+        - description#string#Terms to search for in “Description” fields.
+        - description_508#string#Terms to search for in “508 Description” fields.
+        - keywords #string#Terms to search for in “Keywords” fields. Separate multiple values with commas.
+        - location #string#Terms to search for in “Location” fields.
+        - media_type#string#Media types to restrict the search to. Available types: [“image”,“video”, “audio”]. Separate multiple values with commas.
+        - nasa_id #string#The media asset’s NASA ID.
+        - page#integer#Page number, starting at 1, of results to get.-
+        - page_size#integer#Number of results per page. Default: 100.
+        - photographer#string#The primary photographer’s name.
+        - secondary_creator#string#A secondary photographer/videographer’s name.
+        - title #string#Terms to search for in “Title” fields.
+        - year_start#string#The start year for results. Format: YYYY.
+        - year_end #string#The end year for results. Format: YYYY.
+
+    }}
+    
+    Below are several task descriptions along with their respective input examples.
+    Task: get the 2nd page of image and video content starting from the year 2002 to 2010
+    Example Input: {{"year_start":  "2002", "year_end":  "2010", "page": 2}}
+    
+    Task: get the image and video content of saturn photographed by John Appleseed
+    Example Input: {{"q": "saturn", "photographer": "John Appleseed"}}
+    
+    Task: search for Meteor Showers with description "Search Description" with media type image
+    Example Input: {{"q": "Meteor Shower", "description": "Search Description", "media_type": "image"}}
+    
+    Task: get the image and video content from year 2008 to 2010 from Kennedy Center
+    Example Input: {{"year_start":  "2002", "year_end":  "2010", "location": "Kennedy Center}}
+    """
+
+
+NASA_MANIFEST_PROMPT = """
+    This tool is a wrapper around NASA's media asset manifest API, useful when you need to retrieve a media 
+    asset's manifest. The input to this tool should include a string representing a NASA ID for a media asset that the user is trying to get the media asset manifest data for. The NASA ID will be passed as a string into NASA's `get_media_metadata_manifest` function.
+
+    The following list are some examples of NASA IDs for a media asset that you can use to better extract the NASA ID from the input string to the tool.
+    - GSFC_20171102_Archive_e000579
+    - Launch-Sound_Delta-PAM-Random-Commentary
+    - iss066m260341519_Expedition_66_Education_Inflight_with_Random_Lake_School_District_220203
+    - 6973610
+    - GRC-2020-CM-0167.4
+    - Expedition_55_Inflight_Japan_VIP_Event_May_31_2018_659970
+    - NASA 60th_SEAL_SLIVER_150DPI
+"""
+
+NASA_METADATA_PROMPT = """
+    This tool is a wrapper around NASA's media asset metadata location API, useful when you need to retrieve the media asset's metadata. The input to this tool should include a string representing a NASA ID for a media asset that the user is trying to get the media asset metadata location for. The NASA ID will be passed as a string into NASA's `get_media_metadata_manifest` function.
+
+    The following list are some examples of NASA IDs for a media asset that you can use to better extract the NASA ID from the input string to the tool.
+    - GSFC_20171102_Archive_e000579
+    - Launch-Sound_Delta-PAM-Random-Commentary
+    - iss066m260341519_Expedition_66_Education_Inflight_with_Random_Lake_School_District_220203
+    - 6973610
+    - GRC-2020-CM-0167.4
+    - Expedition_55_Inflight_Japan_VIP_Event_May_31_2018_659970
+    - NASA 60th_SEAL_SLIVER_150DPI
+"""
+
+NASA_CAPTIONS_PROMPT = """
+    This tool is a wrapper around NASA's video assests caption location API, useful when you need 
+    to retrieve the location of the captions of a specific video. The input to this tool should include a string representing a NASA ID for a video media asset that the user is trying to get the get the location of the captions for. The NASA ID will be passed as a string into NASA's `get_media_metadata_manifest` function.
+
+    The following list are some examples of NASA IDs for a video asset that you can use to better extract the NASA ID from the input string to the tool.
+    - 2017-08-09 - Video File RS-25 Engine Test
+    - 20180415-TESS_Social_Briefing
+    - 201_TakingWildOutOfWildfire
+    - 2022-H1_V_EuropaClipper-4
+    - 2022_0429_Recientemente
+"""

libs/langchain/langchain/tools/nasa/tool.py

@@ -0,0 +1,28 @@
+"""
+This tool allows agents to interact with the NASA API, specifically 
+the the NASA Image & Video Library and Exoplanet
+"""
+
+from typing import Optional
+
+from langchain.callbacks.manager import CallbackManagerForToolRun
+from langchain.pydantic_v1 import Field
+from langchain.tools.base import BaseTool
+from langchain.utilities.nasa import NasaAPIWrapper
+
+
+class NasaAction(BaseTool):
+    """Tool that queries the Atlassian Jira API."""
+
+    api_wrapper: NasaAPIWrapper = Field(default_factory=NasaAPIWrapper)
+    mode: str
+    name: str = ""
+    description: str = ""
+
+    def _run(
+        self,
+        instructions: str,
+        run_manager: Optional[CallbackManagerForToolRun] = None,
+    ) -> str:
+        """Use the NASA API to run an operation."""
+        return self.api_wrapper.run(self.mode, instructions)

libs/langchain/langchain/utilities/__init__.py

@@ -260,6 +260,12 @@ def _import_zapier() -> Any:
     return ZapierNLAWrapper
 
 
+def _import_nasa() -> Any:
+    from langchain.utilities.nasa import NasaAPIWrapper
+
+    return NasaAPIWrapper
+
+
 def __getattr__(name: str) -> Any:
     if name == "AlphaVantageAPIWrapper":
         return _import_alpha_vantage()
@@ -307,6 +313,8 @@ def __getattr__(name: str) -> Any:
         return _import_merriam_webster()
     elif name == "MetaphorSearchAPIWrapper":
         return _import_metaphor_search()
+    elif name == "NasaAPIWrapper":
+        return _import_nasa()
     elif name == "OpenWeatherMapAPIWrapper":
         return _import_openweathermap()
     elif name == "OutlineAPIWrapper":
@@ -373,6 +381,7 @@ __all__ = [
     "MaxComputeAPIWrapper",
     "MerriamWebsterAPIWrapper",
     "MetaphorSearchAPIWrapper",
+    "NasaAPIWrapper",
     "OpenWeatherMapAPIWrapper",
     "OutlineAPIWrapper",
     "Portkey",

libs/langchain/langchain/utilities/nasa.py

@@ -0,0 +1,52 @@
+"""Util that calls several NASA APIs."""
+import json
+
+import requests
+
+from langchain.pydantic_v1 import BaseModel
+
+IMAGE_AND_VIDEO_LIBRARY_URL = "https://images-api.nasa.gov"
+
+
+class NasaAPIWrapper(BaseModel):
+    def get_media(self, query: str) -> str:
+        params = json.loads(query)
+        if params.get("q"):
+            queryText = params["q"]
+            params.pop("q")
+        else:
+            queryText = ""
+        response = requests.get(
+            IMAGE_AND_VIDEO_LIBRARY_URL + "/search?q=" + queryText, params=params
+        )
+        data = response.json()
+        return data
+
+    def get_media_metadata_manifest(self, query: str) -> str:
+        response = requests.get(IMAGE_AND_VIDEO_LIBRARY_URL + "/asset/" + query)
+        return response.json()
+
+    def get_media_metadata_location(self, query: str) -> str:
+        response = requests.get(IMAGE_AND_VIDEO_LIBRARY_URL + "/metadata/" + query)
+        return response.json()
+
+    def get_video_captions_location(self, query: str) -> str:
+        response = requests.get(IMAGE_AND_VIDEO_LIBRARY_URL + "/captions/" + query)
+        return response.json()
+
+    def run(self, mode: str, query: str) -> str:
+        if mode == "search_media":
+            output = self.get_media(query)
+        elif mode == "get_media_metadata_manifest":
+            output = self.get_media_metadata_manifest(query)
+        elif mode == "get_media_metadata_location":
+            output = self.get_media_metadata_location(query)
+        elif mode == "get_video_captions_location":
+            output = self.get_video_captions_location(query)
+        else:
+            output = f"ModeError: Got unexpected mode {mode}."
+
+        try:
+            return json.dumps(output)
+        except Exception:
+            return str(output)

libs/langchain/tests/integration_tests/utilities/test_nasa.py

@@ -0,0 +1,32 @@
+"""Integration test for NASA API Wrapper."""
+from langchain.utilities.nasa import NasaAPIWrapper
+
+
+def test_media_search() -> None:
+    """Test for NASA Image and Video Library media search"""
+    nasa = NasaAPIWrapper()
+    query = '{"q": "saturn", + "year_start": "2002", "year_end": "2010", "page": 2}'
+    output = nasa.run("search_media", query)
+    assert output is not None
+    assert "collection" in output
+
+
+def test_get_media_metadata_manifest() -> None:
+    """Test for retrieving media metadata manifest from NASA Image and Video Library"""
+    nasa = NasaAPIWrapper()
+    output = nasa.run("get_media_metadata_manifest", "2022_0707_Recientemente")
+    assert output is not None
+
+
+def test_get_media_metadata_location() -> None:
+    """Test for retrieving media metadata location from NASA Image and Video Library"""
+    nasa = NasaAPIWrapper()
+    output = nasa.run("get_media_metadata_location", "as11-40-5874")
+    assert output is not None
+
+
+def test_get_video_captions_location() -> None:
+    """Test for retrieving video captions location from NASA Image and Video Library"""
+    nasa = NasaAPIWrapper()
+    output = nasa.run("get_video_captions_location", "172_ISS-Slosh.sr")
+    assert output is not None

libs/langchain/tests/unit_tests/tools/test_imports.py

@@ -68,6 +68,7 @@ EXPECTED_ALL = [
     "ListSparkSQLTool",
     "MetaphorSearchResults",
     "MoveFileTool",
+    "NasaAction",
     "NavigateBackTool",
     "NavigateTool",
     "O365CreateDraftMessage",

libs/langchain/tests/unit_tests/tools/test_public_api.py

@@ -70,6 +70,7 @@ _EXPECTED = [
     "MerriamWebsterQueryRun",
     "MetaphorSearchResults",
     "MoveFileTool",
+    "NasaAction",
     "NavigateBackTool",
     "NavigateTool",
     "O365CreateDraftMessage",

libs/langchain/tests/unit_tests/utilities/test_imports.py

@@ -23,6 +23,7 @@ EXPECTED_ALL = [
     "LambdaWrapper",
     "MaxComputeAPIWrapper",
     "MetaphorSearchAPIWrapper",
+    "NasaAPIWrapper",
     "OpenWeatherMapAPIWrapper",
     "OutlineAPIWrapper",
     "Portkey",