bump version 0081 (#956)

Co-authored-by: Harrison Chase <harrisonchase@Harrisons-MBP.attlocal.net>

docs/deployments.md

@@ -32,3 +32,8 @@ It implements a Question Answering app and contains instructions for deploying t
 ## [Vercel](https://github.com/homanp/vercel-langchain)
 
 A minimal example on how to run LangChain on Vercel using Flask.
+
+
+## [SteamShip](https://github.com/steamship-core/steamship-langchain/)
+This repository contains LangChain adapters for Steamship, enabling LangChain developers to rapidly deploy their apps on Steamship.
+This includes: production ready endpoints, horizontal scaling across dependencies, persistant storage of app state, multi-tenancy support, etc.

docs/modules/document_loaders/examples/azlyrics.ipynb

@@ -0,0 +1,93 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "9c31caff",
+   "metadata": {},
+   "source": [
+    "# AZLyrics\n",
+    "This covers how to load AZLyrics webpages into a document format that we can use downstream."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "7e6f5726",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import AZLyricsLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "a0df4c24",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = AZLyricsLoader(\"https://www.azlyrics.com/lyrics/mileycyrus/flowers.html\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "8cd61b6e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "162fd286",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content=\"Miley Cyrus - Flowers Lyrics | AZLyrics.com\\n\\r\\nWe were good, we were gold\\nKinda dream that can't be sold\\nWe were right till we weren't\\nBuilt a home and watched it burn\\n\\nI didn't wanna leave you\\nI didn't wanna lie\\nStarted to cry but then remembered I\\n\\nI can buy myself flowers\\nWrite my name in the sand\\nTalk to myself for hours\\nSay things you don't understand\\nI can take myself dancing\\nAnd I can hold my own hand\\nYeah, I can love me better than you can\\n\\nCan love me better\\nI can love me better, baby\\nCan love me better\\nI can love me better, baby\\n\\nPaint my nails, cherry red\\nMatch the roses that you left\\nNo remorse, no regret\\nI forgive every word you said\\n\\nI didn't wanna leave you, baby\\nI didn't wanna fight\\nStarted to cry but then remembered I\\n\\nI can buy myself flowers\\nWrite my name in the sand\\nTalk to myself for hours, yeah\\nSay things you don't understand\\nI can take myself dancing\\nAnd I can hold my own hand\\nYeah, I can love me better than you can\\n\\nCan love me better\\nI can love me better, baby\\nCan love me better\\nI can love me better, baby\\nCan love me better\\nI can love me better, baby\\nCan love me better\\nI\\n\\nI didn't wanna wanna leave you\\nI didn't wanna fight\\nStarted to cry but then remembered I\\n\\nI can buy myself flowers\\nWrite my name in the sand\\nTalk to myself for hours (Yeah)\\nSay things you don't understand\\nI can take myself dancing\\nAnd I can hold my own hand\\nYeah, I can love me better than\\nYeah, I can love me better than you can, uh\\n\\nCan love me better\\nI can love me better, baby\\nCan love me better\\nI can love me better, baby (Than you can)\\nCan love me better\\nI can love me better, baby\\nCan love me better\\nI\\n\", lookup_str='', metadata={'source': 'https://www.azlyrics.com/lyrics/mileycyrus/flowers.html'}, lookup_index=0)]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6358000c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/modules/document_loaders/examples/gutenberg.ipynb

@@ -0,0 +1,83 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "bda1f3f5",
+   "metadata": {},
+   "source": [
+    "# Gutenberg\n",
+    "\n",
+    "This covers how to load links to Gutenberg e-books into a document format that we can use downstream."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "9bfd5e46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import GutenbergLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "700e4ef2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = GutenbergLoader('https://www.gutenberg.org/cache/epub/69972/pg69972.txt')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "b6f28930",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7d436441",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3b74d755",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/modules/document_loaders/examples/youtube.ipynb

@@ -39,7 +39,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "loader = YoutubeLoader.from_youtube_url(\"https://www.youtube.com/watch?v=QsYGlZkevEg\")"
+    "loader = YoutubeLoader.from_youtube_url(\"https://www.youtube.com/watch?v=QsYGlZkevEg\", add_video_info=True)"
    ]
   },
   {
@@ -51,7 +51,7 @@
     {
      "data": {
       "text/plain": [
-       "[Document(page_content='LADIES AND GENTLEMEN, PEDRO PASCAL! [ CHEERS AND APPLAUSE ] >> THANK YOU, THANK YOU. THANK YOU VERY MUCH. I\\'M SO EXCITED TO BE HERE. THANK YOU. I SPENT THE LAST YEAR SHOOTING A SHOW CALLED \"THE LAST OF US\" ON HBO. FOR SOME HBO SHOES, YOU GET TO SHOOT IN A FIVE STAR ITALIAN RESORT SURROUNDED BY BEAUTIFUL PEOPLE, BUT I SAID, NO, THAT\\'S TOO EASY. I WANT TO SHOOT IN A FREEZING CANADIAN FOREST WHILE BEING CHASED AROUND BY A GUY WHOSE HEAD LOOKS LIKE A GENITAL WART. IT IS AN HONOR BEING A PART OF THESE HUGE FRANCHISEs LIKE \"GAME OF THRONES\" AND \"STAR WARS,\" BUT I\\'M STILL GETTING USED TO PEOPLE RECOGNIZING ME. THE OTHER DAY, A GUY STOPPED ME ON THE STREET AND SAYS, MY SON LOVES \"THE MANDALORIAN\" AND THE NEXT THING I KNOW, I\\'M FACE TIMING WITH A 6-YEAR-OLD WHO HAS NO IDEA WHO I AM BECAUSE MY CHARACTER WEARS A MASK THE ENTIRE SHOW. THE GUY IS LIKE, DO THE MANDO VOICE, BUT IT\\'S LIKE A BEDROOM VOICE. WITHOUT THE MASK, IT JUST SOUNDS PORNY. PEOPLE WALKING BY ON THE STREET SEE ME WHISPERING TO A 6-YEAR-OLD KID. I CAN BRING YOU IN WARM, OR I CAN BRING YOU IN COLD. EVEN THOUGH I CAME TO THE U.S. WHEN I WAS LITTLE, I WAS BORN IN CHILE, AND I HAVE 34 FIRST COUSINS WHO ARE STILL THERE. THEY\\'RE VERY PROUD OF ME. I KNOW THEY\\'RE PROUD BECAUSE THEY GIVE MY PHONE NUMBER TO EVERY PERSON THEY MEET, WHICH MEANS EVERY DAY, SOMEONE IN SANTIAGO WILL TEXT ME STUFF LIKE, CAN YOU COME TO MY WEDDING, OR CAN YOU SING MY PRIEST HAPPY BIRTHDAY, OR IS BABY YODA MEAN IN REAL LIFE. SO I HAVE TO BE LIKE NO, NO, AND HIS NAME IS GROGU. BUT MY COUSINS WEREN\\'T ALWAYS SO PROUD. EARLY IN MY CAREER, I PLAYED SMALL PARTS IN EVERY CRIME SHOW. I EVEN PLAYED TWO DIFFERENT CHARACTERS ON \"LAW AND ORDER.\" TITO CABASSA WHO LOOKED LIKE THIS. AND ONE YEAR LATER, I PLAYED REGGIE LUCKMAN WHO LOOKS LIKE THIS. AND THAT, MY FRIENDS, IS CALLED RANGE. BUT IT IS AMAZING TO BE HERE, LIKE I SAID. I WAS BORN IN CHILE, AND NINE MONTHS LATER, MY PARENTS FLED AND BROUGHT ME AND MY SISTER TO THE U.S. THEY WERE SO BRAVE, AND WITHOUT THEM, I WOULDN\\'T BE HERE IN THIS WONDERFUL COUNTRY, AND I CERTAINLY WOULDN\\'T BE STANDING HERE WITH YOU ALL TONIGHT. SO TO ALL MY FAMILY WATCHING IN CHILE, I WANT TO SAY [ SPEAKING NON-ENGLISH ] WHICH MEANS, I LOVE YOU, I MISS YOU, AND STOP GIVING OUT MY PHONE NUMBER. WE\\'VE GOT AN AMAZING SHOW FOR YOU TONIGHT. COLDPLAY IS HERE, SO STICK', lookup_str='', metadata={'source': 'QsYGlZkevEg'}, lookup_index=0)]"
+       "[Document(page_content='LADIES AND GENTLEMEN, PEDRO PASCAL! [ CHEERS AND APPLAUSE ] >> THANK YOU, THANK YOU. THANK YOU VERY MUCH. I\\'M SO EXCITED TO BE HERE. THANK YOU. I SPENT THE LAST YEAR SHOOTING A SHOW CALLED \"THE LAST OF US\" ON HBO. FOR SOME HBO SHOES, YOU GET TO SHOOT IN A FIVE STAR ITALIAN RESORT SURROUNDED BY BEAUTIFUL PEOPLE, BUT I SAID, NO, THAT\\'S TOO EASY. I WANT TO SHOOT IN A FREEZING CANADIAN FOREST WHILE BEING CHASED AROUND BY A GUY WHOSE HEAD LOOKS LIKE A GENITAL WART. IT IS AN HONOR BEING A PART OF THESE HUGE FRANCHISEs LIKE \"GAME OF THRONES\" AND \"STAR WARS,\" BUT I\\'M STILL GETTING USED TO PEOPLE RECOGNIZING ME. THE OTHER DAY, A GUY STOPPED ME ON THE STREET AND SAYS, MY SON LOVES \"THE MANDALORIAN\" AND THE NEXT THING I KNOW, I\\'M FACE TIMING WITH A 6-YEAR-OLD WHO HAS NO IDEA WHO I AM BECAUSE MY CHARACTER WEARS A MASK THE ENTIRE SHOW. THE GUY IS LIKE, DO THE MANDO VOICE, BUT IT\\'S LIKE A BEDROOM VOICE. WITHOUT THE MASK, IT JUST SOUNDS PORNY. PEOPLE WALKING BY ON THE STREET SEE ME WHISPERING TO A 6-YEAR-OLD KID. I CAN BRING YOU IN WARM, OR I CAN BRING YOU IN COLD. EVEN THOUGH I CAME TO THE U.S. WHEN I WAS LITTLE, I WAS BORN IN CHILE, AND I HAVE 34 FIRST COUSINS WHO ARE STILL THERE. THEY\\'RE VERY PROUD OF ME. I KNOW THEY\\'RE PROUD BECAUSE THEY GIVE MY PHONE NUMBER TO EVERY PERSON THEY MEET, WHICH MEANS EVERY DAY, SOMEONE IN SANTIAGO WILL TEXT ME STUFF LIKE, CAN YOU COME TO MY WEDDING, OR CAN YOU SING MY PRIEST HAPPY BIRTHDAY, OR IS BABY YODA MEAN IN REAL LIFE. SO I HAVE TO BE LIKE NO, NO, AND HIS NAME IS GROGU. BUT MY COUSINS WEREN\\'T ALWAYS SO PROUD. EARLY IN MY CAREER, I PLAYED SMALL PARTS IN EVERY CRIME SHOW. I EVEN PLAYED TWO DIFFERENT CHARACTERS ON \"LAW AND ORDER.\" TITO CABASSA WHO LOOKED LIKE THIS. AND ONE YEAR LATER, I PLAYED REGGIE LUCKMAN WHO LOOKS LIKE THIS. AND THAT, MY FRIENDS, IS CALLED RANGE. BUT IT IS AMAZING TO BE HERE, LIKE I SAID. I WAS BORN IN CHILE, AND NINE MONTHS LATER, MY PARENTS FLED AND BROUGHT ME AND MY SISTER TO THE U.S. THEY WERE SO BRAVE, AND WITHOUT THEM, I WOULDN\\'T BE HERE IN THIS WONDERFUL COUNTRY, AND I CERTAINLY WOULDN\\'T BE STANDING HERE WITH YOU ALL TONIGHT. SO TO ALL MY FAMILY WATCHING IN CHILE, I WANT TO SAY [ SPEAKING NON-ENGLISH ] WHICH MEANS, I LOVE YOU, I MISS YOU, AND STOP GIVING OUT MY PHONE NUMBER. WE\\'VE GOT AN AMAZING SHOW FOR YOU TONIGHT. COLDPLAY IS HERE, SO STICK', lookup_str='', metadata={'source': 'QsYGlZkevEg', 'title': 'Pedro Pascal Monologue - SNL', 'description': 'First-time host Pedro Pascal talks about filming The Last of Us and being recognized by fans.\\n\\nSaturday Night Live. Stream now on Peacock: https://pck.tv/3uQxh4q\\n\\nSubscribe to SNL: https://goo.gl/tUsXwM\\nStream Current Full Episodes: http://www.nbc.com/saturday-night-live\\n\\nWATCH PAST SNL SEASONS\\nGoogle Play - http://bit.ly/SNLGooglePlay\\niTunes - http://bit.ly/SNLiTunes\\n\\nSNL ON SOCIAL\\nSNL Instagram: http://instagram.com/nbcsnl\\nSNL Facebook: https://www.facebook.com/snl\\nSNL Twitter: https://twitter.com/nbcsnl\\nSNL TikTok: https://www.tiktok.com/@nbcsnl\\n\\nGET MORE NBC\\nLike NBC: http://Facebook.com/NBC\\nFollow NBC: http://Twitter.com/NBC\\nNBC Tumblr: http://NBCtv.tumblr.com/\\nYouTube: http://www.youtube.com/nbc\\nNBC Instagram: http://instagram.com/nbc\\n\\n#SNL #PedroPascal #SNL48 #Coldplay', 'view_count': 1175057, 'thumbnail_url': 'https://i.ytimg.com/vi/QsYGlZkevEg/sddefault.jpg', 'publish_date': datetime.datetime(2023, 2, 4, 0, 0), 'length': 224, 'author': 'Saturday Night Live'}, lookup_index=0)]"
       ]
      },
      "execution_count": 4,
@@ -62,6 +62,55 @@
    "source": [
     "loader.load()"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6b278a1b",
+   "metadata": {},
+   "source": [
+    "## Add video info"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "ba28af69",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ! pip install pytube"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "9b8ea390",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = YoutubeLoader.from_youtube_url(\"https://www.youtube.com/watch?v=QsYGlZkevEg\", add_video_info=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "97b98e92",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='LADIES AND GENTLEMEN, PEDRO PASCAL! [ CHEERS AND APPLAUSE ] >> THANK YOU, THANK YOU. THANK YOU VERY MUCH. I\\'M SO EXCITED TO BE HERE. THANK YOU. I SPENT THE LAST YEAR SHOOTING A SHOW CALLED \"THE LAST OF US\" ON HBO. FOR SOME HBO SHOES, YOU GET TO SHOOT IN A FIVE STAR ITALIAN RESORT SURROUNDED BY BEAUTIFUL PEOPLE, BUT I SAID, NO, THAT\\'S TOO EASY. I WANT TO SHOOT IN A FREEZING CANADIAN FOREST WHILE BEING CHASED AROUND BY A GUY WHOSE HEAD LOOKS LIKE A GENITAL WART. IT IS AN HONOR BEING A PART OF THESE HUGE FRANCHISEs LIKE \"GAME OF THRONES\" AND \"STAR WARS,\" BUT I\\'M STILL GETTING USED TO PEOPLE RECOGNIZING ME. THE OTHER DAY, A GUY STOPPED ME ON THE STREET AND SAYS, MY SON LOVES \"THE MANDALORIAN\" AND THE NEXT THING I KNOW, I\\'M FACE TIMING WITH A 6-YEAR-OLD WHO HAS NO IDEA WHO I AM BECAUSE MY CHARACTER WEARS A MASK THE ENTIRE SHOW. THE GUY IS LIKE, DO THE MANDO VOICE, BUT IT\\'S LIKE A BEDROOM VOICE. WITHOUT THE MASK, IT JUST SOUNDS PORNY. PEOPLE WALKING BY ON THE STREET SEE ME WHISPERING TO A 6-YEAR-OLD KID. I CAN BRING YOU IN WARM, OR I CAN BRING YOU IN COLD. EVEN THOUGH I CAME TO THE U.S. WHEN I WAS LITTLE, I WAS BORN IN CHILE, AND I HAVE 34 FIRST COUSINS WHO ARE STILL THERE. THEY\\'RE VERY PROUD OF ME. I KNOW THEY\\'RE PROUD BECAUSE THEY GIVE MY PHONE NUMBER TO EVERY PERSON THEY MEET, WHICH MEANS EVERY DAY, SOMEONE IN SANTIAGO WILL TEXT ME STUFF LIKE, CAN YOU COME TO MY WEDDING, OR CAN YOU SING MY PRIEST HAPPY BIRTHDAY, OR IS BABY YODA MEAN IN REAL LIFE. SO I HAVE TO BE LIKE NO, NO, AND HIS NAME IS GROGU. BUT MY COUSINS WEREN\\'T ALWAYS SO PROUD. EARLY IN MY CAREER, I PLAYED SMALL PARTS IN EVERY CRIME SHOW. I EVEN PLAYED TWO DIFFERENT CHARACTERS ON \"LAW AND ORDER.\" TITO CABASSA WHO LOOKED LIKE THIS. AND ONE YEAR LATER, I PLAYED REGGIE LUCKMAN WHO LOOKS LIKE THIS. AND THAT, MY FRIENDS, IS CALLED RANGE. BUT IT IS AMAZING TO BE HERE, LIKE I SAID. I WAS BORN IN CHILE, AND NINE MONTHS LATER, MY PARENTS FLED AND BROUGHT ME AND MY SISTER TO THE U.S. THEY WERE SO BRAVE, AND WITHOUT THEM, I WOULDN\\'T BE HERE IN THIS WONDERFUL COUNTRY, AND I CERTAINLY WOULDN\\'T BE STANDING HERE WITH YOU ALL TONIGHT. SO TO ALL MY FAMILY WATCHING IN CHILE, I WANT TO SAY [ SPEAKING NON-ENGLISH ] WHICH MEANS, I LOVE YOU, I MISS YOU, AND STOP GIVING OUT MY PHONE NUMBER. WE\\'VE GOT AN AMAZING SHOW FOR YOU TONIGHT. COLDPLAY IS HERE, SO STICK', lookup_str='', metadata={'source': 'QsYGlZkevEg', 'title': 'Pedro Pascal Monologue - SNL', 'description': 'First-time host Pedro Pascal talks about filming The Last of Us and being recognized by fans.\\n\\nSaturday Night Live. Stream now on Peacock: https://pck.tv/3uQxh4q\\n\\nSubscribe to SNL: https://goo.gl/tUsXwM\\nStream Current Full Episodes: http://www.nbc.com/saturday-night-live\\n\\nWATCH PAST SNL SEASONS\\nGoogle Play - http://bit.ly/SNLGooglePlay\\niTunes - http://bit.ly/SNLiTunes\\n\\nSNL ON SOCIAL\\nSNL Instagram: http://instagram.com/nbcsnl\\nSNL Facebook: https://www.facebook.com/snl\\nSNL Twitter: https://twitter.com/nbcsnl\\nSNL TikTok: https://www.tiktok.com/@nbcsnl\\n\\nGET MORE NBC\\nLike NBC: http://Facebook.com/NBC\\nFollow NBC: http://Twitter.com/NBC\\nNBC Tumblr: http://NBCtv.tumblr.com/\\nYouTube: http://www.youtube.com/nbc\\nNBC Instagram: http://instagram.com/nbc\\n\\n#SNL #PedroPascal #SNL48 #Coldplay', 'view_count': 1175057, 'thumbnail_url': 'https://i.ytimg.com/vi/QsYGlZkevEg/sddefault.jpg', 'publish_date': datetime.datetime(2023, 2, 4, 0, 0), 'length': 224, 'author': 'Saturday Night Live'}, lookup_index=0)]"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "loader.load()"
+   ]
   }
  ],
  "metadata": {
@@ -80,7 +129,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.9"
+   "version": "3.9.1"
   }
  },
  "nbformat": 4,

docs/modules/document_loaders/how_to_guides.rst

@@ -37,6 +37,15 @@ There are a lot of different document loaders that LangChain supports. Below are
 
 `GCS Directory <./examples/gcs_directory.html>`_: A walkthrough of how to load all files in a directory from Google Cloud Storage (GCS).
 
+`Web Base <./examples/web_base.html>`_: A walkthrough of how to load all text data from webpages.
+
+`IMSDb <./examples/imsdb.html>`_: A walkthrough of how to load all text data from IMSDb webpage.
+
+`AZLyrics <./examples/azlyrics.html>`_: A walkthrough of how to load all text data from AZLyrics webpage.
+
+`College Confidential <./examples/college_confidential.html>`_: A walkthrough of how to load all text data from College Confidential webpage.
+
+`Gutenberg <./examples/gutenberg.html>`_: A walkthrough of how to load data from a Gutenberg ebook text.
 
 .. toctree::
    :maxdepth: 1

docs/modules/llms/integrations/huggingface_hub.ipynb

@@ -5,9 +5,9 @@
    "id": "959300d4",
    "metadata": {},
    "source": [
-    "# HuggingFace Hub\n",
+    "# Hugging Face Hub\n",
     "\n",
-    "This example showcases how to connect to the HuggingFace Hub."
+    "This example showcases how to connect to the Hugging Face Hub."
    ]
   },
   {

docs/use_cases/agents.md

@@ -6,7 +6,7 @@ These agents can be used to power the next generation of personal assistants -
 systems that intelligently understand what you mean, and then can take actions to help you accomplish your goal.
 
 Agents are a core use of LangChain - so much so that there is a whole module dedicated to them.
-Therefor, we recommend that you check out that documentation for detailed instruction on how to work
+Therefore, we recommend that you check out that documentation for detailed instruction on how to work
 with them.
 
 - [Agent Documentation](../modules/agents.rst)

langchain/callbacks/__init__.py

@@ -36,7 +36,7 @@ def set_default_callback_manager() -> None:
         )
 
 
-def set_tracing_callback_manager(session_name: Optional[str] = None, example_id: Optional[int] = None) -> None:
+def set_tracing_callback_manager(session_name: Optional[str] = None) -> None:
     """Set tracing callback manager."""
     handler = SharedLangChainTracer()
     callback = get_callback_manager()
@@ -49,9 +49,6 @@ def set_tracing_callback_manager(session_name: Optional[str] = None, example_id:
         except Exception:
             raise ValueError(f"session {session_name} not found")
 
-    if example_id is not None:
-        handler.example_id = example_id
-
 
 @contextmanager
 def get_openai_callback() -> Generator[OpenAICallbackHandler, None, None]:

langchain/callbacks/tracers/langchain.py

@@ -22,22 +22,11 @@ class BaseLangChainTracer(BaseTracer, ABC):
     """An implementation of the SharedTracer that POSTS to the langchain endpoint."""
 
     always_verbose: bool = True
-    _example_id: Optional[int] = None
     _endpoint: str = os.getenv("LANGCHAIN_ENDPOINT", "http://localhost:8000")
     _headers: Dict[str, Any] = {"Content-Type": "application/json"}
     if os.getenv("LANGCHAIN_API_KEY"):
         _headers["x-api-key"] = os.getenv("LANGCHAIN_API_KEY")
 
-    @property
-    def example_id(self) -> Optional[int]:
-        """Return the example_id."""
-        return self._example_id
-
-    @example_id.setter
-    def example_id(self, value: int) -> None:
-        """Set the example_id."""
-        self._example_id = value
-
     def _persist_run(self, run: Union[LLMRun, ChainRun, ToolRun]) -> None:
         """Persist a run."""
         if isinstance(run, LLMRun):
@@ -47,9 +36,6 @@ class BaseLangChainTracer(BaseTracer, ABC):
         else:
             endpoint = f"{self._endpoint}/tool-runs"
 
-        if self._example_id:
-            run.example_id = self._example_id
-
         try:
             requests.post(
                 endpoint,

langchain/callbacks/tracers/schemas.py

@@ -40,7 +40,6 @@ class BaseRun(BaseModel):
     serialized: Dict[str, Any]
     session_id: int
     error: Optional[str] = None
-    example_id: Optional[int] = None
 
 
 class LLMRun(BaseRun):

langchain/document_loaders/__init__.py

@@ -1,12 +1,16 @@
 """All different types of document loaders."""
 
+from langchain.document_loaders.azlyrics import AZLyricsLoader
+from langchain.document_loaders.college_confidential import CollegeConfidentialLoader
 from langchain.document_loaders.directory import DirectoryLoader
 from langchain.document_loaders.docx import UnstructuredDocxLoader
 from langchain.document_loaders.email import UnstructuredEmailLoader
 from langchain.document_loaders.gcs_directory import GCSDirectoryLoader
 from langchain.document_loaders.gcs_file import GCSFileLoader
 from langchain.document_loaders.googledrive import GoogleDriveLoader
+from langchain.document_loaders.gutenberg import GutenbergLoader
 from langchain.document_loaders.html import UnstructuredHTMLLoader
+from langchain.document_loaders.imsdb import IMSDbLoader
 from langchain.document_loaders.notion import NotionDirectoryLoader
 from langchain.document_loaders.obsidian import ObsidianLoader
 from langchain.document_loaders.pdf import UnstructuredPDFLoader
@@ -16,6 +20,7 @@ from langchain.document_loaders.roam import RoamLoader
 from langchain.document_loaders.s3_directory import S3DirectoryLoader
 from langchain.document_loaders.s3_file import S3FileLoader
 from langchain.document_loaders.unstructured import UnstructuredFileLoader
+from langchain.document_loaders.web_base import WebBaseLoader
 from langchain.document_loaders.youtube import YoutubeLoader
 
 __all__ = [
@@ -36,4 +41,9 @@ __all__ = [
     "S3DirectoryLoader",
     "GCSFileLoader",
     "GCSDirectoryLoader",
+    "WebBaseLoader",
+    "IMSDbLoader",
+    "AZLyricsLoader",
+    "CollegeConfidentialLoader",
+    "GutenbergLoader",
 ]

langchain/document_loaders/azlyrics.py

@@ -0,0 +1,22 @@
+"""Loader that loads AZLyrics."""
+from typing import List
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.web_base import WebBaseLoader
+
+
+class AZLyricsLoader(WebBaseLoader):
+    """Loader that loads AZLyrics webpages."""
+
+    def __init__(self, web_path: str):
+        """Initialize with webpage path."""
+        self.web_path = web_path
+
+    def load(self) -> List[Document]:
+        """Load webpage."""
+        soup = self.scrape()
+        title = soup.title.text
+        lyrics = soup.find_all("div", {"class": ""})[2].text
+        text = title + lyrics
+        metadata = {"source": self.web_path}
+        return [Document(page_content=text, metadata=metadata)]

langchain/document_loaders/college_confidential.py

@@ -0,0 +1,20 @@
+"""Loader that loads College Confidential."""
+from typing import List
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.web_base import WebBaseLoader
+
+
+class CollegeConfidentialLoader(WebBaseLoader):
+    """Loader that loads College Confidential webpages."""
+
+    def __init__(self, web_path: str):
+        """Initialize with webpage path."""
+        self.web_path = web_path
+
+    def load(self) -> List[Document]:
+        """Load webpage."""
+        soup = self.scrape()
+        text = soup.select_one("main[class='skin-handler']").text
+        metadata = {"source": self.web_path}
+        return [Document(page_content=text, metadata=metadata)]

langchain/document_loaders/gutenberg.py

@@ -0,0 +1,28 @@
+"""Loader that loads .txt web files."""
+from typing import List
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+
+
+class GutenbergLoader(BaseLoader):
+    """Loader that uses urllib to load .txt web files."""
+
+    def __init__(self, file_path: str):
+        """Initialize with file path."""
+        if not file_path.startswith("https://www.gutenberg.org"):
+            raise ValueError("file path must start with 'https://www.gutenberg.org'")
+
+        if not file_path.endswith(".txt"):
+            raise ValueError("file path must end with '.txt'")
+
+        self.file_path = file_path
+
+    def load(self) -> List[Document]:
+        """Load file."""
+        from urllib.request import urlopen
+
+        elements = urlopen(self.file_path)
+        text = "\n\n".join([str(el.decode("utf-8-sig")) for el in elements])
+        metadata = {"source": self.file_path}
+        return [Document(page_content=text, metadata=metadata)]

langchain/document_loaders/imsdb.py

@@ -0,0 +1,20 @@
+"""Loader that loads IMSDb."""
+from typing import List
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.web_base import WebBaseLoader
+
+
+class IMSDbLoader(WebBaseLoader):
+    """Loader that loads IMSDb webpages."""
+
+    def __init__(self, web_path: str):
+        """Initialize with webpage path."""
+        self.web_path = web_path
+
+    def load(self) -> List[Document]:
+        """Load webpage."""
+        soup = self.scrape()
+        text = soup.select_one("td[class='scrtext']").text
+        metadata = {"source": self.web_path}
+        return [Document(page_content=text, metadata=metadata)]

langchain/document_loaders/readthedocs.py

@@ -19,7 +19,11 @@ class ReadTheDocsLoader(BaseLoader):
 
         def _clean_data(data: str) -> str:
             soup = BeautifulSoup(data)
-            text = soup.find_all("main", {"id": "main-content"})[0].get_text()
+            text = soup.find_all("main", {"id": "main-content"})
+            if len(text) != 0:
+                text = text[0].get_text()
+            else:
+                text = ""
             return "\n".join([t for t in text.split("\n") if t])
 
         docs = []

langchain/document_loaders/web_base.py

@@ -0,0 +1,29 @@
+"""Web base loader class."""
+from typing import List
+
+import requests
+from bs4 import BeautifulSoup
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+
+
+class WebBaseLoader(BaseLoader):
+    """Loader that uses urllib and beautiful soup to load webpages."""
+
+    def __init__(self, web_path: str):
+        """Initialize with webpage path."""
+        self.web_path = web_path
+
+    def scrape(self) -> BeautifulSoup:
+        """Scrape data from webpage and return it in BeautifulSoup format."""
+        html_doc = requests.get(self.web_path)
+        soup = BeautifulSoup(html_doc.text, "html.parser")
+        return soup
+
+    def load(self) -> List[Document]:
+        """Load data into document objects."""
+        soup = self.scrape()
+        text = soup.get_text()
+        metadata = {"source": self.web_path}
+        return [Document(page_content=text, metadata=metadata)]

langchain/document_loaders/youtube.py

@@ -1,7 +1,7 @@
 """Loader that loads YouTube transcript."""
 from __future__ import annotations
 
-from typing import List
+from typing import Any, List
 
 from langchain.docstore.document import Document
 from langchain.document_loaders.base import BaseLoader
@@ -10,26 +10,67 @@ from langchain.document_loaders.base import BaseLoader
 class YoutubeLoader(BaseLoader):
     """Loader that loads Youtube transcripts."""
 
-    def __init__(self, video_id: str):
+    def __init__(self, video_id: str, add_video_info: bool = False):
         """Initialize with YouTube video ID."""
         self.video_id = video_id
+        self.add_video_info = add_video_info
 
     @classmethod
-    def from_youtube_url(cls, youtube_url: str) -> YoutubeLoader:
+    def from_youtube_url(cls, youtube_url: str, **kwargs: Any) -> YoutubeLoader:
         """Parse out video id from YouTube url."""
         video_id = youtube_url.split("youtube.com/watch?v=")[-1]
-        return cls(video_id)
+        return cls(video_id, **kwargs)
 
     def load(self) -> List[Document]:
         """Load documents."""
         try:
             from youtube_transcript_api import YouTubeTranscriptApi
         except ImportError:
-            raise ValueError(
+            raise ImportError(
                 "Could not import youtube_transcript_api python package. "
                 "Please it install it with `pip install youtube-transcript-api`."
             )
+
+        metadata = {"source": self.video_id}
+
+        if self.add_video_info:
+            # Get more video meta info
+            # Such as title, description, thumbnail url, publish_date
+            video_info = self._get_video_info()
+            metadata.update(video_info)
+
         transcript_pieces = YouTubeTranscriptApi.get_transcript(self.video_id)
         transcript = " ".join([t["text"].strip(" ") for t in transcript_pieces])
-        metadata = {"source": self.video_id}
+
         return [Document(page_content=transcript, metadata=metadata)]
+
+    def _get_video_info(self) -> dict:
+        """Get important video information.
+
+        Components are:
+            - title
+            - description
+            - thumbnail url,
+            - publish_date
+            - channel_author
+            - and more.
+        """
+        try:
+            from pytube import YouTube
+
+        except ImportError:
+            raise ImportError(
+                "Could not import pytube python package. "
+                "Please it install it with `pip install pytube`."
+            )
+        yt = YouTube(f"https://www.youtube.com/watch?v={self.video_id}")
+        video_info = {
+            "title": yt.title,
+            "description": yt.description,
+            "view_count": yt.views,
+            "thumbnail_url": yt.thumbnail_url,
+            "publish_date": yt.publish_date,
+            "length": yt.length,
+            "author": yt.author,
+        }
+        return video_info

langchain/evaluation/ExampleRunner.ipynb

@@ -1,317 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "bd0b1515-e25f-4707-a240-b5c26d5d33f8",
-   "metadata": {},
-   "source": [
-    "# ExamplerRunner Demo\n",
-    "\n",
-    "Run a chain on multiple examples for evaluation."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "13fbc962-98e0-470c-9467-c5e28db658a0",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "os.environ[\"LANGCHAIN_HANDLER\"] = \"langchain\"\n",
-    "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"http://127.0.0.1:8000\" \n",
-    "\n",
-    "import langchain\n",
-    "from langchain.agents import Tool, initialize_agent, load_tools\n",
-    "from langchain.llms import OpenAI\n",
-    "from langchain.evaluation.example_runner import ExampleRunner, CsvDataset\n",
-    "from langchain.llms import OpenAI\n",
-    "from langchain.prompts import PromptTemplate\n",
-    "from langchain.chains import LLMChain"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "8afb6529-dfdb-4f0a-b19f-a700d80d4362",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "# Upload the dataset (only need to do once)\n",
-    "runner = ExampleRunner(\n",
-    "    csv_dataset=CsvDataset(\n",
-    "        csv_path=\"test_dataset.csv\",\n",
-    "        description=\"Dummy dataset for testing\",\n",
-    "        input_keys=[\"input1\", \"input2\", \"input3\"],\n",
-    "        output_keys=[\"output1\"],\n",
-    "    ),\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "603dc948-0a57-4696-8816-008aaf346538",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[Example(created_at=datetime.datetime(2023, 2, 9, 7, 45, 27, 878780), inputs={'input1': 'one', 'input2': ' two', 'input3': ' three'}, outputs={'output1': ' four'}, dataset_id=1, id=1),\n",
-       " Example(created_at=datetime.datetime(2023, 2, 9, 7, 45, 27, 878790), inputs={'input1': 'five', 'input2': ' six', 'input3': ' seven'}, outputs={'output1': ' eight'}, dataset_id=1, id=2),\n",
-       " Example(created_at=datetime.datetime(2023, 2, 9, 7, 45, 27, 878792), inputs={'input1': 'nine', 'input2': ' ten', 'input3': ' eleven'}, outputs={'output1': ' twelve'}, dataset_id=1, id=3)]"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "runner.dataset.examples[:3]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "11eba157-ec8e-43e9-8326-75099c07e574",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Retrying langchain.llms.openai.BaseOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: The server had an error while processing your request. Sorry about that!.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      ",  four\n",
-      "\n",
-      "Five\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Retrying langchain.llms.openai.BaseOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: The server had an error while processing your request. Sorry about that!.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "\n",
-      "Eight\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Retrying langchain.llms.openai.BaseOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: The server had an error while processing your request. Sorry about that!.\n",
-      "Retrying langchain.llms.openai.BaseOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: The server had an error while processing your request. Sorry about that!.\n",
-      "Retrying langchain.llms.openai.BaseOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: The server had an error while processing your request. Sorry about that!.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "\n",
-      "Twelve\n",
-      "\n",
-      "\n",
-      "Sixteen, seventeen, eighteen.\n",
-      ",  twenty\n",
-      "\n",
-      "Twenty-one\n",
-      "\n",
-      "\n",
-      "twenty-four\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Retrying langchain.llms.openai.BaseOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: The server had an error while processing your request. Sorry about that!.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      ",  twenty-eight\n",
-      "\n",
-      "Twenty-nine\n",
-      "\n",
-      "\n",
-      "Thirty-two.\n",
-      "\n",
-      "\n",
-      "Thirty-six\n",
-      ",  forty\n",
-      "\n",
-      "forty-one\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Retrying langchain.llms.openai.BaseOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: The server had an error while processing your request. Sorry about that!.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "\n",
-      "Forty-four\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Retrying langchain.llms.openai.BaseOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: The server had an error while processing your request. Sorry about that!.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "\n",
-      "Forty-eight\n",
-      "four\n",
-      " eight\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Retrying langchain.llms.openai.BaseOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: The server had an error while processing your request. Sorry about that!.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      " twelve\n",
-      " sixteen\n",
-      "\n",
-      "twenty\n",
-      " twenty-four\n",
-      " twenty-eight\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Retrying langchain.llms.openai.BaseOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: The server had an error while processing your request. Sorry about that!.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      " thirty-two\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Retrying langchain.llms.openai.BaseOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: The server had an error while processing your request. Sorry about that!.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      " thirty-six\n",
-      " forty\n",
-      " forty-four\n",
-      " forty-eight\n"
-     ]
-    }
-   ],
-   "source": [
-    "runner = ExampleRunner(\n",
-    "    langchain_dataset_name=\"test_dataset.csv\",\n",
-    ")\n",
-    "\n",
-    "llm = OpenAI(temperature=0)\n",
-    "prompt1 = PromptTemplate(\n",
-    "    input_variables=[\"input1\", \"input2\", \"input3\"],\n",
-    "    template=\"Complete the sequence: {input1}, {input2}, {input3}\",\n",
-    ")\n",
-    "chain1 = LLMChain(llm=llm, prompt=prompt1)\n",
-    "\n",
-    "prompt2 = PromptTemplate(\n",
-    "    input_variables=[\"input1\", \"input2\", \"input3\"],\n",
-    "    template=\"\"\"\n",
-    "    You are given the text representation of three numbers. You are to give the next number in the sequence. Only provide one number! \n",
-    "    \n",
-    "    Example:\n",
-    "    Input: one, two three. \n",
-    "    Output: four \n",
-    "    \n",
-    "    Input: {input1}, {input2}, {input3}\n",
-    "    Output:\n",
-    "    \"\"\"\n",
-    ")\n",
-    "chain2 = LLMChain(llm=llm, prompt=prompt2)\n",
-    "runner.run_chain(chain1)\n",
-    "runner.run_chain(chain2)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b510335a-872f-4392-829d-bdcba3a052cb",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.9"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

langchain/evaluation/example_runner.py

@@ -1,202 +0,0 @@
-import asyncio
-
-from pydantic import BaseModel, validator, root_validator, Field
-from pathlib import Path
-from typing import Optional, Dict, Any, List
-from langchain.utils import get_from_dict_or_env
-from pydantic.networks import AnyHttpUrl
-import requests
-import datetime
-import langchain
-from langchain.agents import AgentExecutor
-from langchain.chains.base import Chain
-from langchain.llms.base import BaseLLM
-from langchain.callbacks.base import CallbackManager
-from langchain.callbacks.tracers import LangChainTracer
-from langchain.callbacks.stdout import StdOutCallbackHandler
-from urllib.parse import urlparse
-import os
-
-
-class ExampleBase(BaseModel):
-    """Base class for Example."""
-    created_at: datetime.datetime = Field(default_factory=datetime.datetime.utcnow)
-    inputs: Dict[str, Any]
-    outputs: Dict[str, Any] | None = None
-    dataset_id: int
-
-
-class ExampleCreate(ExampleBase):
-    """Create class for Example."""
-
-
-class Example(ExampleBase):
-    """Example schema."""
-    id: int
-
-
-class DatasetBase(BaseModel):
-    """Base class for Dataset."""
-    name: str
-    description: str
-    created_at: datetime.datetime = Field(default_factory=datetime.datetime.utcnow)
-    examples: List[Example] = Field(default_factory=list)
-
-
-class DatasetCreate(DatasetBase):
-    """Create class for Dataset."""
-    pass
-
-
-class Dataset(DatasetBase):
-    """Dataset schema."""
-    id: int
-
-
-class CsvDataset(BaseModel):
-    """Class for a csv file that can be uploaded to a LangChain endpoint."""
-    csv_path: Path
-    description: str
-    input_keys: List[str]
-    output_keys: List[str]
-
-    @validator("csv_path")
-    def validate_csv_path(cls, v):
-        """Validate that the csv path is valid."""
-        if not v.exists():
-            raise ValueError("CSV file does not exist.")
-        return v
-
-
-def fetch_dataset_from_endpoint(name: str, headers: Dict[str, str], endpoint: str = "https://localhost:8000") -> Dataset:
-    """Fetch a dataset from a LangChain endpoint."""
-    response = requests.get(f"{endpoint}/datasets?name={name}", headers=headers)
-    response.raise_for_status()
-    if len(response.json()) == 0:
-        raise ValueError(f"Dataset with name {name} does not exist.")
-    return Dataset(**(response.json()[0]))
-
-
-def upload_csv_dataset_to_endpoint(csv_dataset: CsvDataset, headers: Dict[str, str], endpoint: str = "https://localhost:8000") -> Dataset:
-    """Upload a csv to a LangChain endpoint."""
-    with open(csv_dataset.csv_path, "rb") as f:
-        response = requests.post(
-            f"{endpoint}/datasets/upload",
-            headers=headers,
-            files={"file": (csv_dataset.csv_path.name, f)},
-            data={
-                "input_keys": csv_dataset.input_keys,
-                "output_keys": csv_dataset.output_keys,
-                "description": csv_dataset.description,
-            },
-        )
-    response.raise_for_status()
-    return Dataset(**response.json())
-
-
-class ExampleRunner(BaseModel):
-    """Class that runs an LLM, chain or agent on a set of examples."""
-
-    langchain_endpoint: AnyHttpUrl
-    dataset: Dataset
-    csv_dataset: Optional[CsvDataset] = None
-    langchain_dataset_name: Optional[str] = None
-    langchain_api_key: Optional[str] = None
-
-    @root_validator(pre=True)
-    def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]:
-        """Validate that either csv_path or langchain_dataset is provided but not both."""
-        csv_dataset = values.get("csv_dataset")
-        langchain_dataset_name = values.get("langchain_dataset_name")
-        values["langchain_endpoint"] = os.environ.get("LANGCHAIN_ENDPOINT", "https://localhost:8000")
-        langchain_endpoint = values["langchain_endpoint"]
-        if csv_dataset is None and langchain_dataset_name is None:
-            raise ValueError("Must provide either csv_path or langchain_dataset.")
-        if csv_dataset is not None and langchain_dataset_name is not None:
-            raise ValueError("Cannot provide both csv_path and langchain_dataset.")
-        if urlparse(langchain_endpoint).hostname not in ["localhost", "127.0.0.1", "0.0.0.0"]:
-            values["langchain_api_key"] = get_from_dict_or_env(
-                values, "langchain_api_key", "LANGCHAIN_API_KEY"
-            )
-        # Try fetching the dataset to make sure it exists
-        if langchain_dataset_name is not None:
-            headers: Dict[str, str] = {}
-            if values.get("langchain_api_key"):
-                headers["x-api-key"] = values["langchain_api_key"]
-            values["dataset"] = fetch_dataset_from_endpoint(langchain_dataset_name, headers, langchain_endpoint)
-        if csv_dataset is not None:
-            # Upload the csv to the endpoint
-            headers: Dict[str, str] = {}
-            if values.get("langchain_api_key"):
-                headers["x-api-key"] = values["langchain_api_key"]
-            values["dataset"] = upload_csv_dataset_to_endpoint(csv_dataset, headers, langchain_endpoint)
-        return values
-
-    def examples(self) -> List[Example]:
-        """Get the examples from the dataset."""
-        return self.dataset.examples
-
-    def run_agent(self, agent: AgentExecutor):
-        """Run an agent on the examples."""
-        for example in self.examples():
-            agent.run(**example.inputs)
-
-    def run_chain(self, chain: Chain):
-        """Run a chain on the examples."""
-        for example in self.examples():
-            langchain.set_tracing_callback_manager(example_id=example.id)
-            print(chain.run(**example.inputs))
-
-    def run_llm(self, llm: BaseLLM):
-        """Run an LLM on the examples."""
-        for example in self.examples():
-            llm.generate([val for val in example.inputs.values()])
-
-
-    # async def arun_agent(self, agent: AgentExecutor, num_workers: int = 1):
-    #     """Run an agent on the examples."""
-    #     # Copy the agent num_workers times
-    #     agents = []
-    #     for _ in range(num_workers):
-    #         tracer = LangChainTracer()
-    #         tracer.load_default_session()
-    #         manager = CallbackManager([StdOutCallbackHandler(), tracer])
-    #         agent.from_agent_and_tools(agent.agent, agent.tools, manager)
-    #         agents.append(agent)
-    #
-    #     i = 0
-    #     while i < len(self.examples()):
-    #         for agent in agents:
-    #             example = self.examples()[i]
-    #             await agent.arun(**example.inputs)
-    #             i += 1
-
-
-if __name__ == "__main__":
-    os.environ["LANGCHAIN_ENDPOINT"] = "http://127.0.0.1:8000"
-    runner = ExampleRunner(
-        csv_dataset=CsvDataset(
-            csv_path="test_dataset.csv",
-            description="Dummy dataset for testing",
-            input_keys=["input1", "input2", "input3"],
-            output_keys=["output1"],
-        ),
-    )
-
-    # runner = ExampleRunner(
-    #     langchain_dataset_name="test_dataset.csv",
-    # )
-
-    from langchain.llms import OpenAI
-    from langchain.prompts import PromptTemplate
-    from langchain.chains import LLMChain
-    llm = OpenAI(temperature=0.9, model_name="text-ada-001")
-    prompt = PromptTemplate(
-        input_variables=["input1", "input2", "input3"],
-        template="Complete the sequence: {input1}, {input2}, {input3}",
-    )
-    chain = LLMChain(llm=llm, prompt=prompt)
-    runner.run_chain(chain)
-
-
-

langchain/evaluation/test_dataset.csv

@@ -1,13 +0,0 @@
-input1,input2,input3,output1
-one, two, three, four
-five, six, seven, eight
-nine, ten, eleven, twelve
-thirteen, fourteen, fifteen, sixteen
-seventeen, eighteen, nineteen, twenty
-twenty-one, twenty-two, twenty-three, twenty-four
-twenty-five, twenty-six, twenty-seven, twenty-eight
-twenty-nine, thirty, thirty-one, thirty-two
-thirty-three, thirty-four, thirty-five, thirty-six
-thirty-seven, thirty-eight, thirty-nine, forty
-forty-one, forty-two, forty-three, forty-four
-forty-five, forty-six, forty-seven, forty-eight

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langchain"
-version = "0.0.80"
+version = "0.0.81"
 description = "Building applications with LLMs through composability"
 authors = []
 license = "MIT"