From 53dc15714584f742ef72b96376a75680cf45b6ef Mon Sep 17 00:00:00 2001 From: Tim Asp <707699+timothyasp@users.noreply.github.com> Date: Thu, 13 Apr 2023 10:54:40 -0700 Subject: [PATCH 01/49] [Docs] minor fixes to loaders links and rst warnings (#2846) The doc loaders index was picking up a bunch of subheadings because I mistakenly made the MD titles H1s. Fixed that. also the easy minor warnings from docs_build --- docs/gallery.rst | 4 ++-- docs/modules/indexes/document_loaders/examples/csv.ipynb | 2 +- docs/modules/indexes/document_loaders/examples/web_base.ipynb | 4 ++-- docs/use_cases/evaluation.rst | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/gallery.rst b/docs/gallery.rst index 8eae3eedab1..0725fdb2ecd 100644 --- a/docs/gallery.rst +++ b/docs/gallery.rst @@ -1,5 +1,5 @@ LangChain Gallery -============= +================= Lots of people have built some pretty awesome stuff with LangChain. This is a collection of our favorites. @@ -223,7 +223,7 @@ Open Source Answer questions about the documentation of any project Misc. Colab Notebooks -~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~ .. panels:: :body: text-center diff --git a/docs/modules/indexes/document_loaders/examples/csv.ipynb b/docs/modules/indexes/document_loaders/examples/csv.ipynb index 6d0fecd9f28..5b43f39d49a 100644 --- a/docs/modules/indexes/document_loaders/examples/csv.ipynb +++ b/docs/modules/indexes/document_loaders/examples/csv.ipynb @@ -106,7 +106,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Specify a column to be used identify the document source\n", + "## Specify a column to be used identify the document source\n", "\n", "Use the `source_column` argument to specify a column to be set as the source for the document created from each row. Otherwise `file_path` will be used as the source for all documents created from the csv file.\n", "\n", diff --git a/docs/modules/indexes/document_loaders/examples/web_base.ipynb b/docs/modules/indexes/document_loaders/examples/web_base.ipynb index 9b3feff19c3..d500e77816e 100644 --- a/docs/modules/indexes/document_loaders/examples/web_base.ipynb +++ b/docs/modules/indexes/document_loaders/examples/web_base.ipynb @@ -89,7 +89,7 @@ "id": "150988e6", "metadata": {}, "source": [ - "# Loading multiple webpages\n", + "## Loading multiple webpages\n", "\n", "You can also load multiple webpages at once by passing in a list of urls to the loader. This will return a list of documents in the same order as the urls passed in." ] @@ -123,7 +123,7 @@ "id": "641be294", "metadata": {}, "source": [ - "## Load multiple urls concurrently\n", + "### Load multiple urls concurrently\n", "\n", "You can speed up the scraping process by scraping and parsing multiple urls concurrently.\n", "\n", diff --git a/docs/use_cases/evaluation.rst b/docs/use_cases/evaluation.rst index 60e0da7ed1c..66bc26337d9 100644 --- a/docs/use_cases/evaluation.rst +++ b/docs/use_cases/evaluation.rst @@ -1,5 +1,5 @@ Evaluation -============== +========== .. note:: `Conceptual Guide `_ @@ -83,7 +83,7 @@ The existing examples we have are: Other Examples ------------- +-------------- In addition, we also have some more generic resources for evaluation. From 82d1d5f24efe57215ef79f954ff610483389ada6 Mon Sep 17 00:00:00 2001 From: vowelparrot <130414180+vowelparrot@users.noreply.github.com> Date: Thu, 13 Apr 2023 11:00:09 -0700 Subject: [PATCH 02/49] Fix grammar in Vector Memory Docs (#2847) --- .../memory/types/vectorstore_retriever_memory.ipynb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/modules/memory/types/vectorstore_retriever_memory.ipynb b/docs/modules/memory/types/vectorstore_retriever_memory.ipynb index e44ac4d39a2..a7d54810fd4 100644 --- a/docs/modules/memory/types/vectorstore_retriever_memory.ipynb +++ b/docs/modules/memory/types/vectorstore_retriever_memory.ipynb @@ -1,15 +1,16 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "id": "ff4be5f3", "metadata": {}, "source": [ "# VectorStore-Backed Memory\n", "\n", - "`VectorStoreRetrieverMemory` stores interactions in a VectorDB and queries the top-K most \"salient\" interactions every type it is called.\n", + "`VectorStoreRetrieverMemory` stores memories in a VectorDB and queries the top-K most \"salient\" docs every time it is called.\n", "\n", - "This differs from most of the other Memory classes in that " + "This differs from most of the other Memory classes in that it doesn't explicitly track the order of interactions." ] }, { @@ -60,13 +61,14 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "8f4bdf92", "metadata": {}, "source": [ "### Create your the VectorStoreRetrieverMemory\n", "\n", - "The memory object is instantiated from " + "The memory object is instantiated from any VectorStoreRetriever." ] }, { From be4fb24b325ee7ad30bdfa30f30c19b0682fbf0c Mon Sep 17 00:00:00 2001 From: Tim Asp <707699+timothyasp@users.noreply.github.com> Date: Thu, 13 Apr 2023 11:13:34 -0700 Subject: [PATCH 03/49] OpenAI LLM: update `modelname_to_contextsize` with new models (#2843) Token counts pulled from https://openai.com/pricing --- langchain/llms/openai.py | 52 ++++++++++++--------- tests/integration_tests/llms/test_openai.py | 11 +++++ 2 files changed, 42 insertions(+), 21 deletions(-) diff --git a/langchain/llms/openai.py b/langchain/llms/openai.py index a13beb7367a..ff99ee48bf0 100644 --- a/langchain/llms/openai.py +++ b/langchain/llms/openai.py @@ -476,13 +476,6 @@ class BaseOpenAI(BaseLLM): def modelname_to_contextsize(self, modelname: str) -> int: """Calculate the maximum number of tokens possible to generate for a model. - text-davinci-003: 4,097 tokens - text-curie-001: 2,048 tokens - text-babbage-001: 2,048 tokens - text-ada-001: 2,048 tokens - code-davinci-002: 8,000 tokens - code-cushman-001: 2,048 tokens - Args: modelname: The modelname we want to know the context size for. @@ -494,20 +487,37 @@ class BaseOpenAI(BaseLLM): max_tokens = openai.modelname_to_contextsize("text-davinci-003") """ - if modelname == "text-davinci-003": - return 4097 - elif modelname == "text-curie-001": - return 2048 - elif modelname == "text-babbage-001": - return 2048 - elif modelname == "text-ada-001": - return 2048 - elif modelname == "code-davinci-002": - return 8000 - elif modelname == "code-cushman-001": - return 2048 - else: - return 4097 + model_token_mapping = { + "gpt-4": 8192, + "gpt-4-0314": 8192, + "gpt-4-32k": 32768, + "gpt-4-32k-0314": 32768, + "gpt-3.5-turbo": 4096, + "gpt-3.5-turbo-0301": 4096, + "text-ada-001": 2049, + "ada": 2049, + "text-babbage-001": 2040, + "babbage": 2049, + "text-curie-001": 2049, + "curie": 2049, + "davinci": 2049, + "text-davinci-003": 4097, + "text-davinci-002": 4097, + "code-davinci-002": 8001, + "code-davinci-001": 8001, + "code-cushman-002": 2048, + "code-cushman-001": 2048, + } + + context_size = model_token_mapping.get(modelname, None) + + if context_size is None: + raise ValueError( + f"Unknown model: {modelname}. Please provide a valid OpenAI model name." + "Known models are: " + ", ".join(model_token_mapping.keys()) + ) + + return context_size def max_tokens_for_prompt(self, prompt: str) -> int: """Calculate the maximum number of tokens possible to generate for a prompt. diff --git a/tests/integration_tests/llms/test_openai.py b/tests/integration_tests/llms/test_openai.py index 1ada0ca6094..9db120a5984 100644 --- a/tests/integration_tests/llms/test_openai.py +++ b/tests/integration_tests/llms/test_openai.py @@ -211,3 +211,14 @@ async def test_openai_chat_async_streaming_callback() -> None: result = await llm.agenerate(["Write me a sentence with 100 words."]) assert callback_handler.llm_streams != 0 assert isinstance(result, LLMResult) + + +def test_openai_modelname_to_contextsize_valid() -> None: + """Test model name to context size on a valid model.""" + assert OpenAI().modelname_to_contextsize("davinci") == 2049 + + +def test_openai_modelname_to_contextsize_invalid() -> None: + """Test model name to context size on an invalid model.""" + with pytest.raises(ValueError): + OpenAI().modelname_to_contextsize("foobar") From 70ffe470aa96440e07c5f411a74b7a947b7f3ee3 Mon Sep 17 00:00:00 2001 From: Tim Asp <707699+timothyasp@users.noreply.github.com> Date: Thu, 13 Apr 2023 11:28:42 -0700 Subject: [PATCH 04/49] Add easy print method to openai callback (#2848) Found myself constantly copying the snippet outputting all the callback tracking details. so adding a simple way to output the full context --- .../llms/examples/token_usage_tracking.ipynb | 14 +++++--------- langchain/callbacks/openai_info.py | 9 +++++++++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/modules/models/llms/examples/token_usage_tracking.ipynb b/docs/modules/models/llms/examples/token_usage_tracking.ipynb index 1f8de942098..cbb0c9bda93 100644 --- a/docs/modules/models/llms/examples/token_usage_tracking.ipynb +++ b/docs/modules/models/llms/examples/token_usage_tracking.ipynb @@ -43,22 +43,18 @@ "name": "stdout", "output_type": "stream", "text": [ - "Total Tokens: 39\n", - "Prompt Tokens: 4\n", - "Completion Tokens: 35\n", + "Tokens Used: 42\n", + "\tPrompt Tokens: 4\n", + "\tCompletion Tokens: 38\n", "Successful Requests: 1\n", - "Total Cost (USD): $0.0007800000000000001\n" + "Total Cost (USD): $0.00084\n" ] } ], "source": [ "with get_openai_callback() as cb:\n", " result = llm(\"Tell me a joke\")\n", - " print(f\"Total Tokens: {cb.total_tokens}\")\n", - " print(f\"Prompt Tokens: {cb.prompt_tokens}\")\n", - " print(f\"Completion Tokens: {cb.completion_tokens}\")\n", - " print(f\"Successful Requests: {cb.successful_requests}\")\n", - " print(f\"Total Cost (USD): ${cb.total_cost}\")" + " print(cb)" ] }, { diff --git a/langchain/callbacks/openai_info.py b/langchain/callbacks/openai_info.py index 299a3c8fb57..42005acbcdd 100644 --- a/langchain/callbacks/openai_info.py +++ b/langchain/callbacks/openai_info.py @@ -53,6 +53,15 @@ class OpenAICallbackHandler(BaseCallbackHandler): successful_requests: int = 0 total_cost: float = 0.0 + def __repr__(self) -> str: + return ( + f"Tokens Used: {self.total_tokens}\n" + f"\tPrompt Tokens: {self.prompt_tokens}\n" + f"\tCompletion Tokens: {self.completion_tokens}\n" + f"Successful Requests: {self.successful_requests}\n" + f"Total Cost (USD): ${self.total_cost}" + ) + @property def always_verbose(self) -> bool: """Whether to call verbose callbacks even if verbose is False.""" From ecc1a0c051955f28991a05e0cca24272866787f1 Mon Sep 17 00:00:00 2001 From: leo-gan Date: Thu, 13 Apr 2023 11:29:59 -0700 Subject: [PATCH 05/49] added code-analysis-deeplake.ipynb (#2844) This notebook is heavily copied from the `twitter-the-algorithm-analysis-deeplake.ipynb` --- .../code/code-analysis-deeplake.ipynb | 673 ++++++++++++++++++ 1 file changed, 673 insertions(+) create mode 100644 docs/use_cases/code/code-analysis-deeplake.ipynb diff --git a/docs/use_cases/code/code-analysis-deeplake.ipynb b/docs/use_cases/code/code-analysis-deeplake.ipynb new file mode 100644 index 00000000000..d946f2778ca --- /dev/null +++ b/docs/use_cases/code/code-analysis-deeplake.ipynb @@ -0,0 +1,673 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Use LangChain, GPT and Deep Lake to work with code base\n", + "In this tutorial, we are going to use Langchain + Deep Lake with GPT to analyze the code base of the LangChain itself. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Design" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "1. Prepare data:\n", + " 1. Upload all python project files using the `langchain.document_loaders.TextLoader`. We will call these files the **documents**.\n", + " 2. Split all documents to chunks using the `langchain.text_splitter.CharacterTextSplitter`.\n", + " 3. Embed chunks and upload them into the DeepLake using `langchain.embeddings.openai.OpenAIEmbeddings` and `langchain.vectorstores.DeepLake`\n", + "2. Question-Answering:\n", + " 1. Build a chain from `langchain.chat_models.ChatOpenAI` and `langchain.chains.ConversationalRetrievalChain`\n", + " 2. Prepare questions.\n", + " 3. Get answers running the chain.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Implementation" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Integration preparations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We need to set up keys for external services and install necessary python libraries." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "#!python3 -m pip install --upgrade langchain deeplake openai" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set up OpenAI embeddings, Deep Lake multi-modal vector store api and authenticate. \n", + "\n", + "For full documentation of Deep Lake please follow https://docs.activeloop.ai/ and API reference https://docs.deeplake.ai/en/latest/" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdin", + "output_type": "stream", + "text": [ + " ········\n" + ] + } + ], + "source": [ + "import os\n", + "from getpass import getpass\n", + "\n", + "os.environ['OPENAI_API_KEY'] = getpass()\n", + "# Please manually enter OpenAI Key" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Authenticate into Deep Lake if you want to create your own dataset and publish it. You can get an API key from the platform at https://app.activeloop.ai" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdin", + "output_type": "stream", + "text": [ + " ········\n" + ] + } + ], + "source": [ + "DEEPLAKE_ACCOUNT_NAME = getpass()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdin", + "output_type": "stream", + "text": [ + " ········\n" + ] + } + ], + "source": [ + "os.environ['DEEPLAKE_KEY'] = getpass()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "!activeloop login -t $DEEPLAKE_KEY" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Prepare data " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load all repository files. Here we assume this notebook is downloaded as the part of the langchain fork and we work with the python files of the `langchain` repo.\n", + "\n", + "If you want to use files from different repo, change `root_dir` to the root dir of your repo." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1147\n" + ] + } + ], + "source": [ + "from langchain.document_loaders import TextLoader\n", + "\n", + "root_dir = '../../../..'\n", + "\n", + "docs = []\n", + "for dirpath, dirnames, filenames in os.walk(root_dir):\n", + " for file in filenames:\n", + " if file.endswith('.py') and '/.venv/' not in dirpath:\n", + " try: \n", + " loader = TextLoader(os.path.join(dirpath, file), encoding='utf-8')\n", + " docs.extend(loader.load_and_split())\n", + " except Exception as e: \n", + " pass\n", + "print(f'{len(docs)}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, chunk the files" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Created a chunk of size 1620, which is longer than the specified 1000\n", + "Created a chunk of size 1213, which is longer than the specified 1000\n", + "Created a chunk of size 1263, which is longer than the specified 1000\n", + "Created a chunk of size 1448, which is longer than the specified 1000\n", + "Created a chunk of size 1120, which is longer than the specified 1000\n", + "Created a chunk of size 1148, which is longer than the specified 1000\n", + "Created a chunk of size 1826, which is longer than the specified 1000\n", + "Created a chunk of size 1260, which is longer than the specified 1000\n", + "Created a chunk of size 1195, which is longer than the specified 1000\n", + "Created a chunk of size 2147, which is longer than the specified 1000\n", + "Created a chunk of size 1410, which is longer than the specified 1000\n", + "Created a chunk of size 1269, which is longer than the specified 1000\n", + "Created a chunk of size 1030, which is longer than the specified 1000\n", + "Created a chunk of size 1046, which is longer than the specified 1000\n", + "Created a chunk of size 1024, which is longer than the specified 1000\n", + "Created a chunk of size 1026, which is longer than the specified 1000\n", + "Created a chunk of size 1285, which is longer than the specified 1000\n", + "Created a chunk of size 1370, which is longer than the specified 1000\n", + "Created a chunk of size 1031, which is longer than the specified 1000\n", + "Created a chunk of size 1999, which is longer than the specified 1000\n", + "Created a chunk of size 1029, which is longer than the specified 1000\n", + "Created a chunk of size 1120, which is longer than the specified 1000\n", + "Created a chunk of size 1033, which is longer than the specified 1000\n", + "Created a chunk of size 1143, which is longer than the specified 1000\n", + "Created a chunk of size 1416, which is longer than the specified 1000\n", + "Created a chunk of size 2482, which is longer than the specified 1000\n", + "Created a chunk of size 1890, which is longer than the specified 1000\n", + "Created a chunk of size 1418, which is longer than the specified 1000\n", + "Created a chunk of size 1848, which is longer than the specified 1000\n", + "Created a chunk of size 1069, which is longer than the specified 1000\n", + "Created a chunk of size 2369, which is longer than the specified 1000\n", + "Created a chunk of size 1045, which is longer than the specified 1000\n", + "Created a chunk of size 1501, which is longer than the specified 1000\n", + "Created a chunk of size 1208, which is longer than the specified 1000\n", + "Created a chunk of size 1950, which is longer than the specified 1000\n", + "Created a chunk of size 1283, which is longer than the specified 1000\n", + "Created a chunk of size 1414, which is longer than the specified 1000\n", + "Created a chunk of size 1304, which is longer than the specified 1000\n", + "Created a chunk of size 1224, which is longer than the specified 1000\n", + "Created a chunk of size 1060, which is longer than the specified 1000\n", + "Created a chunk of size 2461, which is longer than the specified 1000\n", + "Created a chunk of size 1099, which is longer than the specified 1000\n", + "Created a chunk of size 1178, which is longer than the specified 1000\n", + "Created a chunk of size 1449, which is longer than the specified 1000\n", + "Created a chunk of size 1345, which is longer than the specified 1000\n", + "Created a chunk of size 3359, which is longer than the specified 1000\n", + "Created a chunk of size 2248, which is longer than the specified 1000\n", + "Created a chunk of size 1589, which is longer than the specified 1000\n", + "Created a chunk of size 2104, which is longer than the specified 1000\n", + "Created a chunk of size 1505, which is longer than the specified 1000\n", + "Created a chunk of size 1387, which is longer than the specified 1000\n", + "Created a chunk of size 1215, which is longer than the specified 1000\n", + "Created a chunk of size 1240, which is longer than the specified 1000\n", + "Created a chunk of size 1635, which is longer than the specified 1000\n", + "Created a chunk of size 1075, which is longer than the specified 1000\n", + "Created a chunk of size 2180, which is longer than the specified 1000\n", + "Created a chunk of size 1791, which is longer than the specified 1000\n", + "Created a chunk of size 1555, which is longer than the specified 1000\n", + "Created a chunk of size 1082, which is longer than the specified 1000\n", + "Created a chunk of size 1225, which is longer than the specified 1000\n", + "Created a chunk of size 1287, which is longer than the specified 1000\n", + "Created a chunk of size 1085, which is longer than the specified 1000\n", + "Created a chunk of size 1117, which is longer than the specified 1000\n", + "Created a chunk of size 1966, which is longer than the specified 1000\n", + "Created a chunk of size 1150, which is longer than the specified 1000\n", + "Created a chunk of size 1285, which is longer than the specified 1000\n", + "Created a chunk of size 1150, which is longer than the specified 1000\n", + "Created a chunk of size 1585, which is longer than the specified 1000\n", + "Created a chunk of size 1208, which is longer than the specified 1000\n", + "Created a chunk of size 1267, which is longer than the specified 1000\n", + "Created a chunk of size 1542, which is longer than the specified 1000\n", + "Created a chunk of size 1183, which is longer than the specified 1000\n", + "Created a chunk of size 2424, which is longer than the specified 1000\n", + "Created a chunk of size 1017, which is longer than the specified 1000\n", + "Created a chunk of size 1304, which is longer than the specified 1000\n", + "Created a chunk of size 1379, which is longer than the specified 1000\n", + "Created a chunk of size 1324, which is longer than the specified 1000\n", + "Created a chunk of size 1205, which is longer than the specified 1000\n", + "Created a chunk of size 1056, which is longer than the specified 1000\n", + "Created a chunk of size 1195, which is longer than the specified 1000\n", + "Created a chunk of size 3608, which is longer than the specified 1000\n", + "Created a chunk of size 1058, which is longer than the specified 1000\n", + "Created a chunk of size 1075, which is longer than the specified 1000\n", + "Created a chunk of size 1217, which is longer than the specified 1000\n", + "Created a chunk of size 1109, which is longer than the specified 1000\n", + "Created a chunk of size 1440, which is longer than the specified 1000\n", + "Created a chunk of size 1046, which is longer than the specified 1000\n", + "Created a chunk of size 1220, which is longer than the specified 1000\n", + "Created a chunk of size 1403, which is longer than the specified 1000\n", + "Created a chunk of size 1241, which is longer than the specified 1000\n", + "Created a chunk of size 1427, which is longer than the specified 1000\n", + "Created a chunk of size 1049, which is longer than the specified 1000\n", + "Created a chunk of size 1580, which is longer than the specified 1000\n", + "Created a chunk of size 1565, which is longer than the specified 1000\n", + "Created a chunk of size 1131, which is longer than the specified 1000\n", + "Created a chunk of size 1425, which is longer than the specified 1000\n", + "Created a chunk of size 1054, which is longer than the specified 1000\n", + "Created a chunk of size 1027, which is longer than the specified 1000\n", + "Created a chunk of size 2559, which is longer than the specified 1000\n", + "Created a chunk of size 1028, which is longer than the specified 1000\n", + "Created a chunk of size 1382, which is longer than the specified 1000\n", + "Created a chunk of size 1888, which is longer than the specified 1000\n", + "Created a chunk of size 1475, which is longer than the specified 1000\n", + "Created a chunk of size 1652, which is longer than the specified 1000\n", + "Created a chunk of size 1891, which is longer than the specified 1000\n", + "Created a chunk of size 1899, which is longer than the specified 1000\n", + "Created a chunk of size 1021, which is longer than the specified 1000\n", + "Created a chunk of size 1085, which is longer than the specified 1000\n", + "Created a chunk of size 1854, which is longer than the specified 1000\n", + "Created a chunk of size 1672, which is longer than the specified 1000\n", + "Created a chunk of size 2537, which is longer than the specified 1000\n", + "Created a chunk of size 1251, which is longer than the specified 1000\n", + "Created a chunk of size 1734, which is longer than the specified 1000\n", + "Created a chunk of size 1642, which is longer than the specified 1000\n", + "Created a chunk of size 1376, which is longer than the specified 1000\n", + "Created a chunk of size 1253, which is longer than the specified 1000\n", + "Created a chunk of size 1642, which is longer than the specified 1000\n", + "Created a chunk of size 1419, which is longer than the specified 1000\n", + "Created a chunk of size 1438, which is longer than the specified 1000\n", + "Created a chunk of size 1427, which is longer than the specified 1000\n", + "Created a chunk of size 1684, which is longer than the specified 1000\n", + "Created a chunk of size 1760, which is longer than the specified 1000\n", + "Created a chunk of size 1157, which is longer than the specified 1000\n", + "Created a chunk of size 2504, which is longer than the specified 1000\n", + "Created a chunk of size 1082, which is longer than the specified 1000\n", + "Created a chunk of size 2268, which is longer than the specified 1000\n", + "Created a chunk of size 1784, which is longer than the specified 1000\n", + "Created a chunk of size 1311, which is longer than the specified 1000\n", + "Created a chunk of size 2972, which is longer than the specified 1000\n", + "Created a chunk of size 1144, which is longer than the specified 1000\n", + "Created a chunk of size 1825, which is longer than the specified 1000\n", + "Created a chunk of size 1508, which is longer than the specified 1000\n", + "Created a chunk of size 2901, which is longer than the specified 1000\n", + "Created a chunk of size 1715, which is longer than the specified 1000\n", + "Created a chunk of size 1062, which is longer than the specified 1000\n", + "Created a chunk of size 1206, which is longer than the specified 1000\n", + "Created a chunk of size 1102, which is longer than the specified 1000\n", + "Created a chunk of size 1184, which is longer than the specified 1000\n", + "Created a chunk of size 1002, which is longer than the specified 1000\n", + "Created a chunk of size 1065, which is longer than the specified 1000\n", + "Created a chunk of size 1871, which is longer than the specified 1000\n", + "Created a chunk of size 1754, which is longer than the specified 1000\n", + "Created a chunk of size 2413, which is longer than the specified 1000\n", + "Created a chunk of size 1771, which is longer than the specified 1000\n", + "Created a chunk of size 2054, which is longer than the specified 1000\n", + "Created a chunk of size 2000, which is longer than the specified 1000\n", + "Created a chunk of size 2061, which is longer than the specified 1000\n", + "Created a chunk of size 1066, which is longer than the specified 1000\n", + "Created a chunk of size 1419, which is longer than the specified 1000\n", + "Created a chunk of size 1368, which is longer than the specified 1000\n", + "Created a chunk of size 1008, which is longer than the specified 1000\n", + "Created a chunk of size 1227, which is longer than the specified 1000\n", + "Created a chunk of size 1745, which is longer than the specified 1000\n", + "Created a chunk of size 2296, which is longer than the specified 1000\n", + "Created a chunk of size 1083, which is longer than the specified 1000\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3477\n" + ] + } + ], + "source": [ + "from langchain.text_splitter import CharacterTextSplitter\n", + "\n", + "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n", + "texts = text_splitter.split_documents(docs)\n", + "print(f\"{len(texts)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then embed chunks and upload them to the DeepLake.\n", + "\n", + "This can take several minutes. " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "OpenAIEmbeddings(client=, model='text-embedding-ada-002', document_model_name='text-embedding-ada-002', query_model_name='text-embedding-ada-002', embedding_ctx_length=8191, openai_api_key=None, openai_organization=None, allowed_special=set(), disallowed_special='all', chunk_size=1000, max_retries=6)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from langchain.embeddings.openai import OpenAIEmbeddings\n", + "\n", + "embeddings = OpenAIEmbeddings()\n", + "embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain.vectorstores import DeepLake\n", + "\n", + "db = DeepLake.from_documents(texts, embeddings, dataset_path=f\"hub://{DEEPLAKE_ACCOUNT_NAME}/langchain-code\")\n", + "db" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Question Answering\n", + "First load the dataset, construct the retriever, then construct the Conversational Chain" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "-" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This dataset can be visualized in Jupyter Notebook by ds.visualize() or at https://app.activeloop.ai/user_name/langchain-code\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hub://user_name/langchain-code loaded successfully.\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Deep Lake Dataset in hub://user_name/langchain-code already exists, loading from the storage\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset(path='hub://user_name/langchain-code', read_only=True, tensors=['embedding', 'ids', 'metadata', 'text'])\n", + "\n", + " tensor htype shape dtype compression\n", + " ------- ------- ------- ------- ------- \n", + " embedding generic (3477, 1536) float32 None \n", + " ids text (3477, 1) str None \n", + " metadata json (3477, 1) str None \n", + " text text (3477, 1) str None \n" + ] + } + ], + "source": [ + "db = DeepLake(dataset_path=f\"hub://{DEEPLAKE_ACCOUNT_NAME}/langchain-code\", read_only=True, embedding_function=embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "retriever = db.as_retriever()\n", + "retriever.search_kwargs['distance_metric'] = 'cos'\n", + "retriever.search_kwargs['fetch_k'] = 20\n", + "retriever.search_kwargs['maximal_marginal_relevance'] = True\n", + "retriever.search_kwargs['k'] = 20" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also specify user defined functions using [Deep Lake filters](https://docs.deeplake.ai/en/latest/deeplake.core.dataset.html#deeplake.core.dataset.Dataset.filter)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def filter(x):\n", + " # filter based on source code\n", + " if 'something' in x['text'].data()['value']:\n", + " return False\n", + " \n", + " # filter based on path e.g. extension\n", + " metadata = x['metadata'].data()['value']\n", + " return 'only_this' in metadata['source'] or 'also_that' in metadata['source']\n", + "\n", + "### turn on below for custom filtering\n", + "# retriever.search_kwargs['filter'] = filter" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain.chat_models import ChatOpenAI\n", + "from langchain.chains import ConversationalRetrievalChain\n", + "\n", + "model = ChatOpenAI(model='gpt-3.5-turbo') # 'ada' 'gpt-3.5-turbo' 'gpt-4',\n", + "qa = ConversationalRetrievalChain.from_llm(model,retriever=retriever)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "questions = [\n", + " \"What is the class hierarchy?\",\n", + " # \"What classes are derived from the Chain class?\",\n", + " # \"What classes and functions in the ./langchain/utilities/ forlder are not covered by unit tests?\",\n", + " # \"What one improvement do you propose in code in relation to the class herarchy for the Chain class?\",\n", + "] \n", + "chat_history = []\n", + "\n", + "for question in questions: \n", + " result = qa({\"question\": question, \"chat_history\": chat_history})\n", + " chat_history.append((question, result['answer']))\n", + " print(f\"-> **Question**: {question} \\n\")\n", + " print(f\"**Answer**: {result['answer']} \\n\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "-> **Question**: What is the class hierarchy? \n", + "\n", + "**Answer**: There are several class hierarchies in the provided code, so I'll list a few:\n", + "\n", + "1. `BaseModel` -> `ConstitutionalPrinciple`: `ConstitutionalPrinciple` is a subclass of `BaseModel`.\n", + "2. `BasePromptTemplate` -> `StringPromptTemplate`, `AIMessagePromptTemplate`, `BaseChatPromptTemplate`, `ChatMessagePromptTemplate`, `ChatPromptTemplate`, `HumanMessagePromptTemplate`, `MessagesPlaceholder`, `SystemMessagePromptTemplate`, `FewShotPromptTemplate`, `FewShotPromptWithTemplates`, `Prompt`, `PromptTemplate`: All of these classes are subclasses of `BasePromptTemplate`.\n", + "3. `APIChain`, `Chain`, `MapReduceDocumentsChain`, `MapRerankDocumentsChain`, `RefineDocumentsChain`, `StuffDocumentsChain`, `HypotheticalDocumentEmbedder`, `LLMChain`, `LLMBashChain`, `LLMCheckerChain`, `LLMMathChain`, `LLMRequestsChain`, `PALChain`, `QAWithSourcesChain`, `VectorDBQAWithSourcesChain`, `VectorDBQA`, `SQLDatabaseChain`: All of these classes are subclasses of `Chain`.\n", + "4. `BaseLoader`: `BaseLoader` is a subclass of `ABC`.\n", + "5. `BaseTracer` -> `ChainRun`, `LLMRun`, `SharedTracer`, `ToolRun`, `Tracer`, `TracerException`, `TracerSession`: All of these classes are subclasses of `BaseTracer`.\n", + "6. `OpenAIEmbeddings`, `HuggingFaceEmbeddings`, `CohereEmbeddings`, `JinaEmbeddings`, `LlamaCppEmbeddings`, `HuggingFaceHubEmbeddings`, `TensorflowHubEmbeddings`, `SagemakerEndpointEmbeddings`, `HuggingFaceInstructEmbeddings`, `SelfHostedEmbeddings`, `SelfHostedHuggingFaceEmbeddings`, `SelfHostedHuggingFaceInstructEmbeddings`, `FakeEmbeddings`, `AlephAlphaAsymmetricSemanticEmbedding`, `AlephAlphaSymmetricSemanticEmbedding`: All of these classes are subclasses of `BaseLLM`. \n", + "\n", + "\n", + "-> **Question**: What classes are derived from the Chain class? \n", + "\n", + "**Answer**: There are multiple classes that are derived from the Chain class. Some of them are:\n", + "- APIChain\n", + "- AnalyzeDocumentChain\n", + "- ChatVectorDBChain\n", + "- CombineDocumentsChain\n", + "- ConstitutionalChain\n", + "- ConversationChain\n", + "- GraphQAChain\n", + "- HypotheticalDocumentEmbedder\n", + "- LLMChain\n", + "- LLMCheckerChain\n", + "- LLMRequestsChain\n", + "- LLMSummarizationCheckerChain\n", + "- MapReduceChain\n", + "- OpenAPIEndpointChain\n", + "- PALChain\n", + "- QAWithSourcesChain\n", + "- RetrievalQA\n", + "- RetrievalQAWithSourcesChain\n", + "- SequentialChain\n", + "- SQLDatabaseChain\n", + "- TransformChain\n", + "- VectorDBQA\n", + "- VectorDBQAWithSourcesChain\n", + "\n", + "There might be more classes that are derived from the Chain class as it is possible to create custom classes that extend the Chain class.\n", + "\n", + "\n", + "-> **Question**: What classes and functions in the ./langchain/utilities/ forlder are not covered by unit tests? \n", + "\n", + "**Answer**: All classes and functions in the `./langchain/utilities/` folder seem to have unit tests written for them. \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From f3180f05f91f5a0bd84d38b90ae3b4ea49689a12 Mon Sep 17 00:00:00 2001 From: Jon Luo <20971593+jzluo@users.noreply.github.com> Date: Thu, 13 Apr 2023 14:46:59 -0400 Subject: [PATCH 06/49] Update sql chain notebook to clarify use of SQLAlchemy for connections (#2850) Have seen questions about whether or not the `SQLDatabaseChain` supports more than just sqlite, which was unclear in the docs, so tried to clarify that and how to connect to other dialects. --- docs/modules/chains/examples/sqlite.ipynb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/modules/chains/examples/sqlite.ipynb b/docs/modules/chains/examples/sqlite.ipynb index a4b33f04b89..b3b23eb43d4 100644 --- a/docs/modules/chains/examples/sqlite.ipynb +++ b/docs/modules/chains/examples/sqlite.ipynb @@ -9,9 +9,9 @@ } }, "source": [ - "# SQLite example\n", + "# SQL Chain example\n", "\n", - "This example showcases hooking up an LLM to answer questions over a database." + "This example demonstrates the use of the `SQLDatabaseChain` for answering questions over a database." ] }, { @@ -23,8 +23,10 @@ } }, "source": [ - "This uses the example Chinook database.\n", - "To set it up follow the instructions on https://database.guide/2-sample-databases-sqlite/, placing the `.db` file in a notebooks folder at the root of this repository." + "Under the hood, LangChain uses SQLAlchemy to connect to SQL databases. The `SQLDatabaseChain` can therefore be used with any SQL dialect supported by SQLAlchemy, such as MS SQL, MySQL, MariaDB, PostgreSQL, Oracle SQL, and SQLite. Please refer to the SQLAlchemy documentation for more information about requirements for connecting to your database. For example, a connection to MySQL requires an appropriate connector such as PyMySQL. A URI for a MySQL connection might look like: `mysql+pymysql://user:pass@some_mysql_db_address/db_name`\n", + "\n", + "This demonstration uses SQLite and the example Chinook database.\n", + "To set it up, follow the instructions on https://database.guide/2-sample-databases-sqlite/, placing the `.db` file in a notebooks folder at the root of this repository." ] }, { @@ -679,7 +681,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.1" + "version": "3.10.10" } }, "nbformat": 4, From c26a259ba6e2682f6a9be9a7f97c95eeacd419a3 Mon Sep 17 00:00:00 2001 From: Benjamin Tan Wei Hao Date: Fri, 14 Apr 2023 11:26:26 +0800 Subject: [PATCH 07/49] Fix tiny typo (#2863) --- docs/ecosystem/gpt4all.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ecosystem/gpt4all.md b/docs/ecosystem/gpt4all.md index 81f073e35b0..36422eb529c 100644 --- a/docs/ecosystem/gpt4all.md +++ b/docs/ecosystem/gpt4all.md @@ -36,7 +36,7 @@ from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler callback_manager = CallbackManager([StreamingStdOutCallbackHandler()]) model = GPT4All(model="./models/gpt4all-model.bin", n_ctx=512, n_threads=8, callback_handler=callback_handler, verbose=True) -# Generate text. Tokens are streamed throught the callback manager. +# Generate text. Tokens are streamed through the callback manager. model("Once upon a time, ") ``` @@ -44,4 +44,4 @@ model("Once upon a time, ") You can find links to model file downloads in the [pyllamacpp](https://github.com/nomic-ai/pyllamacpp) repository. -For a more detailed walkthrough of this, see [this notebook](../modules/models/llms/integrations/gpt4all.ipynb) \ No newline at end of file +For a more detailed walkthrough of this, see [this notebook](../modules/models/llms/integrations/gpt4all.ipynb) From 6be5d7c6129e1ba126798850720084c18b594e10 Mon Sep 17 00:00:00 2001 From: Adam McCabe Date: Thu, 13 Apr 2023 23:27:40 -0400 Subject: [PATCH 08/49] Update reduce_openapi_spec for PATCH and DELETE (#2861) My recent pull request (#2729) neglected to update the `reduce_openapi_spec` in spec.py to also accommodate PATCH and DELETE added to planner.py and prompt_planner.py. --- langchain/agents/agent_toolkits/openapi/spec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langchain/agents/agent_toolkits/openapi/spec.py b/langchain/agents/agent_toolkits/openapi/spec.py index 72f3ff9d61a..b8196239823 100644 --- a/langchain/agents/agent_toolkits/openapi/spec.py +++ b/langchain/agents/agent_toolkits/openapi/spec.py @@ -68,12 +68,12 @@ def reduce_openapi_spec(spec: dict, dereference: bool = True) -> ReducedOpenAPIS I was hoping https://openapi.tools/ would have some useful bits to this end, but doesn't seem so. """ - # 1. Consider only get, post endpoints. + # 1. Consider only get, post, patch, delete endpoints. endpoints = [ (f"{operation_name.upper()} {route}", docs.get("description"), docs) for route, operation in spec["paths"].items() for operation_name, docs in operation.items() - if operation_name in ["get", "post"] + if operation_name in ["get", "post", "patch", "delete"] ] # 2. Replace any refs so that complete docs are retrieved. From ed2ef5cbe4381bec914589eb0e7647819eda74df Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Thu, 13 Apr 2023 21:31:18 -0700 Subject: [PATCH 09/49] Harrison/rwkv utf8 (#2867) Co-authored-by: Akihiro --- langchain/llms/rwkv.py | 76 ++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/langchain/llms/rwkv.py b/langchain/llms/rwkv.py index f4294f1c53d..5c27185ab66 100644 --- a/langchain/llms/rwkv.py +++ b/langchain/llms/rwkv.py @@ -3,7 +3,7 @@ Based on https://github.com/saharNooby/rwkv.cpp/blob/master/rwkv/chat_with_bot.py https://github.com/BlinkDL/ChatRWKV/blob/main/v2/chat.py """ -from typing import Any, Dict, List, Mapping, Optional, Set, SupportsIndex +from typing import Any, Dict, List, Mapping, Optional, Set from pydantic import BaseModel, Extra, root_validator @@ -58,7 +58,7 @@ class RWKV(LLM, BaseModel): CHUNK_LEN: int = 256 """Batch size for prompt processing.""" - max_tokens_per_generation: SupportsIndex = 256 + max_tokens_per_generation: int = 256 """Maximum number of tokens to generate.""" client: Any = None #: :meta private: @@ -69,6 +69,8 @@ class RWKV(LLM, BaseModel): model_tokens: Any = None #: :meta private: + model_state: Any = None #: :meta private: + class Config: """Configuration for this pydantic object.""" @@ -139,42 +141,66 @@ class RWKV(LLM, BaseModel): """Return the type of llm.""" return "rwkv-4" - def rwkv_generate(self, prompt: str) -> str: - tokens = self.tokenizer.encode(prompt).ids + def run_rnn(self, _tokens: List[str], newline_adj: int = 0) -> Any: + AVOID_REPEAT_TOKENS = [] + AVOID_REPEAT = "，：？！" + for i in AVOID_REPEAT: + dd = self.pipeline.encode(i) + assert len(dd) == 1 + AVOID_REPEAT_TOKENS += dd - logits = None - state = None + tokens = [int(x) for x in _tokens] + self.model_tokens += tokens - occurrence = {} + out: Any = None - # Feed in the input string while len(tokens) > 0: - logits, state = self.client.forward(tokens[: self.CHUNK_LEN], state) + out, self.model_state = self.client.forward( + tokens[: self.CHUNK_LEN], self.model_state + ) tokens = tokens[self.CHUNK_LEN :] + END_OF_LINE = 187 + out[END_OF_LINE] += newline_adj # adjust \n probability + + if self.model_tokens[-1] in AVOID_REPEAT_TOKENS: + out[self.model_tokens[-1]] = -999999999 + return out + + def rwkv_generate(self, prompt: str) -> str: + self.model_state = None + self.model_tokens = [] + logits = self.run_rnn(self.tokenizer.encode(prompt).ids) + begin = len(self.model_tokens) + out_last = begin + + occurrence: Dict = {} decoded = "" for i in range(self.max_tokens_per_generation): - token = self.pipeline.sample_logits( - logits, temperature=self.temperature, top_p=self.top_p - ) - - if token not in occurrence: - occurrence[token] = 1 - else: - occurrence[token] += 1 - - decoded += self.tokenizer.decode([token]) - - if "\n" in decoded: - break - - # feed back in - logits, state = self.client.forward([token], state) for n in occurrence: logits[n] -= ( self.penalty_alpha_presence + occurrence[n] * self.penalty_alpha_frequency ) + token = self.pipeline.sample_logits( + logits, temperature=self.temperature, top_p=self.top_p + ) + + END_OF_TEXT = 0 + if token == END_OF_TEXT: + break + if token not in occurrence: + occurrence[token] = 1 + else: + occurrence[token] += 1 + + logits = self.run_rnn([token]) + xxx = self.tokenizer.decode(self.model_tokens[out_last:]) + if "\ufffd" not in xxx: # avoid utf-8 display issues + decoded += xxx + out_last = begin + i + 1 + if i >= self.max_tokens_per_generation - 100: + break return decoded From bf0887c486f87b2d5d1781971579f8843cc29032 Mon Sep 17 00:00:00 2001 From: vowelparrot <130414180+vowelparrot@users.noreply.github.com> Date: Thu, 13 Apr 2023 21:31:59 -0700 Subject: [PATCH 10/49] Add Slack Directory Loader (#2841) Fixes linting issue from #2835 Adds a loader for Slack Exports which can be a very valuable source of knowledge to use for internal QA bots and other use cases. ```py # Export data from your Slack Workspace first. from langchain.document_loaders import SLackDirectoryLoader SLACK_WORKSPACE_URL = "https://awesome.slack.com" loader = ("Slack_Exports", SLACK_WORKSPACE_URL) docs = loader.load() ``` --- .../examples/slack_directory.ipynb | 81 +++++++++++++ langchain/document_loaders/__init__.py | 2 + langchain/document_loaders/slack_directory.py | 112 ++++++++++++++++++ .../document_loaders/test_slack.py | 23 ++++ .../examples/slack_export.zip | Bin 0 -> 3904 bytes 5 files changed, 218 insertions(+) create mode 100644 docs/modules/indexes/document_loaders/examples/slack_directory.ipynb create mode 100644 langchain/document_loaders/slack_directory.py create mode 100644 tests/integration_tests/document_loaders/test_slack.py create mode 100644 tests/integration_tests/examples/slack_export.zip diff --git a/docs/modules/indexes/document_loaders/examples/slack_directory.ipynb b/docs/modules/indexes/document_loaders/examples/slack_directory.ipynb new file mode 100644 index 00000000000..471efa536f2 --- /dev/null +++ b/docs/modules/indexes/document_loaders/examples/slack_directory.ipynb @@ -0,0 +1,81 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "1dc7df1d", + "metadata": {}, + "source": [ + "# Slack (Local Exported Zipfile)\n", + "\n", + "This notebook covers how to load documents from a Zipfile generated from a Slack export.\n", + "\n", + "In order to get this Slack export, follow these instructions:\n", + "\n", + "## 🧑 Instructions for ingesting your own dataset\n", + "\n", + "Export your Slack data. You can do this by going to your Workspace Management page and clicking the Import/Export option ({your_slack_domain}.slack.com/services/export). Then, choose the right date range and click `Start export`. Slack will send you an email and a DM when the export is ready.\n", + "\n", + "The download will produce a `.zip` file in your Downloads folder (or wherever your downloads can be found, depending on your OS configuration).\n", + "\n", + "Copy the path to the `.zip` file, and assign it as `LOCAL_ZIPFILE` below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "007c5cbf", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.document_loaders import SlackDirectoryLoader " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1caec59", + "metadata": {}, + "outputs": [], + "source": [ + "# Optionally set your Slack URL. This will give you proper URLs in the docs sources.\n", + "SLACK_WORKSPACE_URL = \"https://xxx.slack.com\"\n", + "LOCAL_ZIPFILE = \"\" # Paste the local paty to your Slack zip file here.\n", + "\n", + "loader = SlackDirectoryLoader(LOCAL_ZIPFILE, SLACK_WORKSPACE_URL)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1c30ff7", + "metadata": {}, + "outputs": [], + "source": [ + "docs = loader.load()\n", + "docs" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/langchain/document_loaders/__init__.py b/langchain/document_loaders/__init__.py index 956f85f9ebe..c2ea430abfc 100644 --- a/langchain/document_loaders/__init__.py +++ b/langchain/document_loaders/__init__.py @@ -55,6 +55,7 @@ from langchain.document_loaders.roam import RoamLoader from langchain.document_loaders.s3_directory import S3DirectoryLoader from langchain.document_loaders.s3_file import S3FileLoader from langchain.document_loaders.sitemap import SitemapLoader +from langchain.document_loaders.slack_directory import SlackDirectoryLoader from langchain.document_loaders.srt import SRTLoader from langchain.document_loaders.telegram import TelegramChatLoader from langchain.document_loaders.text import TextLoader @@ -138,4 +139,5 @@ __all__ = [ "DuckDBLoader", "BigQueryLoader", "BiliBiliLoader", + "SlackDirectoryLoader", ] diff --git a/langchain/document_loaders/slack_directory.py b/langchain/document_loaders/slack_directory.py new file mode 100644 index 00000000000..718367c4d46 --- /dev/null +++ b/langchain/document_loaders/slack_directory.py @@ -0,0 +1,112 @@ +"""Loader for documents from a Slack export.""" +import json +import zipfile +from pathlib import Path +from typing import Dict, List, Optional + +from langchain.docstore.document import Document +from langchain.document_loaders.base import BaseLoader + + +class SlackDirectoryLoader(BaseLoader): + """Loader for loading documents from a Slack directory dump.""" + + def __init__(self, zip_path: str, workspace_url: Optional[str] = None): + """Initialize the SlackDirectoryLoader. + + Args: + zip_path (str): The path to the Slack directory dump zip file. + workspace_url (Optional[str]): The Slack workspace URL. + Including the URL will turn + sources into links. Defaults to None. + """ + self.zip_path = Path(zip_path) + self.workspace_url = workspace_url + self.channel_id_map = self._get_channel_id_map(self.zip_path) + + @staticmethod + def _get_channel_id_map(zip_path: Path) -> Dict[str, str]: + """Get a dictionary mapping channel names to their respective IDs.""" + with zipfile.ZipFile(zip_path, "r") as zip_file: + try: + with zip_file.open("channels.json", "r") as f: + channels = json.load(f) + return {channel["name"]: channel["id"] for channel in channels} + except KeyError: + return {} + + def load(self) -> List[Document]: + """Load and return documents from the Slack directory dump.""" + docs = [] + with zipfile.ZipFile(self.zip_path, "r") as zip_file: + for channel_path in zip_file.namelist(): + channel_name = Path(channel_path).parent.name + if not channel_name: + continue + if channel_path.endswith(".json"): + messages = self._read_json(zip_file, channel_path) + for message in messages: + document = self._convert_message_to_document( + message, channel_name + ) + docs.append(document) + return docs + + def _read_json(self, zip_file: zipfile.ZipFile, file_path: str) -> List[dict]: + """Read JSON data from a zip subfile.""" + with zip_file.open(file_path, "r") as f: + data = json.load(f) + return data + + def _convert_message_to_document( + self, message: dict, channel_name: str + ) -> Document: + """ + Convert a message to a Document object. + + Args: + message (dict): A message in the form of a dictionary. + channel_name (str): The name of the channel the message belongs to. + + Returns: + Document: A Document object representing the message. + """ + text = message.get("text", "") + metadata = self._get_message_metadata(message, channel_name) + return Document( + page_content=text, + metadata=metadata, + ) + + def _get_message_metadata(self, message: dict, channel_name: str) -> dict: + """Create and return metadata for a given message and channel.""" + timestamp = message.get("ts", "") + user = message.get("user", "") + source = self._get_message_source(channel_name, user, timestamp) + return { + "source": source, + "channel": channel_name, + "timestamp": timestamp, + "user": user, + } + + def _get_message_source(self, channel_name: str, user: str, timestamp: str) -> str: + """ + Get the message source as a string. + + Args: + channel_name (str): The name of the channel the message belongs to. + user (str): The user ID who sent the message. + timestamp (str): The timestamp of the message. + + Returns: + str: The message source. + """ + if self.workspace_url: + channel_id = self.channel_id_map.get(channel_name, "") + return ( + f"{self.workspace_url}/archives/{channel_id}" + + f"/p{timestamp.replace('.', '')}" + ) + else: + return f"{channel_name} - {user} - {timestamp}" diff --git a/tests/integration_tests/document_loaders/test_slack.py b/tests/integration_tests/document_loaders/test_slack.py new file mode 100644 index 00000000000..7baa1319fc9 --- /dev/null +++ b/tests/integration_tests/document_loaders/test_slack.py @@ -0,0 +1,23 @@ +"""Tests for the Slack directory loader""" +from pathlib import Path + +from langchain.document_loaders import SlackDirectoryLoader + + +def test_slack_directory_loader() -> None: + """Test Slack directory loader.""" + file_path = Path(__file__).parent.parent / "examples/slack_export.zip" + loader = SlackDirectoryLoader(str(file_path)) + docs = loader.load() + + assert len(docs) == 5 + + +def test_slack_directory_loader_urls() -> None: + """Test workspace URLS are passed through in the SlackDirectoryloader.""" + file_path = Path(__file__).parent.parent / "examples/slack_export.zip" + workspace_url = "example_workspace.com" + loader = SlackDirectoryLoader(str(file_path), workspace_url) + docs = loader.load() + for doc in docs: + assert doc.metadata["source"].startswith(workspace_url) diff --git a/tests/integration_tests/examples/slack_export.zip b/tests/integration_tests/examples/slack_export.zip new file mode 100644 index 0000000000000000000000000000000000000000..756809ad71938a1460b9428c2f79712cbc514be9 GIT binary patch literal 3904 zcmbW4c{G%5AIEQ7%t+R<3`O=BCM1tUGgEeD*9aMm-B^YcS}3IKOOZW;>|2&>B}?{Q z7)^<636HWSdT;fdH}6B|dEfWUx#ryS$9(VKxqhGP`}uyR%r;PSXi4^U7qQvkq^zt6>x@gxM> ztA%UOFo(O5LcT=LJ1oYp8Ebp=4Pw|2gm)D2kNBqA_nKK2lnckzixAs!*%?`n%I?yS zJ{*4@)^(v}4h?ZCa8rOMKX&VL*J@)-_{u9{v=Tl>?WRBr<&HXfPtF{2t;ir}qjqks zmT;G|>%NM;lgDspkdX=Rq19NG`&XY~tO9~=WiB0GVpB^xh@oup9!2?%kdtBI!lm&x zW*M2ta9f%yBO?)2s01rJ?ySPVOb-ps;*R&+5*KYZnV7{t)Bstr&-jz+ zF?LOQLZhlA7iM~*q8r6soija@deF0*O)MuW&c^2cX-r1(M`}^jv8)-BZByvi=h}>X z5s#)|XOH4oIyh9alW;PA)KBzd&qq)v=o=MfhjJJ!Q?|wGV7X`Xl|IyGtY&|zl04XX zW)?T(IcxOx%#@%}6|Tj>?iyW~*Lu0=%5==vY=PtZsq$`xw=yPwIGRDA&ffGh6s==+ z3~t>-9vM<$I+%`W%5gL_s5NALa4I1B;>CzJPch0fT&0pIY>F&8Csy-a^*vk}EO^d3 zFQ;U`;(Gx>ZSvt3LglpwmcvOCTZ41B%FQbz$z=V^3<$L3VV zN0a!ex$d0xn>Vs6pYRmNBVR$9xMZrK@8YP+bC7ABaRUqG#_}@zZ(EO=)qFZfqOYo7 zoVdhLEKzfQG(L4-qI;L{`8J5VEdsyv$?W@)&lT>gJlY^~P!Kr)fLVqZ{&@K7iwQ&y zcNOd8;)sL)g`VUO>h>AQST5JcH_H_jnZE$8bIOaTkm?}^wLM-E}m(AD{#6d40zPPht%)M2cuaz9qF2IAGI z8p~~K*KH<~TgC?)h7|^VWTwsJ@j=G*EaZ~wss+7H(d^m=BEtlDHoCzUww%_Gsi^kN zMH%(+9D881dRlRJ4ZGJIzl^FEbPeyg_Hh!ZxB~4Cg$Kl0LfvJT$r8?+Q8y6O4=Kg! z)oZMdme^AuU)Br6@8j6egDNDU?zG5g@oLDW$}n!2ew%+;R#)XI0t@P-bF*(qQmbne zUxI{q3#AuNN*?llb78fyP{(YRRQ=FJTh5OkTGRX=|vDge&wdd4`JrNdb<%x1D` zqn7S)r>?z!2x}=BVj$t{hdX<#DA=0sf*o`3y(olDF*K*59b+6Jg{?42qc|hsvL?Rk=9=l~_exJ? z|KwV=*0|t$ii@$|%!xJqjl#Tzm*a@P=iT<+;v7asJ5J^RpkiC2=&QF~L3u8}k5#u&ZnG zTkSqA(+L7Pe8_;~HVQ-;3DkKG`f&xcpBjbeXC;}hmh=X!p9+ywU!BRF5YtSZmwh)! zueeQr=P&45S{nGXAmBOTo5cpzFZYlHNl=`*P~-~wP%gL*f+X1C-ElTbnU}qf8jj*3(n4L`9;zu(lmbY+Pr{7pJh~%J zTCMy!25C`ViWO5X zp6^%-PAjr>|+Nndd5&G82bPxSn!> z-b%6IpaApUys~eYZi!9*9K42kmZ*byEvxFcB0eTE%kY#273uF<(Hv=8E3havqZEJt z(RJh2eEQ?D=UlH4%OM#zN3|AVE}PI95{*uZ<0%eKMcH(&xc43t)87|4)`BO$)+e8o zWinZf>Y4R;$IjSGa`dxej*=l&JHIKLVKo^PZv2yTe6ssV<%gHnuGxW-h?HsOr5<}V z9rHLodX5kHE|W5hEf=PuZipf04Ufq4+73@2dUd`6|$Su7=2R$=vY1r1K11{SsmMeoVZ2mRSh=@%=9jnRtv|Tpy0BRUS+{(0b4~2^>gu(x9oGgZ zRZ|7v@}6nD3Y;5Xj(jy<)c(1cu% zx6KXnOvwqam=;uRx!MQN=qfr4@y}wf~7Fj^U)v$DU z+CbVf+C;!~h)Z$#8}A+7^SY9ul?PKTsI%=SUIfi1UJkK&pV<}7)c-=GkKpq$xs-pbhHDWS-l3)3naVB0Op0gw*nMxTZ$*qkOaHk2SFv!4uGN>!8fV4xKt=K+)iXsYfgA zhx#jxa`JU;PgJv(ppoXv+)l-|kl?7ZP|N0GBYPdyQ__#=wfl>^KZOT9X(IJ4^bL1B zY|A1n`Fv;B)}Wv~O!d3#0mqNj208{n^xb~^+hu{~=Vf76WZRbfUHtng2>?vM3nT&L zdUqHERky?7o~pl#f;$VX*+X@g#epx`4u*TaWbe}HfSsimfC= Date: Thu, 13 Apr 2023 21:38:49 -0700 Subject: [PATCH 11/49] torch 2 support (#2865) Lang-chain seems to work with torch 2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0754d68df21..7acd501b730 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ spacy = {version = "^3", optional = true} nltk = {version = "^3", optional = true} transformers = {version = "^4", optional = true} beautifulsoup4 = {version = "^4", optional = true} -torch = {version = "^1", optional = true} +torch = {version = ">=1,<3", optional = true} jinja2 = {version = "^3", optional = true} tiktoken = {version = "^0.3.2", optional = true, python="^3.9"} pinecone-client = {version = "^2", optional = true} From 016738e676a83fc1794e0551d85561fc870ce378 Mon Sep 17 00:00:00 2001 From: ecneladis Date: Fri, 14 Apr 2023 06:39:20 +0200 Subject: [PATCH 12/49] Add GitLoader (#2851) --- .../document_loaders/examples/git.ipynb | 199 ++++++++++++++++++ langchain/document_loaders/git.py | 74 +++++++ 2 files changed, 273 insertions(+) create mode 100644 docs/modules/indexes/document_loaders/examples/git.ipynb create mode 100644 langchain/document_loaders/git.py diff --git a/docs/modules/indexes/document_loaders/examples/git.ipynb b/docs/modules/indexes/document_loaders/examples/git.ipynb new file mode 100644 index 00000000000..ffebd95d2dd --- /dev/null +++ b/docs/modules/indexes/document_loaders/examples/git.ipynb @@ -0,0 +1,199 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Git\n", + "\n", + "This notebook shows how to load text files from Git repository." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load existing repository from disk" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from git import Repo\n", + "\n", + "repo = Repo.clone_from(\n", + " \"https://github.com/hwchase17/langchain\", to_path=\"./example_data/test_repo1\"\n", + ")\n", + "branch = repo.head.reference" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.document_loaders.git import GitLoader" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "__init__() got an unexpected keyword argument 'path'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[3], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m loader \u001b[39m=\u001b[39m GitLoader(path\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39m./example_data/test_repo1/\u001b[39;49m\u001b[39m\"\u001b[39;49m, branch\u001b[39m=\u001b[39;49mbranch)\n", + "\u001b[0;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'path'" + ] + } + ], + "source": [ + "loader = GitLoader(repo_path=\"./example_data/test_repo1/\", branch=branch)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = loader.load()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1040" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "page_content='.venv\\n.github\\n.git\\n.mypy_cache\\n.pytest_cache\\nDockerfile' metadata={'file_path': '.dockerignore', 'file_name': '.dockerignore', 'file_type': ''}\n" + ] + } + ], + "source": [ + "print(data[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clone repository from url" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.document_loaders.git import GitLoader" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "loader = GitLoader(\n", + " clone_url=\"https://github.com/hwchase17/langchain\",\n", + " repo_path=\"./example_data/test_repo2/\",\n", + " branch=\"master\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = loader.load()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1040" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/langchain/document_loaders/git.py b/langchain/document_loaders/git.py new file mode 100644 index 00000000000..39a9235e26e --- /dev/null +++ b/langchain/document_loaders/git.py @@ -0,0 +1,74 @@ +import os +from typing import List, Optional + +from langchain.docstore.document import Document +from langchain.document_loaders.base import BaseLoader + + +class GitLoader(BaseLoader): + """Loads files from a Git repository into a list of documents. + Repository can be local on disk available at `repo_path`, + or remote at `clone_url` that will be cloned to `repo_path`. + Currently supports only text files. + + Each document represents one file in the repository. The `path` points to + the local Git repository, and the `branch` specifies the branch to load + files from. By default, it loads from the `main` branch. + """ + + def __init__( + self, + repo_path: str, + clone_url: Optional[str] = None, + branch: Optional[str] = "main", + ): + self.repo_path = repo_path + self.clone_url = clone_url + self.branch = branch + + def load(self) -> List[Document]: + try: + from git import Blob, Repo + except ImportError as ex: + raise ImportError( + "Could not import git python package. " + "Please install it with `pip install GitPython`." + ) from ex + + if not os.path.exists(self.repo_path) and self.clone_url is None: + raise ValueError(f"Path {self.repo_path} does not exist") + elif self.clone_url: + repo = Repo.clone_from(self.clone_url, self.repo_path) + repo.git.checkout(self.branch) + else: + repo = Repo(self.repo_path) + repo.git.checkout(self.branch) + + docs: List[Document] = [] + + for item in repo.tree().traverse(): + if isinstance(item, Blob): + file_path = os.path.join(self.repo_path, item.path) + rel_file_path = os.path.relpath(file_path, self.repo_path) + try: + with open(file_path, "rb") as f: + content = f.read() + file_type = os.path.splitext(item.name)[1] + + # loads only text files + try: + text_content = content.decode("utf-8") + except UnicodeDecodeError: + continue + + metadata = { + "file_path": rel_file_path, + "file_name": item.name, + "file_type": file_type, + } + doc = Document(page_content=text_content, metadata=metadata) + docs.append(doc) + except Exception as e: + print(f"Error reading file {file_path}: {e}") + + return docs From 04c458a2701301648ef3d7714bd38051852d747d Mon Sep 17 00:00:00 2001 From: sergerdn <64213648+sergerdn@users.noreply.github.com> Date: Thu, 13 Apr 2023 21:49:31 -0700 Subject: [PATCH 13/49] feat: improve pinecone tests (#2806) Improve the integration tests for Pinecone by adding an `.env.example` file for local testing. Additionally, add some dev dependencies specifically for integration tests. This change also helps me understand how Pinecone deals with certain things, see related issues https://github.com/hwchase17/langchain/issues/2484 https://github.com/hwchase17/langchain/issues/2816 --- poetry.lock | 264 +++++---- pyproject.toml | 3 + tests/README.md | 7 + tests/integration_tests/.env.example | 9 + tests/integration_tests/conftest.py | 36 +- .../TestPinecone.test_from_texts.yaml | 489 +++++++++++++++ ...necone.test_from_texts_with_metadatas.yaml | 489 +++++++++++++++ ...tPinecone.test_from_texts_with_scores.yaml | 557 ++++++++++++++++++ .../vectorstores/conftest.py | 53 +- .../vectorstores/test_elasticsearch.py | 38 +- .../vectorstores/test_pinecone.py | 279 ++++++--- 11 files changed, 1985 insertions(+), 239 deletions(-) create mode 100644 tests/integration_tests/.env.example create mode 100644 tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts.yaml create mode 100644 tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts_with_metadatas.yaml create mode 100644 tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts_with_scores.yaml diff --git a/poetry.lock b/poetry.lock index 197c8d5b8f4..434a6bccba5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "absl-py" @@ -743,7 +743,7 @@ name = "cachetools" version = "5.3.0" description = "Extensible memoizing collections and decorators" category = "main" -optional = true +optional = false python-versions = "~=3.7" files = [ {file = "cachetools-5.3.0-py3-none-any.whl", hash = "sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4"}, @@ -1338,20 +1338,20 @@ files = [ [[package]] name = "deeplake" -version = "3.2.21" +version = "3.2.22" description = "Activeloop Deep Lake" category = "main" optional = false python-versions = "*" files = [ - {file = "deeplake-3.2.21.tar.gz", hash = "sha256:3ad6f81f666ba9e2951aff697babb845249b54dfdd72f22e3c7ef5838ae2e9d7"}, + {file = "deeplake-3.2.22.tar.gz", hash = "sha256:068280561366dd1bd891d3ffda8638ec59860a23b9426815a484b0591ab467a6"}, ] [package.dependencies] aioboto3 = {version = "10.4.0", markers = "python_version >= \"3.7\" and sys_platform != \"win32\""} boto3 = "*" click = "*" -humbug = ">=0.2.6" +humbug = ">=0.3.1" nest_asyncio = {version = "*", markers = "python_version >= \"3.7\" and sys_platform != \"win32\""} numcodecs = "*" numpy = "*" @@ -1955,14 +1955,14 @@ uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.17.2" +version = "2.17.3" description = "Google Authentication Library" category = "main" optional = true python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" files = [ - {file = "google-auth-2.17.2.tar.gz", hash = "sha256:295c80ebb95eac74003c07a696cf3ef6b414e9230ae8894f3843f8215fd2aa56"}, - {file = "google_auth-2.17.2-py2.py3-none-any.whl", hash = "sha256:544536a43d44dff0f64222e4d027d124989fcb9c10979687e589e1694fba9c94"}, + {file = "google-auth-2.17.3.tar.gz", hash = "sha256:ce311e2bc58b130fddf316df57c9b3943c2a7b4f6ec31de9663a9333e4064efc"}, + {file = "google_auth-2.17.3-py2.py3-none-any.whl", hash = "sha256:f586b274d3eb7bd932ea424b1c702a30e0393a2e2bc4ca3eae8263ffd8be229f"}, ] [package.dependencies] @@ -2064,14 +2064,14 @@ grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] [[package]] name = "gptcache" -version = "0.1.8" -description = "GPT Cache, a powerful caching library that can be used to speed up and lower the cost of chat applications that rely on the LLM service. GPT Cache works as a memcache for AIGC applications, similar to how Redis works for traditional applications." +version = "0.1.9" +description = "GPTCache, a powerful caching library that can be used to speed up and lower the cost of chat applications that rely on the LLM service. GPTCache works as a memcache for AIGC applications, similar to how Redis works for traditional applications." category = "main" -optional = true +optional = false python-versions = ">=3.8.1" files = [ - {file = "gptcache-0.1.8-py3-none-any.whl", hash = "sha256:953662291819471e5461920c89367084f905237a8506f1a1605729f3e633f147"}, - {file = "gptcache-0.1.8.tar.gz", hash = "sha256:23200cc0783776210cce85a588ae68222d522ce9456f74b7836945ebe8b15820"}, + {file = "gptcache-0.1.9-py3-none-any.whl", hash = "sha256:c603dea4a6e835032875ccc21a9205bb4d8ef0f2696a8df3ac13ac3a060c4cf6"}, + {file = "gptcache-0.1.9.tar.gz", hash = "sha256:b92f30e4d8bc22a017a71fb4801f469739d4c83734a874fea51ca6998ef833d4"}, ] [package.dependencies] @@ -2954,14 +2954,14 @@ websockets = ["websockets"] [[package]] name = "jina-hubble-sdk" -version = "0.35.0" +version = "0.36.0" description = "SDK for Hubble API at Jina AI." category = "main" optional = true python-versions = ">=3.7.0" files = [ - {file = "jina-hubble-sdk-0.35.0.tar.gz", hash = "sha256:f0826dc6d22aa8da983e4f67af8fa5b78dd852274da71adea7a38d9620438b37"}, - {file = "jina_hubble_sdk-0.35.0-py3-none-any.whl", hash = "sha256:957e4ce5e9400ee27aad7a6e728172cdcffb8bb6667e430fa3c97f000650beb3"}, + {file = "jina-hubble-sdk-0.36.0.tar.gz", hash = "sha256:ba1a72c7a5c14963fdad9af1ff4c3bba26a03ddcced08111bd11a95b153249ec"}, + {file = "jina_hubble_sdk-0.36.0-py3-none-any.whl", hash = "sha256:56db142147d7c72142ed1b6505020c3b4d8070b1603d07ff796e56d491dde294"}, ] [package.dependencies] @@ -4794,14 +4794,14 @@ tests = ["pytest", "pytest-cov", "pytest-pep8"] [[package]] name = "packaging" -version = "23.0" +version = "23.1" description = "Core utilities for Python packages" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] [[package]] @@ -5188,14 +5188,14 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "posthog" -version = "2.4.2" +version = "2.5.0" description = "Integrate PostHog into any python application." category = "dev" optional = false python-versions = "*" files = [ - {file = "posthog-2.4.2-py2.py3-none-any.whl", hash = "sha256:8c7c37de997d955aea61bf0aa0069970e71f0d9d79c9e6b3a134e6593d5aa3d6"}, - {file = "posthog-2.4.2.tar.gz", hash = "sha256:652a628623aab26597e8421a7ddf9caaf19dd93cc1288901a6b23db9693d34e5"}, + {file = "posthog-2.5.0-py2.py3-none-any.whl", hash = "sha256:3d3ece06c3fe4135497c239f211c82344962b93e4fd4ba5afd20bd4ad12d77be"}, + {file = "posthog-2.5.0.tar.gz", hash = "sha256:7601ef75b483eb67a6229cafec02da9624f6d46df61066771ca9e3f986284fc3"}, ] [package.dependencies] @@ -5309,6 +5309,22 @@ files = [ [package.dependencies] wcwidth = "*" +[[package]] +name = "promptlayer" +version = "0.1.80" +description = "PromptLayer is a package to keep track of your GPT models training" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "promptlayer-0.1.80.tar.gz", hash = "sha256:1012018ed3e4bca4f0d9c9164cb00b7ca0936cba6a40d4de53e87ef08fdff62f"}, +] + +[package.dependencies] +langchain = "*" +openai = "*" +requests = "*" + [[package]] name = "protobuf" version = "3.19.6" @@ -6228,14 +6244,14 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} [[package]] name = "qdrant-client" -version = "1.1.3" +version = "1.1.4" description = "Client library for the Qdrant vector search engine" category = "main" optional = true python-versions = ">=3.7,<3.12" files = [ - {file = "qdrant_client-1.1.3-py3-none-any.whl", hash = "sha256:c95f59fb9e3e89d163517f8992ee4557eccb45c252147e11e45c608ef1c7dd29"}, - {file = "qdrant_client-1.1.3.tar.gz", hash = "sha256:2b7de2b987fc456c643a06878a4150947c3d3d6c6515f6c29f6e707788daa6e7"}, + {file = "qdrant_client-1.1.4-py3-none-any.whl", hash = "sha256:12ad9dba63228cc5493e137bf35c59af56d84ca3a2b088c4298825d4893c7100"}, + {file = "qdrant_client-1.1.4.tar.gz", hash = "sha256:92ad225bd770fb6a7ac10f75e38f53ffebe63c7f239b02fc7d2bc993246eb74c"}, ] [package.dependencies] @@ -6473,14 +6489,14 @@ files = [ [[package]] name = "rich" -version = "13.3.3" +version = "13.3.4" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = true python-versions = ">=3.7.0" files = [ - {file = "rich-13.3.3-py3-none-any.whl", hash = "sha256:540c7d6d26a1178e8e8b37e9ba44573a3cd1464ff6348b99ee7061b95d1c6333"}, - {file = "rich-13.3.3.tar.gz", hash = "sha256:dc84400a9d842b3a9c5ff74addd8eb798d155f36c1c91303888e0a66850d2a15"}, + {file = "rich-13.3.4-py3-none-any.whl", hash = "sha256:22b74cae0278fd5086ff44144d3813be1cedc9115bdfabbfefd86400cb88b20a"}, + {file = "rich-13.3.4.tar.gz", hash = "sha256:b5d573e13605423ec80bdd0cd5f8541f7844a0e71a13f74cf454ccb2f490708b"}, ] [package.dependencies] @@ -6675,49 +6691,57 @@ transformers = ">=4.6.0,<5.0.0" [[package]] name = "sentencepiece" -version = "0.1.97" +version = "0.1.98" description = "SentencePiece python wrapper" category = "main" optional = false python-versions = "*" files = [ - {file = "sentencepiece-0.1.97-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6f249c8f1852893be86eae66b19d522c5fb30bbad4fe2d1b07f06fdc86e1907e"}, - {file = "sentencepiece-0.1.97-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09e1bc53178de70c557a9ba4fece07364b4416ce3d36570726b3372b68aea135"}, - {file = "sentencepiece-0.1.97-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:667193c57fb48b238be7e3d7636cfc8da56cb5bac5559d8f0b647334e1175be8"}, - {file = "sentencepiece-0.1.97-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2780531985af79c6163f63d4f200fec8a28b70b6768d2c19f70d01568a4524e8"}, - {file = "sentencepiece-0.1.97-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:205050670c53ef9015e2a98cce3934bfbcf0aafaa14caa0c618dd5667bc217ee"}, - {file = "sentencepiece-0.1.97-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28b183dadef8e8b6b4645c1c20692d7be0a13ecc3ec1a07b3885c8905516675f"}, - {file = "sentencepiece-0.1.97-cp310-cp310-win32.whl", hash = "sha256:ee3c9dbd558d8d85bb1617087b86df6ea2b856a528669630ce6cedeb4353b823"}, - {file = "sentencepiece-0.1.97-cp310-cp310-win_amd64.whl", hash = "sha256:f7dc55379e2f7dee86537180283db2e5f8418c6825fdd2fe436c724eb5604c05"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ba1b4154f9144c5a7528b00aff5cffaa1a896a1c6ca53ca78b6e74cd2dae5244"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac3d90aee5581e55d029d124ac11b6ae2fbae0817863b664b2f2302e966ababb"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c27400f1ac46518a01c87cb7703650e4e48728649feb115d2e3f1102a946a42"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6e12a166eba75994ca749aadc4a5056b91b31405f805d6de6e8914cc9741c60"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-win32.whl", hash = "sha256:ed85dff5c0a9b3dd1a414c7e1119f2a19e863fc3f81da525bf7f885ebc883de0"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-win_amd64.whl", hash = "sha256:91a19ab6f40ffbae6d6127119953d2c6a85e93d734953dbc8629fde0d21ace66"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bae580e4a35a9314ff49561ac7c06574fe6afc71b821ed6bb00534e571458156"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ad7262e7530c683b186672b5dd0082f82719a50a500a8cfbc4bbd7cde5bff8c"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:620cee35279720016735a7c7103cddbd9b84fe5e2f098bd5e673834d69fee2b8"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93b921b59914c0ec6697e8c6d5e6b44d99d1298fb1a0af56980a79ade0540c19"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-win32.whl", hash = "sha256:9b9a4c44a31d5f47616e9568dcf31e029b0bfa776e0a252c0b59247881598b09"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-win_amd64.whl", hash = "sha256:f31533cdacced56219e239d3459a003ece35116920dd64b2309d4ad047b77644"}, - {file = "sentencepiece-0.1.97-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7d643c01d1cad13b9206a276bbe5bc1a468e3d7cf6a26bde7783f945277f859d"}, - {file = "sentencepiece-0.1.97-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:542f1985b1ee279a92bef7740ec0781452372028ce01e15aa88df3228b197ba3"}, - {file = "sentencepiece-0.1.97-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93701da21fea906dd244bf88cdbe640385a89c45d3c1812b76dbadf8782cdbcd"}, - {file = "sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a51514047b964047b7fadb480d88a5e0f72c02f6ca1ba96258fbbc6e79274a94"}, - {file = "sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3ae2e9b7a5b6f2aa64ec9240b0c185dabe597d0e787dc4344acfbaef1ffe0b2"}, - {file = "sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:923ee4af16dbae1f2ab358ed09f8a0eb89e40a8198a8b343bf54181482342721"}, - {file = "sentencepiece-0.1.97-cp38-cp38-win32.whl", hash = "sha256:fa6f2b88850b5fae3a05053658824cf9f147c8e3c3b40eb64539a976c83d8a24"}, - {file = "sentencepiece-0.1.97-cp38-cp38-win_amd64.whl", hash = "sha256:5137ff0d0b1cc574751d178650ef800ff8d90bf21eb9f71e9567d4a0548940a5"}, - {file = "sentencepiece-0.1.97-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f92876271a10494671431ad955bff2d6f8ea59baaf957f5ae5946aff56dfcb90"}, - {file = "sentencepiece-0.1.97-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:35c227b6d55e473033db7e0ecc51b1e99e6ed7607cc08602fb5768132543c81d"}, - {file = "sentencepiece-0.1.97-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1706a8a8188f7b3d4b7922db9bb00c64c4e16ee68ab4caaae79f55b3e18748c7"}, - {file = "sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce61efc1862ccb18856c4aabbd930e13d5bfbb4b09b4f111081ac53a9dc62275"}, - {file = "sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a78c03800ef9f02d320e0159f5768b15357f3e9ebea545c9c4ba7928ba8ba254"}, - {file = "sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753b8088fd685ee787d9f54c84275ab347de558c7c4ebc6accb4c35bf7776f20"}, - {file = "sentencepiece-0.1.97-cp39-cp39-win32.whl", hash = "sha256:24306fd86031c17a1a6ae92671e76a350390a3140a65620bc2843dad7db24e2a"}, - {file = "sentencepiece-0.1.97-cp39-cp39-win_amd64.whl", hash = "sha256:c6641d0b7acec61fde5881ea6ebe098c169557ac9aa3bdabdf124eab5a5592bb"}, - {file = "sentencepiece-0.1.97.tar.gz", hash = "sha256:c901305e0a710bbcd296f66d79e96f744e6e175b29812bd5178318437d4e1f6c"}, + {file = "sentencepiece-0.1.98-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1daf0a79cd953e4830746c41e92b98a2f2e9e5ec0e90a9447aa10350e11bd027"}, + {file = "sentencepiece-0.1.98-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:57911445fc91c80d59552adf8a749af9205458920a7328f3bd7d51308658bcd9"}, + {file = "sentencepiece-0.1.98-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3f9239785849ed1f55a825bcc282bef1a6073f7431cc535bdc658a94873652ea"}, + {file = "sentencepiece-0.1.98-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:467740ef8af170e5b6cfe22c272114ed930c899c297619ac7a2ac463a13bdbac"}, + {file = "sentencepiece-0.1.98-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b6f0b9ffb601e2699e265f3f20c353ec9a661e4b5f0cff08ad6c9909c0ae43e"}, + {file = "sentencepiece-0.1.98-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6150ba525fac4fda76f5c4777ae300597e70cef739ed2a47cea02ff81a88873f"}, + {file = "sentencepiece-0.1.98-cp310-cp310-win32.whl", hash = "sha256:58ca96d73ea0e5575e3f6a9524449c673d62e6ecee3b2ddd5bfb4f49cb315c0a"}, + {file = "sentencepiece-0.1.98-cp310-cp310-win_amd64.whl", hash = "sha256:8abe5c4c034e497e69f485dcd2c0e6bc87bf0498ad5aef5f539a7d0f9eae6275"}, + {file = "sentencepiece-0.1.98-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b6ed62f89c0bd25cec39a7075f6b9354fe4c240ed964e63009d77efcf29c34e9"}, + {file = "sentencepiece-0.1.98-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c2d9a74986d3716dc6961e9dbae7a3b25bb1260118f098545fd963ae23252c1"}, + {file = "sentencepiece-0.1.98-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f7dc2fc175623529fb60a2799748f8877cd48c4541b32cd97b8523465e88b69"}, + {file = "sentencepiece-0.1.98-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64e32c55d04a2e21f0c2fda1b7a3dd108133ebfb8616b52896916bb30e4352ed"}, + {file = "sentencepiece-0.1.98-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:443f32e94b18571231b02a51be173686114b5556b5edfcbf347fb63e7bd5ddc6"}, + {file = "sentencepiece-0.1.98-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:558a373a8660bdff299d6c133c2a4f4fb0875e9e6fafe225b8080ecce8a405f9"}, + {file = "sentencepiece-0.1.98-cp311-cp311-win32.whl", hash = "sha256:fcf100268cefe1774794b18cbaf3065e2bf988f168a387973eb1260d51198795"}, + {file = "sentencepiece-0.1.98-cp311-cp311-win_amd64.whl", hash = "sha256:05b4eecbece0606883cd81ed86bb3c619680bb570b997b236533ec854d64a575"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:35af00f5103a4779694fedea41b6e24947a9ed81166efe63864ab1e781d70a66"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2766cd708e9fc2b5b9784a990a8b303b9e0b9a69fa482616fe86fa538daa1756"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b2531c0e9cc8cd404fabd856d80d695b373371c00f1fce29c06f41f3f7429d87"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffcc78e80c55eab67ee3439ade493607a4e37e1f0b82b168ead3debf9eaeaabe"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-win32.whl", hash = "sha256:ef384b31ec7a06a9a6aba42e68435f3f3b38809aa65559ede3658cdd446a562c"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-win_amd64.whl", hash = "sha256:e7a828f1fe2e51d2d9e5e9b3283d4006f1891efb02a3d9303ed39ddafdd9c864"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8663be00a68098f85d6cda1f7041a27de05c320e433fa730ecb1156a8304f21c"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf05611089a075b78d353720ccc3a09a78e0846332cff0cc78fda8b2383626a"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11f410cc7eeb3e1cfa8d92d128b568e5dc7829b7904b164499fd0209316ec2fa"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5ea8fb2c68073fe25a08a178eed269ed382fba074ff2ba4de72f0f56d86630e"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-win32.whl", hash = "sha256:fa13a125417d28e84fbdebcaf6aa115e4177d3e93aa66b857a42e7179f515b88"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-win_amd64.whl", hash = "sha256:e54aa70b574eee895d184072d84e62824f404821e551a82c619c5d4320a93834"}, + {file = "sentencepiece-0.1.98-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:515a971c2a157647ca0e60ce3c435f4b43cd5c9f5862159cfefa0b5b4d46d3c3"}, + {file = "sentencepiece-0.1.98-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c23c3a562221bc40eaae42428fcd8e607e0f084ea8aa968ba3f1a7d0ea975807"}, + {file = "sentencepiece-0.1.98-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c067ba22be8edc699f6365e01ec15046bf3563dbabfdc052ecc88e581b675cba"}, + {file = "sentencepiece-0.1.98-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12c913493d6ebac86ee7ae109e368522a5a365a7b150d4d8cf845599262d2b21"}, + {file = "sentencepiece-0.1.98-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:720f827dc69ee24951ea4f51b9fb69cc56890a7190fc52c2c0da2545caab1760"}, + {file = "sentencepiece-0.1.98-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:918b4caf18b2f73c302c4e197d1c2dafba39eb143d16b4590930b45f15042fdd"}, + {file = "sentencepiece-0.1.98-cp38-cp38-win32.whl", hash = "sha256:2d50edfc4649a1566b64f1a8402cd607e1893bf8e368732337d83f00df62d3fa"}, + {file = "sentencepiece-0.1.98-cp38-cp38-win_amd64.whl", hash = "sha256:7425b727c3d6b3b7bad0005a3be316078b254180b712d73955ff08cae3f6a385"}, + {file = "sentencepiece-0.1.98-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:00b2becbd7b98905a6de9695cf8682abe0d510ab0198e23c7d86fb2b793b6ae0"}, + {file = "sentencepiece-0.1.98-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7f71c4bdedb797052fb2ccad0871c2409bf6f812cb6b651917c55f9e8eced07f"}, + {file = "sentencepiece-0.1.98-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7287461d2346f530928ab187f0834cb15ddfbc4553592cacdcb6470364739ec6"}, + {file = "sentencepiece-0.1.98-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:472ad943eaffcb6871ece56c7850388e7b8722f520ba73c93e7a6ef965453221"}, + {file = "sentencepiece-0.1.98-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b7e23aaf9d5afd91ca13550968bd17f0c17b0966823188ad2a50c51544cf8ed"}, + {file = "sentencepiece-0.1.98-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b0ce9efc790c209cce2463058855dceb21438213d2ff13cb5a565d52a7efe25"}, + {file = "sentencepiece-0.1.98-cp39-cp39-win32.whl", hash = "sha256:8b50cbe8e46204eff7aa5a663af5652c45e7807aa560d08e5f5b10c60e795a49"}, + {file = "sentencepiece-0.1.98-cp39-cp39-win_amd64.whl", hash = "sha256:14841bd2a3d77c4dbba58f02488c374866551e428d755e8d473d82325a0a94f3"}, + {file = "sentencepiece-0.1.98.tar.gz", hash = "sha256:947cf0a4b8a480510d560a922f8256f34e93984a86cf870be4d05731f59fb28d"}, ] [[package]] @@ -6809,40 +6833,40 @@ files = [ [[package]] name = "spacy" -version = "3.5.1" +version = "3.5.2" description = "Industrial-strength Natural Language Processing (NLP) in Python" category = "main" optional = true python-versions = ">=3.6" files = [ - {file = "spacy-3.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:428925faf26a7c7a9564431d7a505af7816b22b5c68b240bbe073ae928e9ef36"}, - {file = "spacy-3.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:63bc19f4b5fa5f806698e7d16828cacbfefd0ab44f770e0b2a1a0509dd07f6f9"}, - {file = "spacy-3.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f5a1073cc7bb9896624682f6a5ab29c2d3d2d935cb36f88b25cbb01f12b57ef"}, - {file = "spacy-3.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb9af95d1c06e23e89731d61f3fa5f28583684e10bd3d29d9e7bb161ffe02df9"}, - {file = "spacy-3.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:dec30afd4916cb4f02449ccec94e2f8a3eb929686e9f96bd74f51f4c07d75577"}, - {file = "spacy-3.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d2e256c44241b9a2ac3204659891d332d370dfa0e39917254574bc1ffdfb079"}, - {file = "spacy-3.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d337054213f837ae295431a35638bb469c4e4796f6c5ff17d2dd18d545615a0e"}, - {file = "spacy-3.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab9bbd8e34bfabd506f74d2739c6a4e47c899fd7d3f1648bbffde0c16b8a339d"}, - {file = "spacy-3.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5ab0e2b406b3953c5975adcc4ac09bdc8fbcb20dd9a2a8ea2774b4d83106c24"}, - {file = "spacy-3.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:9cbec19e55fcdb6e4be220c6b6335af96c374a7ac76dffb15f9da95c9d39ce62"}, - {file = "spacy-3.5.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92b590c1c50eb421b6aaa0373b37fbdfb290a130771728e8d06159517cc120d"}, - {file = "spacy-3.5.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2525bc1ec9e784597365daf245f65b9ca9fd8a25fa96f9c7a6b7bfd5048b87bc"}, - {file = "spacy-3.5.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e3f113cbf4331052622ec5c27e581751beba5c62e9af2f21d2798db50a41e04c"}, - {file = "spacy-3.5.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c9e93851a210ccc59112243fc74dcac82191383e7654731c2842326f7d1eb1d"}, - {file = "spacy-3.5.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ca30de7f82ab97e054a457eeb424060091b609114ebf7c90ef1775cac40fe04"}, - {file = "spacy-3.5.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3083ccbcc52102bf53ff797233ea90a7d2b01c3853d811272ebc63de0aff4df5"}, - {file = "spacy-3.5.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1e795b3f85f229ea54ff7f91e15fb5d7afacec5e5fca302dca1bc3224547e4f0"}, - {file = "spacy-3.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fa47b47142883891252dda54da7a79055cb4e703914a90928c2fbe5bd058f4ed"}, - {file = "spacy-3.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d92387989fe9c3bebd60faaeb590206e34ca9c421a52460a058ee5050d9fc8c6"}, - {file = "spacy-3.5.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1067f7ef0e87341cea2c3187f9b96965f4b0c076a87e22c1aac45ea5586f856"}, - {file = "spacy-3.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ab781021e896aae4a0f9f0a5424c75fc5d6ef4c20f56fd115e8605484567fd6"}, - {file = "spacy-3.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:c43b2597649549e84ceda7b658479e28c6e66995ebd9a61e0193b0c0dceffe50"}, - {file = "spacy-3.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9a389f850ab1a3f17e6beb90fd92533bad21a372979496b01a99ae1a9f3e96e3"}, - {file = "spacy-3.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8ca295e8381a0273b6543c1389275af98878a43ab70c781630277e49ce978f"}, - {file = "spacy-3.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a62a458c88c296234471fe540fe5d1ec763701d2f556870512143de8559286c0"}, - {file = "spacy-3.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04ad29a306d1879cafe23e4e8a613046f62d81ceeb70e6fcab3fddb4b0fedf7f"}, - {file = "spacy-3.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:c4be3508c9b4109afe3e5c7fdf91b9d7153ec2227f24270625caee96651fa9e2"}, - {file = "spacy-3.5.1.tar.gz", hash = "sha256:811ae1468c58b97fc9aa31187d6b55317784258f0a47ebf69d81cab639e3fa15"}, + {file = "spacy-3.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f5ac232dda9aae44caef639243695702f2e15d78c2b0e05ed6d3368386b61bd9"}, + {file = "spacy-3.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b7a1b43b778ee8f61dfac34d47b795ee1cf1d8dc26f66c178f5900fca8dd331"}, + {file = "spacy-3.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b76dc9a14dc36863828d982354c025a833ee89a763c22bb83ef6ff51da4656"}, + {file = "spacy-3.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1af8aae58c201ebc038be74109b8a88fbc417375122b92c82af4d7373e1dbe76"}, + {file = "spacy-3.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:ee629cb45e9cb378bb0bce7d740758d1761f04da1e1463c496bddd93a1738c9b"}, + {file = "spacy-3.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e3f6a44d697e8e8a304fc2ed46f3e9c564ae7b148c0500ad35c56cda285e1b41"}, + {file = "spacy-3.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:eb0b7ddc37ff28ba2e59fc3b88f411d570ac41b64477f56500b4ffdae582d242"}, + {file = "spacy-3.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5fb40480dd611d31149b6b122c99a22d46e15d6b367da36e41dda73eaaf3036"}, + {file = "spacy-3.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:572d01288e40bb57adf1a41949776e005bb0d3c19e9e830f5718355ac891ba44"}, + {file = "spacy-3.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:7ac4cde010cd09775f659e57ad005276f4c52657af039769433beefa161e0288"}, + {file = "spacy-3.5.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52056e71e8e1c35911494493cc3395e0d1fa4a9c7113ffe210424694dc9b7a09"}, + {file = "spacy-3.5.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d296e80cb69861eaa4fd3f9f7f4be6ac812f6e711e47ee4b61f4c41df0ab2d0"}, + {file = "spacy-3.5.2-cp36-cp36m-win_amd64.whl", hash = "sha256:97aa6b954964f5bc5d5bb6c12e3de81c053e66c7f5e214d5e4fde8ef467cb2de"}, + {file = "spacy-3.5.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ddc1f35289b848a544bc2429497a5b11b0c60c01220efea38dcbe9e2779894c6"}, + {file = "spacy-3.5.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fcdb5005230af1dd269b19b41af25c657b88932b2b371be63321e079c9f04c8"}, + {file = "spacy-3.5.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2bf6a44e5a87fa4a8be57cd31494609af2bc9d249a2f477d00ea8f279b59751"}, + {file = "spacy-3.5.2-cp37-cp37m-win_amd64.whl", hash = "sha256:f4f8c5e070bef0e4cd897d0c2f54e78bd386e83afd1147221ca22ca1c3a1eea1"}, + {file = "spacy-3.5.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:655a2a19065a129468b47a3d56e96e8404e952f6219d42f3248e6075a0e43eaa"}, + {file = "spacy-3.5.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:dcd10594641306cd697540fa68ea24aa5400556f113aa229834e803f7d8fb9cf"}, + {file = "spacy-3.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:728d84a2d47d53a54a05934790ff089d8ff09982ba5ada5ab3fd89c70fc1db63"}, + {file = "spacy-3.5.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3b06f48231c537fd05db60d050baaecbdf4322a8cc58cb540787ce8817d3817"}, + {file = "spacy-3.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:4f1c1c1283da6c8206e06adc459a96abf9e3515b3c8e08e4ff722a80c5692d6f"}, + {file = "spacy-3.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ecdba339249873d7f70af5da075aaac0acc15dd93db4167c4a0514a5219e2c2"}, + {file = "spacy-3.5.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9eb26967fd66e756ba3c90f61d3803de845255b6a0b7a08a5b4e044d9b02029d"}, + {file = "spacy-3.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b1a4ed3ece6d763415e596ac312e1f6870171e557b6d26daf35a0206cb76f2a"}, + {file = "spacy-3.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a168f76105c21dc4aab9f2dc9c4b1662f86910cb7799b40dd5aac8840888f017"}, + {file = "spacy-3.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:f0e687e6f1d40f3ed5f250f7242cf0c1058aa396338c341fa595700b57410f43"}, + {file = "spacy-3.5.2.tar.gz", hash = "sha256:22c1ffaab285b7477003d4b5b038414cc32468a690d479015b9a698c531c813b"}, ] [package.dependencies] @@ -6869,24 +6893,24 @@ wasabi = ">=0.9.1,<1.2.0" [package.extras] apple = ["thinc-apple-ops (>=0.1.0.dev0,<1.0.0)"] -cuda = ["cupy (>=5.0.0b4,<12.0.0)"] -cuda-autodetect = ["cupy-wheel (>=11.0.0,<12.0.0)"] -cuda100 = ["cupy-cuda100 (>=5.0.0b4,<12.0.0)"] -cuda101 = ["cupy-cuda101 (>=5.0.0b4,<12.0.0)"] -cuda102 = ["cupy-cuda102 (>=5.0.0b4,<12.0.0)"] -cuda110 = ["cupy-cuda110 (>=5.0.0b4,<12.0.0)"] -cuda111 = ["cupy-cuda111 (>=5.0.0b4,<12.0.0)"] -cuda112 = ["cupy-cuda112 (>=5.0.0b4,<12.0.0)"] -cuda113 = ["cupy-cuda113 (>=5.0.0b4,<12.0.0)"] -cuda114 = ["cupy-cuda114 (>=5.0.0b4,<12.0.0)"] -cuda115 = ["cupy-cuda115 (>=5.0.0b4,<12.0.0)"] -cuda116 = ["cupy-cuda116 (>=5.0.0b4,<12.0.0)"] -cuda117 = ["cupy-cuda117 (>=5.0.0b4,<12.0.0)"] -cuda11x = ["cupy-cuda11x (>=11.0.0,<12.0.0)"] -cuda80 = ["cupy-cuda80 (>=5.0.0b4,<12.0.0)"] -cuda90 = ["cupy-cuda90 (>=5.0.0b4,<12.0.0)"] -cuda91 = ["cupy-cuda91 (>=5.0.0b4,<12.0.0)"] -cuda92 = ["cupy-cuda92 (>=5.0.0b4,<12.0.0)"] +cuda = ["cupy (>=5.0.0b4,<13.0.0)"] +cuda-autodetect = ["cupy-wheel (>=11.0.0,<13.0.0)"] +cuda100 = ["cupy-cuda100 (>=5.0.0b4,<13.0.0)"] +cuda101 = ["cupy-cuda101 (>=5.0.0b4,<13.0.0)"] +cuda102 = ["cupy-cuda102 (>=5.0.0b4,<13.0.0)"] +cuda110 = ["cupy-cuda110 (>=5.0.0b4,<13.0.0)"] +cuda111 = ["cupy-cuda111 (>=5.0.0b4,<13.0.0)"] +cuda112 = ["cupy-cuda112 (>=5.0.0b4,<13.0.0)"] +cuda113 = ["cupy-cuda113 (>=5.0.0b4,<13.0.0)"] +cuda114 = ["cupy-cuda114 (>=5.0.0b4,<13.0.0)"] +cuda115 = ["cupy-cuda115 (>=5.0.0b4,<13.0.0)"] +cuda116 = ["cupy-cuda116 (>=5.0.0b4,<13.0.0)"] +cuda117 = ["cupy-cuda117 (>=5.0.0b4,<13.0.0)"] +cuda11x = ["cupy-cuda11x (>=11.0.0,<13.0.0)"] +cuda80 = ["cupy-cuda80 (>=5.0.0b4,<13.0.0)"] +cuda90 = ["cupy-cuda90 (>=5.0.0b4,<13.0.0)"] +cuda91 = ["cupy-cuda91 (>=5.0.0b4,<13.0.0)"] +cuda92 = ["cupy-cuda92 (>=5.0.0b4,<13.0.0)"] ja = ["sudachidict-core (>=20211220)", "sudachipy (>=0.5.2,!=0.6.1)"] ko = ["natto-py (>=0.9.0)"] lookups = ["spacy-lookups-data (>=1.0.3,<1.1.0)"] @@ -7234,7 +7258,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\""} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] @@ -8096,14 +8120,14 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6. [[package]] name = "types-pyopenssl" -version = "23.1.0.1" +version = "23.1.0.2" description = "Typing stubs for pyOpenSSL" category = "dev" optional = false python-versions = "*" files = [ - {file = "types-pyOpenSSL-23.1.0.1.tar.gz", hash = "sha256:59044283c475eaa5a29b36a903c123d52bdf4a7e012f0a1ca0e41115b99216da"}, - {file = "types_pyOpenSSL-23.1.0.1-py3-none-any.whl", hash = "sha256:ac7fbc240930c2f9a1cbd2d04f9cb14ad0f15b0ad8d6528732a83747b1b2086e"}, + {file = "types-pyOpenSSL-23.1.0.2.tar.gz", hash = "sha256:20b80971b86240e8432a1832bd8124cea49c3088c7bfc77dfd23be27ffe4a517"}, + {file = "types_pyOpenSSL-23.1.0.2-py3-none-any.whl", hash = "sha256:b050641aeff6dfebf231ad719bdac12d53b8ee818d4afb67b886333484629957"}, ] [package.dependencies] @@ -9002,13 +9026,13 @@ cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\ cffi = ["cffi (>=1.11)"] [extras] -all = ["anthropic", "cohere", "openai", "nlpcloud", "huggingface_hub", "jina", "manifest-ml", "elasticsearch", "opensearch-py", "google-search-results", "faiss-cpu", "sentence-transformers", "transformers", "spacy", "nltk", "wikipedia", "beautifulsoup4", "tiktoken", "torch", "jinja2", "pinecone-client", "pinecone-text", "weaviate-client", "redis", "google-api-python-client", "wolframalpha", "qdrant-client", "tensorflow-text", "pypdf", "networkx", "nomic", "aleph-alpha-client", "deeplake", "pgvector", "psycopg2-binary", "pyowm"] +all = ["aleph-alpha-client", "anthropic", "beautifulsoup4", "cohere", "deeplake", "elasticsearch", "faiss-cpu", "google-api-python-client", "google-search-results", "huggingface_hub", "jina", "jinja2", "manifest-ml", "networkx", "nlpcloud", "nltk", "nomic", "openai", "opensearch-py", "pgvector", "pinecone-client", "pinecone-text", "psycopg2-binary", "pyowm", "pypdf", "qdrant-client", "redis", "sentence-transformers", "spacy", "tensorflow-text", "tiktoken", "torch", "transformers", "weaviate-client", "wikipedia", "wolframalpha"] cohere = ["cohere"] -llms = ["anthropic", "cohere", "openai", "nlpcloud", "huggingface_hub", "manifest-ml", "torch", "transformers"] +llms = ["anthropic", "cohere", "huggingface_hub", "manifest-ml", "nlpcloud", "openai", "torch", "transformers"] openai = ["openai"] qdrant = ["qdrant-client"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "26b1bbfbc3a228b892b2466af3561b799238a6d379853d325dc3c798776df0d8" +content-hash = "3c8488864a754852fdec3e56dd5630ed73852ec2120a94cfe22537c075901b24" diff --git a/pyproject.toml b/pyproject.toml index 7acd501b730..d3f34c3ce7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,6 +104,9 @@ deeplake = "^3.2.21" torch = "^1.0.0" chromadb = "^0.3.21" tiktoken = "^0.3.3" +python-dotenv = "^1.0.0" +gptcache = "^0.1.9" +promptlayer = "^0.1.80" [tool.poetry.group.lint.dependencies] ruff = "^0.0.249" diff --git a/tests/README.md b/tests/README.md index b0296faf77f..0ccd5374757 100644 --- a/tests/README.md +++ b/tests/README.md @@ -39,6 +39,11 @@ cd tests/integration_tests/vectorstores/docker-compose docker-compose -f elasticsearch.yml up ``` +### Prepare environment variables for local testing: + +- copy `tests/.env.example` to `tests/.env` +- set variables in `tests/.env` file, e.g `OPENAI_API_KEY` + Additionally, it's important to note that some integration tests may require certain environment variables to be set, such as `OPENAI_API_KEY`. Be sure to set any required environment variables before running the tests to ensure they run correctly. @@ -54,7 +59,9 @@ cassettes. You can use the --vcr-record=none command-line option to disable reco new cassettes. Here's an example: ```bash +pytest --log-cli-level=10 tests/integration_tests/vectorstores/test_pinecone.py --vcr-record=none pytest tests/integration_tests/vectorstores/test_elasticsearch.py --vcr-record=none + ``` ### Run some tests with coverage: diff --git a/tests/integration_tests/.env.example b/tests/integration_tests/.env.example new file mode 100644 index 00000000000..6b39afda058 --- /dev/null +++ b/tests/integration_tests/.env.example @@ -0,0 +1,9 @@ +# openai +# your api key from https://platform.openai.com/account/api-keys +OPENAI_API_KEY= + +# pinecone +# your api key from left menu "API Keys" in https://app.pinecone.io +PINECONE_API_KEY=your_pinecone_api_key_here +# your pinecone environment from left menu "API Keys" in https://app.pinecone.io +PINECONE_ENVIRONMENT=us-west4-gcp \ No newline at end of file diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index cdddb4eab2c..8fcd9e0c35d 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -1,10 +1,31 @@ import os +from pathlib import Path import pytest # Getting the absolute path of the current file's directory ABS_PATH = os.path.dirname(os.path.abspath(__file__)) +# Getting the absolute path of the project's root directory +PROJECT_DIR = os.path.abspath(os.path.join(ABS_PATH, os.pardir, os.pardir)) + + +# Loading the .env file if it exists +def _load_env() -> None: + dotenv_path = os.path.join(PROJECT_DIR, "tests", "integration_tests", ".env") + if os.path.exists(dotenv_path): + from dotenv import load_dotenv + + load_dotenv(dotenv_path) + + +_load_env() + + +@pytest.fixture(scope="module") +def test_dir() -> Path: + return Path(os.path.join(PROJECT_DIR, "tests", "integration_tests")) + # This fixture returns a string containing the path to the cassette directory for the # current module @@ -15,18 +36,3 @@ def vcr_cassette_dir(request: pytest.FixtureRequest) -> str: "cassettes", os.path.basename(request.module.__file__).replace(".py", ""), ) - - -# This fixture returns a dictionary containing filter_headers options -# for replacing certain headers with dummy values during cassette playback -# Specifically, it replaces the authorization header with a dummy value to -# prevent sensitive data from being recorded in the cassette. -@pytest.fixture(scope="module") -def vcr_config() -> dict: - return { - "filter_headers": [ - ("authorization", "authorization-DUMMY"), - ("X-OpenAI-Client-User-Agent", "X-OpenAI-Client-User-Agent-DUMMY"), - ("User-Agent", "User-Agent-DUMMY"), - ], - } diff --git a/tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts.yaml b/tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts.yaml new file mode 100644 index 00000000000..d3af2d8a814 --- /dev/null +++ b/tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts.yaml @@ -0,0 +1,489 @@ +interactions: +- request: + body: '{"input": [[831, 677, 31172, 220, 19, 370, 10718, 830, 1484, 762, 3443, + 22, 6043, 2437, 68, 22874, 346, 22, 65, 21, 68, 1927, 712, 2689], [2059, 7341, + 527, 264, 1912, 315, 658, 10753, 677, 81, 3581, 7795, 32971, 555, 264, 7558, + 321, 351, 61798, 30535, 11, 4330, 311, 8254, 342, 484, 1776, 1220, 389, 279, + 11314, 315, 279, 2010, 11, 323, 281, 1279, 278, 66079, 430, 527, 539, 75754, + 311, 279, 2010, 13, 18766, 61535, 527, 21771, 2949, 279, 1206, 1037, 24082, + 613, 318, 269, 4055, 320, 269, 24082, 613, 3893, 8, 323, 527, 279, 13219, 1912, + 311, 279, 426, 4428, 42877, 320, 66243, 323, 24890, 570, 4427, 8336, 13334, + 279, 4751, 330, 939, 847, 1, 439, 459, 42887, 5699, 2737, 69918, 3697, 315, + 921, 2159, 14172, 339, 9891, 320, 11707, 321, 351, 61798, 7795, 8, 449, 264, + 44892, 12970, 79612, 11, 1778, 439, 6409, 65, 86815, 82, 323, 53265, 582, 276, + 17323, 13, 61536, 12970, 523, 2159, 14172, 27520, 598, 1778, 439, 2493, 5670, + 301, 1815, 323, 25227, 3205, 355, 1176, 9922, 304, 279, 60434, 1122, 26572, + 320, 19391, 12, 19192, 11583, 705, 3582, 1063, 31376, 1534, 523, 2159, 14172, + 339, 8503, 12970, 29505, 527, 439, 2362, 439, 279, 36931, 31137, 869, 12734, + 320, 21209, 12, 14870, 11583, 570, 578, 24417, 6617, 61535, 320, 9697, 613, + 5493, 8, 527, 3967, 505, 279, 23591, 84474, 11, 922, 220, 1049, 11583, 13, 220, + 71923, 2134, 304, 1404, 505, 279, 2678, 50561, 74265, 939, 847, 320, 36, 14046, + 2985, 46109, 281, 5515, 72, 705, 264, 5655, 9581, 9606, 430, 374, 1193, 220, + 1114, 2960, 86366, 417, 320, 21, 13, 22, 304, 8, 304, 3160, 11, 311, 279, 51119, + 44892, 320, 73262, 2910, 77152, 3666, 355, 705, 279, 7928, 7795, 304, 279, 1917, + 11, 902, 25501, 13489, 220, 717, 37356, 320, 1272, 10702, 8, 304, 3160, 13, + 2435, 527, 1766, 304, 682, 52840, 323, 527, 4279, 311, 43957, 709, 311, 220, + 17, 11, 931, 37356, 320, 21, 11, 5067, 10702, 570, 2435, 8965, 656, 539, 3974, + 304, 80744, 11, 8051, 1070, 527, 264, 2478, 3967, 20157, 11, 1778, 439, 279, + 17231, 44892, 323, 279, 15140, 44892, 11, 902, 649, 387, 1766, 304, 2225, 67329, + 977, 323, 80744, 8032, 18, 60, 71923, 617, 264, 18702, 315, 2761, 14991, 294, + 4351, 645, 430, 36236, 872, 6930, 505, 5674, 323, 79383, 304, 5369, 311, 18899, + 872, 15962, 30295, 13, 2435, 617, 12387, 7437, 315, 8454, 481, 18311, 13, 220, + 26778, 9606, 527, 72627, 56217, 11, 902, 527, 44304, 430, 527, 520, 279, 1948, + 315, 872, 3691, 8957, 13, 8593, 10507, 2997, 279, 52835, 44892, 11, 6437, 44892, + 11, 2294, 4251, 44892, 11, 296, 29886, 44892, 11, 270, 86524, 44892, 11, 323, + 24354, 2025, 44892, 13, 220, 71923, 527, 10791, 555, 12966, 369, 44892, 13339, + 477, 44892, 1913, 19724, 13, 9176, 44892, 22673, 527, 21699, 555, 3823, 7640, + 13, 8876, 220, 4468, 15, 11, 44892, 22673, 617, 1027, 11293, 555, 220, 6028, + 13689, 10213, 505, 927, 69, 11218, 13]], "encoding_format": "base64"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2798' + Content-Type: + - application/json + User-Agent: + - User-Agent-DUMMY + X-OpenAI-Client-User-Agent: + - X-OpenAI-Client-User-Agent-DUMMY + authorization: + - authorization-DUMMY + method: POST + uri: https://api.openai.com/v1/engines/text-embedding-ada-002/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA5x6SfOCPLfn/n6Kp94tt0oEJOHdManIkCDg1CtARFBAhiSQW/3du/R/q7u6qle9 + cQGoIZzzmw7/9R///POvLquLfPrXv//517sap3/95/fYPZ3Sf/37n//xH//8888///X7/L+uLJqs + uN+rtvxd/jtZtfdi/te//xH/95H/c9G///nXGq/29CCdT9myjY4I5mtrpr6cN9lihJ9EBeJlpLgU + pmHpKmcH9VrYUWz4RT2nOijUhxUesD7Jcj2tA/MFHzG5IPb5hDVzHu0CwHMMMda2B29xM7sB/Uaq + sWNcZsD2+c6E6RFSpFZ3H8x77dyBSbsM1KezyMlnnCToq0KD9yHAgO9mVmjXp82p/SCyx3mMerDd + 4Ab5q37IZuJlI3zJPiWV0PXWZCxLCmuJzQhc9mtAI5ghQLJX/V0fq0egYBPSofHoAWHAeSAbLxiM + RwcHj7kB8xi2ZyjNU4ed++XDF3H9IHAaH5ioPX4CdspdEzjXJKVutGjZmx76M9zuHJsGrdLFlLu3 + BfZ9tCFMq1qLw2A2IQ8SjxoGI9bCSDKCMnnVpLGtLWf5nSmavBIuGJUVHVg/iApEWlsh8ECXbCHv + LoHV2zzQfSe+sl6aIYRm44hYf1bYWpI3FYFWNT7249MIZm+1K+GVLjpZU+hYs7GoVzgO7hrvZvOe + 0WoHS2icoo7uP6EYc2w0Kojvr4hGKzv1SHTpEnVMoxtFpfAYeLaTKijrsk1WilxZ33rx4Uupn0S4 + 7oxh3opVpO4uyohjersOY5h7FRxH60iWtEus5VneEIA1QNSPZwj4lj8lDXKzJ2Ibva05UpwSGs07 + wsZGftaLHXYJbN3jhtqmZg6iIL4iiECa0ADPLpcOuQbVRtkNGF8FWk/iSRZgtVbfWD/uam/OdsgG + UignFL+YPfATjHyYLdc3Tsl5G/MUXkd4O5YzNgXtXPNACF7QOL9TNC8+s2huDCmgsGiJ2N66eqp0 + RYDh5IpIyD8nj/bOcIUfpzoS5VbgWKbHYgEuWPfUuRU4W4bxnUB/WLt4VzZztlwz/aU57krA/kOB + MXfUUQSq6QpkylYUTFI4JsqRPBDWj/JqmErvMYKwsK5I8IOP9dsvAIzIxS45v7NpdStGYMtmR11z + 0DiddnOp0Vx/4qsRkmwWAt0Gv/U+C1/ypjEIbOBsECHy6XDm9Gq8rjCeGhfrwe5tjY/aMlfMe1k0 + t1AQs/WyVsByP2eE77rOmmvzI6r7blHwDw+4cm6vkAmfN/XpKuPsclevgGr8Rv2sPnL2rN0Kuunq + jBaV0oHFNSpVED+u5PMZy5itp57BYn1/YJNIF+vDkAghm+YDxXOcWPN4GM6gdyqDojzphrnc+Q1M + 9M8a7xL3HTOU2zpUzbePw0nIvDmUQldDbqnj2+KQ/36+pTXuqXMPd5k8+54Dx8FZY6w9bx6z77IC + vbRi2K5hYU2hlrjqLAwnInSB8T2/UtVWjT8YJcHe4iZzGli/iz1R1fMznt0NV+H+ECCy9M+xblv3 + FIIbP37IKmHdQGv3HP36jXS3g8nnTDu6mrwcGmyv2kPNV7Hlgvf1ecPO7b3w8fzMe6hbpY5R/jlZ + tLWOoaZyQPBeK+Rsrr0zgWTMVxhP8dl7e4pZquK0GDQod/PwQVLYgU4pb9jPcpQtu0stqu6my/HB + KOqMRXegwJVUL2QppMxj+/joQmEdXLFxZ43FkpecwpejrbA+CZlFpFkUoFq1NjUj6WHxzs2ZGk8v + FyPlM1msnRoGwZFcCbRQkPW3ok/g9VkbdNc1OPvVKyRZU2ObWRoYV7Opw0h7pTR8Z5+B4OKdgJWl + m/jsN5fsr592svoi8v0CwNh3mQsZ+qy/+H3iM40uPsQwKum2ha63HncHW/18RIDka/PMxupRhpqy + HM7UPQ8XzobmuUDkxzo+kPMlXszLKMLXmX3oFjZZtviJY4Nn0sR0r4h6NjkrXAJ1DiNqt7duYCZm + KWyuqwtS7uM5/uJbCB6ie8f+A72zSU52NvTgscP6nSkeOd9CBOwt2VGrp0ZN4FgmsNyaG7wLLYOP + yR2kEIpzj61VefWmEAwROBb3M91dXSvmzLd6WO74hjqTeos/v3o5CUqCwO7VxMvYAwG+z8sN+2y3 + GhhfpBT2nbCnptD1HlXirQOhQRzsW72W0etuVYKzsVqhIdaXuMnaM4FHzc2//OADAmdQgE112NGd + Zt6yLs8dHZJ8c8LOYkA+x/bFh+bFuKJNdRf4qCNH//HfX33OLhiZyh/1DbvpK8p4atQK1LexTtG1 + mgciMr+B725vYetxy+vlcJoh+PGNEcX3YRlIgUAPrRlb8k2qabZSc9B+bgHejeAGmLJAFZjvMCSi + dVvqodj6HXA+H5tERep587JXcjil5IJNNw5q7vIy0So5uBAVDRswWsdnqOKpjfDhrnpxJ4tNBNYP + u6b+Y1Y428c3F5qSM+KHk5XWOPZeCTw/VrB32a85q1+LAn1PjtB3v+s52KgqgDZXkWDv2rrva5bD + ZscO9Hq8BJyPbrZAOqoaGcAp4kvbawWEkXtGc78l9RyDPoQx2tj4h5fLqQpeUCQ+xj8+n4Qo9jUX + yD1apKdTz7eVWwE8Eom67hlnc7WHAhig8fzTJwSsTyOQZtph5K0MzqLHyICMbYvqUzVY8ypKiWru + WpdiR+AD62td1774hC16U+oxr20dhoNo0ofm+0O/YsYLOh8kkfdj7dTjPj460G6Bj216a4alrg4N + dB3mYCPaijXfBKoJc0PAFDtyaC3rsT+DG48/RHZEki3Vuxbhs7l42AczA+PSXXvwiMcL2QSjE7Pw + rKRqbkCMXtDa8K59bVQ44z0n3RyW2XLKjBcMhc1I3fPZyxb6MW2tWhcXbFnefSCP8kjg1BNMd7mo + ZoMub20tPVZPBEFuxzS1cgYozFukWbkZ8+uKFzBazR210s60psrOEMxmeCHQ3OiAm+tshBEyFVR9 + +Yh3brLA07yv8U6/LhmX42330wdE8JNxYHDqGNzvdR3Nb2Gw6EE2XxC5lY6dZxHELH/GjraVzSt1 + V7qSzTycEgBjc0EjPdmcPZ6RA9Mq1Yg0m/eYv7dmB6RH+UTrC9lby/vdS6r0gBS76HWOJx1QBrRa + FZD4iFqLfrZGDyfvBXDgB8AiweUpwqcctNTYstFi6Kwo4DZXH0Ltz8cb2h0w1c0x7LEFvGngQyCY + QHlODjpd1v3AquYVwVrZ1xTnrKk52W1MwFGnEUV7d4B19NHDoC8dpC7nOuPEjSN4KoP9n54jZthJ + cPDvK2y7Wjlw5ZiHqqMsJSH+bjvwpxUxQLRLRjZrdZvRCzwSLQgOIt4qbpBxvNIgCFR4RouUvi0e + yk4DN9E7J4LyCaynhuZcw+guYF1XdIuzeN8DsDgF3nlQ95h9LJm22gYh3Y6PjFNPUSEQzJeNg3g1 + g5E1XQirJgFE0LaZNxjzBsKCjw98+OIb2d7uCJpNa5LNly+46js9FIPl8NUDT29JbuEITkjXqV/X + mNN6azlawcmDGJqw9SQjfJ5hEHgitbOW14s8fkQIlteI3p07x8tCEgSHjRjTu7jzhjk02wRKj11M + 91pxiZdn5e5grGOZOutin60f+gzVTGw50dqVx2cYOA3UxDmlXp0LGdPorQfgOF4xXm8Hj8NjFv3q + lVqLZFmTeChFWEtJjVb252CJ336D3SHb4m3pxhlLrmUI9x+8o9iprEEmu9mE10UevvyzibnBKxHe + ayJjHGwv3mTS0IEvZUWRMnMORu/aMzXWA5kaVXr3+OoYRdoJnxekrIs2mwLF1SGwDBdvx2MCGFD2 + JkiI6eB9N5VgFg+lBC8PdqX7KwjAWD+PDpAXr6G6JsjeJIajBHa3oKBuhWvAVouqqKM6PrBL7h1f + 6nrJYdPVBrYWqfY4VmZBDVThjDFjes0LK2NgSMMPNdN0sqbSuxBQyZ83ERRZtKad4uvQOXz8n56w + aGN0qeq4moBWdrWt2f68qFCX2grbtWcMsoY/6s//0MCrLcDROkRaZTcWAWTTcHbNDzpY7ssHB15d + A77xDy4sKtEi7BOU9fwwexFqXE/RInWjtRQ1gLCWzjV5DST++tldB+ttjrGbOmY8J1rmwDlX7ri4 + 7p4Dz502hadwuOND6ojx/NxrBXxuN1uapS8969f9JoefWzohRTAUPmzVtwROj3NPXeFuc9Y0QwSe + 50tMFjeeBl4ITgO+9f6tZ8pn+SQJELkQYaOIbzXPhEMD922+I/xM24FYGXqBeq2e6OGo0oxc3kSE + 6dP2qH2JDt7YNj0D0qN6knkjG/UEo5jA87yi6G1B0ZrtWa+gbB4kUn79zYLLwgfj0FpkHa/HYWT0 + 7ihuv1voIXi/wEL7RYBffYsARiAmirx/QUd9lkg94xaMl5eQwuXOPjR4zDs+IzAw+Kgjg+r34FWT + 9bwuoHV2XkRwssVa6lrN4Z2PFAd5otZknFcQwBic6d55PAFhYy1B05Zc7Hzrly0078AqOY9I0cET + 8AO8j6qzV3yqByz3fn4JZsbOQlTz/XpWvdAHz5P5JEowpvFaubSiGnhrE/txzr76aFD/8gxUfmJr + +PnRKbgAAh/9KRun5hlCNNIdWh1jwZpMKXd+eo/ut6aczQ99A+HrrW6xvfcOg/zxtz28UqbTnO0e + w1evqDARlBf2qXKN+4HtbWi+RBMtZozrxc38BoaQFHgrWh8wwduDwMPGL2hQ90a23gSH5ucHqXMw + rZhcK/sFvQPaEcmDusU24nuBhy4xv/3nAIbTqIfD4G3p3hFJPCSC94JsOgMaeKsYLHzWIBxT0iIN + ZmfrT6/+8qT9tXnGPLbODPZqNOPD7a2DDw3JGc7380C9ej9zfjcqBabWJP/tB4NTuWi6bDF6aPAJ + zBLiuUrvp5gIYcYs4me2Dc2dA2lq4okPq2sRbkDkNnjLjhH/8ncP3U2fIzG7+cPiFG0COenX9Nff + 0y8v2e1VkR5OYxNze50h8Kunr5+PmZPmHfjtRzknpsdj461APxUIRfP24f3yJlWwLzb91Q+XA7VS + vv4cLUU6eIu8LgggmXOiX/8QT1pYnTW3r3VsptuyZu2gqfCHp18/DyZTrEP42aRvoikB9vqx58Iv + ryBrq3xZtFophVo7W06tb96ynKrtCypmK6K1b53AtF02VwjrDcKtLrJsedcbCOfVvsYWiGaLHdOf + /lh7ZG5ieRjMdUZgoB1ef/U56fTaqU2LMFFu7wjMrXca4eIrAbWtiHpLXRkvePRDh96+eDBn+gIh + FoySerEeZWRprwgGIH1h/ev3WJybFXw3y5G847K0GKX3Dp6R6qPPoXhmjAxAhRovOvrjsyUrnhJU + K0fCB8FowDKOowhjenL/+veHL+D5ajwkBwcFEG19QvAyaw8csCQZlqxMyS+/o+Yvz2rrTQGLmLT0 + EIxd1rFpDGHvlAa2Lt5lWKb2SOBJDzDikTRZozp7lUpGEHzx2rfEeVBVaJ5Ig50g7LM52TAFRkhX + qHGsnh4D0tmBQSoU1FrZnUe/eSUYwDtGGtwd6gUqcgF54bf0lLiXmD5WcwEON7/BzvweOXOuwwK4 + gB4UJZ+Lx17NsPzh+0//r5UoH+FZ/wTYOR1egAoJrDYjfDxI7xoVn17BvoFcvwlIpKXrMXEO9N/6 + iCrdd5z76ysBX/7BWEtPA3fVVwKQYDDs7YYPWG5Ffwa9O2F6uKlTPOfmIKkKtxZiknRv9cI6J7+8 + le79yeGkfVcJWASfYqvqdGuxotsIvv9Pt1lr1Mw7b1TYXLUL9iz0yPgPH2KGW/K6eO+B7e+CCs6G + tkIzSW/WnNl3BGRaHb589QK9kXg79YwUH7vRPQR/fr5pmIH1Oams5fJuJHgi+Y5uPbhYs4AUAXpp + ybCxZb7F5GFO4YZbFdkILzUbX8+wgwqGmG79IwS89wMXDhspplsLJtZXH0ZQeUYq1Z/Cw/qrd/3s + 5dQVXk4m+s9Lp7JcnegWk4MnzZ9D88tXsS/XH/76DFoKBzV6UvuLL5vLytM3d/sQoM23vrmu0gQK + L+7SoF0NfNbQXICvv6cB2wUDq583BxqJlGLnYNYZ92FOfnqYmk38HmY1yhG4WccNmoBixpJ/i30o + 0hXG3rCngLWvWYVNsxhIvofXmMfqR4RHZDho9E6Ec5hoJsxF74TtukQePyqbAq7EZcG23Ho1l445 + A75aXgmbE9MaB1r0QE/SHK3ZceFzZcc+tC6PM/Zl5Rzzw/pIYMqPN+qcLsPXTx8d7csnRFImwmek + 5T18KduYSGVzzGZw0gT11foyGeO9xeWvvtR+eOfj+sSX8v06q663Von4WKfDsrvlBDpqXZI60tgw + I+nagXm1rb96cQJTudLLn3+m4Td/mcz5oMOf/jiSdGPxz/bQw7uZBmTK+jqecjsncPZNA28f667+ + 5u2qpuqij30+R1xUet2E8+o5Yy/aED7KYvOdO0gRAt/6pT99dj1ql69e7K1F+jg7eOfhk6jNuc0W + /6RAaKxHgk0zjb0Rp1EHt84pIqvbllvLLd/kwFN3KbVMGgzz6nYe4bF4nLFLXn42mbOhw8tzb2FP + 1p1YLF9cBQhIFTWkbV9/n1cOv/WC86Gfs08boBf45jlo5X9aj7+3bgd++aiHFi9e6+KwwOMY6vQM + m0/89dMNdJQkIvA735H2uVDA6qROROqOJpdWzGggC+bjr3+8n36GzebrB77Pd/m00Qjdm3+mhb1r + h7lcORWMi4Jg/5sXT1dQMhiiDcTGSfaGpa3nHOamyMnpQlpv+uHlMWhUwlHXeMy91hH006qkPu5v + 8bzdaFdYK9saB+384iQpcwKazXrAJuoUbznkawEap7CjVtTZHitfQIU+nL55Kjx7azU42HB3KQqK + 8sz3OJIPO5gU6khUdTlmo3F8LlCIzQi7wiYBi5evC3jwbpCiJGi95ccPPjxeiWxPLpe+9adqUZNg + LAr7Wp6CdfXTw1S/fZJ6vu5lAWxCKafb8arEzE/jHsjGMcLO6S2CMb8rym+9+PTtn2FxOgU8t4X8 + xXsf/Pkv1doIhJ2C3bDciuoMvdSu6GMUPjUTu6sLq/1nTZ1vns926cmFhehOSNlcDD5boGFwOKQB + PiBHifu+Vgr128/URcONf+h+zv/yZKuQPt6iI6X44S/6/l+9RDnPN6rpCNgImOstcS0IUFzZA7Ut + 8vBEMD1D7TsPRPIIWsA3uFfVcJBM6sl7j3Ml3rpQqHX3Lx+eM10VoOdWCjbTNPA4dZ4KKE/6TM1K + E4aptqNRY2hYk84ISbwM5OxvmoMMUOCtOP/TKxzvPdRHRsl7XJ79Xz/Qn56di5VhasQd91TfBm7N + ktcqhcuqfqKhWbYxm4dFhUYipjQ2pdkj3/nQX77m0VyNZ2uzVuHPD97l1Z7PcuDuIHhGJqKBn9Z/ + 88VvfkEkEch8lqMTgr/8YKdPAme8NyrQe29OjZtAvVGicQ9DzRnxfipANmUrU4fn0pdpdI4P9XoO + 35L6WIM7tkwtqZcN82yQG6WB7r/5YLsDOvjTR5MAPL6blQLu1W2FIOiDmOaCYcPsWR5QeAqaejlm + ZqPtZSXBwaXfZrISBw503aeDf/57fu285qefieqeacy6Lu5gtVXW6OvHBvIqQx+upO0LW9GzHMR6 + EJTf/AjNJ9ZZdA8LBL5++KtXPvVinjiE9t5UsL9ayeCVNU0ItThn1Ke1B5awloUfX9HDnbeA/PBk + 0k4DPWzfO8CFBJbgq7eQtl+twTj1nvnzu9hv8z6mDu9EYDQHm3rS5gUWK0MNpH4u46QtR2vq9jyH + CeYhPpC7Cn75jpII+Zl+118vKPMauODVCrHbJxnGrrcroK8jRsShpQNPFKWA1jvU6WU+IE5zp73C + EI4F3rGrHU/f/EV7bsEWI8aWmggkHOHqjR2k3PmNf/OuTtbWQYeDehWARfhOOvRE6rHfnhowK1Ey + /uWJ28S9ZBz6jqPqidiTIR0s/qLzroJE4x/01Ts19dY3pLrO4nzzqCOfEi124TePo76nV/Gin8QC + HvFJ/dM/TBTbCLZtHyD5mw8Pv/nifW09sXseZPDNV0x4KvH+m3eU9XdepahZdJ2wuzNsLv3w7Ksn + ySUIYTybUu6q60c10wBmkkWSW0jgv35vBfzP//z/eKNg/f9+o8B7X9dEYruOT9mLRQDvLy11JvLI + Zv6ZO/BCF4FuSxRmY6QdETQudKS2vyDO7LZ/wbOYHQk/98J3mp8j8NiRFm8lqYnnw7MVoDFJKd6W + NfXm+zN3IXmOJ7yNjSleLGg7UNSXM3mKgZWx1yMVYZq5UqBuxSJbruNugdfi7uC9sLI5c/xrAYfr + vFDX19thSR5chIt58hGbhdcwP6fWBsknXxEJCiMfjduzAVcDGqhF7W5Y1N18Bu88wdQ1gFR3R+VQ + QTJHIjUfxyZe7OCZaGYNn9js4X2YqpWN4Ak7PrWWqQVMTSZX8Q9JQ/27Y8ZyKAEVTua6x3advjhp + Tk6v3tq+od46PsRsJ996wDT4JMv39+fwE+40XU1cbK3Uvl6Af9Ch4Qs6AVYp1zM4VQx2h0KiBpRL + zk7hR4DqsfUIKIzTMId3t1N3LUbUIteDxfh471TQs45MytJnY0/3KTDUU4Xt6kM9drLzEtaP/onE + NOtqvlmYu8ncWCQQkWhYNp/dVzFfb6Q5Gwb/aL4gwCtSPvQo70lN5S5kwH2qb3ow6iobt4WwwGvQ + ECSdDQNMzbtzoHTeiNhc39/D93478GaRgYNQm+Mlc6yzluzDHh+lQR4Wcx4ZFJT5Q+Rt18fM4esK + ZsWww6aQG5zT9VMFYnz1aSrfPtnop7ccxDFLKQ6VT0bPjeECx99wBDT/aC3Mzq5KykQbZ751txZZ + UcKN+OwZ0sBby3jhVw14g0eApEQ8eWtvlZbw+CEV9s6vF5jHD46gZlVv9H44WU028KZAH+xTvGuR + PXDuLB0UV/cFm2nJBtpfZvFvf3ZaaoHGn3wB2IKyYDRdscU7cVWqt6u9p8lpHix+/VQMqMv9jU1h + LodXQw8KlJwhxN4tfIBp1ywuXLaijZZz64BpueYlQOI048PsumBOH90LbCPxRbGpsprZoxpC8iQn + 7Oyr3qKuL6Vg7+w7bESSMHAoDiMcpoONWL63s/kzDlcguemI0ercDvO7DAuwqB2m6WMo6iWMSwXS + s2CQi0GrYRCReV4dU8eiD6VxreXJ0gWGcZPj3e3dc945LgOnsHCo2/hNPXt2H8KjvNpR42TuARcO + fQlssM6o96gNT5oPww5uijLDhyFVhvH9WXxoe2lFZL1aA45aXwKbpxtRvR4Ciwt2JAFVDq9Eq5rB + 4tOoNFD4YBcjTYQWi3VVhQMyd9jsDn1dINa8IDq9OH74q+fAOhn4CmjpEevobdYMRLINB6KWSH47 + r2EW+k8FVDm6UqMZnvV4SbwGVP11jcODOIIpO042TGB1xsa7EAEdkxOBN/1T0ENRJUNHW72A7SV8 + 0svjusSLsSojrZzoHi0aXluLljEbhrrQ4OBN7hZxX1n/V6/aPpHjJS8UFxqTmOLgFDFrfAqoAe6h + fWJT0HvO9dcnhbUXY6pPSetxyiIfvg/khn3sHup5/4BXcAsDjQb2NoinubnqIK7PGwKvvB6mUooU + 6E/RhfrThljj4C86PB0XC8EdtLxZvVxFaDhegB3/2QxzsWwi+DGSGw2S59dxP1JJVdcVpPvDNvOI + Bh8pJNwM8R7Bd7aMs9vB85Fn2N0bg0WuI1rU3fu+UAezcGAeJCJ8aCGi9+PNiQey6BEMddhQrz2x + msH7uQB75J/pgWsap/Vln8PLJsjxvhuDYe1LZIRNVJzR5tW+60VqMxcu8bDHPn50Mf0EUQGD9LOh + +HpSLRprsgvr0ftg9x64GZeOAwRavLURE54fzm612sEejh7ShkMYLxLvXvDQaAPpAGfW9NCuncpP + N0DtofMyseQUbeoqvFJ821HA6uba/K0/uMFg+PEZSD7Filo7NGdMuV8hlBukU50uIB7laG60XBkF + arbWifNvP4FHHND/xkMvoxL0FH0gEhlOGdPWmzNMTtsJfc4fn/NqvqbQnZ4RvY22FE+yZibQu7hH + FNb4bC0avFzBD+90VSu/+FA20DtaNZLkfh6m0ixTVdMfiBpw51kzezqJSrfdjurraqrLcKoc7dy4 + E7Y+xBuWBS1MMx9vhLfbI4g5qHcN3N/ZmlrLMQRs9CwB8qz64J8eGPt9n4DQJCb2r2bA2Uw3C7Bv + +E0aeRVmH6cudBjU14J6ziOxGHkqC9y43gHvZJTWa1nyXzCo04K6Sb/12GqTClAu5jN2+maKebQb + dHjcVwHeAm9r8eqqlfC739hC83ngaEzKH15iZzVeAIv3rNN+eB7I47qeTYAq2O/cF92djSdYDpFQ + gk/lc7ozNbHm0rYo1SPQ7zgh7B7T0DqkPzxB8kM8ZGS7ei8b1+8V7BRuGX/5b4EJLM/4FACF0996 + qtE5Iu1svfl4Ps09mEtXxrZiXwd2DbbSH18c1MPeGg/blEBqNhQ7OLnVXDSgCp9tssNGda44+/V/ + efR16tyjKB6PLynRwqesk3KTnbL5QOsCEj/vqHmjKphS/54CNIQ6qnNKM0o/efHDe3z74tP0QvwM + 0+v2ge1bHwG6GdQrJGk8UDdbJLDwKElhwxOGb8dayUbOsxRekfqhNvNCPud4ZQKZXNbYVazW4uJJ + t+GtWn9nsxnIBvMZRKDbAEpA8Hbq2ekzBlD0zAlTNR2IxWcvgTlsrkQcie4x6q9K8O7kC/afhmvN + 6PY0ofa6rrDZFYd4/BzC9Pf8qZ+PXiblu3sIz3EpU1xJT7C0cdSApdNvSLhGr7p3mgMBtcAf5Hky + Wz7GYHDhNCoHesttPZNvWRtqo9ju8GEsg2wJmr0PC9l18NY1Z+/1w+9v/WG7pXW9HB6eAOSCn9E3 + 1LOIqg4i/K13f3guFnsEzIcNFh9Uj/jH48uyXVQsJJQGXvCw2K1XUjhtgYH1xxnFEjyXKbR2zQX7 + ZrSt56IZRDDcmzeSA4kBvt1/EEAd+iDFVMZ6PCz7HKBqCxCUaq8mT9ghKL7Inn71m8VzdO2gbEaY + aE7qW4yvShGSYKmpt71U3vLGDIKHiuTv/U7xe3+Vut9+oNFrynredh2CXcpmRFtPr9lxtpi68ZQ3 + /avX9LRygSxaFhK1hweGAxpzKMFDiVpJ74blMkEJPB8gJ8srrDh5tJ8cNmvbx+5+0YEMatTA3RX2 + WB8V5n3Cu9nDPc8tmn9eOJ73V6lXT8+kwnuhnjjZfNAIxWfHcCFxM+ZGZ+kQ7RP8x69iEjgJAOpp + T73inGSNL7oESkxIkHhAW87qJnxpyjYRkGTdvGH+8fUn6hPqPquPxXebqYPHLZuR+qwO3vISsgWu + 5dMJo40rDEx09QhsnGwi6j1w42WfvBrIYN8QdaEDX/xNmsLxVDh45x8pnz9tTKB3NGocVOAAJvvi + 2eqXrwgYk8matdJ0fvdP+scg1KTbPmx4WrVbtMltPZaSq2BDq15t6EHg9sCLK/viWxnQO9tUw/Kp + Dx344WPQX9WayOU9/eEttYTrw+OHU+VrqRvldN82Ciebde5CfV1fyLPBfT0O+vgCp+e5wr/9m27p + VVXvT92kUdCNwxwkxzP0962LPXm+1/1mnbiaGmCFKMBU48mMPznQZHNNuspyPDqEcAelT91Tq3sK + nAXHdwevy1XF+0BinPAouapp+xFwwF41YEfFKNVXkk/USWjgjRUelr9+MNTXKia59xLU1gQYCUGu + 8WW8tgzexwDhLENmvBwLJ1J/fKsL2wXwWwsJ6Hv7hHehexk+1nV2gOt3Ct1/jDHjY1sq2k0OttS5 + rY2ab/dPpMGq97BzKk3O98I6US+9oFDz6J/qMUwLAZ5f0kLEKKg8Ztw+DdznhUp++L0UZz+BWWze + kDC2m2zJd/dIHTFTqX2v1YFgEEB4Cq4MsW99T8ekEJWyZA8aoZuZrcGTIa3vdyfqvL3UYr20dWFv + 7QPsGWadLSwsJW0C7oT3H8OP2W6wfS0/Vx/q8LDn8y4dR/Wr9+kXb71Ff2pEvjhLjdHY3jImOakP + v3obac0j4suo6ClMNTZSO6hf9dhUsIArPtb0TPPBYqAxI7hrA0SYbqeA3bzLAuxZcKmV+K7FTfGa + g1Opt/j3/U9zi4gWjfxI9fnz5l//e9a+/I/jr9/tpOCpqD99vwVlzWldBAj+1qtnyzueH681g19/ + SxNljTKeMaJD/Fnd0bzqOaeMvFyY5ZuevHiq8mW7mhY4XPmCZnlPBhqsnQTWrITUnT6XjMm70QEH + V7WJVL1ab/a8Tlerz07FjoLbeLILR4SH/wUAAP//VJxb76o6Hobv96fYWbdkRUGkZd9xEhCQgihi + MpmAB+SkCLTQJvPdJ/BfmWSuTZQU+b3v87ThdOGJcT6vXQpYY8KHyway8E9XJHIBQ0M9E80uEpdp + qZVLM8/7G+doAWqGDoTAriKSvc9BN/M1hD0LXcztV4M+r3+/jaNi47/7+pzxic2o3Je4R2otdYD2 + ncSJDvNzYs38Mn0nYMKI54i/bswnoy5QHrKpR3dMq/pVUnbwMDh9XAPZrG7Lynu2DnxbvYau6YFn + tQbMQvomYzr7llXGVsIzgPqH9kR/8SXoq+iQgvl6yIE8iDumo2IuPILc7osy2ot2CkgkfskzF1fZ + dKGmtOQ5pl1y6ib56qVglLkXFl4XKRqLE7iJt2ZnE92xrvponJ4puDvKlrjnS+JOTZ07MspNb5kP + Ge4n7SNvs6eBrPga64M1hQVcHRwwr4cLeH03QNA8Ew5pXrZi9Bh4I3xK3mb2D0gfX3GbQ7fpRyzP + PEAlqRPA3O+Is+GVrKfS1YZDL+2JTXqVjT5aNZBxztMvnqEZ8ap/pVDW85r4fuuUOFMPDdypebs8 + 72CkLy6ArGUH5LxM6g5nSerhcH7mxLSTjzsGQ+HAIhViokH2cNlBChRYi1eB7Of1H07tuYJF74RE + BVPv9g1RJXHpA3N/0Td77alJcL9t0P447vTNB9kFUF2/It6mzV227aQUvhteIYu/WGesxdLsr/74 + nPadYRhejB4diCa69P3MlaVv4s1O3rsTDWO65BVKMOjdyRn4BIwJ5vFUP05sTG9QAkeg85hTz6ib + vo+3CIL1Q/Bp6oaMXUJLBPrVPxADj3I28c25ABt+fJLMuq4ydscbDs4+DL3LYXLZTdN9MPsg5GZ+ + EXXi8LmBZoyuCGE5cPuyCZoffvXc8MWY3vUf0AftmTj3wzeartahh6nbXMji8/o6fMbg7JyvaHke + fnisv9cyctqRdOxzCjyYP0UDXYdQ1+kn7AsYXswei5s97YbpeTwCvs2PyM/kuhtYAUQ491vcVu9d + x3DnFNAixugLKL8y7NzvEpgyVmG2ueyir/ggJ0BL1CA1X18yXErUht9o9UZIe1y7aTAnCdoTWWH5 + kOcdw0fyhbJkemj/gUY2Onr1gRPOUn+18B7pAxtU/MVCyzzsRN8SoDo+QrzeOoNO1dz6wtXYeegg + xixjRa1BuT+m0/I8uTNvFEAlzhnp6HDo6PJ/zWglkmhzqbNPvK88WBbHhCBDqaPWP5YBEA4PHVnY + ytnk7ScOSnwOfTb3e9YF0IBtmWN02JaKu87Yq5ePwdHBdPYfY0TuWLwkZ4049/aVjbbiasBBGOFq + 7vOCv9/nMF+pGrL6+hyxbfL24dxHyaG8KIApYexBcRdzyFttmo6Z3iiCleJFeFOxHeC3x70Nva4A + vqSCuKOzDwCFsJXJjr6sEltFbsoFN1ooPApc2bf7JIFmwn2JZolp9H75DxvsjSkmmjLa3RRtQgPM + fRRZgmKXTP242pIX5KB4u5I3yiCHV73nSTznf38ynQaeg5tN5v5Wtl/OLmBfa29kf0O/pA4/UrDM + Q/tquIB5ZenD7Y2uMHc0QTZEdJ9AvZS3/qbfP7Jh8av4ftYw9bIn+PEFYero8/qpEZ/ovQn8DbOJ + vrtnJT1/Mx+mbnUh+6lRwcZUyptE4/5BIum5yX7ud24/M7RbGT0YpZw2oBgeHkGdYWeil4Y3mGTZ + NPPP16VJ748wT1QJHWziMF6FsAKzD1z6EWgMNRNhRsocrz2KwbT6sIcMdrsM6Vyycqfdq7jBTUZW + yLDvgT5Fm6spifJHRZdW9TJWS/semvvUxZPtlGw0mKiA+8EIkHcZMOsz1vZQ6CITS3NfbcMijaFd + ShDz6G5kVLZ7CRLBWKF9WIrRoGwqUWp40yO+mRm6wB3WDfxwuED2qOZs+hSeBi99f0LR4MKMDbUk + wIcq3QhiR59N4ekhwH5dm8gTSrcbubUeQPAeQqIcNRaNd25vwFraH5ERimWGdd5PoFk/KVl8IUkb + /wvI0xLwxndJN/p3YADtEUVYCs5mthHTcwBojB9zXuqADTVdQ7PXVGLtX0d9JG/7Bl1gaAi59kWn + xqGNoVTDO/7M8wevkoiDB9+eEFpdZZ0isOPg7OPI4rvmvKnA7A/x+KjFbgL7eyz5wn3pd3pE149W + gCnJbRL04qhPyK1zmK7MN1KmA2Ytd3nE0EYrB0/XS1PSdTnlix8gVgGzCK+eYb/0LSwLYMXINSPB + wmtoPzUvJggnxIGX+tkiM+x1fckH6IFditSkGcHYbfEI6zFQibJzSDaZjWSDjVIwst/dHH1DxtQD + UR4WSLNEKZr2qv0B1u0mkZtgrbt1Fe0SuB3xhNcK2pdCZrsxvLap5k9aOnVDEMietL2NK6I75ahP + 2V3KJXpOy9lv8WByzCSGey45oVNstqCfLrEEp/V8Imj1JoA1venBiG6Tn/lOBzkw5cVXBoeEB0u/ + BZ+7YBPzgT4R5Q7rCko7U0d68rLB1LKXs+Q1PuNIdQdG1BjwiX8hdpD6GT1wXwO0deCgoDbzbkSv + Ww9uIubQT54neajJu9CyyeLbJ1q1HEw49T3zVlrSanetlr7pZ7PfmL6cUsjGBB1/5E2hZHAYOGl9 + NEM/83RZp6UkOdBt8Oh/aogYBQwbcBwMyyfkaulslKYGDqIV+6CpJDDWVZEDbwgu6DLP0/XasQPA + ceJl5t8iI6A0K/jpWpd42tuIhOjVSvB0lXLklxGnf1OkmvKRigoxFUftaKqBL9A3akCMuMgZLdq7 + AhOVU5f9pG7q9CSAKrHPJFwf9Ii59TWAc1/1V1TYMKINxx7O+YmUmQ/oM+96qZbcI/JnPmOl8DFh + mKRvshvMU8Z7QtND+X2LkYM6uxsv2f4ESzdEyD0z4g5WvxfgaJxLZM88SQjfSpBshGzpD6A/VKMA + 3LMQE6Pz1tn4lCq45BuWZn8893FuO/c7v5/9Nn2nQQxZcj8i724XEb5d8xucfQVuFOdV/lzfBvkb + nwdurTfHW+ov/tfvHf7LpphBCFAr34nGKQ6bStF8ALsUITH4sI4YQ2IKqnzFE9M6RhFdeHvYbVW/ + HjaOS13MFHhY4ysWQ7dyOyCEFAI5faD9WyAlfUdps/iDn/nw40cWP3hRo64jhxdvwyXv0HooovHi + AxFuO730t0bMd/R4O/pQo2g35+GnG+vweQIzryK1GlbulJz3Ply51ycxm7vPWMKLHnBQj5CtWQIb + tt72AetbjPBkGcQdYw3fpLXTvZGtt0XE5G4jwdmPIEN/RCX/Gt7msl/l0/Ey6dM1DSTZCoQDfq8Z + AJQdjB5Wpr/HNIXvsrkaLwzJ0fWR6+l3l21ubizVst/6Ek4HfeK3HwyHiw/R7FcZnn00TGgqEWve + Xxm3Rf6B/OZyJvvqXXe9crpKyzxF3v7Duf3ii/lqeC7+gDHZtEX4/cRboqXw3Y343MWQudBBu+om + gKEMXolMm0BG1ry+lD8LCuyFU4F26G5E66msTuDNW5XPn24Km2b/Lb8MHPz0dcJs+oUzzyy8DX7y + IHkVObIqVoNp+f6zc7liuu08d+Y5A35BGsz5D0oqqe1n4UXkptuyY0XC5+AwRAQdCufkTreb8QC3 + a6MQtYg1NrzW1Rq+eHc/+8GVO8z5DD24FvC4a5rsm/TmCG0PMITWid7Vy7yJG3sgaqfkEeGTFkP7 + +FZ9ytMqm+SrkcCiNST/nT8sd72WKYUPPunRsn89qm5jSMt+A2rSL/vuErv/uT/O9aSVbIN4CX5G + 7YGFNFbK/iZOyY8PNxTUdpMjaWvw50TBX3///a/lnQXN5/6o54MBw2Mafv/vqMDv9J7+Xq+F30T4 + ebcB7tP88eufP4cQfrXdp2mHfw+f6vHuf/3ztyj/OW/wa/gMaf1/H/w1/95//vovAAAA//8DAJhN + TqA3QQAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 7b6d341b6aff7597-DME + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 12 Apr 2023 17:31:33 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400 + openai-organization: + - own-45h3iv + openai-processing-ms: + - '35' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '3000' + x-ratelimit-remaining-requests: + - '2999' + x-ratelimit-reset-requests: + - 20ms + x-request-id: + - bc8dbbbf8fc5a0f091c40599c9dcf214 + status: + code: 200 + message: OK +- request: + body: '{"input": [[19, 370, 10718, 830, 1484, 762, 3443, 22, 6043, 2437, 68, 22874, + 346, 22, 65, 21, 68, 1927]], "encoding_format": "base64"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '134' + Content-Type: + - application/json + User-Agent: + - User-Agent-DUMMY + X-OpenAI-Client-User-Agent: + - X-OpenAI-Client-User-Agent-DUMMY + authorization: + - authorization-DUMMY + method: POST + uri: https://api.openai.com/v1/engines/text-embedding-ada-002/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SaWw+ySrel779fsbJu6S8iIjVZd5xETlYhCGqn0xEPCIocq4Da2f+9o+/O7u4b + E4GgYs0xnzFm/ce//vrr7zor79fh73/++vtd9MPf/+N77HYZLn//89f//Ndff/3113/8Xv+/K+9V + dr/dik/+u/x3svjc7tPf//wl/veR/3vRP3/9Pe1sk3n+9eR/gpt1R6OnYZYc2K4Tp4cTQKoIBdMW + aYJ4ZOQRZMp4p2LrEnOMpJMk3zX8JNpnr5WcCVmteFrskiATpXJ+z9cLxJ9TQYKjbaPp9f4c0MYe + D4RAW2RjV9p3OB6kPduBvPU7fe2KShQ3G+Y6m2s5TXyRwrWpLIL38Rj3stcKwHwlZBajGlqJxjFH + F6hz5s+C5a/qXQZA9HtCoeBaPMVqrgB1PI6nh9h3VMSCArvVeSQb43zP5gHyCpQUVXTtaUU8or4X + UbEPVmTHKtSNQRrZcNCjPRUFpY/H0Q9nNQi3PZ1yOSxp+XIFpCZPhhfj9YKG1Z60kMTtnW3N0c9m + Gp5s8E/Cja6dDZSTSo8yclQF0f7RXMvlaViGEDUQ0DKJ7Xh62hmA0dEZz4o1ZH2eNNJ65pqM17Yk + dMOYb2WY6VbD79HNu7mvJ0e9XtUlCRR52bW34R2gIn8Ssj08HTS3iduDaxYZ0XRWmTMpzB6abCcR + rN7zbpDusvN7HmyjXA1zqXqRDMZYt+y0Jm+/T6LljLpCd5nn6p7P9002g6p9ZLbrgHT0stdS9HwO + DSPJKUO0MpMLXA/tjhD72GaF7b5tOLT3I55psOZ/zlfO8c6804l142NhSurSWGbEH1gVzytvF8Jm + Ec1Mk+rKZIl8fiFDu4nMTJwgE/2FaINtlgZdl0szliCmNjK0h4iXRCrLSbrLHgS9EdFZTz/ZrH6S + HI43d82MjcYzesF7ce2ruCduVJ5M6gGTUe1XLkYofXDuYLuGPve3eFU0YczrKr+gyzMrsHI8+N08 + 9QBoF/tPChsx6/hOCmU19zcBbY4q4xOrswu8noPBtNQj8bDIYgmdk8OV+CHKuvFXTyBqGrmB/DFn + 3VBmtDhmHpWPwTWjGF8juBwcg3hHd19OcxWGYIqXLcPpUJujdq81VETVFi+Tjd7N7/lwgXNqWGzj + Ggdz/Iz+C1pvsMgpll/ZZG1nA3zkisT6JIus99XdC6LbUcXsGD9LvlWOznoeshNG4q7w6d0y8WJt + Pc+M7LTcHGPPSdHvfkYCbckvunKFffgJybaeZH8iDbZhcZovzBL3KZ9n270itLWuTCM974b3SYuA + S3NC5+vFzubHca7BSt93ogfRLmOR6gcAzHZoJaRxObXGolYmtjCYxhZhPLoJpYoOKMBKtNmh4f66 + GMCW9pGQoO3N6VXfRJR8dhuSHJRtx2cICxXlgk+C4tbH4wlbDpgwMLLxPhvE0xle0D+LIzGytMgm + PehqhJG0JLuwAERndSmjDdnCV3+gZOlo24iJJ0Y2+OrzGe+NETmHfom7EGUlj4w6gut1saQg13o5 + 2mgM0XVVBnjC8jKbHG2vQPWKNswXd4bJNfdxB5GFnOz8+4lPyqEplO0xelA5cuSOnudUhKuMRTy3 + 8pOzSh8N9IheFC8PbOimaCfkCtqwgGxO+NENpIMKKfqC0QUrQz4Pa8VCvXhOieO+CzQOVZ3C4f4c + icsDB9GjtwDQN3ZKc69mqN3zoFWWrugR8712S5HMkoDCej4TLWNPk0uzq8BnGRvMPo6J322mpgdG + J8K8TsFcMtIigPODAvPRakBN2QUHuO0viOHKGrrOfQoW+umZihrX54UbOvC88onpVy9H06nDI5Ja + /0FMWUH+OGnpCMi+iHh9Lp6I3i+NBwPdWxjdTzl6JeE2hF3rhcy8BKHPLTEMlF30WGMl6FKTZXA7 + QXRIBzrRpWBOjXNoYZ8qxz963h6b5gSVSynbnAU/G+/d4QJw8fbMn85zRndKPyvLqdPZRn5es+kh + u1dlNJIV09ZwN3lIxxmczelGiK1L2WfYH0cgiyrCY2fbHTOC6aBotnBif+qXiJoNxV55EINNXck2 + lGO1cntK3MZ6ZoN01WuwuRGyHRzP5liuSgpbxb8z55Lm/hwndgWCWmb4ldA3H9WDOaNzkl6Jb7of + NBsx79FAYwur9mXpN+mjECHclRcSLO83NAZx+UJpmAWEXGZeDviCWtTnEiPbZhv40yV6U1hfpJw5 + t1Y2e3paiXB/FDIxiFjwqXcdDRg3H1RePc7+UI5NiiK14MQFAcrJ6lYajD15k2AS94gnj40D90cu + syT/6P4Uv653gGnhEH0kqTk+rFoB88NlZr+eczewQb7AruYaXTtVW/KPpApI2PTvP/2JJ8ixlWAE + j/iRpGfzI0pq5KvygXknI+vG2/EFkL8ERJy15X/1dB4R7fSZ3q6XMR6qqIyQ1Stv4uzVezmWq46i + 03uFsWI5WswXW8WGpeFzYlsvl4/W4XxC+wf6EN++PfjEOldGbk+flMtKZk7mafuC8HN2id6Plj9P + rLyCNRcb5l67B5/X4qUFa/KWlI9BF//5vQc93BPbTHzOC/fkgMCqmDjx0H75pJYhSrSE+Nc68ecZ + kROonYvpafSf2ZjNYg5qe6uofNd6f6iIOEL9cEZmbqqef57nNwWtWR6JH3XbbKzO7xq05ONQ8R1u + 4mnbF6d1bHox26pHlM2zqsqwSc4W5oHllLw/dxFshI/HnNJinDVZdfnpGSHX46Yb5PZqwepcDyRY + nc2Ofs4oBWsSb8SNSW5Od+doQxBuekYkPej6hjy01V65C1hMSh+N7+vVU/J3XZLH9fjuGvRu7/DJ + zjHTpFtg8vxhUGhZi4gRyhWnjX6J5MmeAYvhgfrTW0AzquTiwAx3rfp0vdId1Y6WFzy3xY4P4uqM + gSdoxRxz+/H76VwGahg9IuJ6mhHzWeI5JMvVEouLUIsnRzsr4Fc7FcvHoxTTo7cSgN7GiKIJn7KB + DeMJxn69w3zU9/EsIAvWp49skwylDzTdRFahtcoeOA33ZsYVYR+AzvGB4G1qlMNhmZwA5eDjhR7b + 5SS/FhbUw5QTosdVN5ufzls/drVECLiHeO7zoUZlqN3oivqM/3gXfZ8PMdz1zZzE1T4Aoz9dmC4U + x3LqUmahhXGLmdtYejwWpRjB52lwLPSFG/NNcw+QfZZcttnWYHK7YhTppnmncPI8NOan0wE8mX+w + tL41aHKXRxFEi93Yt19mY60HFfoshZYFN0y7+stn6CFJ6q9/+eMyoDbkjyRjnrE2sqk7JxVMUugx + Y/LNjs/izgClW92IWy7L+MdTaGVRCcOXd8d90VygCOBDNq/94A+nrTvD/Do5xKRznPXPiB8gWS6X + zHd2Ph+foSXBdvIws0j2Mns3oT3gapcyPZ4eZv/jqV3sPukjskrU3tGiRsvjtsUQ3Ka4/n7fP3w4 + n6u0G9PeC2C3sLZMa1Ff1le715Dz+Yi4Yp5jMvHpCDBbu4h50wq6dzcXBngJ0fAo3QKfB5cbIJo3 + ES5eKpj8Ug8Aex1yvD4ft2j5tGMBVdfPwPCbrsvxhfUe1sxmxOuYyHvYTId1ts7wl9dEn0ssO6Ht + eK2YneRPczYHx4NEzQNmfHlpFYcK/f1f9P0wmDm9rJOI1urwYN5sSB0/U09Q4HxvqNQgLxtFNNjo + gQ93dk1RwOco117AQtFhvmNMnG+aFEOGrvaX57yOL7MO5PYdZcyNStkf98XzAmVo3BgBV4ynETwR + Tv7i8dW7lz87fL5CVShLrJxo2bF0xBb6fj5WVnMR849I7pBgQyDm4XOPOZJ2PSjpusJnc9DKQdlz + A+X+LOJZT7fZfLFxCtQRM7Ybl+tumvgqhc/u6rOvHqPh7u8q1J9XC2aL58ofQ7wGWF88mWiLvuFz + Y2kjtHOJyHbjldkf/XbUXUxleb33+0PmRmtTPR+ZdpcX/mCzUIGt4t7x+l5tuqm9WhZERbykkogD + v86GYwG9mKXE26qVP405UX78zsyYihmXz7kEu4W9ZbawepZDWD0q9O33xEi3r5L7WNHgy2NE25j3 + 7+8UFLjtpJi5sWWb01nvR/C0vfvV74DzPoURsrez+N5P75ZEdCwYZSf4w+OzgAKAXeuEJHEN0ezX + m50AaUpjsmuPN84eWJ+BFkbBNpsdNXs9XOQoqV8JISh7ZXNZyfcfrzJLd4qSPsjHQysRW3gxfg5o + Ss2PDLESP9k23oZ8/Lgn+8frePYuvjkMGhToebudidcLEWpqkFtUoHnHLFeaY5pYk6Zqjc9YkI4R + +vEv3BWLMF17x9/3cvTz15jv73k8Gp/kDldHF5idJFt/fDbnEG2NQ0308RKaleJ4DvKEs81+PM5P + l6xA1baR8Ljfv8v5tVqfUB6aFhVZWaHut954u6Rsc3E3Ma90WUOb1OqZvZKPaJKubou2Rlqz7SJ6 + mrOgtRp89Yg48npvDsqhySGERGO4u7/j6bpKbXh7yKFH+b7vpnK/kJUVfb3w8zWk3az1hxGMm23Q + yhX0TvKAKX+etyUoQcybjJ5QKJlrKpnexKf9Sc7h2Vw0rPp3Gc028WzopSsmyXJnxfQ6Zj0qfftN + tjdL6GpVWYWQ5Eig66dy60YB0xF25LX+479XK+16AtHvA/Jg5YiGsyDg//KbX7/cH1fUU5yDviMb + yuqyIzooSsF8Af/u9+2XAVLHmRL/s287JkiTrX77D4u+ejOZolGoysgT9vVHcX/Vt9qv37PN4cXL + Vr6JFHyrx8ybd2E3O5MpQFVWtz/8MKj0IUMMhzPTrrXvj2zVhHDcrZ44nRfYHHetnv94gsXLrDGn + bnkqIAuJg+VXo6Mx9rRUTT7KQGUsauXqExpXkISM0mbdvsqZn0dFdaJGpcpN1bNZ66+zcgseJhVd + ac4Gz90e4MvX7PZcFH7/9D6SMomKgucvfzdffoV6Tbc/Hux48/RPyk+vdUXuOvZMVwGcoujDNnFO + Sr5MsYAw8nfMvElXf/iuf+X5ZA1GdYK7urg/QpTka4Ftljsr49uzU6PDembMlcMeTWcrjqDJfYpX + Ozn2581SrZAxti0V1HteDt0yLBA6HAEv5SfE4/4WXODr33C+OUfm6PVBi/ThJTLH8jufTr0oQITZ + Di8Pr7ird6FqIXRIgNlRessYIlEA9WsM8ZidB3OezQtAun5HpLty4vN970VIejqEXEp943f0vQ3Q + WtVXxN0Gc0fN9twC754p8evc6aa9stWUjboumPP1p/2wy2TlyztMfyyqjpMFThHzg5qFaFUivvDq + F0zPNMaoHa8llcszhTCWCuIfWfdbDxFMme5jUdKLsjdVfoHxGFPmA9qjIQieGAKlM8im8VI0Ho0a + wyN/SmR7vsxlGxjjQc3fbUml+8tC0zcPQ/IozFgqb7Sb5dOxR3wrMLLT0ymbNsbNgLO3vVEtTZrs + Dy8tBqPBE0t7c/7qDdhcC//kWXx3ZHfZi5cj22VpH1NpHUTIV5UDXX/1Wzw/eQWBIQZED26fbLpE + Qw8zN+QfT3Ts53fjk+0xHZnvsjdaXwB3TnXMlUfPp7k6RbATLg9Glt2p/Pr/HHmLOSX2a1WV8zY3 + NfRaHZ7MVR9tydzXq/rVG+6/+ak0zmeA7mXrZNMkI5rrw/2wdg3Xp6pqPs2Kn2UFvryPc4nn5vjl + caQJHOMxMD6IVo5y+uUdX3+to/Fb36qOSsDTo5m7SbrqLXzzDyqKe4nP8VH909+pXKwTzsX75Ypm + TnMWFDzPaCg/25//oc3HzWK+r94hCOc4YM4xgLj3mtT71T8xN1OFeHAL7uuf37Uste7YJ5IFiCuq + UEVVWn9sjkW+9lrxwR7QGnETNesrFHlJsOC8tW405fvlT37aWJNVrkohteHnz7+8FPPxsK7R7OVn + ZvDsWPJ53WpovZGbbz0GiHvIKxDeL7b/VZ+TNEggp+eMbfn47CYjj22wTnXKtoeU8m++Zf30mOld + r/kr371p6Os3qHI6kU7c1I4EVIIzibXCNbnTyzYo01Vl5iPbIeXm5hgJws4kW+eOMhrpDwcY5YTm + iyfhc6VVMjIFa8Hs4QJZv78kMkRxt8GKQZ1s+bjiAg5a1WDpOCbmK20yir7rkxiT1pR8sTFf4B7n + lhjSRc8k1TzlsO8kzPwJn2J+uhkazK+Lw7y5tH7984LOr1YkuFWP2c+vImUIBhJrQWK2g5jYa9sK + Whrqr96f41tkKb/1Flwudz4V10lWXUPkxBUrDU30GF2hK+jETDeU+dyRvods/zqxnZ7u45/e/Hjo + W68v9OW9A7osu4mKX54fl+7iBfXLqr/+eYHaDd9a0Fw/Nh2/ejIBFxXYpq8Vxdu06Ob0VF9h8Pob + Mw9qlA1LQw3Qq7i0zBn9ZzxFzfoOPz33Un4rRybOswIbfCF6oql+LQwnDCszqYj7zWvHj91UIJxB + JI6Vvk2J3dsCEvAAAzZK/+cP0U8Pd+1RRY3+Hi3oPBz9WR+jYw0zkm8nj04+bjJ+sWoALZe2xH71 + QTmUgyLCc5U4xOvrO5/wrXXQTOeaTlXkZFxH8gFMO3RJ8jCY3/tGUaP9+7mlH5q6fBWpPkbffIRt + 1YHE3Mr8EVQ9zomzarlJ2XNvKIbj+H/82gzpoYJjY+dUsljJp3ZTeVAV8vJXX+b4zX/RcGrejHz9 + klidhxZANLQvbxrmyuC9BDefZyQw8rKjh67PkTvVZ6YzTTbnItY18C2KmdYnr24I+IFCjkKDHK6B + 3o3B2amQkB0stmNT0vFIufbwq0diH71sHLz3CTDkN+bc/cofmrD2fvrMzMX6Vc40DC31GsP7T37x + kc+5CN//l2xNtMlWt6qQkH26rxnR1Kjjxiil8E2oiVvTXbZM9qcTElWC6LQUt/zpfcZZPXub2+95 + IPrlYSD3QSReKqxKpl4ED8TLAZh3PHRl72Ym/eVVVDyfph8PW0jagkO0/DaZk2zmFWTeu2a2Ww9l + b8r3k+JpMDFdu5w5u6+6Gf34RNSdohtd8eSsFzrv8Op8iTr2nVf99JHsrm8vm796sP7mh4x850eT + skcGOFB2bIvxx5wOy+SCrFObMiOUbTRuKZmV73yLmOXmzCeRbegfPzPt+7c5K57cgtasjmzT5C3v + Uf8S4avvJIj3GpIqrVLgPbMF5m544uxov2xo+drH4out4hbHU6R+81z666cimIaofvPXn350U7Ob + 6l/9/nifz2I7Sb9515/52SwsmfDjRbzetTTmYdKnaJjLA5Z220VJD3zUVFEMOmJcL1U8ct3zFON4 + LSkUdZUNN/HzAnk7dwTLpECf3zzvp2e7If2U87ZYFxDhYcfsuVFMGlmbFvpcZGRDFaNc4S6VZYzc + HTvzoObteb6LMD/uIbHu16Sc5i2VgA3ZgjmnTMnoV/+U1Und0NWhLzjtvC398QDNZMMxp1UseEiS + bBuzZ1XG09K5BKhcKz7btcURzQc+GnAtPgldPk6m3zvWe4TbTozZzW9mzhOkWShfKzPxVlcad6Vw + t1BmhNq3n+Vl880/f/xG34cX76br4lkBNtMjVmSpihne7mXI/H1IcHF5llPWKR5syAaIr+7fiHny + VYPyJRyI/+OtRbATQMxriwr9UMZT397zP3n97mNu+Hd+OQLfxlusGsj0V+sPCkG+XTwKC1hnbeeR + HjXPQsAIrXacH16tg6p7+ybW/cPK2Tu1Mjiby4356n7DmV19qPJm/pGiuivKPhsexZ/53c8P1GT5 + viu//OOnN8PZyiLYj7NN7LspIZYGug1Pl3h4FmbVp8ELS/CdH5GbPTYmjXLnBV+eIjsDlf70zedU + 93SNMEwLnw894wUMHr0RfxZeJg3lplV+eYxyU5/Z6J4SDGonSezrr8rBR3Xwh4cJPOpsesMmVL/8 + QvAjW/HGnA4auPNBx4v9uuHj0cgxlK4+ED2Xx5K9u4sBb3FLiLs7XLJhZyU9HHHpMt/mMeILI7Hh + efd1OnpRxvnp5hnA7c+DKts5MMcm2uQQGN6LWAndIPZMFxjNPtbZ5lav4jlNJgEJAjGZJV5DJNGp + ekGTu5ToS/GDeguLF1i6kvfrx+g7LxYU+zPtCfEnw1zx3b6A7/z856f48puf/vIEds/bhI/Ch2MY + nVZggVP72ZyXaQV//3YF/Oe//vrrf/12GFT17f7+bgwY7tPw7//eKvDvy+3yb1GU/s2kPzsRaH/J + 73//81+bEP5uurpqhv891K/7p//7n7+W8Ge7wd9DPVze/+/xf30/7T//9X8AAAD//wMA/dfjMeMg + AAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 7b6d3423ed247597-DME + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 12 Apr 2023 17:31:34 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400 + openai-organization: + - own-45h3iv + openai-processing-ms: + - '24' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '3000' + x-ratelimit-remaining-requests: + - '2999' + x-ratelimit-reset-requests: + - 20ms + x-request-id: + - 2edcb14cac190a015b9514ed617d1cf1 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts_with_metadatas.yaml b/tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts_with_metadatas.yaml new file mode 100644 index 00000000000..8a6941c6110 --- /dev/null +++ b/tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts_with_metadatas.yaml @@ -0,0 +1,489 @@ +interactions: +- request: + body: '{"input": [[831, 677, 31172, 272, 762, 14087, 68, 17, 64, 25350, 1774, + 1897, 51542, 9081, 19272, 1135, 65, 1774, 67, 6069, 712, 2689], [2059, 7341, + 527, 264, 1912, 315, 658, 10753, 677, 81, 3581, 7795, 32971, 555, 264, 7558, + 321, 351, 61798, 30535, 11, 4330, 311, 8254, 342, 484, 1776, 1220, 389, 279, + 11314, 315, 279, 2010, 11, 323, 281, 1279, 278, 66079, 430, 527, 539, 75754, + 311, 279, 2010, 13, 18766, 61535, 527, 21771, 2949, 279, 1206, 1037, 24082, + 613, 318, 269, 4055, 320, 269, 24082, 613, 3893, 8, 323, 527, 279, 13219, 1912, + 311, 279, 426, 4428, 42877, 320, 66243, 323, 24890, 570, 4427, 8336, 13334, + 279, 4751, 330, 939, 847, 1, 439, 459, 42887, 5699, 2737, 69918, 3697, 315, + 921, 2159, 14172, 339, 9891, 320, 11707, 321, 351, 61798, 7795, 8, 449, 264, + 44892, 12970, 79612, 11, 1778, 439, 6409, 65, 86815, 82, 323, 53265, 582, 276, + 17323, 13, 61536, 12970, 523, 2159, 14172, 27520, 598, 1778, 439, 2493, 5670, + 301, 1815, 323, 25227, 3205, 355, 1176, 9922, 304, 279, 60434, 1122, 26572, + 320, 19391, 12, 19192, 11583, 705, 3582, 1063, 31376, 1534, 523, 2159, 14172, + 339, 8503, 12970, 29505, 527, 439, 2362, 439, 279, 36931, 31137, 869, 12734, + 320, 21209, 12, 14870, 11583, 570, 578, 24417, 6617, 61535, 320, 9697, 613, + 5493, 8, 527, 3967, 505, 279, 23591, 84474, 11, 922, 220, 1049, 11583, 13, 220, + 71923, 2134, 304, 1404, 505, 279, 2678, 50561, 74265, 939, 847, 320, 36, 14046, + 2985, 46109, 281, 5515, 72, 705, 264, 5655, 9581, 9606, 430, 374, 1193, 220, + 1114, 2960, 86366, 417, 320, 21, 13, 22, 304, 8, 304, 3160, 11, 311, 279, 51119, + 44892, 320, 73262, 2910, 77152, 3666, 355, 705, 279, 7928, 7795, 304, 279, 1917, + 11, 902, 25501, 13489, 220, 717, 37356, 320, 1272, 10702, 8, 304, 3160, 13, + 2435, 527, 1766, 304, 682, 52840, 323, 527, 4279, 311, 43957, 709, 311, 220, + 17, 11, 931, 37356, 320, 21, 11, 5067, 10702, 570, 2435, 8965, 656, 539, 3974, + 304, 80744, 11, 8051, 1070, 527, 264, 2478, 3967, 20157, 11, 1778, 439, 279, + 17231, 44892, 323, 279, 15140, 44892, 11, 902, 649, 387, 1766, 304, 2225, 67329, + 977, 323, 80744, 8032, 18, 60, 71923, 617, 264, 18702, 315, 2761, 14991, 294, + 4351, 645, 430, 36236, 872, 6930, 505, 5674, 323, 79383, 304, 5369, 311, 18899, + 872, 15962, 30295, 13, 2435, 617, 12387, 7437, 315, 8454, 481, 18311, 13, 220, + 26778, 9606, 527, 72627, 56217, 11, 902, 527, 44304, 430, 527, 520, 279, 1948, + 315, 872, 3691, 8957, 13, 8593, 10507, 2997, 279, 52835, 44892, 11, 6437, 44892, + 11, 2294, 4251, 44892, 11, 296, 29886, 44892, 11, 270, 86524, 44892, 11, 323, + 24354, 2025, 44892, 13, 220, 71923, 527, 10791, 555, 12966, 369, 44892, 13339, + 477, 44892, 1913, 19724, 13, 9176, 44892, 22673, 527, 21699, 555, 3823, 7640, + 13, 8876, 220, 4468, 15, 11, 44892, 22673, 617, 1027, 11293, 555, 220, 6028, + 13689, 10213, 505, 927, 69, 11218, 13]], "encoding_format": "base64"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2795' + Content-Type: + - application/json + User-Agent: + - User-Agent-DUMMY + X-OpenAI-Client-User-Agent: + - X-OpenAI-Client-User-Agent-DUMMY + authorization: + - authorization-DUMMY + method: POST + uri: https://api.openai.com/v1/engines/text-embedding-ada-002/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SZSdOCSrel5/dXnDhTK0JEITffjF7aTBQbrBEoKtjQZgJ5//wNfCuqoiZGiIkQ + sPfaz1r53//1zz//VlmZX/t///PPv++i6//9X/OxW9qn//7nn//9X//8888///37/P9W5p8sv92K + 7+O3/Pdj8b3l47//+Uf4v0f+36L//PPvkkoLFhBjh6bHdqEiVfh8WHiw0nY4y6cHvD12ZrgLv+1o + qbcPLA9LlZmW9TEm1Wo38u1jnglBiW/wTXM0kc16DbN3FXBuHlcCajV+xEvhMqFms3MTid6KlqgJ + WvujmJk6VJ8rJWohfRH33fMVreX9goX4Wvl12YYNdM8DIgZpHm3f9eNH0dK7zXz7rKCJXI8YIulB + CN55bTZwfb2BWEkLOmz9Tdnp71xE2UG3saS+DZ+P7nIhX7tsh0V0H42e6rUNlyHVmfd8WMbae51M + yA6qTYwHC/whOTsYlKGKiUNXr3LS36qtTHenoEMhbRHnOUtRVaQRhfTe8O55v+whvnwVKnRvnfcx + SxcwoCLHaF8fjMF5yA2wN8EUHtXH4NUCIhQVxxUWJ801eu+rbZRJLxLivbylP7nZIYL1s9+TcPPJ + 0ajbrwGiMw1ZOClBRqsbTWCVL87E1gwznkrQKhT2w4aQc4iyodQqE3pLdCkP/Hs5lILTgb1RziRo + dknW3VPUQebYLxZ+WtHg6S3HqPBQwTRFlPzezO0JLUX7wrTELX2+1x0K6RA+mXqXWczXDj5IZXNz + 6DiEHzQ6uvNCEWxMcr8+pLg+2ooOsf964GVUauUYP1GD5vtltnbZoOHQP/cKFe2EyvmzK7krfAG6 + y21NtHudGkPo7V5ocRQ9RqZo3Y4x2y9AzTuLGYetFovkTga0n04+8ZpFOK+/fADtmwex0ouJhrtX + CyhYNDpzAh758/UrhOtSImH0LuM+ekcRuId4gakmXcshDAMHLvd9hYVMpWVtZgcMPA1kYqUvZnTP + 6UjhNd50jCTHjZm+yAYYKZvo6quRUnhfPBup95Qy/fVIWj5KHxXUnFpEe409Gqf8USm2NRjkHkmi + MeEaCnm3fXZ4Ur1Fy+o8ekB3ua+JZrOLT5XpY8KpYi4WNsHHGI5vZsvr4JmS7bkUUPP8phh9zRCY + +4RXyV0oHGVdZIj4zpGioVlpFdJltSOhv7cyqtyfFQjPQGBm7o1xt+ObK8zPh7j7dsUfmlLIy94S + XJaKg1x2eRWraKuVJh3HSzLXj9rBV/c0ot/1IeNmFHXoNEYiC9PLnXPzs+1Q02Kf4S7Z8d7udzY4 + JPqS+TuayGmw0Y1nPm7hJsa0cGsVdraTEq24VXF9dL8vkJP9SB+n6zce1S5yNkK8UZm1s7/+oIYg + Qm8/JmLVx6MxKcU1QNNYY5Kct8dsUo/KBOfS3hBd9fJybPAtQYV0ZEy9wKIdfTfpoC5CRqyFXPuD + Ni0clJ5rgWy96GxMyJswqMPDYZZ2NxB9yKdKPo17kfzqbZJcgkHkCmXhblwZ/OnvKahXVcIfdLD5 + YF+PJuI4TsI1whni2NdzYG9p/adHr91n5ynhkQQ4VV2jHCRUm6iMCo+4BnuXXVYqOWxVMaOv5jvy + 7oMiQZFaIpLAetdoaqajCO2tE4m52yC/62A3oW8++SxwXkX2PiejDNfoPuCpPdjtSDYHCtIN6Vhc + CLk/nrHlwP4mAJufnz8k1/cJClARcbzV3eB20+9RQl1C8FvSjTF3Z4U6nBKG64sUU4t5E7qlzgpf + 4lPF+TIzHVj7jkP09mCXLOCPRDHTPWLY1yJ/VEdljz4ivhNHgbs/rqNzhNQxGNkJ3XdGLya7K+LB + xibpV/Cy0TGRA3stmIi9eSSIP/2UwsJ77zDa5zUfo0oXYOHvZBZwrLViez5j2Gz0AjfiILe99MYd + LJh9Z77oRjxHya4ALEwT8Y2+5GNAU/VX78zPdjni6Xk0od5+tlRoJjBow9oIbfaezgyjFFoe5raM + 7lTeE+sjWvG4lqsAsj7KSFCjErEVPwlIvSeUOK+ubjuupBRlzhgyf8dP6HOG6QBaqXNi3eiOj1B3 + e9C3hfZX7121n2QkyDeXbTUulv32aOkQnO8VM/ZuFA+/eriYxoA33uERT+39ksJRxDnxkhKhaWmW + OmzyKWBh2d3aia/cAUb3iYivGpu2as/JFX76XGnntc/MKKLwBNkiRHl+MrZATEYRyCazeGwhTuXI + hNvy1JGtuQVjNCQrgUsmc7z2bSHu36wMoLnva4J3x2PJv0HWyHZvrMhWeBV+J66NB+zWcGUJ9caW + 9vXVht1+ehPDCB/GEJw3GyReTjZzg3PUDveUU/Qc25AEjnzi3H6rDgrOt4oYgb8s2e2uYRjORoTF + qnz6nf+OAligrvnVd9l5FqiwMlTx7/kOxfluQveiA4XP+PTHo62oCJqzPs/PBacfYycg97Bb0ME7 + ovhvHqenqCDZamGXU3J3HWkoYCLXVfE1xsYZImC4MWmbjdtytSxvtmy0jkWIgJ2WD7vJAcX4JkyX + 5CIeP/fLCczBvVL5XgjG0DRXGe724UPcX7/oGk6gvVQJsfA+4cPY3oS/+9dG42V0V2chgCmEMlEL + fOCj70YUXJQntLkXB386puUH3P2mw5OxPPjjTy8Fp92SmYc4u4ttgIzWs+hAxXs53ckmkJEU1syu + pFU7rkS9UrJCK4m2F99GJ3+2L8B6tGWRuuYZE/fchMU5vOL16Wz4dXTeALQ3NST3RdGjeT56QNJ6 + T8zv/sW57EoneMLGYnoRcGNMkNMgNuRbuiKO6E/i0rzCQ0sXZFsIZdvX9UEHy08OjIQv1xivjdmh + hffd4UGBl9FMe6SDu5c7WmbfJBsmbzIhC1Yt822tzrh0jirlKzQdcdV3+bv/D7xeDmYmTkpUzbyq + HOLYwLAPhbJ/szZA0bkLqWTIDPWZZaS/58m0i9eUdXq87eFcmhtGvPST0ce6MyHZfmxav+1vPI6w + EyGReUrCjaX6wqF/RrCqgp4Fi5WJ1ucTNuGlnwIsu1ctHtw4OwBfUOdPb7iOnoFSeIHE1MJo2ik2 + tBc67vQvpmbxKNmPD/xBvjO8Uq8lF8tggbY66fGUbcJyVJlHUf0QbyQsT64x/XjA3izPuBoPQkxR + cilAvBzsub9Ug3+DuEL5ZdcyVVb8cuV9XVnuAtegAk4MxH1jOkEVStqsrxs0nbwdhXVxQZTzx4gG + objakC/uBdOD64EPm8v+BJ/jyLBw0MRs2C5aEbC+39I+Qms0frXKA9o4IgvSg9ryZNXsQfuOMtGH + r9mOr/Caoy7hGHPindHU11cTZQfK6Ggzyei+/Vgob2wFBGt0nw2LmyWAZ6oWCZekKMcff878R0W2 + CVFXRepBOTD8ItZ0c0pecPsDm/t1RezzUojbz1lNFElbPilfTpXB4u7roMJ9RMTrt1Xchu0ngIPX + nzDsdS8eqvfyAGemXjBvkkVM7d3lA5fRbJlPDltjILuawuk6fpmBl307928q7eshZ8ZycoxGf6um + 4pD9l7bN2cjWRCltKFg6/Hi15PZ36qA+BHtmOsThAnO3NjzaMzALDXY2vOoY4LOd9swD2pbDeBML + sLfrBVPHu9EKngU6unuZSZfvW1EyRH/+IcTsx1/jOT/rSN8+NOJ3bTXzhpfDpuM18/JTkdEXbQ5I + +lwZXmkNNsRZj6U6OGOCF3GaDTvp9kHFaWUyQ8p1tLL7iw0Pc+IzT+jxWolBRZeafIlGuiKrDf6I + AFfe9/d//ng0vavkZ/Q7+zXH+PCVNimG3ayx7Hxefqf1FxWc92ok9ie10PQStjnYvbYiwdK/ZJP+ + nRpIB/KkY6CvDXrZPUVke6XHwptuoMnUt3tUSGdGpWODs8li+gCJs2dki+irHEKQA1icCk4M62m1 + Y4HOD/TINQeLm0fC/67vus2JedQhfHSFZyB/pM8BIylXeI+jSoAivhEWOCwo38M5KdBn1++I6+FV + Ro9nSQdTY2T2C0YmHm+5B1lhlCw4nAiaVCfP4XycTCzH0Y3zVT9StHDlivjf8NHyp4JOcCoXOpWf + LYv5XboN0O/zA90wlfLhvG7hdz5dX9uq5RkRN1CVvMAbVziXQ6k9TGWuH3LbRE02bSUrh+06e83+ + LotHlekUdsLry+KZT7tiNwLctK9LxautttTvdx7MfpXpxVmJ+2l82xAgEJnjHdRMXJy/OQIzL9g2 + TCXOe6/RIUTbDvNGrzmvaJOCcPjUuNuHQlsXm24hJ46SsODgW+34OOUBlL1zYE6sTG2nvFUd0WZ9 + pzP/8qfRfmzI/T2mx4e+jodn4YvSWmk4HrbtM5uY32zQcRJq5qD92++lvbdHU719sl8eMImdo0Ma + divMe62I++6RRrKjtjJtZ75huRbpykj7aT7fMrj3bB+/PIBZh8jlw8zHwMb1Ba+w4vMxGIMXtFNg + UXn2N/3XXQqIKZPMLISin/8qoHup25/fb4fr6eTIl+muYNDHrB2qyDlsJgM1Pz/ddj+/KakvQpej + XBr0UOkbtDgKHrMO2cFYdY99BHEStMS8Lbe+0I+9DnN/EBufCqM76ckAu2h7oZN7g6zXXjdAk3BI + 8ftMdGP4zWe/Ok3McMSpHbXd5QFB0T0x9JbdrnJwP7/3xTCqH9lA7poO/m0r4UVaxfHr+P6aAOL+ + RRe746qkP38kGWCSo9ZQow4/dxEqTXmwMDP3SMjfVQ7fzl7SZbzBGc+1RIVNZ6lsq90Ko575B245 + AvwcjZfPywKo/ONx4xjHxtjxvIPseZKZhRUfjdMrHGBrvQvczXnH7MdSOJrplZHJ6mPeonMFKztt + SXDLjXhakTyF55u1zOTMioXbSj7AkpIjXv7WXyergeMVHBatuYN+8wE+ouQwff9mvOvIKZJbzexY + +lpNPtXT+CQ7GXZJWD55zFDn2PBdHY/Ez2CHJufjqn/5h+5uv7yf8qr68S2b86ayA6sVUYisjnlJ + mfHxdnYiGNfxnQqZn6Dh6ogCLI91/Vdv1f32idBVgAXzxNT2pyu2AsRBB7w+07s/bovXCWol/zKv + ddcxneevMkb0RAwj8eLprN8LQJunyoLe+sx6dn8gpbI/lKFylXVnQUohLTY92yKJcjbzpLx43Ka5 + //bt8GSlA69vYRBDqnuffje9Co+mjkk1+2Pebt4m4NU9I8bjpfG1WAaAnsPiRekiluNy5g/wxGIg + zkz91JM3V7Q1g4h5Rh5lkxpFFeTJV6cAmoiG3efioI+8/DBne/wivvkWj5/eM7vM+phuYrSAvqU1 + MZbKlA2GiWzgxfAlmvzsjGHXLiKo1A+mQIwRTZKoU5i+lDCPoAGNwaKU0bvcmXjYqvSnf1eoZXP6 + 452Zfz9gXj57+qqzhz/XUwP8tLaI0Rq7bMirTIfeLiY2z/+seWwXf/3HnDt+xg0oqQBSG4pEUzOO + mLLTcugl40Zp5JaowyyV0czDxHmtY76+TmED5LKoSXhmbzTE2DLhVILOvH6/b6dJfzqgSyeD2TyU + +St7ejK6ZIFHsB/IMdM1OwW9T54sm/0pvdgrE83+mgSz35i2/HFCZn6+MBy5BuqdbU9Bap4hM8+i + j9abb/MAHsg2ldR36bPdPv1ITNXORBeRy3u1SzxwqRDiTeQofueUffDjNRY6js6HuX5kgvuBWHhL + 20nVpQOSuC3/5QnT+NifIAp9ZebFkg+nfdP8+IBYGzszql3tU3gjZ8vu7bhFlSgkgGZeIH46PIyZ + /yLZXn8s5qgnVlKnQh+w9ykmYX1/ZcP1lDsgYi7ifpoKfxpQAujXr5pLd+WUtWKCyt47MJ3DLmbG + ADn60GNG8IKX/p9fnHmf4IUTGdPVqWzQ6dab5w3ik0jlDzTby4cY87wYbxdZRI860vD4nFYZezqV + A/f2cWP2ouZoHJdBCj/9t9NY4tRTbw06bK4y2wosiEfdPR+QHizHmf/CdnUfDyIc82PKvPwZtMKr + swuAm46J7Z/dttOURobfPMXbdZhtOpJHaC29jkQN3brlpN8FCq6cL5v9UDn85qUQyypz98dnxt0u + if78ku33C3+8ClIOB/dg0kW0fPp/ehcrSUF0YykY09JsVTiNyzvR7jlDU+x8KLotKoNs2Tcp+SHw + E9h/rf3s7998+PmlYHlqZ39/9Se+htOf/9V7fvrpawUsgw1Wpkk3aK2zQV5VuMfDnMcN5O6qP/0j + 9kWxYk4Vr5Kf19uRmZm/4dOdDIEyNu2BDnf2Ngbl7ehwvC4cXA7+2x9M4enAKUuPtLC5Hq9ldzwo + 1+rAmQbNN+ZePizkY35Oaf2+Zmi8XfsUEAoC5mhBh3j3rip0bbuKufmkoan2akCXIdFZcrHsbJrz + Onh7/ZkO8Y3yAXfVCdL0rNLPxjPjOsxVTwl3DiVuT/cGt78yhdPHUDDwi89XZ6fS4b1EGEtIY6jP + g/gBy/7RzXm0bLx/+U+bGQ1FTX/mI/s26V9e5YtL6vMEbRMwhCQjftOvEa0SN4Lb8tAxI/tu4mn2 + l7Kj1vL/4f+T5R8gCASVXFcb1DKX6YJ83PUR0feDbAyliCI0zwfmyNcvn1DYDdCshRO5lSfXn5R7 + XaHX5Lb452/5gkcBVKUpEW9vBsZKKq4ezO+PGcOJxYMmb06wzX2MsxQ12fqJ1Ekhl0hl1sQVY/Lv + kijv1YdLjOfbzcbvTlahic8O+9XTxPDxClro7FnstFU29SJP0W8/R3+eVqjNQE6QOH2uxBeFEg3V + 8X0FSdtv8DCuEn/mjwFFMnpSJOU39JcXpAU2md7kdTy8d2QD+eHxIu6WAKfvSPJQUF48Yh66OuNN + 3E6wi6wL8/W13fLdMfSgoHRLLJ5H/uoyZBv47VdYphChoSqDCZHPR6DCYKVzfpRgUKvaJL7kuNlY + VpkNQeoJzExXr2xIx+sE4RGtsPQEsxSxUO/R52jZc/7q8tWKJzZ0Kin+8p1p/Yz3aJ6nzJn9+NQL + qghaKCASLNvCZ2srHmCBVI0k6vOCJjU2AoiD2CWq4N5aPmpVBF5URczafAw0pAI5ofn9sRgv+3J8 + GpcJVu2nYOR9K9pJSH0PoK4uTG/quJxQZpow6yfBZu/xOg+yh1yFOGGGtU4NRibxAN2ipsS69k3G + 9ybaw1d3tL/8bNR2uwd8+mYillnb7Zw/iX/5O4g0QeJ3NV0VZTLeeDPz/bhYZJ0852csXFzecYXZ + XobZ79A1270Mbq9S/bdfQUhEW0TDbT+gX15u3lwXTVWiRcr123jYea05n3rBEWTpPdiY20XvV2dh + TGD2q+THP32C1ArG/vRgM6/xv/03otIDXV4fUvad80uwGdPoclH0vHvvtjK6Gh6wX14z87CqzPxG + tme6NIZjIRQwOk7NrunhUXbDRvGgvekhfZfrjvM7Hxrl5zcRbJ98kt8bEUnGwsSboLK4QHZPCk3+ + UpnppPG8n8QbYLgy5/2mF6JXHGKZL5ORnVKHlOuFqA9IwUJE9v3tyccWRRvYa1KGRU3SyoE+Yweq + 0paIbYZbX5wiNYcTuXJKSzQavZk8K8D3j0yfP16a+V65TV1B3FmfhofOKtjc8xXRXrDxKXM2J5jz + aNok4TceLkF8hTo4Ylo+B972P3/8l5ee36d2HdfC6+ensSymH4OJj30KECUmCXyJGLNfv4JM1Ixo + Q5llPL2dsPStL5xCs9tk/fSMMViJemcGb8F4v7uhUar0GbBQW25bvj0/BOh7C8352DPr7EtToMi3 + dULSIOa0SpGASvNTMUKHRzb+9PkjBneSHjHmnWFyE447FhHvKUrtpPf3AanGLaTKbjwaU5W4e4hk + 6Tnn2XY2JI3ZgHtSvF89oOnMMh2cTJ6YdSu/ZbdcXl/o9fIwPc/5AW26TacEYVGT7VVdtRWerCuS + bsGFzHkiH3NwX2jeT/jljX7f4GP6p9f+63VFA+hShFaCR5g273/ymS+QvU8wcdCNxHQVtAJqVmuO + x1/eL5dHXT77oUC26kR81kQShs32O9HRvrzKX3+jF16c2P8AAAD//5ycW4+DMLal3/tXtPqVaQUI + wea8cQsQLjYJCSHSaAS5AiGEiw22dP77iFRrpJHmaR5LqUoZ1/ba31rbFbR/rgBT+5MAjvCVocmR + bIsP72cHyf7tYzc9V8EI1kEDzxE6Y3v1OnBerGoB6HYgo3ld8mrmKbnD3RM/yUrK1YShr1hCYh9S + Aj6RXi3z/VL91+9WwH//j/+PGwXS//tGgZDUMlmpUsvpYROn4OwkH+qb+0fB9+mrBoIWCHSnGXE/ + euoLwVn+DNTzO8TnsfdrOI+PPVFWkVBQqb764CEePtgs+yZh7XUtQKI5OfbtEw0mcoM+fIX6ERtI + HBMu1UcPfmeWEhqNVjF9PF+E6eNajzAx7sGkRA6DWiB5GDsXm89um+Wwm0dOUYY/PQenSoTIvQRI + 2vC6ZxU628BvLwYBLzZa9IINAiwwOahkV6fnWmHkYJJbTMNKlK3u9NqVMCt9kUbbvEnmbv86arat + vLA9nG6AGp8jguP3GlK3lj9gWm8CUzG+zzf15ouZiPcBqDDppQ5HQ/xO6J14pXqXUEPdW7lLGBAv + HVhv65IIq22TsHp8Opqr6D4Oq7GrWAs9CO2dqZO1tSSQ4boc4NHrZLo9eU8+68eLAL/HlU/miJ36 + 6ZCrTxXQKvp7fx5LY6u6XOnJ63btCgLX6xxY8Fti47ynwfx6wytcb5sSKR+1rfgt2LNNN1NO+GF7 + 6Jf9nSA39YCUK3eVNMVJEOAK2V96O+4IH3gUQzDY9zeNVKUMBiVCDG6zkiIZEDMY9yfFgzd2E7F1 + Obx7rnw3A9hmjoG3pTQn0xhUqVbO1w4/4HYNePgJJxhudx3RNnKXzGf/3UAQUQcbaDT4JD5eDIh7 + O6QX9/oNSG9driDYTTn1b+8WDE318sFjestIkb97i9ePLFTSpLbx+fq5WSy1lONGX5kzWsOd1nOo + HBowHIIICVN9CuSy9Z9QGA4l9ixcA3aq6QHejukdtSTJ+VBuvgqMczfH+nC0AS8vrIVVYTAcNGAq + htwwRLiVpi/dJRsffI6PQQXvV8axZQTYYgThVg0OV5ee36veYuTKBhC9bm9s3soqaJJhp8CvS2Ns + rtxHQa/pwYdTaZtIjUSvH+rpmoFCPLGlPvyexYe2BRtrqKmRaFM1gcqPYcvICZvr95fTKGjuYFTP + LUYtEsC0V4IBovq7RVPI7II1cp8Bs0gHbAbNp+eb8Fn+1X+uKPeKqaUuQqrcOlJ0XlV0056lKzcN + Lbrf9b7FY64yuNqVVxzye8d/5xO0xHcpPshNNZ+MLob9dHao/hpcwBXoP4GVSwUNgWgE4i3vHZjQ + Z4FNRVLAeDLKGDqWWRGoHyTAOwPK4I7kPbXZJUo485kKzFq/kKmRemsytlkHi7T3sXOroDVXWifD + 4HB3sTlkZ+tuDWkNQZpxfGn1V8+yELSKsnX3GN1zs+LG17VhpspPtLlkdT+p8eYOzjsho84xfPHx + mfQdSHa1hM/sOoBx70Y27Cb5jE37IgIaS+8ObjPtRnfF8Qi+b0u/Q9l5vmix2rFkcuXnQbPLzw6B + eCVZ8/40eXA5/zjA2s0i111AwFbfz0i5XNYJa7+KDwX/WmBbx5NF+V1ogLjCL+xveMdZ1H5z+FF6 + TCNx/ATzb798n1yw+RF31Xz7Dhl4Hi4atY5xxImRtjrYqI5CmGZXPXlyU4Fe2JxpeNqOnBiOaUME + VQtpR9MqmNkqIlSfj+inzwWzp90Biqf2Qp29oxbD5V6Y6utLIDW7qABjsH3cITLzGFu717uYNsRv + oRgVOUbfXW9RN78zVdE1Rm1exv1UdoIIt9cW0cIuXN6l1pTCn17avJwqvk2cFLjRM6XIG4SEKuP5 + CjUW3bBlP6JezNl9gHmdn5AGjXc1Kx3w4bAqHOwdd201vll5h9yPVBrc36pFemeNIFzTLzZ2vR/M + og4geGejg8Tb48v/1pvZ1xApbhonM1srNRzuxki+Yzwlw05TBrXxNUB3vAiK9SHH6aa+hGfqxhYF + c3LzGrjTw4aGxjHqeda1JvC6w4rqu/dccJAof/q9IBmoRnO3bzSd1yuK1+2Jc386HMC53RLshnRO + mLGjMowv7UDYB5wC9nF3Ofyc9zJqvDKymJIrOUSb6EDP+UlOyIkfjpBcd4uT69KEnepPDJbzicPW + egVTUU0N3Kq8RtPXncForHWiji+MqBnSwOKHTXZUL/HkUkMebOujzqanSaE54t/z8u5wmDQfagjv + IgkkzBrSBraklqi1a2MwlV9LgC+Wf3EA4KOgaOiOQHjmJl72o5pT+SuD6PV4L/sZF998L+iwBvqd + 7gQ/4fOWZwy2J7DDtqrl1VrphxoaLrnTHVS3AQdWYcLwJqXYFMmYzL3sQ+hLKMKB/N5aLHTHJ1QL + dYWDXZ/287u0n3Dwahv7rDkD7gVTq4VdV+JQP0h8dmShhJ2U19QmhxeYBOGegWGnM6r7RKymc49i + FTnwhu/VfOVjzHcptABzEDDcXUHdPGWb23hXsHcKnkt9TARWuZfitA021lCcoids6jpB0kF7V2N7 + fRGwOiwTAu+b9cxJT/Jfv7DMyOHj2VMJfL19iv2Vdak4EUIZEmdwcOA3JZ8RvXeAVFCn0dc6JOPx + 2Ry1Xeg6pL41p4J1jN9h2+kttYyD2hMbRzn4Cm6NMr8jYHy94R0qq7eNL0F17AnMeArV8/zEqHWT + gojIz2D3WLIKXspgck51Dss6nPDhlCgFfVCQw3K+d9QPWcxnDSAViE0lYRsoDWdCqNv/4S0dw759 + m+MB3OJ+JNzvvGpuw2ACQzPmZMJfHcjP+iyDXdBlBBb3FRhS7/EE68/6vPSrHWct1AVYy8oK2xHZ + VQSe9Tu017uUbuE2KNaI3EI4H49rGvXuC0zj5nD/6QUSNFhZbWpvGPigoCDP1/Dhw2rufaio9Y4W + yqQX6xaeQ42fLWepp6iYL9QNYaIgD5vbYujfP/0+jd0NR+u6qqaF34CluCckAymqhnzqRfhbr5N6 + zOKCNnlwjKcn1f3PN2Crl9ypufak1EHgYfFUanO41y0DB2hEiRiwKYWpr56xP0RbzvktGMBeEj5I + /kYTmMXVzgRKc++Q8JaGitrTMgE9XDQkPz5BRYbGQ5BumUstI6DW9Hh6LTx2JibK1wqtv37tmssN + 0IXHeJAY9l/9uZVIeT0dmxYu+oVaZXzymQ8KgjG4cvR6M72azMvxqP70GG/vWT9dN9QHYVNZiMtf + H7QLf0BM3w3qIrHtmWxDFchFdSWqKpUV/SibKwTrKcT68veW6j1p4PUidng730jxTcKygyt6Nend + veKEy41rqu06LnFYy8QiQS0McCqfE47tk1mxtq902GEdU4skx15e+A3AcHYpTvdxX2tdTqCWdUek + Lvu56F2tHburhtiwD/opizQbWlN6pG40fy1O49sAN1k8I1jEHmBvo5fhQxxP2ICW0HOk6QcQGwUh + 7FeP7sZu4CcXPkSUxp7zyerusHRKDy/6z/nJTAgMqlOFdzLdAXqcfFHt2ndGNDwMnJtr5sFvnGlk + PGxXFqnQw4arD7YRUCY9kaUS2XAOXYX+8aSzphBAWYzo6fYu+2mqLgMId8kOG/1O5UOyJKgr5Hyp + +XjfARfGQ6h9j+WVLvzEf/4KagzfSK0ZXULT4/AEyWHh2do4FgM2Lgc1FCaDFmY89BO5Goc/ftql + h5vVbWfR15b1kKneLv1z4euWNBJpP6rXD6EDHUiVR0fdty1wdtS3LRxUcYND5zbxcfASUc03koCt + UKrA1D3mUN1nw0gdr4wC8toWDFoGNXA0f1bVMIJtqkouQIixTqtY1q8H+Os3D7Eyk7kXM0fdCOGZ + hmXPwKRdIAN5nZ2wd7mci/4lxRAYsbih2xMeihn1k6Kp6LulnukY1WwLe1+7D06AdzEw+SzcpEHF + 1FeoJdhHTjYTEWC/E2aivNoy4Pv028CLJQCiDtGSsB+GI1S1Q4Z4dt4UXJYHQR03ikpxBNRi/PUz + bZ4oEmQ6BUPbzfVGfeoPuifYLOSh0ZG28Ard1WpucZ9tfeiRdYgN91IV/KQ/Ze3xMUds0VOY8Idp + x9pWYl+qJ3bH2ceDsXq8fWdq7YVPMG2NligLH2P/8boE3FyrHvxQx0HrbXHg80+/Fn6gqFnXCd1n + 1zt0LL2iNwv3Flv4HZ5i6UC0Ncv7aX92ZaDvU5/aY+xbzL4pV2C2dYO3+b622gcuiVY9gj11vfbN + WeXPqSZDRcL5EL1B30vGUzXehUOtZ1Nb4zq/Ifi6bibqlfM7maL+NEB6V170Tg6o4N8H0uGxOd/R + RvA5H5uz7cONpF3Ia6nvuQUjg+jtcrRWN6QfAqYcofxoAXUqcv7lAx547NItWbHsEzC4XmfqTm5U + jOz7J6G8yEQYcyzRAEgczDZtHDjcrfHHk0W7oVoJ/cd4pBa9ngFz1fVVnUZYos395QK+dhe++MCE + ppdr3PNO30G4rvb+X17A74/bdVM55hpR/XQq1hehItobNz22pbEHvAxwo+xt/0mjjSaBCUiFA6ey + nNBcCw/OSuV510i9vpKVEb8qDj+wW3JzG/t9MPLa9TY+LJPJxBnmzGouqCnV0BMLamyvq2KqGY7h + /SwM1NWDOhjxIbqDZT305+e5vI2d//Dyc8Q9n45eDhxR7Ghu7lfFrAFHhbVe9mQdKceecW3IwVdO + KyLvHTWZl3pWLtnLo/itX6xp6Gj+d748BZzBdLk+fe3R++Hv+wvSPctWO4WJjXexnla0l4zyd36w + dzkEYL0ybhBMa1vDDrmtqsl+DhOM9voaR+seW+z02j1hVcUzWQWlZ7HVM2DgYpXRkm/ogKTpzoPH + 9WFHvf3a4POc0Qb6Q/lEZdk7yVr5bgi8ru2a/viBelpUwuh6/WJflcyeeTcSQ9PFEY62rxmQo5AP + MK0fTxocwzbgz0OJYJX7KfUe/Q0wFU86tCpNXvhc4sOWSTWsm/ueOjTrwaAez0ihppuilVfrlsiK + laNeifTGXqJvrfUmbEugd/eaOo/nM5hfcp7Dt6Tpf+uXz+1FUKmDJCI4+bpioAIEpnM8YLdkSvDH + U4PX2ISTcBdweE4JdNN2xnczGQKWz6cYWBCJRPoIx2ouxIEBGq1kIrhXXLCf3wsOqoRWB+1gzVH4 + UcChVyNqh1wrmBJuUxBv7QfN7GZVMBUrS8Jfr3B3reaApdsKgctGfeHow6uk15PsCU6sv2DcmTEY + 2Xpq4GiVDV74mbN3AlvQDKcjNW7X7i9vgqhGZ4ojkYDxQjECt+/pgm3Nk8CfH6N01nB4K2n/x08Y + QBsfz5FlzSt9KKFaNwNR3zoDxIoOJXi/nwds8Ozdj2kTKHDJM8g1dbf9/CVqCb+BSBEz0aUaF/0A + i98h8FbYVn/I8RH0EDfYdl5pP2ShGUJMPw3eGdml5+7XkGEAHxqRtudnP2kF7eBSf3h3yeyeK31d + w7du5WgKNxTw+hGHIC65ixFm76ozdh8Zbq55TDatO1rTQ193sIJFgCMz4sXcyybUNtlhplZ2vhTT + syI5cND9hCNhFfXTT4/0/aTQ61t5g77YiiG8vZqMRou/7ecTjwF7IhujvfbL+/YCLNtJQzyGtJhC + B9rwEw0jNsxYD+TV9Bo0Zgo+gQnsEnZVRqIkaGdSly0T6cVPgaw5JKTmZVxJsnN5wo+4NXEwvU7V + tNQ3fF/lN0V7TQc8gbINZ9kXsL89N4C76ayDHtkxWXvtFqwxvnjQGXwNicv+Mldd3YHknSANEXet + Mbo8HS2noovvLRL4n553rd9RHI1V8g5bFIJPfzlRW6y9noNkgmC1Uibs3O9eMi15A9xmqxvdWuE2 + kfBzekLqhBLNzGPWE3fXNTDcPz2KS0nj/ZKvQul7+ODlf32qqT9OMqiq8YH9gxaAKRArBG86WhHg + PkExMn93hb+8UMLre0AXvwv76eSQRa8XfxtAoAuNRdH8MBKx/AYCkN6BR209LarZGgGCqa+caTBf + DSAveZ56Ox7vNLtc1sXPbwIgugV2l/yIBVvW/embqW68QlHj+QqPt37GOFl9wSyZRITuFKl4e618 + Lur1dQDJgcZIQbNa1EpQKJDN7pOwOSeAhZl114C4LbB9ileL3hwyOI9ghUPNjq2pe2zQj/dwpohh + 8eNJOH7vIRFVWlVLXqED7yLucWRnpKK34DJBXFGHCEv+8l1nXQoffaMS5tzsYt4eruqfPwuHvWKN + r7d4VXeyH1JvI9uW9L0fGziNQokxWD+rGW6uJkyi4YjzcAOLqbdzFSa0LKjznLA12TGSIYhGB3u7 + V9DPpVfFsN9KB+rWGk8m19t4v69xkJ+rYFj8OlT0FVv60yYYBx+RP/3jz5H2zI+BDrCSJGTd+k4h + p9I2BL88wQolC7BMLSe45M80eHz2nEtxdoXfIDPwwhfWxLJLCh+H65M0Usd6+uaJ+utP2Ew8zZrA + eivA3Qm/fnxTjOOgXQGgzoNMK1/pWVtEjgroK8I4Gq2EYbqRYRTXO3raqNOSV7yff/UZCDq1Wnkr + pFApzwHZSE1T8YWnoIsujOL5lnOqfOdB045PiazoYVUNzvkRw2tYT9jbr19cXHcPAfDDdYM9v7IS + ZuNtBjN9LvCSp/Vz+hEmKPutTvGY0oLt134MDMFhf7wnRTvfA2P2emEj+KgJ/zRtC/ZjrNLYJ2Iv + zutTBpGWzoTF1q4Sx6BP4elYWn950rB6RaH662/B2Z+saTfmk6rYXUUdmUtgvvltCuVZP+E833cF + mRVH/uvngXuhgHm3JoaOtM8WP/wGEwe6o50/vU7S5ee5OvseWPwW3QlWm/A8Fmu4oncTO4LuAY7x + ciM2yRRy2B6NYMz61x0s/pqG4gcVXBZVG5zko4+zGj573myHCazEu4B99hiK4dYbpgZD7lL/xR+A + 49dXgGZ8av/yqFk1vzXoq5eKzkcfFDMBz1Jb8gQEdr1c8U2MDmq5iXp0aiDkc1+pPux8dULDmGDO + EyjY8PsuFHRd9HB2tq8OTkZwRqv5pgI+WeUVeGF9Xnh6C+SD28bg9qoznPbaC4z1vqlhBS8BNZ2b + nYgw26k//sFG9hKs1gpfjgbLWKe7Vjd6Tp+gA2vHiKnPsiefI1/TYZLmxm+e1HNfV2LooOuJ/vXb + 5nSJIST+E6mJvU5ok5oDLN2ixtHiDxYem1S9A4dl3qVxVo+tA5e8ghrP9bEQc5YOcMlfMDJ8Dyzn + 5wibzRthozdpQX71L2unCi9+KSGzsxFg7voF/uU3JAt0BqK3n1JHcMXFX9UQVh9RINriV+aoyIWN + tRWf6NsIu4qJ92cKN/bugL1XW1o0n6crXPseJWR+vJKJA8+BqcMUNB2dkjefS45gu+cdeg/H3mJF + c1XA6/2+/eWf7L6SfZBbOqR+2L8TjjTvAHZhJdLIjJKEi8aYw+Y8u+h1u/rFPLFEh805vZDfPPE7 + nwwZppDcMTpHtGJLPwKGbY3UqD7vYt7gqwx++eA+9Xow8vvWg213SykWqzLhqz1QfvuNVrEj9X88 + emWrLTXiVdsv/J2B5XlwkOWrgOWHC4JImx8LXyE+uXIbg5Wox9jRQ5kvfv0O3ZzFRFRlGjBruNfq + Mq/Fun0qq+mCPwJc8mcctHbCRVasHbh5P3ukNMrEJ7rXVY2uG0SIKoN+yk1xgN8ReWTz3dW8nqo9 + gWeII7xt4BXM8BMgdcbsi8R7O1osCzwGkz6H2L6/XtUwdJ8M3odyQ3fLfIXFh2cLHyI9UefrvAva + WBsV7gPwwt6j18CvXsBqOj2wn6yqajqxVoHeutzQLQ0+PbMffQr1/dHHv7xyyDbzUYusScPB1+l7 + HtipDhc/hwP3YCdr9VmH4OQFbySvTL2aX86Ua9Unj7EOt1uLLP0YNiYL6TK/AnNQt1e46Pl/9Eg+ + vjLNkZKMrIYoDLhv9h68YRbTUAtAxel8qeEjUg7Yy49VP1XRuwXkGlBsrvQETHJ6PABSCTr1dWRV + lAi2CJ8r7NFfvyaz+HWgn4gikZbn+6YfeYIqdjm2b48dqCUzbjSPweFv3ktepw2Bsb7SEZ/OdcGo + ccygdxoYemWJG0gP4yDDN657HFmpXM35XtbV7nHVqDFGXdX3pTJAqpgQGwSbyXSqJBlOAF2JyjKd + U6l8xYDTgVEztr79tH8zBfznRsE//vnP//n7zIKmvd3fy8WA8T6P//4/VwX+nd/yf4ui/G8q/322 + ARny5/1f//WfSwj/+vZt8x3/19jW98/wr//6pwK1v/sG/xrbMX//Xy/8Y/l9//2P/w0AAP//AwDm + daMVN0EAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 7b6d34be9caf16f4-DME + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 12 Apr 2023 17:32:00 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400 + openai-organization: + - own-45h3iv + openai-processing-ms: + - '1177' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '3000' + x-ratelimit-remaining-requests: + - '2999' + x-ratelimit-reset-requests: + - 20ms + x-request-id: + - 3301ccd4f99ce79864700677997b0a0a + status: + code: 200 + message: OK +- request: + body: '{"input": [[831, 677, 31172, 272, 762, 14087, 68, 17, 64, 25350, 1774, + 1897, 51542, 9081, 19272, 1135, 65, 1774, 67, 6069, 712, 2689]], "encoding_format": + "base64"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '164' + Content-Type: + - application/json + User-Agent: + - User-Agent-DUMMY + X-OpenAI-Client-User-Agent: + - X-OpenAI-Client-User-Agent-DUMMY + authorization: + - authorization-DUMMY + method: POST + uri: https://api.openai.com/v1/engines/text-embedding-ada-002/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1R6Ww+6Orvn/fspVtat80YEpWXdISBnWgREnEwmgIiCiBxaoDv7u0/wv7Nn5sYE + JGLbp7/T0//4119//d1mVZGPf//z19/v1zD+/T/We/d0TP/+56//+a+//vrrr//4ff5/TxZNVtzv + r0/5e/z35etzL+a///mL++87//ehf/76G0JJouZzOIPpNUsDEMa4ocb2k/ZTfI9LmCZCQq0i/WQz + 2kov6JoPmaKj2qiz7Q6m+IncK0YZc9TZnE8yKLqLgZqEuGz5zDsO0HN/QcsYcqAvdmlysMeix6qC + eDCzJ6dD/cURbD6mD2Bf9qmBr+obiu7PT9+X89jBvqkh1ufsmQ3G+GwkewQ6PeXGxlms/QVBfip9 + fDpZfbZcLWOAvtJVZHe67tlw7jcL0HtdR9tvrgCWHbedKD0/AYKnflaH/fugQNZtjhQ7vsY4HzUy + VFGrY708u9mkiokNo9oMsL3162q+1JMucVutJhOIDDAFFm6A4TdnAjbHjo09uCHIalUk07lQGY2D + dAM1p7ijPf8JGWt3XQerfe+RwxG+2ZSfYA5MNO7Rrr9YKr01z700bsQEm+/7Bkxnl4vgC57P2Fji + Akz2qx6gUBCPqmHigHF3IAlUbs0V44BpIfuejxNIzOiAXcEB2fLBiQzxhXcJdxEfKhveyQBjeEuw + d7neMvp+ZgNUSrGmxj3i1VmtEQI71lcUucUho49Yb8ARpDeKv/kLzKxICJwM60Hx8UvD2a2L/GDQ + r0bEu9aA2T3uW1C8zBOOgblX+3s2KpD1SY44Vz9WC3CdBew1lFAUHfeAWcUcSC4RYzK9yqGaBc6A + sNl8eXy87VJ16s9nDnTf2Ka4QEK/jJKygf170Kn5+B7VHXzTAeRC42D97Hjh7DS3F2wF8YHVlNP6 + +WCnMngNi0K16eI77Er7AWiv8IBNaLzYgJjvw/Ua1fL2/qee4S1rGsSPX8K+l1MdQDuuRWwFG1KN + u/zSwcK1LDSVvVUN7saZYH99LGSJK1zxJOl0AO8Foe6XJYB9DroMF9KcsCHTEUxbX26lw65Vcdx4 + vDp7r2ERTTa36JDXG0AvzM/hZXrssRy2mTPkRmPCrXi1kGTBRl3gsBXFz3C7YWNId87X39kKOF+/ + kOqCX4czgoEpJXIFsfkQSD85j2MEhqztsV7bp360xmcLT6bPURuLczUisc3h99IpWOFZVT07SRG3 + WxVa9CzxQB2oVvngHDOdABEn4fRk0wQn9Dpi/SpP2VyxiQObR8RRGwyPagmvHwgmTnGpe+rP6jhy + Rx1uTLnFsuSfwbwRZQUobe+gj9ILFeXfLYSjPGTY7vJv2C3Dp4XmPf2QpyJ8QuZcn+bhyziZ6tbz + 40y1l/PwjqMFy84mYpP4cn0Azk+EfTm79GzoxgW+7t0eK+JSqNN1L9VgCHeEugd500+xuG9hNMwU + axP6ZrPYIxOE/o3DVi1fq3X+bLjLOYOeLEXraV2dEvFbEx67/fcYLppJEWT5OFKvaXfqtP+8CAwy + boc+gOpskfR3BD6Xk0fUZJM7bF8qL4iE955sN0Vf1fx4tCX+menoqsdqyBLrZgL32tjYtHdvNjhb + qYBiFOSk065LOJq+v5eaTc9jxYi/YMVTHqKzu8dqqwGHvJMz/2e+T46bgbfBzQvkv86IoBPo/TJu + uA7maaiiRe0Lh3nzyYR6BiG1wtMlY87+EsCUrwGWD8UjXOLOQ2B2PYSddquok3ZtN/C2FAlF00MM + h7tmL+DgPzcoiPOWLbsnZ8Iacjo2QnJiQzr4ibQDIqAGMvxsSV6SDt5Z8cD2tS/6KbCMGqRHbab5 + fjmrNPafPvjhbZrFdjbVTm/CnZ7M2FmGBDAvsAlMhWOMFiP+sqmVFA6y+ihSM9KPgB/vgg1PD/uN + yPMGsjGuyQDX91ErpVn1ANy5gOCsM2xnuKrmIelkmBiFSnXZfDgT9p/ab38SoEmgor3TR+AMlyNV + 65QD8/dtIKAvaYjNIDmFzGWmC7Vvm2GrNCtA6nuzB8/WJNj1jC+gx5vdgDL3PHrU3iqoKxJE8PYS + 2a/+GUPuEMBk1x2xetHbcEidQAT+/m1SN8/4kPbeRYEY9h9q7i9+OP3qwTYeE+K9tAznzD+k8HVH + d+xADfQzz6sKRJ8OUbuZ7z1D43eCUX2C2LCR4PTV28yhZX9v5CX2QjZIV3+BXrjR8fEsNdkgv7fr + Nuc1itXxlE0DLk14oGjA+i4FjK31D82acIirNE4dXgHzYaC9vti23Uu45IKriHtX2GElvrycEWis + hhOCOb238dxTMR10OAfdGyvHtFRZN7QcKNSNQT354vdL2VYLODXAw3bTxWzFt714+lzale82jCbW + 2YbmxcFoZ3jPbIRH34XRMvWIkx4ndbwJDoScLe+wFbEzmzzxIcNqjxYyL9HTWbwvhKChHwWr42Gj + kmvz5IB0uBzI5nkD4QzLtIPpB77w/VDq1aTTzD98onbBxVv7qHM4yO4f/uyPJ4PxzuzpIlVMHcts + NjNWNYsJQ7pNqDYZr2rmuVsMq+eck13qcOpE9XwDV72Aj49Ly5YH2iRw3OwTLM/cTWWH/D5BMRqE + la/qinrL5k99Y/tMI8aGt08gsgqfPO02ymZNYg00zvKAdqkTORMoriUsuquBHXvZVEN8AyZg7qIS + 4eE/2CLifSlevlZPsRXu+pmSVysN7vOJjez8Dml2FFpYqNCgcZkwh+x5VYM7ZCVoV/UKaLVrC6FQ + DB6+E20E83WT65Dl5wDrdKzZMoJbAKOcO1HPG5k6J+KegCEUNSISj8+m4yvK4Z3jJewEcdWPP73n + f8uIGpfKUufGi/YgY1mAlpV/ulGyN3DVh6Rzj0nP3o+XDAN37CnS91+H6ebUSr/6RMvlxQbtem2g + FkSYqq9r7fSPWK+loHIUBLh0F9JH0LvgWGse2bszBYPGVymcnTihpg2Hque8MYCPR8JT6y01YLhs + cvnHh2RQ6UddLG7mIUe3KbZzSXZ2SnP0oY39kRovTgM7WG1kaALbRod4OqrzovUR/ADdxI7OanXy + rrMrLVJ7oKiKu34yxmcNNqbSom8RltUwLCcIabk8qL0T8oqJAuRBlYIOCQLvVT/8A99Yv2N02Foq + 23W8DC+Tekf9VuKqYVIODSQVNPHxIMgVO7mRDAzhNlC8KZxq91Butijt7yrZPg9qxshjCeAcnmTs + ftkeTEHzJHAUnjzZnfzF+bO+77B/0SPvRow9iRLA7PKlaGN/eWcpKBChFOgq6a61ANgXJzo09ZKn + 9iWV+9lVuxjy013EXvnW+jl0YQrumHmIB2bisHyTu6BwCCGgtw4hDR7Hl2ReLIz1HgTZzy9Aw/I1 + 7DzzV7XEPS+CUC5sciieyKEMl5GE6+KNsXcyVbbqU1ilMofdl7VUHTTLRLo+QUWWlZ8GTdQnMGSu + j81nUYddJccmFE9WgAC62yED8TaCvhPFaFtcNuEQPW4NfJhaR7VHaKjMb24djN/PD7Uabex/+kWk + 5fRY19Ou+vxealJz5TPy0WM1E0YtRBCDZabGqm/+8JXkuGeqgdmshLoWdLjqM2ryNz1bhLSWIa/E + AT22Y1+xweAbePCrDXVPd7Xn3i0QQcSwvq5fFY6hYxOIoedSrV2mnnkLzwPz1MrYu0ZtPyewS+FY + 4o6u89ePb5Im4KVHA1rnT929Uqc4dOoVYxfbabaMuVcAxz1pVE8XBXDY/+qQaxGjplspIWedMgjS + 4/aDFU7KnO8PD6tSbJAgUw9M6PrhDhNXfCh6a4b6XPWHlNXLAe0spe4pNxw0uBvm+acPwFyzTwH3 + 7m73x3/MsAw6uHxwRfii46uhP595YD0r5w9/Lpr5QcCrwET48YvAMpPXAAkgFLtjUVeMPEQfBqLC + sGU9TtkPD8Edzx7aRfcrG/XrV4ap38VUWfU5S6yzKYa7TYQE+yIxkor7CZ7MEVNjCFH4zoekAJd2 + 9LFyChgYes5SoDN/MLb6XnWE1Ch0aMtOTU13xmBOY1RARxUVJDbzvZpO3ExAQ+0Pti9p2U9l6qRQ + fR8kMjdbqi7ZXhrgW7NDAoaJqtP0hDKUN41Ctvtv20+PezPBiPVPxOblGq71Jks//3mJcJetfJxC + 1QhrjJ63LJzFz2uBWPc/9GJ4x3A82IEMU0otMgHv6JCXckawfmgVPWadpBJnPunwYmsC1b1UzoSS + XWNQS8GTmmg8VOz96JQ/+hHE8pf9qSc1Tgnqj+Eu+15OQyDer6eUnkZ1ddA1cv/gubpVmUPze6kD + Te1LInSFo9bdIdZhsnv35C7nQji1G/g6rOuFWLR99ksUpHuw+gMqj/CdkZ9/SmTjSbF3ap2pjFsF + Kra8IOGSv0KyK+1cXP0aee+LPSDhlW7gU7EYxTx3CheL70to5tClnoYsxj7JWYTQUBM0P21XZdtv + Xv/wkuzSQGSjwGEInttCpB7eYzCwq/yCV0czfn6/X4r5XYvrfkTbpMyyxfYTuG8+146I8vsBxkU5 + dPCtmSEBK96PbxIkoPtGNj2lTqTudsniQ2Tte+yNkuHs4pekwKR5CdjazC+Vqr45wfsuTMjyFKBD + d91GBiufoWeFFHUqRaGEp6ZYqHfzl54B7pbDd5Y/kHQo9V744dO6XlStT2W/mNenDh9WBhC7Wglr + HoNhwpMUV+THh8Nrlgi034OGAyxPaptbDx467vNBsaAEYLfWPzwy50iW0UXOn/zAhEeZ4jSu2Jd6 + ow29xdQQ3B1rZ+qEIRUn4y1Su03CcJ5NNECPKeKffGJBhjfBRD49Ubv6m0nWpRSyPs2pfnbGcCZM + GCDd6z3Gz4MaMrNGMVz571dfFTeXdgSv6SdGklKOIQu2uw5at8ik10tlOXNcNwRCeDKoG82TOqRD + kojv1zRQX7dmMBp8VYjzC9nYLneMDf5g2jAaLhFW+uLcTz1nyfAeo/qXN1WkeyctbORNR5X9s1Tp + lfYEoPI8UEeTsmrK36YPs6EqyW5fJIDxps7BsfQ66rHFVvtNr/ugZNqWehbUnYm7nHIQeQQgsLBH + v8xTlMKV/6l6uPCMfM5HIq34hA2pttXFM7cF5I4XhcoF32RrXtUCXuBz0nPpLht/+v9A3eFP/kI+ + s8SJ/fW+0NPEBf1ktqoJs0pU1zxkzIbRGzX47p4hrs3sAiawvciwerIc4/3ryHZ1NMqgXLqalGxu + w2q/dTdQ17oJe/pZDMlZbEtwPO0xdWLD75eclS0smb4lkrPl+xlfDz4oRaGhsuy1zlztXiVUi/dE + 5TAdK6K62QZqwdJheR6XjOkfoMN8Lzf4+LQHlV0umwh2ju2SpS/m/o+fFt0NplZ/mPr5+6x4sM4H + 4kKBZEuwf+fQmU0O41yTw3V/vCCwYod0uVQ6i3W9Erj6a3wE4tlZHk6m/PzAH333hdVGgWJEBOq0 + 25fa0Z3IwSA67rEhSFw2WM25gBwvuKS/HSrnl9+Bk3/MsZPBsBJ+eSRq9S/G9PwG86xfVr3YKlRn + xRn88ZMRUFSqWh2tPox0CyCnycbyi4gqGeQmhSrWnjSZFp/R7LUzAfdZEuyY7SecEPMDUFZVSm1h + PDnjLr930H3MLtX0hwO4U2CXsL0FOpE0o3Kowmfx4Zq+Y4y6g60OxX1vQ87PXbSjHgQk7jwXwgsx + qMWQUs23xHqJXnWY8MnekIzVys0HumiLK78Dhw0fJYVRbUDCb1jF5lVvAGGcjxip3j3s1/3ww0ea + +Y4O2rOVQPDjX/nqluoySjYU35miUXvRaUU/qdPA6cIjfOSsOpuONTJhOxkvVGvVy5n40TIBBO1A + MT6e2ZzGeg48T48oqu7ncFj5AszhM8M6xlU27cJwAYGmIawXFWbL7mgqEJwrRHabELCl2XXNT69h + D3evkP3m/0FKGW2PE++Qd9yaMGm0nBrajXOWecpTuI6fYnE4qJRU9w5cO3N18h+3msjwKcFuYDM1 + BN7rOfPL8XDN36i5Ud2eb7HeQSdDCGtyZ4Kxk2wR7u7Bi6qU97J9ijY+sMr2guWy//aLXsyu1H2n + N139UDWfmR+Dyt7I1Ghfz4y97/sIbitFWvOrjfNbL9gKuUYgc8sf3vDwziUvfLRbLpyPHxHC8y57 + 4BX/nDlg+gZEQFYxxmZSMUHrc3ixnwE1QvJma77e/PwVVp5C7rBWyOM/v4c5IVYnA10myDJOQNtp + q4REvuJcvI+vCU3O9Annn55y3OrxJw9Z/XMrvt3LhTrLsGeTKvqmJKlZRA6weYcTMRMFFofijMab + 8nbm/Xh0f/kMaV4XJeTBePSlLthz9Ff/y4qHotZVZ9JWbdbPfe6lcLpwiGKaDCv+tC3Ib1z706fZ + YkALAkfdKzRf8655NvUBduk3I/AjEMZsMYlha9uQPJfMYN21nmxpaaIRa5fhXC2xYy/QLhhEgACH + 8W6/V6ClXj20UbYUUFmqcthu3R7Ldc5X9Q//DSEbyDbuE/WPPlr1CfaaJ8nm5X3NoRj5OVYbLIC1 + H+BCe8z7X97NmHFy9uJluu+psm918Mcv5op2xMGP/w01cEUnvJ2xurUPbNWPCajfmwvV51f7y4sn + SO5ajMODYWVTnFg5EE/vEcHf+g/xFEEjifYYbTlX5Z5CbsOzaY/UcjKqzmeWxH/4NFjELuPboVyk + 5prIVAlmSWWCcljE3ijNNa8wAbuOogyXhpnU6IakmoXLqYS9JAf0hqM2m7K26sCQdT22BbD0XzOw + a3C5dTl2V32wAG+Xwwmnwk+PgLE6eHtgHMGLCPblDubibca/vICquvwNGWNb7k9eYtwyyEjkfxFw + JcnGCjp8M/bDn9PGu1H9rOn9suYx8OroBrZY7ju8v3H2sOGV2y9/dpZ7lPOgm/WFcJWTgjEcTARF + b3fCx+tiZUuvZTZ0ZpujOGB1tuDXwEMkfPZI3PpaxS3gmwLaPnWszrXFeGYmNiw69iKzRLlwXvUF + 2LtZShWO50Lm+WcFekwW8bFCL2f8PisOCkZyxDcIbj2LbpX2y/OxRbt7v2yOif/HH54SU+2Zhrcp + kBR4oMXbGdkUNF8CXR5VdMWzbIpvwIZdkN+o4u/CcLbd2oTJw+2wNZZ62Hmt44v4bidUW843RsQl + juCaf6/80DnL1+kDeMwnGXGrv52d5lzCOC4YXv2bQ9u44H/XZCmVm8O3JMilxVLfaH6eF/DT6yJ1 + vA2VJaUOuygIROgc+A9ZqF+rzIK28md8aMsNDlGQ1wI6nQlVNncLsFo5+5LnlRT99Bg7XpO9SMtI + W/OOse++1lzDxTQp9g5l05Pu7bcw8oKSKhUI1Nlgsi5dgzQi+/i6BU1pPdZ87bUlYB+Sagz86wKu + nQ2oE2myM+vXpyyt+RE+Tu9tOD9O3As6Tvulj+hZMrLd3m14KV+I9NluYKu+6aTrAVOyj/Tn7/+J + oH8THYmDfWL8ik9wQuWRarwdOsvrGXa//Gv1n+9slLZ3XQwTd6bnR44rYft57cHrtD/j+9V7sjlW + 5D000fn+X/2w862SYdSLB2zIreHsirccwzLKG/IS66ka6/OxhdlgnEhJ1EFdtsVTkZ5b+YHRik8/ + fQjRNt1hR8T7bPzt73iM6j/9k8WhYfKrd/IJIcuGH59pncZj9XCJAccEroXzdzOjw059M5Lvghj+ + +ncy22N1ycRPAoVGTn9+M5uHQ2wffnkRWN9HH0GFfn6E6iGSWG1aMpHm7ubS42FjZH/G6/TWHgkr + v5EUdgV4yLGKjzcShqMQZhyoFNJSA7SlM/mDbEutsH9gH2SoGrZU1eDt5PjY6a3D2t/FA+ieT48c + 0jZibPs+IBix7xMbpqP3bBbqBmbD6US2eTCCmeed1T8qC1Xz74dRuoEDWPuxpMiGOqRrP1T66VPX + ughZH+3fJZhGM8VojQvW/TcAWebLP3g28N4uhpUNZWz6rxz80b9aEGNqr/3Pxe33MtgYtYeNu4dD + cgizPVj7g0gqhFYdUsQHohJ4O+zykgcG73pAMIYCI+Kvv8xJmQ3C6xJTjPE2m3zUKICevxck7ESV + zSkrG+gNloldT66c4ZtmHdxI+hVbkAsYG3VNBGsej5ZTwNjcGpsCPj5OQaRVz8/7qW7gQW4isu9d + uWLupdHFv3+nAv7zX3/99b9+Jwya9l6814MBYzGP//7vowL/Tu/pvzmO/zfl/5xEIENaFn//81+H + EP7+9m3zHf/32NbFZ/j7n794/s9xg7/Hdkzf/+/9f61v+89//R8AAAD//wMA+SttleMgAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 7b6d34cd99e416f4-DME + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 12 Apr 2023 17:32:01 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400 + openai-organization: + - own-45h3iv + openai-processing-ms: + - '22' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '3000' + x-ratelimit-remaining-requests: + - '2999' + x-ratelimit-reset-requests: + - 20ms + x-request-id: + - 3dd93c4866672c1fff861e8379706c22 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts_with_scores.yaml b/tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts_with_scores.yaml new file mode 100644 index 00000000000..fbd26183d9d --- /dev/null +++ b/tests/integration_tests/vectorstores/cassettes/test_pinecone/TestPinecone.test_from_texts_with_scores.yaml @@ -0,0 +1,557 @@ +interactions: +- request: + body: '{"input": [[8134], [2308], [43673]], "encoding_format": "base64"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '65' + Content-Type: + - application/json + User-Agent: + - User-Agent-DUMMY + X-OpenAI-Client-User-Agent: + - X-OpenAI-Client-User-Agent-DUMMY + authorization: + - authorization-DUMMY + method: POST + uri: https://api.openai.com/v1/engines/text-embedding-ada-002/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1R5SdOCSrPm/v6KE2dLR4iIVPLtUOapCgUHesWgCI4MVUDdP9+h743u6I0LIBQz + s54p//u//vnn33feXIrh3//88++j7od//9f3WpkN2b//+ed//9c///zzz3//Pv+/Jy/P/FKW9av6 + Pf67Wb/Ky/Tvf/4R/++V//fQf/75t4tnhEe13HSruD0Kyljs78xfRm80UrkpAEf2xCybid2YYlNE + 1ZKazOptNR7rNHCQ7jYSXfmsRLx99RjcKy6xrFOrWcZ7M1FGQeqJnvctmq9H4qFOLkSysSy9m4/Z + oYY2sGpmmocq7qSVICH59Vgy7Ypyv39vrUCxg+2N/H5/urwKD06OkFB06atuvgnrLRzUYiS24uzQ + tLod3yhIhJp57LjmQ0RPdxiL6E62cb/ppquc7WFQxi1z39knfr1iPUCXVbBl5UvTEV2VvYyarWoS + kl9Rzr71AM2iBFerw5i37F3JaAuXmODReHCGaOKhfhJfLDhkD86zScmQJh0UohfYzmcf2FPJNNEj + hHiezpdm5iC+LzWGRb9rZrlf3+G8upxoJyS3nCl928MSr3Z0QU9JPpWXpQXHWDSwNBij3t/nW4Ks + U/JmdmU9/akppwJQcPWJX7KQ08Ul6+HzMXRy2JHcnxP7AZBdHxYxBXfB3+d4q6HUVijBdWTmq1vr + OiChsPnr52Q7VQLe+x6xQ+gcct5v3AIlcftinl2F3Vx4IUbTAWtUQsFGl2jliutIf29JfOVRN8v9 + 9IQ4B0wiOXV1/nnMs/p4vs7Mmp593CZlFcFDOTf4eU/eca+n1xbEsg6INT2DeCwnVMEWipgZxyzS + p26zOCqNNUdM33XXrr9dnAStQ9Vh/uTquoQ22RuCBGpyia97fZp3awnKz30gYbuDbgpeqYgEg5+Z + 3Z2EbtaPYwKv9VElBqYamvvVSYF5UVS4OelvPukhFiGSxz2xU0Hkj9+8wDE+0OddunezcdNEqGvr + RAe54k1/O2UUyQvsE585q2bWoy2F2Lp+qJq0djwenamCkwMJcV+T7Y/o4wBsptwmnrNt9Vl0fRku + +9qlSv2c/UEPLRECo/aJPapxN0XBWoOoXh6Jqe1szqVwHcAm1zmuj/nFn2PpNa5pujiyrVm2nK1I + +4ZGtg943bS3fKbVRoTurZXEuWea3/vTOkC6QBiVg0cS88SRz+C9nxGxnnBBbM9ib2HSPGBmpeF4 + Vj5PCc6vrUa8NOecR90Ywc24bmnzQTc0X5KSonMTLOkzuEIzwQkBivhu+M2HP5lXIQPK5D2+F7aH + +K0YFNDM7fLb7z5vP7UkQ9o/X3gdzO/uI10fAdiL64muXPuVt8YOPHhAneDl9RTo90TaKOo7Czvi + SnbDp9SpW0iVdUj27j1qvvUVYFXyjvhPJdCn1uKVGnrGzHSTOB1/mQzAsPYl7YLyg4aQyTJaid6C + YE+K9Lk4vQsIF2TLSHJ4N/QjrzHKnbrGi7Ky+BSfFUCv7S1mblce47GQPx6syqkjupHmzXyIzC2a + wtcH8+E1+XOC8yeY/TDTw2E6xXOr1K36estPLEeeitq7hzS0WeI9lmbn3fROGWE1tUhCJWVqm+nm + b2Q1N9iJBGrndPOlqTOw0N0jpKyenA1uVYFy+NyYmUdWTLmdFbDriUOdbE/RPJzaGWXxeU/MYXij + yUvqCPqgP5Lks3f9KTJSAebP7c6CrRY1M8vNFhLxdiAEJ6XeimSqVbETZ6YPr53PD95NBOlMDCok + jcdHkTkj9HY44dXFVJqxu3sFHJjjsdN3fudqmBX0fV+6LkeE6Hfe0YKcQ3a6hx2neJDvcJmGErPz + 6o6mo+jeoa3mgm3p/NQHnp0V1Ckepm1wt2J+UHGN7s6tZ9o2rHxePwwBbgAWM+byldPiVBVqex4o + I9btrY8fb9gjU0xc4rzue/6iqvSG/LW7siA6Dvk8FisD2Xzf4CXKmpwHWA7QSopG5q53EuLdqO8V + 8kEKVVKpjSf/Ms9QLgQNq71fx53g1hJC+AnEH18WEvPz8QgJyQ2GsbpEzw21zmjse0Q2z0hrxDbv + MohpfSSb0qzjDypkD4qwnTFaRm8+Hbq6BnN+M1bYxkef1JfTwp17NW0Fe9D5ZtpGsCZtjUWlm9F0 + M4oIre9JQcJVkfHPuPACEIzDhjYXqUTTNqst5Tefxns1N1Q/jkeIz6+M2JVl+VJIrgHaRHWK19pW + Q6OYrWSkDBQTV8FuM+1PboXSwokYORc4npttDHDduuKPj5uRyILwqx9xD5roD7/5slJty3Zkd+N9 + yXsB/PmckTJpX/GcHDYYLRlbMGdXv/z+6Kxr+J0/y+FhzD/HpwEL+rjhue2qfELJzoJoZcjELccc + jeXEK3TokgWxw4/e8PPrI//0AlXVxbaRHvnk/PCOLrNsnXeSrd+BV9OTGJpSNfxUphlCEaX0/tm7 + +rRolgpK20Ii++v12A1698aIvZyEfPUAX16kSoLvfQx9anRvbGUR2B1Z0oOtSqi7prqG1sGOUs6t + ym/JGzLAsnfBSrI+NeNqeF1g2dx8ugejjadF21XwkJn71495TRsN3pX3xlLuuoj71s6B8rWOsRQe + 3G56ztkFytQ5Mnu7OPqv6b3fqtM8KBi4EPLvfEdINh5HQlb13LWX3E8Utnp2dHlytWbe2l2LLEvZ + MTe6rNAYD60I+9f+gPmVjx0toln8Tm3ODqFQ54N1YyI64woR662tmvZ1LTSQhPuJYLRSOrY6b3pA + d4jIj3/nzyY6Iqm/PJm1E/xu3ovCEUrlmrPA+uy7qY4rAYqZ69/6a00XHZYC2lTCyIJDrPlzH2YR + 8pLJJG76kfTxi++QuncHryW7QTSxHwKS3l1O+X1T5FxE7h3elfPGNY0+aAo//h5tKhiZbw8bPrbJ + clQ1/ZQT410U+vBhpIfUlinxb7nq08yqj+Di2WJbntQ5e2hxC7tBrokzeC80puW+QrHj63/6cZbq + mwXdQemxQB9SN375E7yWbphl9HbHd0/Rgi9fEt8aiT5mj+mpJBJmdCraDvU93YPqegLCcpjfcu7R + xxG9ouUaq7dbwke8MiuYs/HFjNOp5tN7uOyRNFIXL374ZazbQkHL7EV8X3w1M7jrfp1pksdwfJ31 + 2YvkGkZfCtn3/OmT46UjLOalwTZF63/5Wo5gexozZuyXVjesRQsry9gQif1Ods2f/jZ7NuM6P0do + bLLlE8gFHHa4wKGZzVrZoy+f0sZIUTc2tw7DMYlyqgh1003jx5bRT4983tkn56yIjJ8epdPmbscj + 5I8eFjvhiNW2j/2RTqoIzzLdfvVXxft9t87gvjgcsThvRzTJm1sN0jk0SCDTJfriGYZ11ezofH0W + cTac2hEYMZ/EuktGM8aXo/eH/0F687u/86OpEGHeKFLOROQ+0Xl7tPEIppzPi4eWALFQTj87aaNz + jZ1FEHrRJKZjFJxToz+iL58xEvdWLC2e6xk1n+P521+mf/WIgz6rpUY0jr1m0nDlqELbJySQ40XO + pXDCsOfoxnR61/kyD8IK7FoMmCfUejM7qzyCOsxK5vvp5DPDbBLQDmXA8NZ6x1N7dPDqasKCOUak + +0trVXtwufQqC09ipnPrnCmwqjeYuU7U57Oo9SIa7ksL841TIw7qWwLEypbt5WvF3459UBAcdwcM + y+2z4U7dZbA12w8xBuT7f3gq9JLJcJc33UjwdAReoxOV6mWq8wHCAMVqHJMv/uj9BYwt+uIfrj/x + Sv+kyE5gN3sms756j3rRWKulvX0QW2mcWEqH0IAvXtGq1YOcBYvmAtQPUlKWGfFnRUY12sHnyrzQ + 2najsa4vgLToxLT46aJZiosziMqLUWCb3B8mBWtK7IolCXN05bMq2BKaZqYQN5idbg6PhogkkFWq + yhXv+s0BZrRctsDC8/HYfOr4DagQFREvZIPpA0pSA0XaDjPteak5HZu+RxqSrmSL3x+//8TBEdRd + UtOFZ03d9Op6CRL/sWbGw2/y0X1VVO15taHTeeMifmviGrhOdTwN76vfp3fHg9R9OsTvNjs+as1Z + AxwrQCeDKvFUuVGh5v1sUKWPe8SDRVOgbz0xrN0gZ/54HKHM9zbZbo4PnUbtQwLk8QnPpryJeXRQ + BXhAlbAscBkfbexXIA7QfP3dI5+SRi9gkvGVheRJm3FtBxTVc0WZ9u3H/Hy6LVomI2bu1TF1aVX2 + Crye85JsdSFvRu+Teb//wwjPe9R2BsdgnM+Y2Pptzuevn1MV+4iwElrbZm62OaDuvS2p4O+sTtpI + wwX1pTMQQ3KlZnquHQuwXjwYJucgZ3gfGxD6c4CF2xKjyVq1HmoX5obpZ/vWTZRre/CLjYVlpfD5 + n98nHn3SGvzI50YQCbC4GzbDz/nWDC3J7mCsIyAb2AfdYL0PhULL+UbCL95yIYqKHx9964Gbqdq2 + 4l8esD8fGj5H7SDBWzx7VHn6qOOtNCpA2PnK3IP6RBPPzrLinGqT6XC+591XT4BOtjmd5sf0xY97 + AoYVlcwJ9LM+V1IK6Mv/bMM1Jxf19NTCrix8En7zhmntOU9Udv2Fih/RzH/4jXh0IFQ6tBnnh1iT + QaKOxiJm39Gcwm5Gz9kCYrmj5q8CLGNweaH9+DsebktSK9fncsDzvO7Q5+a7CjzszKCS5nUxLU7v + C8ifHSHaoskR7QyEgVf8ietcGfx5t921AEKyZdvKOfM3UwoZFdITY9Qctt1P36JUQSExhpQijnWc + wbf/FOT4Gk+1Zfawv00j8dRe6uheFBJokFx9/de7mTeGM0OZekfmTpcbH7v+U8FyXdjE0AzeDZHa + 7+E+lQ7DnjTqs10KBpI+5kwN6R7qU5LdM3j3p4K5oyV3c9i8DQC3dckmPttN/5SCAKKVJTO3kwM+ + LDEzfn6MqmzsujGvlhnwW6gy07nG+bzbpm+IEivFS2URIL5pkzesg5gy3RBnnbudd0GPlc5YCK80 + nv39B+Bc7mSmlbHRcWHjj4o+nm7M76ubPzfoGf3yL6ZrQ5DP7F0pPz4irqrIaNb6RECYdClttP0S + TS3ZP2FN3jXJ7b2fj69Q3sJ+d0oZToRNPO2jXEFsud4xDRWSP8YPf1TU5hLTMdpU3VDHFfzxgTPt + hm6Wl1ELdeFnWDg9V6gtNtQBR156hJ3Oh+6nr9HDPhvklFU7nT8chSL86m1yvFAev395Bg0Tj8pr + hXVvSzUTcKf5SYdkCnXWjXoE+cGndJGmG1/6+fPa0z/MyYZHzpXyqMEXT+jaj9Oc6+LDA7UpYpJm + Rt/0P3+uX4uZbfaaEPOHFr8hLs4PZkf82PFD7CiQ5ROiSqlGPpe0A6D7dHUolO8pZ4Hj3eFqtyld + cE2K27G5U1iR0qTjfC678Rx7WzRs9wJxpt745SPzL9+g0rV76ty0k1mVi7dO7EXQxpPeuM7/1Oub + xzHVXp8he9Q67WN07OjWVPD6i8/k56+Xo9R5iDVTRZymHPwRVTcFDoOW/vm9PqnI/ZdXMuyu1byX + xNKCagwr4oVW3fBr1AjAHheFWdlraPjq7Pbg6fKJbLDUxVPr9AJqp4Az1+lqNPmPKoNvXsp+530Z + fvwIlKHHxKrCuuHCRp8h5FvM8C3ed9xbZgIYhtezH/5NTbkuYB0rJlXNi9pw5e63f3pme+J2vvzl + qd/zQrxsV/ERxVYL1e3S4NWbOf7QHLIzjILYM2dnR90LLTcz+taTmaA/Gma5FgZ91hC5EF6j8SFm + I9LnLcJJUd9iXq4wVR5K1hAnPYDPPHDuIApnhRGWvHx+UK0ahM7rmC1tAn2wbi8JffGAnZVVnQ/j + YotRIO1exO6lkE+Ve778/AgWIF3pw6QhquSfNGLaeXLib/5wgcY0JOJbB7mpz/enBmh6FF+/NCB6 + 5koPdDy2zH1NL/+r/yQIB3tH52Kl6TytYA+/fGFrbsV4PNiaBdmtl5kXLwU06qu1pgjbz4vetavn + rzaTt0faY92Qb36a3+vregZnuVmwgAshkjeXXFq/nuOS+K/Cyeeb7J/BTQeZdvWZ8TFcJga64Oj5 + 9RspnzNDlgFNrwKrVh7lYsKSPWLbIsc/PvoIU+rB51NlROPjkw/GkL5BOz1Vgm/044/90rmvv/nY + d77ChiY4voNZGS9iLZaNzufmIMIunyfM2oh2o5bJAgyaGxKnRiP/5W1IfdwNqnvxTZ/i6/uJzLES + 2Ga4reNfHqlkYXkgRVLGfu8QAJRUMceTZenNfFbeCXzzBILlweZTuttQOMmPBYVffvUe9x4qlTKn + 42rVN4zPbf3L07943fNR5bsLipJ6/Pu+8Vs/KC5NRSd50L/vX0roGQgdM0yn97/66YJiQRiYPeyM + uJ8PzR3qO3a+eL2OeeWzGb58SIK1G8SdP15moEzZE7crpbwXmTPDLw9dfvX/KpmFLVjGWLFtfh75 + R02HM5zvCsH54VbHtNXdGV2Gq8LMLFvHg17kNRqSjH3z6MaffW+rKHHLfLItVpU+VYreQ1i0Cn4u + h73+tgVhhP2q78lmsVqiuYPCgBfSTiS1g7jr9dWkgVlZL2KPtar/6eFv/sTCqLk1NDahgqRO73iS + /SdnUjgF8M7XJvHGbN+Np935Dkut7rGAVkozfvP1X57DvGynccbXZw+metfgleOJ3XKJ2d9+4e/8 + re6TpQB+UZv4v31EvO7e4Ob5npFtVse8tPIL6g5yz9Kv3uaGsGyVrz+gd7Opm0mz5wqgrVfMv+Wl + P6dUqP70QMhGv+lRjFvUnZ4WwxzL3fTInr1i+iuDeXdzE4urgRVQL/w1Xl0Xld5nx14BU1g1xFEO + li8h5Q5w+HweWE4Phc9bmkpIeJ0qomtDn49too5oGVsi/div1u+cXJ3/5lFX1kuf50p6AcVS7sT8 + PHV9nlVXgfNF3bPYSPNu1MKSgpQdl8xwWyeWkNILCBQV00V+RfGUkvMeXS5UJcR0L5zueKCgy75y + mU/QEH/zZQH9/JphPBkaXyv1jX71dPR49D/feiMsTi3RzFPsT61myvB5nT90nX6O+qyGxRNO5qNi + prIIOK8v1+NPD1FhTe4+TytxD7JaT8xaxE/OYy1NoC2qNTE+4iOeTsLegNuZ3lkoo1YfgqoPYMwe + HjGc4dCNEtpS8Bqzw7R8NXkNcMxgHIULveVPrg+T/TmisFUrZmr87v/2O3Ce1JmOVblqOvP4KdB6 + SheEDKDyIUurGRXSHTNiLnz+zasV+OoTPJnRsfvNG/r6JxY4zg2NyVMN4HiZlsQwN8/4D4+W+7bE + QAZf54kzntV6XCaYe/HNH6+qekGfI+y/+6ky7/YrJUDPxcJjG7ML9flw3VH1kClvOpluxMfq8Ilg + Xlyqv7yfpZ84QRMZMtp9/fv4yyMvNVRsH/e3hltXp/7zZ3Z3ujRjdbjt0WnER6ad9mt9qvjFQMeo + jMjWu+nxdz9wQXMh/88+qU/LrAZ45xbB5NzHDKcahbLVDaa1j3czMWoYwHFYMn3j1Lw/5zcLchXn + dNGFm3xOIR2Ry94icYPT7M+Pg16rv30LSdObPu1tUUS+Pd2JE+a3eLzPnyPkVjbh/ssvfIxoBGvp + EpKwvtjNLUXkCN/9Lfk/AAAA//+cWknSqzCTvEtv3RHMqFgygwEjDB7wznjAgG1GCVBE373D3/uX + veoLaKHKysrKShuCylrjxyCD9lqVeNURGaYQf1y4nx+A//YRkUukG3TKZonn371uxuc8//MvsbFd + U7S+iyWAR/SNsOmYh2xmF/em/upH9SzMMtYsrozaXfOkOhlJRjwmXyG8k5Lav/m4+Nbog7Q7HnDc + PxJGvYDr//wvbEnUDOe+uujw889J/9N7f/4HfJF5xtGf//0oSQmL5WJsrmc8zNQIGvAFi4vR5ykM + 05hqj//sL+uZDlTYNA9FbLsiVgXFGET75YswWKeG6kEx1N1u6Q/g8Kct3grv7zBXbc5rPz4j8+dR + obkv6hy22+wVy9fp/Xf/4sGZuSPhdbUcVmRce3he1gD7vvIOZynSZcR774S6bZWheSMEKlyu5pMa + khdbc1CNFTrsSzFG3vRia6bdcxUOb/QPb6PqiwFopfOkIU/Gek1RDbCeL4xUP//751fniNqfKL4l + S40YCfcbEO2Lgn/6cfh37wwOB0JUt1ZRe1KaGRbjKtDtT2//7sc24rdpT81+0LMFTkEJqis31Fmx + YzEemAn/9ZcK+J///n8kCoT/O1GQ1oeWCB1UxdyUdYy6sUxolm3fYZPP2gHZYvCm5jp6oWhPsotg + I9jUCqQinNR1n2u3fn5g92AW4eqc3R4aexMRLXaFbH0QvwEiXHQcvgcVsYutmHAhBo6j3NJq8uaj + Flb0sKhHfLloHVtuETv/EgvmIR0W/RioqnTOdLxt5CRkg50EGik+e7pr3C+ac7Wq4F3wUyyYhYXW + ZHdtoeW0MxHmaM9Gv3qc0C7YGjQ+DW49kpuhImPiHGymuAsXWUU+mrgDpvsD+4QEVWWiWWRbUOtl + d4g9z8oBGbpXxWisnhbVtIsNPbuuOHq+Ttm89pSgw9svqbNNW4uds1lGurtM2GaOkpG72J+gvD0F + GgqGU5AaPjlwWArJKGdcNrlDmmjd9S3FcpmP9YBNPvr3H5FAnLC7cdoJljXAZA3Dtia+G/pQvG49 + vdy/WdZ+4lsCnZ5ivH0YA+tNpZWBXDYU25S/1d0h8lPobviC3b6rrHlOLjY6ON0V6+h4HVapEz8K + O48ODc+1wuYSZTLEjrylD2XaINpFigzqpiXUddRruL55u4XlqO6ow7hjxlB00QHSt4UjwUsKthX3 + G/Ca6Imf17NiMYznhxZg6xBzr/ierctJTGC9vHqKX8Ia0vCbuoD7NsZ43JTZaAXWAb739k02zDqx + 1eruDSrKVqFY/V7ZJO7MAOkOB7FGm7Ie3k1WgueeEN4dG5JN/hFSxFhwwla4Z8Ma9jGooVfnRLEN + ZViCwLrBfrcu2OXqSzHX3+0JhLNaEN4gjbU6Gdeg82T5eCvtpnBWBTcFdrJranSqMiyEu93g/Dgy + ava8MIxB4uWwe39TImuBkM0i5kzoiktLlFOdItZZNkHy5pbjLIUWzS7xfGCp9MTBwzuxyflwK3pt + dUL9b1MUjBRngl7vdMZGnewHdu/9BGbx4sXi4k5sZeHuBoVW8IR7dFY43s+JDTWTGmp671dIhCiR + tYdcv/BOuUr1xAXjQ7Ujk4vnbndBc5MRQNbmllL/2Ils+cg2gJCTCFt7VAzjdah6zv12H2y4wztc + UPEFWIVlxLEybdiUv+wYJR4xYil4DsWqDIGOHOfAUzNYZmuMzRTA3YlSrEpZHi4zG3r0Ol8ivNWF + YFjpql6Rdhtb7MqNi2a/hRtIubvg+CIda/bNqxFtTuadDNwjQf1ef1xhFMWIeqSa0XRN7RPQg+xi + D/qFLbdI6VGm5wXO1bAaCKeiFMIqlHH4SXE97/zF1uSHlhLldBKH6XJ+6vD6qCmNbpWFVt1veRSY + 6QX7g/guiLFwJ1Sk7Eh1fy7Y2n7yGI5xeCfn7563Zv59N1FlVCvdNgNj603TH1p9flfUs4w065wP + Nyun90nEuB3Foo9I94CnF4k05Hd8QXl5F8MeXzMavN+UtcEFPuo1Prjx9yP9EgnV4wOX51El1T4u + 2Fpz2g3OthliS71/i/ljUxcR/otjwZfxML6UJIVkjxXq1uE2bLNCyeGz2T6pU8kGEpbJKSGtKgl7 + peNka1sbPtTtGODnAwc1Vfb1CCUKg7hphd5aNdjI6jUiDXYi+1GvnDTG4O4sl8zOxsmW27ZvkGPR + NObGjV6QwukTUKR1R+PhfWdLk6y2tq2/9g+PRjjfz4kLQYzreLqbXtE+HcOHPc4zes81pZ7ghnJk + IL7D3nd/CNcueo7o1sFEg+pSspmkBwJ1fslwxC25Rai4faCjpds4pndarN+XMKq/epMRwgTN1zZr + IKHKjnpPVlhsUHsZ2VdywH5699DUHLb+v35ia6gVyzpsVej5uMbBm/OLOX3ZLnivJaLx5vLLjTBT + VhNTKqiefZ+MNIYP6hg0AXZ34Ymt71lo0Pu6C3F8uWvh0t8PKVha/6LOb56uWyUWkaoZEd7G8r1e + Rb10tc1La/HOu+2y+ennMdK6ZxNDbt2HxV3mK2x9aGgyjaU1f/LtAzQZehp+lKZooZJO6N2Ne7w1 + zA8j+AhXJCJzE7+T7ZhRbmmTPz77zcs5ZKR4EnRU3gQ71X6yVuW7ghoTxcVu7vloFfXWBjPYeUQu + OSObq4qI6Nb5gMOnsSLy5u0eHPtlYWPivsVnud+uaBT5CO+FpLFYISg2XPuPj93enAoSaN0B3rsH + pp77vbBZrPdX7Y9fAvQ2LTEv/ATofDrT3Sm0wrUwsxj0s3qMFd/1MqkQFBcsLrWwW4edxW5P3COV + ezzjaT8CowiXNuyjEGNDvrFwLmpdVfv1bdBf/TLJKs0GWXTVqB9evIIcBHmjJLWf0lg9sHr+7IcT + 4h5lSfWMB2vqKgKg8K8t9m+TlE3RJv1AaQa7WPj9JxtnjaD7pkBYj4RuWO3vBeDO4ZEagf0jCM/x + 4enrX2qlvoTetmTZSPLfA/ZTSc9WbRckCk1vDOvdd80YfaoRiPsowakW6/XMhPIA863J8U57vgtq + DCxH3Z1HRJSXOpyrVQ3AqfIvYWpv1d9BUzbqou1b7N10qWbHI1Toox0Jdl4SP6zRMldIVPGOgLd1 + rcnuXVGdZfFKPnXYhfOe+L9EnS3GQmRvampQ86YoRRnSQxVW4fTjeyUPbBS3zDPD1Y9eCagar2Ej + /3SoLTcVaKYAGO/iKbAW5yOtsH3HH+oFKQ7pYBi2GqDPjobPYijW1HiJYJUNxdFBioeFr9oAcXdC + qe5snEKIW/4DA/Lsf/iYHppCgC+OFQ0Su0a9F95F9OtX6hjXU02iRS7/6R3dDV/DnPPRFWwRTjQS + DtGwWpSN2vNtX3G0++zRKn1oCUHX8XSXa5e6F2BXQfiJJurW+TtjW8p4NfTP27haUzIwNT608Bib + Ew0Kv/nx/VFHpV4NcYfTU8ikem21gRx8WkisGFhQcQ0KP/FEGp/TGZ++bBt+/0Mx577CFmt6pF0c + 0cLGU3ILnuQrD3y8pzg8a2mxiFtkwyq+79gxi5r99OYJtY25I4IZezVvW1ME9QXb5CGSyJrHh1P9 + 6VsaRspqrahqE7AMp8NO9dhm6+A5OmjdvcF21rX1MnmbBLrrVyLN1T1kxD2GJggLDTB+4OCvHj7a + da1HbQOVGVvO+QP0tRSofpVv9TIe1go2JzEmnTq4oXS/RybkTOnprx9qumd8CstyeFD7tXmj+ZBG + MfSZh3DgVH22HkdrRW2j7/CFVhIa8/4ZofwkSTF/zZJ6vLbFBwrXvMQbRUXhcpAnEcHRPOGgu7+y + FV/CB5jiqlMnmQ1rSc+Kq1xDldHQaYWMCqtawe9/Y+m3P0xd59gQbjcDQdMpCZdoOZ1AuoxZPHzG + NlyjRa7++I/G0lFngr5nIkAZzvFKlDpj2IQYDO0RUl/ZfDJmvI8putdJhWON89mSfGRAhu5UhG6e + YzGlAfCwcV9n6il7d5Awlh9QUMGjhslYvbwtjYfISksaHfeeNWmX8YTmoOPi2WkiRMMSbZA/OPrf + vMjYN7U/EJuDQ7fjNkLS9+mLUEcPn3wOWMpmvXBFCEbP++vvUHAfDx0xzivJBn/d8G+ewN97xVhx + IXEyqVF/eCDqXz2Wsn6g8VQfsVemxjAek99+tm6B4t88FzV3DOC5CCe6tds4m4+J3mhVQg0ccPkb + jeH+KqqXhPvps/2hnsXMdUHUvZZGX6YXS+DCFTS1iXBhdmYt6Pke4ONssphIflPPm+u9kVk2JPiH + t2KdyFmF/WehVE8Mue5Oc2Sjw5ZgGo7V8z/60zHRK5ZL7lXQe68nmnC2+58+0NFiz80GkHg2sI3O + VtaNRdJqxnissXlxdwXb63IDYc2udPsaX2E3HtTqbz5gW63XjJWGJ8N9th5EPGCpmPvPN4XDnKk4 + MvsnGrNiuaJJI1MMwL3QDEq7AjnXN+o4O7WYS9kWISbeMV7776WYZtVYwfwmKnZGkjP2DDQX/fQA + dsy2DJePHAHw8uVI47duMXGvn3JgDDyyBkFR/OMfHIISS8dtVvzhDcJtoOKYWSKbZLu9wgROGA9T + HWUrZ5eVxqP2Qu/a08kkb4NG6OeQxKtVZsWiUvMAv/2H7sTCZqK+YR8UTKSjbnDRs8EzAxFGIe6p + kf8SHG6euhp/02zsBc8hm9LCD6A0DjdaCLExrNJ8I/DDT8xfy5FN/i3NtUi7OkT+9ctyGgUdjGl7 + wNGHo8U/fKxFGJE1aNj/AgAA//+kfcuygsCy5fx8xYkzJU6ICFRyZzwUkEcViiJGdHSADxR88KoC + KuL++w3c93b0oGc93W7BKjNXrrUyC/ngsP1b+0h1yUwNF+13rRFT1ZQhJ3v5ebOanz543nY9Mz/L + J5p2nlVBcxIeVFuKgzWAtKOaYylf5ncM5XS7XExIvGcnKro+QeNK+XQg30eLbKLEbaUiH7/K5qBv + 2fY7PMrp+44CcCbboss87Mo+UcZIC1jxYKlki3zcY19F4yToZKvKZr58afcvqqLNm/lBd0QMma2I + krf7oQtBWaPJJXICRzIt6OLxXfJxag0ZWujezClvQ96nnyb54TOV8oKjqad3FcJyuyAzv50nqHYH + 0G4Hj4TQijnXJFWQo8dzwLL+tOIJP+Uv/PwXlztPn/XLFKNq7ebEty/vfBR23htauR2ZKT8Fi225 + p4K8D07EtHrSju7ZP4CEJEaw9GCcW+fuAnO9Iv4tE/wxfQQYtDB84jEEg48L55r99Ctz8SvJZ/1+ + gSdsd1RC2g5N33LrwZavPRIcmn0+nZLXANXi3bKQeV+r9zXigu19OOXF9WiNaaOkKPGfDl2lny7m + d7z2tFVWrFl6plXZ0yAd0OwX4eWom0jiu/YNBnwzdj2eJos7vqyDEWVngtnK4aL4Opo//UTrdb73 + h2Y4XNCQCRLZDMEODULzlX94zn58m2bueNDIFyOsfTri03yZyAhl8sCs/VH0aS+0gXq7OhEdxLLK + RzN4eGqv1DIdonxnTb98gcV3yfSFqHNp5kOKiyWBOfN6viwuPFQoj+9fvPJ620goTZYrtn6cdj71 + 9OGmlf73xFxmijldiEcX3Q+8xGJ86fJhaj6dOsl4QbYSF8tBbLkHYd04ePm81fE481H0HrMKi2oi + +SN6XV0kTPs7RWFv+XyMpgPY8oCpmM8TOIfN04VdsCVkvf+iuO/SbYX019UkRf2ZcvbO3gf1qQ2c + HGryLF97p5+Q26oj8U5WEg8s/rrw44/PvRmg8WLHBeCn6zPsH8Hqhu0EkH48ylypCPPlFegewq+U + MOPjllZj3+QASrE+0nFMp/anX9C1ktfkiKLJ6mzkZ+DVrYj//JJPkH4BhNUaT8rwaaetKdry4tox + Zun6yh/knXmD92I0aH7/OiU/5E4AtXzNibVnQTt8IYzUPmoQndI95NxZ4ifC1kUidnN5oY4oigtB + e+0IeRVOXplH8MBM1y0ey2gsh+dLk3/5RYJP0JYD9YUKPufXQH56odtXuYtmPc42puHH4tvfYJj1 + IHH9wPfFrt0O6jPqDfxl5iHm79M9Uk9GJWN+5r1f73Pd0y5dpjPduC8sWtp5A6KcH3/+A18dchKg + w5OFVNr0V85bbRRgvf109JOK8wmL0HNh9r+IWVR2vszS5gIltzjZznx0UJd4DzfluGchyEM77LMX + wIwfzHUcLaZHlokIHnzPNjV7W+MPj2b9hevZDxlNImNI68og+51xbydT+aogXf2MLoNdxns81QMw + TbGwsDoWfPYDAvWCnxfmTuqGs+t7xEjL5RVx901lvdPD04NBFjMW0KizZr/qhqQouxNDsRw0Vs5a + hvIcrnG7xI+Shf5VAGHD3hSWzpDTGFkNTJfpzw+L+XH/ugHBWUDRQq7zKXz0GOy8ywnZRK92emAt + gzg4vn7xGDNjWbzR4eUVJPwoftzmiu6iXz22zQdFPNVICq0ax5TrGWtpMc+qhMH7wP7q9ScfJ4jo + rqVi1m9/foiq/vDXkGuSL3VcDTD7Z1T+WgZajd4FyyyOKnIlzrPtCkO1YdbzVO3qKFem8avDaulV + c/3X/ZXjD7r2SAONOWk2WD8/ESVubDFDJhLvCuEpwGrqd0y/e3a7fF6eKgRre8fIxtrGXG+YoDzv + CmE/vTSEgev98RVLkaa2l7bIVnLtLNL9nmwtvmrSAa6VHdPlUza4NOMPDNdsy7avTZdP6H7G8ISF + SbaJ1s71xHzDjFd0MOPBGuJ8TCEgpMGjXLN4MpVChj5eY3IX5xMIl71WoezrLokRPcCa+UH35we4 + +1WR9wvnmMEoKB77+f+9mKoAZZrHVKPlzG8aV4elPXK82ntFyetAUeG611qyPiuy/8O/P791czvn + bZ9eNe9Xb9l6HQv5uFb3LvQ38qGLm4njGT/WMKnNnpHVch1Pp7Mowa3PHIL948WaeJ5O8B3GLV3E + NsQDddkTkVq6MrdYPOLVUkkbuJFnyazJ1+L67aYH2Ax7Hw91lvjD5Zx36JF8cjbzYT7jzx7I5+z9 + 8ZHJkI8XGM7HA6btS5vxZ/RUdVcB8RIp88fiFTTox48NOKxbyfGvIrzCCyFHQTqX3fYtS3B+2hti + WvHJ4sa3OkDkdAY7r7Jt3BzZXtTO7DuSrbZa+s1+PchwYrecDn1X+BM/qF/o+MulFe+oNYVfP0Hd + uHhj9fZgOXvKdI2et7hn5NVbLfrkygTbRHSJ3ne6JVld18CGLWIMF1b4XfNmEfrp3YObBpZ0a8an + Nus5dv3sk7YvybsBdlBtYqZ7iPvTO1HBvT5jRk6vZzsYl32kzXoFLxBmba2RlQfirXpTJGZD+fOb + oUWbNQtDRbH4TqsvPz40842Rd/rtqKrl8m6TWe/6fPUxKBQ354QnK1757BYnEhy8Qif3zfWBxjYR + XNjVxJz1r9uOsUEamFY5xo1Qiajyrv4XLTpVIFg4G7F09gDU2U9l64d34MPztVQBG6sDC6nm81X+ + BQlhY3lghiy15fjjw0xz9sRsjRFNh2meKJv9YVmrej6UH+Pw86/xAZ3KfNgY9Vv97pUXnYpT4/cH + JMo//k4Z1/ZoGKyDqP38kp/eHGd9CI9THmCpIdtyei9TG4RExMSUWZYPVSKZYH72KiPuRSyHtZ64 + EOtZjidlu+ed21nFDy+Zq1Uh//lVyIp1kwT28egPUnlOUfaFNz4UWjrre/UJbNeaTN96h3hMq3yP + gqZFVMzUVy79+klHtt8wko27dvTCrFA3Vr9nW4OUVgddVaDHh1psk39ePv3puZmvEHNnLFo+6cwE + EceM2eXr0TZBm6Xw3ik3LFRnYvGlkn6RczQjeogsI++jspb//E9PPLxarulfDI61WbGNK5Ny5ns3 + MCtDYvrbaPhYTPocb0XM8KwvmfdcvX/9G/qL986K6guUUrQiZlnKvF/uLxJs3/qJndrkwZmtr9I/ + fynYIChn/dioP79i9p/4aAa1BzQ9hXhxM2nMl9K2+tNTrjI2/jDrYXjlJhC9v5ecatbR/uEb+9Wf + sdxrHdRXw6Pj8lrk7LNfVwCbR0SXo/5EfUEVCd3nidgfPo4H+SWB+apTKs39ur/+l7Dp30Qf2j0a + 8MKX/vzLjYk/7VQHpw7usTZh7XPfo+5rlheIzGVOgtn/W+2GV/D3/Rn54oB+9QOZ7+sWl1+3iamk + F2v48UP3SISYncXX9xdvzP0IHBUmSUw4F5eSnJMkKafZ70dbbntU0KagZA+keWjWT7N/9/SnfHIk + uDWCyDbxcxXP+jBD9vd6IpZhVHmVLLQAzX7IX/+mS5tTAFHRLZhTtkE7PfbBDRjsHgQXVOHTqcgA + 1t/YoNwwqniYemkCA18X+Nff5c7mrKL79rtlv+v3KXUviO1v/OfPxv1h4QuQBk/Mtmvj3A7AUAWe + cftirn9axE+e1Wn7xW4injDufDrvJ1oery59NM6SD/ldEX/1kS5fTZr3Uuh56LDtCDnN/skA0rlD + Itw8FsDiwTtQignuLzv7ez/fDT0GRXLbuX9249NKmkREz56A5c94asdLpFNAOX2wtQbXkprSTkQ/ + vivO+SA+WbtGMEYpFa5H3C6nqp3g548Z99U7nzQd63/xECzpy6J2XrtwYpec+R2sysFL+icc3oPJ + Ei+oLL4qygrm+9Of3mI7dZ9p/x8TBdL/e6LgyDPGnNdr44t1c0/Q6uLa7L5o3HyqvNaFj4HWjKwM + v10p19qDvU1kRrb6qhz04djBtl3vceU1A6KTVqsw2sqCqsq94gP3uhQVjAVYWgbnfKon9wZQXb7E + Ozh92dfMw6h0pivb8MPDp00ddWDsQpOt8+UJMcPgNqAQp3//30VOL4FxfHzoY1wqaMqigwn7PB3J + 9nD++lTQ4y+aHKYzA5V3ztuHWiFVdNe0ve9j/g0cZKK7+CRkG4Dj820S7NHlVb3Z/oZfLbXibwp7 + Q4kJXlZtzutTm6GFQE54B5qOxrf+GIDl04HofhO3fZWuMBSsD1iIbYmzvKreSHZOa2LbQxgPIl02 + cJU3NjElqW9ZExJT3dTbjipPrJdLN10U8Kj9lL6jl+1ztyi+mlbYA/PWBkP8XZsycppDhaVbHPhs + TLUvfCN7wSzu7P2xHx6NqryELdHzYRdz5Vq7IKsHg1y28dcfdR4LsNQeBtGHe+DT7iYegIVCRLyt + ffbHT1x0sA0jjMdxqfBhuPiqejP8nG1T8Y66QCoa+IiqQRWvnR21gcjqGIlfCrvlFLON8ZIRjlYG + CZXYRrzFvEFmcviS+/0WcF4V30QTFGFDF4vmm3c0rg/oE75vTB9SpW1Mvb7BVC9rYkOdxd14lQuY + VnXPMIIn51O0UVHt74CZuVbkjRrZBTzrkuDlyd2Ukv8ZK02qpIqso/u2pd0NEtjJuUgMcvdbbgxf + ipTjc8G8EY05b0Kio6F+bEjw2x+1ad/QG46ANXajFj1nQwr29JLx9KoOFl1fPEENlPeI6fn+zIco + qwY424TjZjJL1JePt6BuZTWh48dsYt4gaJAhOxnWznrKqe5UFZAmDsjaRYk/Hq14gM61gXjb0C/H + C48bMI23wvDrhPxJXJ9kuAaWSMwAOn+IV6oMUiVWzNMr+hfP8DGUNV1U+iqnSvZU4Vy4F3Y/qUrc + c4giTQrBJ3fRPvrDZi26kN1rgdhFtcvHdlUN4LW3LdvcRzOW3Hw5gKT4Wzw+JNoOC2LSxVrBC+aO + 8MnHZ3T5QmoID5ycdY3zXXO8oKJwT8SRhs4fcRxLaHDSCzMlKSwn4msU6Vx08aoM5zPgI7VRGeVf + sl6uCsTv3vGA+M4PSHCR3rxh4F2QEvGITu/6mrP+7IkoZacMy8rZjfkesQm+GjJxH91o2/llTeGr + vK/z69+4c8Kkg0A4FOSUXG8+22uHDPqkM8gxe73LET24qOWvgDFzvv6k6oUIYXT32Fo04nasmRfA + MmlqYlL7Fk/v97pB+vGtMr94PfNCrc8C+GE1Uel+//DReog3FMnqjpBNU1s8i7M9BFNtswAxzvvL + 0o3QjFdYMdC+feXaYGqH6UNpsxmaclw8aAO8aRqKHuXZ+l1P/ebDmbjj8sC7R2J7YL++LluHplsO + SO0iRA15QcLUfPr8/Un2yAjUC0XHKuTcO8kycmi4YBtABzQ44a1Dnvi8sWC3nHIuroZMjdxnRrz6 + 7Phcis8AbkNz4hbj3AHQGxt2SXfEsmu++LDfLXXETvMZw0Bqyqlw7jZIJ7NgVnZvrDrXBl3zbaJh + KJ12Xq8eAbRZzMKz48d04PtO++HTnM/5VA+5qEaBuMCjLWHE2aNPQcoHwg5WSPiwHjVP1RfrlOS+ + 9GmnjypjRLXSpldv+MZcGzIVlBdsiUVux3IYPOUN0hTesNTsV/l4PXWATsA/dKyzpz9sEvsCS9Zp + zCnDPp42bCsrv3xmw2C3Y1LvvrCTzyLzabBEXPYarE786FLleLqhbte3FDGz05l/C712mRYPGf2u + tyVDaw1fkXdAFpcN2ejNkDO3+H7h+Hq3xK3jph1j76aioRoTRpJl5A+/+Pzea3HOf8ma5G301hb1 + /UuXiMmoXb0jVcNNcicb8dy0o+YoE2IhRCxqJWZNxWpMtKx4xiwU7s9y+nhXQIeLeye6d1vxjq3U + Cq4Rr7CEbYR6Caz1rz5g7VEv28kNUhss8VkSfFVVa7pL604lgSUQXQgnfwqGiwuhQE/0fU+P8VTO + WQqsk/Hiwfb+tNq7B3RLkgPRpfpWdtWaNoorO0eyiVZ6KT3Y9q1w2aqII94//mfaqQUUyVdm11v8 + sfjGyp6wUyKX6CGSOb8a0wD65fli+JgyNFlxkcGSUQ0vHfPDp7QsPLhXmw2xr9jk3avpnqAu0gPx + PgaZ41lOodu9c7JZ3jji+BwdYGc8LiQ42Kty7K7PDspLETBf3x/iaaNfInB4qpC16b7LSZf2GI2v + 6sSwPnz40ETXCD3MnU4yMZBadjuMHTwt2yDrt9TFI3lnkxolyY24M940Xox1MAL5gocQpYiaOzSo + B8v26Xi/SYgmTbuHtRIsiKepl5Kv3qkKsaMrxEiWqjWOG3cNpvusCBmGd8k14xVB6GTLfrQ1I+/6 + y24NB2Hn4mLZfnjH3t89XJxTSkf1lrSDt0gzMA2f4aYK/Zz7NyoikMeQhIvrOae4Ppvw+7zGXM9Z + NWAdJcHVYnrsiO1wk1cAU3BhLMhuzOpx9fbgLKo+23TL03wmLDaRfLE5MVdIQozi9wWuzBxp6zU+ + 6sPrEMAW+EgFJr2szglvFOluoLOTd9ujfnX7TKBp15rh7XndDnM+gRjiK14ugjMar/a+gXvAn8wr + hiYfx0M/qFvDb+jUu05Lw8mdgBZeiLlexZwr14cLTZPMTwByS59/+X4NqnypmX0cH4g/ZW1QH9Fm + zTYfXOadlmxEraWvK/E3p2vLfnxK2eEMC2gZ5aOplZ7myblCJwu/ONW3OwyG6/bME+PaH5YxBpUq + +IEVfv6U/JmcvR+eYPX3ef2KHzTZGTC7Joy3QyAVX2QE1z3B4v05n3l4yKCs6i/xmtcTzQOABazN + 5EM2BlPayVeHAix55ZIQh9u4c/mlgnvS+WQ7vMq/evXH3358criHZYVmvPzxN2uw602mVq57YFsN + F/FQn0MTYg3t8ECdQzzsd5qJ9qd1wNytDiW3ZWn9yy/mVXaKfvVUWV08Gy9f92/7zT6qiKKSpXSZ + 3US/f0aHBo6BE7Bf/HOzulN0bXYmjVhwbflBOu9hegFmmwda+vVlTU0084deXQw176ajO0HpHGNm + fdw3589PeVOcaGUTj5xu/oy3ATpXd4Poqn1va9fpMey1MsMqwAaJWVI8YXoJGC+s/S4fC3tzg+cr + aGb89a3RRFIBeLHGzCDp2RpmPEKOdQmI0zQ078syruBb+zFWjTr1p0sqVWhf3W0s387fdhwnw4a1 + ga5YvB2e1oAXVgB5dNUJSeBZThoz9nCurBUeN+OrnO/3BHv6yBQtTsDZOmcCOtFFSZXqvuF8ydwv + GLsmYK51dhFfqM81moIbY5brOrF47G8FZJGzpZro+FZ/4fEXnhfvyML4vPFHpsoiQHX70i6opZKf + KmUC21ofWUL2p3b5aBIKuSLMZ5jx2loa9l5Aoa21VDRdu5UuThfB4rKmLFxclZxjJAraXRbFX31F + I/uMMlSBwZj9GHprnJJ9pmn39kXCkcV510m5h0bntJ35vYFWubtUwdeEmo7ZPrcG3A+yNvNjYmzG + TTvVQywhP08lsv6gY9uNfRJA63qE4VhPrX604hvkdXkg1se1uSh91AIMfmzpEGhPa3w0SfenD86h + ekWDAFaHTm5xZH7tdGW/bYUbWvh1xcJ7OuTcGIoObkfFY9uxzPJ+yYtCE+VzTmZ8mfE8w1BG0kRC + n979iZ7GQDtengkWj+0r7n/xc87TjllcX7Td6evqv3ie+b/rDwhSGfZJa+JH6n7bCVXqGq0rOSOn + dRiiYbwOT8jNLxDcNNucF8FWANmZMNsKcd1O43GRwYyveCozIx7J6mWDQ+st+dNPn4c4gBBZV4zG + OkEz8zvAjNdsw1a6taqK4qAdnGzAaHPSWjrwrEODv1vRVXK9WV/poz7VwiaUmLnU82Y5T9jM6yPb + 7f3tMz3XPDBFt2PZZFpcWmfJAdRAeuBFUSY5z/Q8QWWTRHO+Hi2+R58JgtPNxZH3anLOy7UEnXzj + VEnFBaL7Zrhpl11kkBMZDZ+tA3MPbWA0zLzhV1kDt11YvN4hRsL57dNlcxy0jrM1ldjrVY4R1Q5o + MJueGPN+Dp3zesKhhpK4RonKQS9k86dHmE/jb9wZG+OACjOJSchu2Kf9etpDXZceM7GxzPvKqWQw + w32MZfVVWWPYX1WYdsGdhUog5pSR7Q2ai63RVRmG8TAd9QEKrYyohA8FYuGp7ZR1niEsf7ERs2ML + MhrVxGZucNXa3nl8IthXcsTuc/4P2dvGEApxSOWPucmXi8e7gTR7tWRjSJ0/ZAckI7suS7bNmtDi + k15gGCl5MCfWNYu9T3IFgYYS9uPzfNdcb+jAdRnvmlRG4yl8CtpcT+f9ccvaKXMZNia9/fHxFeQs + +fFl5hhLyZrqw8uFQEgK5qnXvB3Pn7uk2q/GJWvr8Mw5bpoGjMWFMP3dxDF9UTGD6lhijJZDnbNT + GbroaWsXYhdtyTnc3eqXr8SHUkbj3ioE7ae3lDKuUXcnKYbFtHCYc8JPf1CMzwV2bKqZr4RePgrR + JUEnY1vgVW++/T98iY7oQPtbcChHrbmlP32IFe8KvBPnEwc6TwWytXTERyiLm+YlccTwwr/xkatl + o1bHB571oY5WlmLtNWbuJaY/3a01GzcCIovbhmyDpViOir1PkCtvjsyMD1ZLefSVEWfp+bcfcVfx + SwQrJzsxO3rZ1iiv+w7tZFEgbogf7edyfdzUKDncWFhUXVzXwkf+8bm/+J/r8RsxpUqp+Chv1oRl + oUO//FyI9tIaywcFFBfPlFnRuLZ4upI7xLQ7p5y8XH8qVkoi80B8s/WMv7wJHROufHpQjZ3yuX7u + 9mg1fY4YejXP+bEFFQWOLpD7g03WpEb2E7h8oXQ43rt42Md1g3B0XjEzDndWI9rBE4WvxqLqyax8 + fslBQshwRGanpuOLc76p65Od4IWyj9C4tsNJ3SU7nzwq/M4nIc7e4CzWMblJ9aEc4ndRoSsfHmTX + 3P14IOXhiXaaX2KlXR78YeXdAyQm0cTsY8Pi/uOcOri1tkW1Of+m6AYNPLS7RgFJdsxv70AHK9kd + 6VNx9mjaDo/kV8/IdfZXJj2VZHXmh4RElW+JItW+yBbohTjn+ycfpdMo/eKdLjXzk/MkwgFSZeOO + s/rsWFROhQiC2v9gRTbXbc2UAKN6et2xyp3e//Fj1d0FGhZ2iKJRX+oVqLxTiZfiwpr1sgqBSc+0 + 3IZROZ2d8aK5TZfPeqYoO6EOBniYsY6jVnuUbZpdM/WHp9WrGcqpkYUMkobuMN3495Z/wgOWF5F0 + Y4b+2qKxVXYpbOVrQCxYynGVVHSCfWUA86Xlvu3alyvBXc6b2X95xmNw8SuYJ1WIB7DhXZiNAhwX + 9pYQT1qjcZMpCWTmbtNTqRbb0ZUyEeRXtSde1HTtuHamDMhR+DDnAt98GJkpqsFrH1ChNx8Wx+f0 + AMax/NBBCPeWSPUmg1qg1uxf8rZvjroH+Oi/Z/wpYhauqwnN/s+sN6Sy74I0hZ9fKeVIzGkbPs2f + v8ZmgInHVjlnSBa+b3JflEtEJe1AITnCk/le03Iel1kCX2M14vfQMH88hQ1Ad0Q6FtZ2kk+mIE7A + 7i1jJj2UMavGaX4m33BhRL+F+UezdjICrQAS1Peyffuf8Q2J697oM0WrnO07z0azX4mrWl9YdXiV + McqKIia/+jJus86FymEeLY73IOfH/lFoJDAEtmnaKp9+fuGbDRnbC9nD/9NvMg0vzHPUqzXO/BS5 + u8Zh3gNsfzVmzwh+eKA96mMr7fg3AOK6Jcm9pkXD/H2jdlVPLIxuuJVzVRng+cIN0Q/x0A5psDto + h0IleETNFI8/vRQ1rYUlGAI+4w1FXnvZMveLjXxJVr29musXXWzKpBx6FunwybSU6dvCsIbHaRqA + MmaSjYstxKsb7+BRPPEvf60JPFmH9iXElB8DhHrjvFMVfpQR86+ukUufj7KH5f7w8xNPOQ+e+hqc + aGnTSYhpS1UBPLTMFhhrotNaI3nvBzCZaJNoWS1jts4/ApAjfIi7G1o+LZaLAOb4I/bI5HjmWyY8 + 7rVCNofDox2nendQT00j4EGqhXIMtsEXbYrnk9az3hqJcqbwOl4RW/tM9Ws7OgjgvKolVjUVWjp+ + hQSyeyvQjxCG/rDryw7N62c6C7Synz5V98sPthbOvrUaxCoBv7r6dFRfW4stPCKhlR9PWDrrV87N + d5ConVvYLF+cLnx5GA4TBGZMmH9si/kZJvUa9Le+ZuZyrHIWjK8BoHAtttvid9lrnaWjb5VXzKvP + H//nX4Eg7BtiJ1JrsUHsEuXHH8hIrzmXvWeAdkns4yLQnrPeznX4+annIuX5dycIutoliUdIs4N4 + aj7VGrJ8KpkL0jafhKqT//gI3puPvGdKECBnt4+o+oC33y8e9Av3nXD5xV+5nPkC0u71izmaa/03 + n51Wbc+8ZTrN/IY28LBfLywuVzoXj/r1iTKzIwQz6eUPt0//BWNxI5Tb4MdL7gsmmv1EZut3Nx+n + T0dBDc4nejqmImerqtrDM+99FtgoygefkxR++l7zofKpXfWFOvtZtOT6vRy9+GHC5Loes6owioe6 + OR2gcaaUOclV8Jsq126wioznHD9GK8rDda3e5IvGfutn6CMnMPuRMz5Y7XIVXi/Ik5cWCbh98ofc + Oz0BGJWpOLQvf9BO0009cFMmm+Nejjnyribk3OTM2Z637fjo/OSn1//8V2nDtir8+DDB8YtPutO9 + 4VNdVsSwJcxnP69CP33ID86+HJef+xPlTtrip5AZvnSX1hSqHa7p0sUlGpefUwE7NtTE0tt33k0V + vYCm+Gu2aaoRPdx09YT84mbMzpEYj9tLOkEq54wKQ8ra8ePcO1Tk2YcFVfzhw9l6VKC47obo3suL + 2UGsvoDCIGWXvExzNrZQ/fCWjjP/HC7X5AZP87uiStb0Ppv7W1Dx6YKXBxvaZvWgOnIXF49Fmvv9 + 6fECsDN1VIikV9m8rcIE2UYqXUTXtBxIeXkiwqeWbSb3lddH4+OBX919YrD240/eOxDASuidmV5r + I2m+/m/9eFDteznrO6ruDHhjeRjatuqk3AWliShzJvxAf3zgQsOBbYZ9atHhqWfw61eGo/Ti46a5 + myDk3QovEUt5ZUzeAer5marByaDWuLTwHlX1ilPYBsuSBTzLIHeylnmXesWb11CbUAudRfy5P9jX + zAyQn0s6CYurwsfdUAvQ7aocl6wZremjDgEEjikwc3h92yE8bxLwki4j2Tmuc3p6oFnvPM5Y2bsm + 5/l7F2iW9l5RZR03+bfgjxs6LtZbEsiiltPjORWQCVzCdRwG/up88UwIrXVEl8vXO57kbVrBGpsG + Cdo68bueRSaszOY741s7n/hJ1+AG8omEr8H1OyOJm9/6iR5opjXd4+8a9ppQMcKuqdVLzTBAydMX + Wyshy7nwkgoIua7R14Wdf37ngK7npUy2x5eFBp87Gai7945sdstTPOsPEemrvqRs9mfo0lE8NPNp + YgXNWL5knidw5acTCzdVV1J2+7ho9peJ74pC2atdGoCZtSFZZ3ND+Sy8v7CqS52iOEh8vrH2T22O + X3Js2nW8uiipCybVJhJasZMPjzUS1a0sJxQd7lXZq1rzRD89jnnAW9qv1Qie0Q0x/2an/iDUwQRW + bq3n1+OWTfo3AJEiILgMv/mwPWeFhmt0JGEi2dabqOUB0qSL6MM1G85/9//5a2OyzPy/fsis77Bc + OfjXL5XhKDQLOrxejU9fBs9AoZ/4r9/Tby/RBHN8sy3bH62hy6oL3IxtTuyvW/Gh0ZI9iIajUKTY + K97LPD+gnp18iuJ4nohfKQcIiueVDizQ2h9eavJlzcl2ke7REG33N3gnnc02HdRtxyreoMe9Vagi + Sd+S96cbhptbROywPvtc+sWTyIaIDhTbaPw4J/o/zyj4xz//+b9+v4Lw/l5vr3kwoL+N/b//z6jA + v7Nr9m9RlP7NpL9fS6BdVtz+9R//PYTwr7r9vuv+f/ff6vbp/vUf/1z9TRv8q//22ev/+vM/5nv9 + 5z/+CwAA//8DAAle6cmFYQAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 7b6d3f542dba9d8c-DME + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 12 Apr 2023 17:39:12 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400 + openai-organization: + - own-45h3iv + openai-processing-ms: + - '31' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '3000' + x-ratelimit-remaining-requests: + - '2999' + x-ratelimit-reset-requests: + - 20ms + x-request-id: + - 067bde68ac014668b13968776001a588 + status: + code: 200 + message: OK +- request: + body: '{"input": [[8134]], "encoding_format": "base64"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '48' + Content-Type: + - application/json + User-Agent: + - User-Agent-DUMMY + X-OpenAI-Client-User-Agent: + - X-OpenAI-Client-User-Agent-DUMMY + authorization: + - authorization-DUMMY + method: POST + uri: https://api.openai.com/v1/engines/text-embedding-ada-002/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1R6XQ+ySrPl/fsrdvat80ZEpMt9h4CIgN0Ifk4mE0BEQOTL7oY+Of99gs/JmZkb + E5DY0lW1aq1V/R//+uuvv+u4SJPv3//89fc7779//4/p3iP6Rn//89f//Ndff/3113/8Pv+/J9Mq + Th+P/JP9Hv99mX8e6fD3P39J/33n/z70z19/b4tRxarabbpl4Cwc9eakb2bxoUZCPhQJuP6HMz3Y + LlxeWYsSVRBt2SZerI1BDsBDnnFdUd50DzSYa8BQwCzGS9FaxVIa3qX63Uc98RZa547P+/yCtMSU + CPY6A/Fe2aZwOY0527z1j9E8r+kMEXctsz1HsUubwylRXw+aEydKpHhaD8Prq4ZULpIsHlassUB8 + b5wYuD6iEb8uJzST85xt5PXK6PnhWoIdeiUxlsUmHg68DcDV4jldobEoyt2r8FB04Bv2XHQmYq7k + KSgPNybZ6wPqGPrudDjNA4Ir4yDcrtttErRWoxMx20slejcwLcTq+s0cM62M8X5uczSnmxUx3rsd + 4tEez9T8muzJjn2ckO9gBei52ejMQccuHOPLvQTDnlk061ZZ3F+J2kPTiCMdCQvRsKdnC861tMXi + eeUhO+tcQuEia5jeHit32GubBHRn7hLDOx8MioVTQ9HeDHLpPrErTH0LgKrVjniXuRYynR4BRff8 + S8z5dhvLXr2y4VA9SqZ/onUodmf/BsvLzWfR/ntG4tDdM1Sdo5pZwjx0/Hp/OOhVUo2qlrsxZG1G + bis4KTo5vmsfiax8VbC1NUyucNkb46sP6FowcWXWvW+M+mr5PhwfZoLrw/ZT0BgTCpIZYUKCzCvG + oOgySOZlwDbmwTdGbUYi9algn3mjl3a9nNs2Gp3jlhmVpAvJ2aklJOz2IlG5OIqRiBWFd5BQYhoU + XDHT7hlKNp8b04NuhsR70Hwox2BNDnmpddxntQ7zPJnj3BSNwcMx7X/rE++BurDyDm0EoIUeLR9t + 2Y1HKeMgSvygTCpF0bd7Z0ThvvLIQR6XhTj4+ghWHjKqzIklRnXcJDC/9CFx5qddN24DG2CRhDvi + FtumGPb7TgGFzLYUPc5j3K9KWYKI5y7RKynsxpt71+AaHq9E559dOOjN3YNCcBvPGErd8a04p5X1 + ZBdmr/NO9OalrWG6xitmvWJ+oBsFzKX0IM5U34wHKxOJet5ReV+cQn482yfAZesTQqXEpRcWOnPW + GR4zUx+HvOqqEfpvqxHXroQQwZJ70LHnidYAuSteyiNHQV4y2jcZhAMDF9AMHTkFKdrE4yuiERzk + 7oz9Y+R0wyFdz6A5yBLRyxdF9Y7ICmzbqsLztHq7dRu8PVicnzGV733jtvUHHFid0xsWn8gryur2 + Utdnvu+Ju7sWYoyRPsK2Xx9IfJkfDYEWNxWOm2VHzHPpGUM1F8naHLSBmUt974qPIAoU/djSlqM2 + Zvq+5kiVnBnZX1Q/HOtVncCRPA1m6nldMCmtKdrP8xyvLier4Cu5s9FWPgds0x4vYnzmewz5+d6R + 3dKLwzHZni7oY7AeL4pkcKd8zqGdb0d6PmjXQtzGvFrfnOSNf/vz3d5dDVW67ONZfW+K7z728Vom + hk+lKmwL8XI2ypoOzyuxaWbHvBiDHH54YsdpKXprzBJI/SZj9mG7K+jTUhOY8IiG3GIuvxKVImrY + ISFRUaNxK482MEW6kDPe2Whki70KRX1+M7vkR4OvtG0LhfK6kH2vpKJJVsd03cnJyLbyy0fDIx04 + LHRk0LEuXWNcfmwFpnhg5X1RxfAd1AQuuuew+3oTdpyrQYsk84apMBW1o+HelX7X7HrKW8Fwb5eg + mN8TZlP8x/N7X0PQRDH74Ss764qMjlX6onl7tAzxeuARRfTRM5KEWcyTtznCycsshp/fj/vbj/Uz + Gb7MOTp1KFoCMsqQ7ZDDt9iJsuoqDtpykbGds//GgjsfD10H64VHqS06IVb2DU39hO3jQo4Hqzjl + qryJFboKjm04qpeRgq3iDV4vqtJoPCYw2t3TNbHnxEJSu7xEQIS7ZeasXKOPe5Nt1DceIju20Qr5 + dkIpFEV1Jc5Yv8OmXtoWrItoiVX6boxffgA6nRh7GmljjMnVbsEo1YJ+Xvhr/ImfauMXXgKMaHjU + iYeC5haTDdlHot7nkQ9K3A+0ksJHJ3gwWCpVm5a4j/MYslujOYCyXUzsl2S5i1Cam2g1kxMs4Xrj + jrPFx0ONmx7IYbmyBW9mKw9JDx4w93XDBb9KhQLpfSGIt6gkIR7Hp4PsEJfkYJ1FTK0kSOBQZzo7 + kjYPe+f40AGWSUQC/vmE3Ct4hOLW1Fi4kKuYYnxPIZ7rc+I+9UMollDZELnfAq+OXdYNB3ewgL2k + FdmuHzEaNSI4iufanFhjagih0zugwt5LVMGjXiyicPDgJmkxlWVN6Zpzb/Qgdas3sexrJrjh3R2U + a2lP+yPZG4O8fEfodVFkElrvc9c3ka2jvpJOJGk3hpBTyFSIgu8Gr0quxU3vRR641zih6aVYdfWG + hz6yr+uRLvhw71o26y/Q7qoYS5v4GoputsyhPqo3ejTKRgxih2qIt0+HbPF3LIRUGzbUsdXhRZfv + uzGINibEyiHE6+XKRkOktBFkx/I89edH/FboqK9P68OARdBhg4eHb4Ye39WFOMwdURfdHEV13ZZT + 6f3VBG/smKL7t/KZtckUdzzvVQWGs3PGUrjk6CtTHZBHeczuu6Bw2YSPaJnbQCzjIIw6dRMbNl5y + IS5t1Y7t1E0JlS75xLMrw+Ccc4w0CN4Mp5rnDvsQX8ATLGZ6kgYdP2WaClQJDWI5+5lRSaGcorpM + OfOoqblc7toSKafVltgfYyGmfqXCKvFNvN5YRUf7tYVRtfukVDl9kpi7dlPDYWUADircIFFvuwv6 + 2PbAdsN1IwQp33x9zI2EuA+UGN/LfF6DXGmUOGyAjs7y/PLrt8wruryj1sagsHyWBTHvnzoehyj3 + UB4aJrNdtBI8kQcLHl3OsDo7yLFYRysPJv7zw9uOrz+mDtL9eiDk6WExLlzDUqf3o+IT9d1vf9ej + kBGeXc2XO0C9uKBZt51jJfuexLjniwSG9FQzXXMKY1Se1EJ1bu3xMke1MaoXtVdHtf0Qi8afYpzb + Tr0iy8phe8sdxIRXEZQPx2MbzRGC3/2GAxH7Lfv1P97Oax8eN7gyrAdWzDbL7UVtRaYQ4otj8Yd/ + 3xTR4A/3/W6Y4XMLdW7u2dPj55BLlvOnn1F2vKN46n/O7/fozOsKJE6rpYTyxSfBneo08TjKmQYW + Qzs6Pyytgs+Hcwbm1jrhBf4G3bh9HySY+C8xcf0y6GneXGA2bh94tsVDzNePVwX4srCIZ29RV788 + NYBbRY50ceEUFdk+koAk+w8xgq0ZjndDxmBFYLP96eN2U7xM0ALlhWcZLF2mkH2F3H2wxyI8LpEQ + K+0GYj63Kb2aG4OfMluBeaWY5LDYJgXPOETIxyvOHGVmFQvn3VD0w+tNNxtEd5AQoOx71oh3fTsG + d2a+uf71V8/U57FoXy8HPuvwxcjNNw15z9cJLPz+wOz+YBSj66ETKEGeMptZo8tIbJyAbRr7D98Y + zr1BV+LhzdkOfXUkRbKuQxDcZkwzyqgYqRTJ8GoemGkLt++Gg+r16P3eWHixT3M0WAtb/vERlqg8 + D7vIulQoXK78ib9UxbiNuwj016whu1PuoFFFhYJ+9bCtVkXH/dMxgI0qIqqo23vIj/XBQ4rJToSo + 6qn4BlspQgcrFPi7shXRrD5XD6KrY7Gd2M/dFo08X5+U9FdPdihXn4MJibGe08+0Hr2dRArK3r6T + h5ITdwitLkXqd58xY73SO1Ge9ak/lldm673jjpeTd4NpPboijwf6Nuzrqx8MD+K6UWbwu31VUb4l + KiHf2u4GY721USZ/txSYL7r+IkBGRIuA/fCg84+1gpa3cYlXXsnCP3x4XG8wM2sjN77jzitRJl1S + YtpKg5g68wK4DuaLrul+6Liv9zPItweV4YIX8fD2+bie+AWF69txhakUKUTpEWGVDclPrzhQiNEm + G38VhL96AdhZClUmfBERzpL1T/+tmdx3Y2oZt5++xmtTxx1ltiUBBWoTHG5K8bXs7fjbD4yOfBPy + FzqocL7VJ/ZUPkxwqUUZrFuvINuL83Y5VsIb4FP6ZPsZpQVfxUmEbmuPss3n7KJxrzcyaq/cYwcE + W0Me9/0MmJ1KxGuyJOSfmeNAKJw/ejXuzl4YAOpOHiEUj/GoHnx//bqoMh41xzDG+Aka8gW90cUm + tjqpfD0C5OGyJc7Hlgv+2w+jVAqmlQl2v51b2GAEFxerZ4HjMX1FDirmZ515j/OrGz8nDcPyuHYx + 1zrP4NLrbsGQXmra+Y3vCpNtdHj75o6Z728m6KmMSmDIW//0ZPe9b86SqgXqixiP9hwORecn8MN3 + Ih9xwY9py2H3UHQWWFlp8E//kGFWgEOHukSIf+7+DBqzfjHD7Sv0w3/VPAfbKV/zrpatVQSLu5pR + +QpDwavcPIG9l55Tv70ZAkWOjezz22U//i5X/JPDtN9kI2Iz5trmVqF2V8aUh7ttN0raqkRk/yYT + X46K8fLmPZyficZucVqiEV1fI4r2FyD23dXcRb5VMHyvxZztd0FhMPel6UiXVzVGyoPG7Y/f1zdq + 0uWw7opvk9QRpHxxJPaoP9x+9+owrCJSYZpdvjG/+68RfvxMU6Mk7I7vREJhMDtiubd1tFRyK0Vq + MT8Q/ZAwd0i8NILJP6Kq8ngaYzNuaxirzfCLd/fdh/gE/aUu2KQ/xDBDNwq3tr38qTfelvfsx9+J + d1yMiL3C/gLa+WgzO4+YEIvrQUGy1zbUCR8HYyjupQNWs4vZfn5S4vHu2SZ8tqNDPPlrGX0fJj4s + IFWY0yvY6IHNb+jM3Z6um6zrxKd5pxB8zmu2ey/D/9IbVYJPePnCB1eQuZQBvpMv05L7IARftgG6 + LpaM2RMeCho4Ggzhdsm8AExXvML+pNIderHdFTLErzPrBCdmpczZ7zx33AbaDMDsj4SY35U7rrKz + g35+Qrv/LtBPH8H48gtyqhsXjYt0N4OPU9ym39MEv0qdinZBc2QbT5bdIRMJV6XHGNAZ3WfoW8lH + DSL95TP3qX/jcf/g1U8/T/xzGbfKk5rQ3zcOyUV2/i9/zFczk6RJeDS4BY6Kbqq0I5EbcdEFm4ii + cF96VC47Hncnfr7Bjy9nu83B6KNI+DBDIaeLIdi4i+LeO6DLqGZa673jMfhuAVp829HhqdzjMTbf + Aaw8MyQ3cutD9rg3OiTE4+zHB8dhHt6gMG4V22n3izumUKtwPTczurwsSDe8i0WNnm1s0R9fZWmh + Zr/8oYMny0ZTlSWFoBt0ulp/HrE4f1odHUprTX7x4WW7pHA/YY+OoVeFw/T8+tfvzUlf80RemdBo + JMF/6qOOVyeIW0uj2e5zQd9FzKrVKglMooeeFcrwQgG6NN8X2c7Krzt01aDCV/PuzPCQjOh8ltlo + 8isZZmjmUokdMIz94kF2xiEPOX8aKtBtrjLC0DcU377pf3yK7Kb1x6efqOhilgPbD0ne8eNZu0Ab + 2YK5bqS50/o+sIt5IBN/LcS6DNs//I9c90H3i+dPTzCnyj00bqz7CU5ebtFZkK3D8aPHKppn1o1o + srLrZLJOHNhp1opovpSJsR4u1Q/PsOpIO7c3BucE587r2aHgJ7c8maODJr9o4pNVyNLAwmDuEkR+ + +kFM+gNFozri+MhfoSjdTEfPISqJpxIV9f3bLqHwyhVzL/dPPO7PFQUpu3S/ejF6CG8W4ndt99OP + MXs5GxWtokNFdi881X93+8MP8LjrlwUrV6hVmXQ/MuIfdmLQsZPCwekV4q2MT5z/8lF+vRu8/JTU + 7R8bp4ZbIbcMy2OFhv5Tz+A4kgOdJ6EW/vGX8f3wnd5X+vE7C5qrt2LWp5yhQTMUUH/43Lwvjitt + 3NZCqbopiaM85m45oj0F01W1P/1NmfzS1da5LYjh3/bxMG/jBHh2pvTNXSaE+JxMdKF2xfCluBfD + 81lLwJxljOWy82M5fko6ynf+DcvYU+NmFzUOxFDfiOW074L98uf21dfEXOqNO6weEl9p+NgyLzt6 + Rb/fGRmsRVmTXVYXIf/0Z+mP35ZP/tv0PjMo861HyORn/vw25IIW0HiuZgUv8S1CDS9nbMuHVSFi + ye5Vub6fSZTsSdeb2cFE7BsLLGnEEHytKCc45/qd7DbZLhQ7dVMBMeUN5StsowHUV4ve9HCncnXr + wqlfVyg7Vmc21ZPBreuRInS6MCyNd8PgsOlGiFxWUP7IDIPb/neG4oXcT3qzd/nzoaU/P5s59ksP + 2boMaziG6p4Z98/K4MuSjLARazrxlSCsV+VMhgk/CV50i46WX0WG9fbzpeN2nhRSvJ/pEOo8+4P/ + LckPN2CbzsaR2r0KOvnL6MzmKnOPZCXYo3BTdFtj+tNv7lD4G6o+hXAJaYenGHtRcJj8RdyyISmm + 9SVQVWiJGTgLJCIAEwJfu5JLNTu7DC18BV4f60O2NV2HIi3GHCZ+y4xl8Qr7aN4n0C5WT4w6+y3o + xjqeYDO8d+TXz8TnUpdA5xbDy/ugFkOeP3O0fun+T28bbAhrB6Z8xTN2lmJ5mc51SMHU/9SfFO2t + GTy36e6HVzF9dl0NBXOPzD1Z+c+fz9F3f+vZvdBsxJfR+6JO+UprPObFQC56CeH5orCDhR4ud2Oa + weQHE40u3JDWQ1qhWVRt2WFlK2isPltTFdXHZO7EZ6XHiWRQrpdLzD3/YVBFBRl28bL841ct3+bW + g7V3v2P0QInLf/V3adiLOPtd7w7K49Cjhlcz+omUDjWWfRgha2f3if8t4iFZ3VPQb1VJtvtUF+PF + aWZQPx4BS1QSoSFovxV8wVlM/qEdSixNZPQUg0tVyNA0H7lF6BFfZgT7NPvz/5BCYMu89vMtJn9Z + RfJWWjPXb7gr4t2hRhP+Er1xJNRcNTqiqT7JQXoGiKvvtwLPlycoOgcXg59wUkGMmwf7zXPGMSQR + sG7jUdE4pcvnXnkBspoNzDpoleDFoTlBfK4R2SyGd8H3eeDDhLdsk8xb8Z3mcxCmL5cYpXbpBscJ + Zn/8pfYht12qz+QUFqp1pPn2PoreHFYRcvnwYvvsW7ryorpLsG9X/TQvkMImb5sbcrX7nPzmRcxW + eYv28omwnSE8Q0jGcQYT/8dStTrF9HOyMZr0ODOS9IVGiR082LKzREgGVSgccZbgWetPPGxV1xj8 + UbutJ3zFwjAyNEzxQ+uX5jPnY1+67senh93VZdhCXjg47xdddwZtqHyo/IL7p7sP1jfP2H7KTzrv + hYT8+TGg/VlhxlA7Bx3uj9uT+bHzEj/+DbtlsCNkf3ka/LkZKILUOrMdzVaGuGUz/6cPf3zC+MPX + m5l3YW71mcVU65wcMpltiem6tKC57VdgqsRkGM/qYmBgAKzPQ8q8SY+zyY+FJ24jCpa/QTx83iW0 + UW6C/PyAscNGvh7i2YY4lvsyRustKYjoQ0Gs9/gqpnhfYJcvdfz5lNgdUI19WPj0MOlDU+RexgLA + X7kgm4rlhuj8TvnNW/BcPtKOmfp2Bk5gIbJ9e3a3IM0ugcmvwpKe28XQJPXtx18nPzFA42w9WKDN + Oo/YaDyFI2qrWo2PB5nt15uw4I/TLkNrTXoyL8y/op/6xZ/9dzhyYiE+iQm3+fv8m2cVkx/XImEP + KnGKaoP+8C/v6Bq0ziOCRJffMKjDeCWu6jSh2B1wBrP1BZODcyMdL8Yog/XgbYgz+XF0z9cpHFGc + 4ddmSRELWHVZTfXzZ569GG+KCpM//Ud/dMdYvf36A9Hv+afjm0qR1lP/pT//hJ+a4gTz+Pn6zaOM + 0e8vCmRVcaISjFknhodTgXK/OET35Hc83Kmv/OHnRnw6uTytWhXsvfxku1lzEEJfQoTWjjZitahe + QujJt1bPbK0SI/WfxlccLxZc/PODbV/7PuRuYJqw3rUupT7N3OGcNz6aVZcBPxZV6f6pr998WU/d + bzfljww+KB1dTfPq7rGTeliElvTzh4rhnpQl+vGrjT7TDO6vnOSnN5ibjaYYOAgd/v6dCvjPf/31 + 1//6nTCo6kf6ng4GfNPh++//Pirw7+gR/VuS5H8z+c9JBNpHWfr3P/91COHvpqur5vu/v3WZfvq/ + //lr8ee0wd/f+hu9/5/b/5rW+s9//R8AAAD//wMACEOkc+EgAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 7b6d3f5cea949d8c-DME + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 12 Apr 2023 17:39:14 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400 + openai-organization: + - own-45h3iv + openai-processing-ms: + - '271' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '3000' + x-ratelimit-remaining-requests: + - '2999' + x-ratelimit-reset-requests: + - 20ms + x-request-id: + - 029a9d93fe8a07946d39f3e957b33c11 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/integration_tests/vectorstores/conftest.py b/tests/integration_tests/vectorstores/conftest.py index f7171a1a076..9687af92ced 100644 --- a/tests/integration_tests/vectorstores/conftest.py +++ b/tests/integration_tests/vectorstores/conftest.py @@ -1,15 +1,49 @@ import os -from typing import Generator, List +from typing import Generator, List, Union import pytest +from vcr.request import Request from langchain.document_loaders import TextLoader +from langchain.embeddings import OpenAIEmbeddings from langchain.schema import Document from langchain.text_splitter import CharacterTextSplitter -# Define a fixture that yields a generator object returning a list of documents +# This fixture returns a dictionary containing filter_headers options +# for replacing certain headers with dummy values during cassette playback +# Specifically, it replaces the authorization header with a dummy value to +# prevent sensitive data from being recorded in the cassette. +# It also filters request to certain hosts (specified in the `ignored_hosts` list) +# to prevent data from being recorded in the cassette. @pytest.fixture(scope="module") +def vcr_config() -> dict: + skipped_host = ["pinecone.io"] + + def before_record_response(response: dict) -> Union[dict, None]: + return response + + def before_record_request(request: Request) -> Union[Request, None]: + for host in skipped_host: + if request.host.startswith(host) or request.host.endswith(host): + return None + return request + + return { + "before_record_request": before_record_request, + "before_record_response": before_record_response, + "filter_headers": [ + ("authorization", "authorization-DUMMY"), + ("X-OpenAI-Client-User-Agent", "X-OpenAI-Client-User-Agent-DUMMY"), + ("Api-Key", "Api-Key-DUMMY"), + ("User-Agent", "User-Agent-DUMMY"), + ], + "ignore_localhost": True, + } + + +# Define a fixture that yields a generator object returning a list of documents +@pytest.fixture(scope="function") def documents() -> Generator[List[Document], None, None]: """Return a generator that yields a list of documents.""" @@ -23,3 +57,18 @@ def documents() -> Generator[List[Document], None, None]: # Yield the documents split into chunks yield text_splitter.split_documents(documents) + + +@pytest.fixture(scope="function") +def texts() -> Generator[List[str], None, None]: + # Load the documents from a file located in the fixtures directory + documents = TextLoader( + os.path.join(os.path.dirname(__file__), "fixtures", "sharks.txt") + ).load() + + yield [doc.page_content for doc in documents] + + +@pytest.fixture(scope="module") +def embedding_openai() -> OpenAIEmbeddings: + return OpenAIEmbeddings() diff --git a/tests/integration_tests/vectorstores/test_elasticsearch.py b/tests/integration_tests/vectorstores/test_elasticsearch.py index 06865a5a788..b79d2b6bf7e 100644 --- a/tests/integration_tests/vectorstores/test_elasticsearch.py +++ b/tests/integration_tests/vectorstores/test_elasticsearch.py @@ -21,6 +21,11 @@ docker-compose -f elasticsearch.yml up class TestElasticsearch: + @classmethod + def setup_class(cls) -> None: + if not os.getenv("OPENAI_API_KEY"): + raise ValueError("OPENAI_API_KEY environment variable is not set") + @pytest.fixture(scope="class", autouse=True) def elasticsearch_url(self) -> Union[str, Generator[str, None, None]]: """Return the elasticsearch url.""" @@ -34,15 +39,6 @@ class TestElasticsearch: # print(index_name) es.indices.delete(index=index_name) - @pytest.fixture(scope="class", autouse=True) - def openai_api_key(self) -> Union[str, Generator[str, None, None]]: - """Return the OpenAI API key.""" - openai_api_key = os.getenv("OPENAI_API_KEY") - if not openai_api_key: - raise ValueError("OPENAI_API_KEY environment variable is not set") - - yield openai_api_key - def test_similarity_search_without_metadata(self, elasticsearch_url: str) -> None: """Test end to end construction and search without metadata.""" texts = ["foo", "bar", "baz"] @@ -67,15 +63,17 @@ class TestElasticsearch: @pytest.mark.vcr(ignore_localhost=True) def test_default_index_from_documents( - self, documents: List[Document], openai_api_key: str, elasticsearch_url: str + self, + documents: List[Document], + embedding_openai: OpenAIEmbeddings, + elasticsearch_url: str, ) -> None: """This test checks the construction of a default ElasticSearch index using the 'from_documents'.""" - embedding = OpenAIEmbeddings(openai_api_key=openai_api_key) elastic_vector_search = ElasticVectorSearch.from_documents( documents=documents, - embedding=embedding, + embedding=embedding_openai, elasticsearch_url=elasticsearch_url, ) @@ -86,16 +84,18 @@ class TestElasticsearch: @pytest.mark.vcr(ignore_localhost=True) def test_custom_index_from_documents( - self, documents: List[Document], openai_api_key: str, elasticsearch_url: str + self, + documents: List[Document], + embedding_openai: OpenAIEmbeddings, + elasticsearch_url: str, ) -> None: """This test checks the construction of a custom ElasticSearch index using the 'from_documents'.""" index_name = f"custom_index_{uuid.uuid4().hex}" - embedding = OpenAIEmbeddings(openai_api_key=openai_api_key) elastic_vector_search = ElasticVectorSearch.from_documents( documents=documents, - embedding=embedding, + embedding=embedding_openai, elasticsearch_url=elasticsearch_url, index_name=index_name, ) @@ -110,15 +110,17 @@ class TestElasticsearch: @pytest.mark.vcr(ignore_localhost=True) def test_custom_index_add_documents( - self, documents: List[Document], openai_api_key: str, elasticsearch_url: str + self, + documents: List[Document], + embedding_openai: OpenAIEmbeddings, + elasticsearch_url: str, ) -> None: """This test checks the construction of a custom ElasticSearch index using the 'add_documents'.""" index_name = f"custom_index_{uuid.uuid4().hex}" - embedding = OpenAIEmbeddings(openai_api_key=openai_api_key) elastic_vector_search = ElasticVectorSearch( - embedding=embedding, + embedding=embedding_openai, elasticsearch_url=elasticsearch_url, index_name=index_name, ) diff --git a/tests/integration_tests/vectorstores/test_pinecone.py b/tests/integration_tests/vectorstores/test_pinecone.py index bcfe4104dd1..4a6a8fb1df2 100644 --- a/tests/integration_tests/vectorstores/test_pinecone.py +++ b/tests/integration_tests/vectorstores/test_pinecone.py @@ -1,97 +1,208 @@ -"""Test Pinecone functionality.""" +import importlib +import os +import uuid +from typing import List + import pinecone +import pytest from langchain.docstore.document import Document +from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores.pinecone import Pinecone -from tests.integration_tests.vectorstores.fake_embeddings import FakeEmbeddings -pinecone.init(api_key="YOUR_API_KEY", environment="YOUR_ENV") - -# if the index already exists, delete it -try: - pinecone.delete_index("langchain-demo") -except Exception: - pass -index = pinecone.Index("langchain-demo") +index_name = "langchain-test-index" # name of the index +namespace_name = "langchain-test-namespace" # name of the namespace +dimension = 1536 # dimension of the embeddings -def test_pinecone() -> None: - """Test end to end construction and search.""" - texts = ["foo", "bar", "baz"] - docsearch = Pinecone.from_texts( - texts, FakeEmbeddings(), index_name="langchain-demo", namespace="test" - ) - output = docsearch.similarity_search("foo", k=1, namespace="test") - assert output == [Document(page_content="foo")] +def reset_pinecone() -> None: + assert os.environ.get("PINECONE_API_KEY") is not None + assert os.environ.get("PINECONE_ENVIRONMENT") is not None + import pinecone -def test_pinecone_with_metadatas() -> None: - """Test end to end construction and search.""" - texts = ["foo", "bar", "baz"] - metadatas = [{"page": i} for i in range(len(texts))] - docsearch = Pinecone.from_texts( - texts, - FakeEmbeddings(), - index_name="langchain-demo", - metadatas=metadatas, - namespace="test-metadata", - ) - output = docsearch.similarity_search("foo", k=1, namespace="test-metadata") - assert output == [Document(page_content="foo", metadata={"page": 0})] + importlib.reload(pinecone) - -def test_pinecone_with_scores() -> None: - """Test end to end construction and search with scores and IDs.""" - texts = ["foo", "bar", "baz"] - metadatas = [{"page": i} for i in range(len(texts))] - docsearch = Pinecone.from_texts( - texts, - FakeEmbeddings(), - index_name="langchain-demo", - metadatas=metadatas, - namespace="test-metadata-score", - ) - output = docsearch.similarity_search_with_score( - "foo", k=3, namespace="test-metadata-score" - ) - docs = [o[0] for o in output] - scores = [o[1] for o in output] - assert docs == [ - Document(page_content="foo", metadata={"page": 0}), - Document(page_content="bar", metadata={"page": 1}), - Document(page_content="baz", metadata={"page": 2}), - ] - assert scores[0] > scores[1] > scores[2] - - -def test_pinecone_with_namespaces() -> None: - "Test that namespaces are properly handled." "" - # Create two indexes with the same name but different namespaces - texts = ["foo", "bar", "baz"] - metadatas = [{"page": i} for i in range(len(texts))] - Pinecone.from_texts( - texts, - FakeEmbeddings(), - index_name="langchain-demo", - metadatas=metadatas, - namespace="test-namespace", + pinecone.init( + api_key=os.environ.get("PINECONE_API_KEY"), + environment=os.environ.get("PINECONE_ENVIRONMENT"), ) - texts = ["foo2", "bar2", "baz2"] - metadatas = [{"page": i} for i in range(len(texts))] - Pinecone.from_texts( - texts, - FakeEmbeddings(), - index_name="langchain-demo", - metadatas=metadatas, - namespace="test-namespace2", - ) - # Search with namespace - docsearch = Pinecone.from_existing_index( - "langchain-demo", embedding=FakeEmbeddings(), namespace="test-namespace" - ) - output = docsearch.similarity_search("foo", k=6) - # check that we don't get results from the other namespace - page_contents = [o.page_content for o in output] - assert set(page_contents) == set(["foo", "bar", "baz"]) +class TestPinecone: + index: pinecone.Index + + @classmethod + def setup_class(cls) -> None: + reset_pinecone() + + cls.index = pinecone.Index(index_name) + + if index_name in pinecone.list_indexes(): + index_stats = cls.index.describe_index_stats() + if index_stats["dimension"] == dimension: + # delete all the vectors in the index if the dimension is the same + # from all namespaces + index_stats = cls.index.describe_index_stats() + for _namespace_name in index_stats["namespaces"].keys(): + cls.index.delete(delete_all=True, namespace=_namespace_name) + + else: + pinecone.delete_index(index_name) + pinecone.create_index(name=index_name, dimension=dimension) + else: + pinecone.create_index(name=index_name, dimension=dimension) + + # insure the index is empty + index_stats = cls.index.describe_index_stats() + assert index_stats["dimension"] == dimension + if index_stats["namespaces"].get(namespace_name) is not None: + assert index_stats["namespaces"][namespace_name]["vector_count"] == 0 + + @classmethod + def teardown_class(cls) -> None: + index_stats = cls.index.describe_index_stats() + for _namespace_name in index_stats["namespaces"].keys(): + cls.index.delete(delete_all=True, namespace=_namespace_name) + + reset_pinecone() + + @pytest.fixture(autouse=True) + def setup(self) -> None: + # delete all the vectors in the index + index_stats = self.index.describe_index_stats() + for _namespace_name in index_stats["namespaces"].keys(): + self.index.delete(delete_all=True, namespace=_namespace_name) + + reset_pinecone() + + @pytest.mark.vcr() + def test_from_texts( + self, texts: List[str], embedding_openai: OpenAIEmbeddings + ) -> None: + """Test end to end construction and search.""" + unique_id = uuid.uuid4().hex + needs = f"foobuu {unique_id} booo" + texts.insert(0, needs) + + docsearch = Pinecone.from_texts( + texts=texts, + embedding=embedding_openai, + index_name=index_name, + namespace=namespace_name, + ) + output = docsearch.similarity_search(unique_id, k=1, namespace=namespace_name) + assert output == [Document(page_content=needs)] + + @pytest.mark.vcr() + def test_from_texts_with_metadatas( + self, texts: List[str], embedding_openai: OpenAIEmbeddings + ) -> None: + """Test end to end construction and search.""" + + unique_id = uuid.uuid4().hex + needs = f"foobuu {unique_id} booo" + texts.insert(0, needs) + + metadatas = [{"page": i} for i in range(len(texts))] + docsearch = Pinecone.from_texts( + texts, + embedding_openai, + index_name=index_name, + metadatas=metadatas, + namespace=namespace_name, + ) + output = docsearch.similarity_search(needs, k=1, namespace=namespace_name) + + # TODO: why metadata={"page": 0.0}) instead of {"page": 0}? + assert output == [Document(page_content=needs, metadata={"page": 0.0})] + + @pytest.mark.vcr() + def test_from_texts_with_scores(self, embedding_openai: OpenAIEmbeddings) -> None: + """Test end to end construction and search with scores and IDs.""" + texts = ["foo", "bar", "baz"] + metadatas = [{"page": i} for i in range(len(texts))] + docsearch = Pinecone.from_texts( + texts, + embedding_openai, + index_name=index_name, + metadatas=metadatas, + namespace=namespace_name, + ) + output = docsearch.similarity_search_with_score( + "foo", k=3, namespace=namespace_name + ) + docs = [o[0] for o in output] + scores = [o[1] for o in output] + sorted_documents = sorted(docs, key=lambda x: x.metadata["page"]) + + # TODO: why metadata={"page": 0.0}) instead of {"page": 0}, etc??? + assert sorted_documents == [ + Document(page_content="foo", metadata={"page": 0.0}), + Document(page_content="bar", metadata={"page": 1.0}), + Document(page_content="baz", metadata={"page": 2.0}), + ] + assert scores[0] > scores[1] > scores[2] + + def test_from_existing_index_with_namespaces( + self, embedding_openai: OpenAIEmbeddings + ) -> None: + """Test that namespaces are properly handled.""" + # Create two indexes with the same name but different namespaces + texts_1 = ["foo", "bar", "baz"] + metadatas = [{"page": i} for i in range(len(texts_1))] + Pinecone.from_texts( + texts_1, + embedding_openai, + index_name=index_name, + metadatas=metadatas, + namespace=f"{index_name}-1", + ) + + texts_2 = ["foo2", "bar2", "baz2"] + metadatas = [{"page": i} for i in range(len(texts_2))] + + Pinecone.from_texts( + texts_2, + embedding_openai, + index_name=index_name, + metadatas=metadatas, + namespace=f"{index_name}-2", + ) + + # Search with namespace + docsearch = Pinecone.from_existing_index( + index_name=index_name, + embedding=embedding_openai, + namespace=f"{index_name}-1", + ) + output = docsearch.similarity_search("foo", k=20, namespace=f"{index_name}-1") + # check that we don't get results from the other namespace + page_contents = sorted(set([o.page_content for o in output])) + assert all(content in ["foo", "bar", "baz"] for content in page_contents) + assert all(content not in ["foo2", "bar2", "baz2"] for content in page_contents) + + def test_add_documents_with_ids( + self, texts: List[str], embedding_openai: OpenAIEmbeddings + ) -> None: + ids = [uuid.uuid4().hex for _ in range(len(texts))] + Pinecone.from_texts( + texts=texts, + ids=ids, + embedding=embedding_openai, + index_name=index_name, + namespace=index_name, + ) + index_stats = self.index.describe_index_stats() + assert index_stats["namespaces"][index_name]["vector_count"] == len(texts) + + ids_1 = [uuid.uuid4().hex for _ in range(len(texts))] + Pinecone.from_texts( + texts=texts, + ids=ids_1, + embedding=embedding_openai, + index_name=index_name, + namespace=index_name, + ) + index_stats = self.index.describe_index_stats() + assert index_stats["namespaces"][index_name]["vector_count"] == len(texts) * 2 From 0226b375d967f30b0ba27d061d465460d016e76c Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 14 Apr 2023 00:52:25 -0400 Subject: [PATCH 14/49] docs: Mendable Search integration (#2803) Mendable Seach Integration is Finally here! Hey yall, After various requests for Mendable in Python docs, we decided to get our hands dirty and try to implement it. Here is a version where we implement our **floating button** that sits on the bottom right of the screen that once triggered (via press or CMD K) will work the same as the js langchain docs. Super excited about this and hopefully the community will be too. @hwchase17 will send you the admin details via dm etc. The anon_key is fine to be public. Let me know if you need any further customization. I added the langchain logo to it. --- docs/_static/css/custom.css | 4 +++ docs/_static/js/mendablesearch.js | 52 +++++++++++++++++++++++++++++++ docs/conf.py | 5 +++ 3 files changed, 61 insertions(+) create mode 100644 docs/_static/js/mendablesearch.js diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index 673008a063b..8e2ddc2c92f 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -11,3 +11,7 @@ pre { max-width: 2560px !important; } } + +#my-component-root *, #headlessui-portal-root * { + z-index: 1000000000000; +} diff --git a/docs/_static/js/mendablesearch.js b/docs/_static/js/mendablesearch.js new file mode 100644 index 00000000000..e3d2c1369aa --- /dev/null +++ b/docs/_static/js/mendablesearch.js @@ -0,0 +1,52 @@ +document.addEventListener('DOMContentLoaded', () => { + // Load the external dependencies + function loadScript(src, onLoadCallback) { + const script = document.createElement('script'); + script.src = src; + script.onload = onLoadCallback; + document.head.appendChild(script); + } + + function createRootElement() { + const rootElement = document.createElement('div'); + rootElement.id = 'my-component-root'; + document.body.appendChild(rootElement); + return rootElement; + } + + + + function initializeMendable() { + const rootElement = createRootElement(); + const { MendableFloatingButton } = Mendable; + + + const icon = React.createElement('p', { + style: { color: '#ffffff', fontSize: '22px',width: '48px', height: '48px', margin: '0px', padding: '0px', display: 'flex', alignItems: 'center', justifyContent: 'center' }, + }, '🦜🔗'); + + + + + const mendableFloatingButton = React.createElement( + MendableFloatingButton, + { + style: { darkMode: false, accentColor: '#010810' }, + floatingButtonStyle: { color: '#ffffff', backgroundColor: '#010810' }, + anon_key: '82842b36-3ea6-49b2-9fb8-52cfc4bde6bf', // Mendable Search Public ANON key, ok to be public + messageSettings: { + openSourcesInNewTab: false, + }, + icon: icon, + } + ); + + ReactDOM.render(mendableFloatingButton, rootElement); + } + + loadScript('https://unpkg.com/react@17/umd/react.production.min.js', () => { + loadScript('https://unpkg.com/react-dom@17/umd/react-dom.production.min.js', () => { + loadScript('https://unpkg.com/@mendable/search@0.0.83/dist/umd/mendable.min.js', initializeMendable); + }); + }); +}); diff --git a/docs/conf.py b/docs/conf.py index ae3924b5905..087b2d20056 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -103,5 +103,10 @@ html_static_path = ["_static"] html_css_files = [ "css/custom.css", ] + +html_js_files = [ + "js/mendablesearch.js", +] + nb_execution_mode = "off" myst_enable_extensions = ["colon_fence"] From 74abeb8c53646c6e879eeb97ad03b1a81846ddcc Mon Sep 17 00:00:00 2001 From: ecneladis Date: Fri, 14 Apr 2023 06:56:17 +0200 Subject: [PATCH 15/49] Update output in Git notebook (#2868) Supplemental to https://github.com/hwchase17/langchain/pull/2851. Updates one notebook cell that I forgot to commit before. --- .../document_loaders/examples/git.ipynb | 38 +++++++------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/docs/modules/indexes/document_loaders/examples/git.ipynb b/docs/modules/indexes/document_loaders/examples/git.ipynb index ffebd95d2dd..87a6feb8fd7 100644 --- a/docs/modules/indexes/document_loaders/examples/git.ipynb +++ b/docs/modules/indexes/document_loaders/examples/git.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -34,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -43,28 +43,16 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 23, "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "__init__() got an unexpected keyword argument 'path'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[3], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m loader \u001b[39m=\u001b[39m GitLoader(path\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39m./example_data/test_repo1/\u001b[39;49m\u001b[39m\"\u001b[39;49m, branch\u001b[39m=\u001b[39;49mbranch)\n", - "\u001b[0;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'path'" - ] - } - ], + "outputs": [], "source": [ "loader = GitLoader(repo_path=\"./example_data/test_repo1/\", branch=branch)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -73,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -82,7 +70,7 @@ "1040" ] }, - "execution_count": 15, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -93,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -117,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -126,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -139,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -148,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -157,7 +145,7 @@ "1040" ] }, - "execution_count": 20, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } From dcb17503f216a521400e1fc4aa2c82162ae849a3 Mon Sep 17 00:00:00 2001 From: Andrey Vasnetsov Date: Fri, 14 Apr 2023 06:57:05 +0200 Subject: [PATCH 16/49] Update qdrant.py (#2750) At the moment of upload we should already know the format of data, therefore we can skip the costly pydantic validation. --- langchain/vectorstores/qdrant.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langchain/vectorstores/qdrant.py b/langchain/vectorstores/qdrant.py index 33eec89e303..1526379e885 100644 --- a/langchain/vectorstores/qdrant.py +++ b/langchain/vectorstores/qdrant.py @@ -81,7 +81,7 @@ class Qdrant(VectorStore): ids = [uuid.uuid4().hex for _ in texts] self.client.upsert( collection_name=self.collection_name, - points=rest.Batch( + points=rest.Batch.construct( ids=ids, vectors=[self.embedding_function(text) for text in texts], payloads=self._build_payloads( @@ -314,7 +314,7 @@ class Qdrant(VectorStore): client.upsert( collection_name=collection_name, - points=rest.Batch( + points=rest.Batch.construct( ids=[uuid.uuid4().hex for _ in texts], vectors=embeddings, payloads=cls._build_payloads( From 8a98e5b50b0093f6a517b59df88b6ef202e411dc Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Thu, 13 Apr 2023 22:01:32 -0700 Subject: [PATCH 17/49] Harrison/index name (#2869) Co-authored-by: Mesum Raza Hemani --- langchain/vectorstores/faiss.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/langchain/vectorstores/faiss.py b/langchain/vectorstores/faiss.py index 9b139807a51..4157fa9db68 100644 --- a/langchain/vectorstores/faiss.py +++ b/langchain/vectorstores/faiss.py @@ -373,39 +373,47 @@ class FAISS(VectorStore): embeddings = [t[1] for t in text_embeddings] return cls.__from(texts, embeddings, embedding, metadatas, **kwargs) - def save_local(self, folder_path: str) -> None: + def save_local(self, folder_path: str, index_name: str = "index") -> None: """Save FAISS index, docstore, and index_to_docstore_id to disk. Args: folder_path: folder path to save index, docstore, and index_to_docstore_id to. + index_name: for saving with a specific index file name """ path = Path(folder_path) path.mkdir(exist_ok=True, parents=True) # save index separately since it is not picklable faiss = dependable_faiss_import() - faiss.write_index(self.index, str(path / "index.faiss")) + faiss.write_index( + self.index, str(path / "{index_name}.faiss".format(index_name=index_name)) + ) # save docstore and index_to_docstore_id - with open(path / "index.pkl", "wb") as f: + with open(path / "{index_name}.pkl".format(index_name=index_name), "wb") as f: pickle.dump((self.docstore, self.index_to_docstore_id), f) @classmethod - def load_local(cls, folder_path: str, embeddings: Embeddings) -> FAISS: + def load_local( + cls, folder_path: str, embeddings: Embeddings, index_name: str = "index" + ) -> FAISS: """Load FAISS index, docstore, and index_to_docstore_id to disk. Args: folder_path: folder path to load index, docstore, and index_to_docstore_id from. embeddings: Embeddings to use when generating queries + index_name: for saving with a specific index file name """ path = Path(folder_path) # load index separately since it is not picklable faiss = dependable_faiss_import() - index = faiss.read_index(str(path / "index.faiss")) + index = faiss.read_index( + str(path / "{index_name}.faiss".format(index_name=index_name)) + ) # load docstore and index_to_docstore_id - with open(path / "index.pkl", "rb") as f: + with open(path / "{index_name}.pkl".format(index_name=index_name), "rb") as f: docstore, index_to_docstore_id = pickle.load(f) return cls(embeddings.embed_query, index, docstore, index_to_docstore_id) From 705596b46a9ad2a5c6ec28a85678493d2dfbea14 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Thu, 13 Apr 2023 22:07:58 -0700 Subject: [PATCH 18/49] Harrison/fix create sql agent (#2870) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Timothé Pearce --- langchain/document_loaders/git.py | 2 +- langchain/tools/sql_database/tool.py | 30 +++++++++++++++------------- tests/unit_tests/agents/test_sql.py | 18 +++++++++++++++++ tests/unit_tests/llms/fake_llm.py | 27 +++++++++++++++++++++++-- 4 files changed, 60 insertions(+), 17 deletions(-) create mode 100644 tests/unit_tests/agents/test_sql.py diff --git a/langchain/document_loaders/git.py b/langchain/document_loaders/git.py index 39a9235e26e..155767629ec 100644 --- a/langchain/document_loaders/git.py +++ b/langchain/document_loaders/git.py @@ -28,7 +28,7 @@ class GitLoader(BaseLoader): def load(self) -> List[Document]: try: - from git import Blob, Repo + from git import Blob, Repo # type: ignore except ImportError as ex: raise ImportError( "Could not import git python package. " diff --git a/langchain/tools/sql_database/tool.py b/langchain/tools/sql_database/tool.py index a9ac6981b51..3921b43a2fb 100644 --- a/langchain/tools/sql_database/tool.py +++ b/langchain/tools/sql_database/tool.py @@ -1,6 +1,7 @@ # flake8: noqa """Tools for interacting with a SQL database.""" -from pydantic import BaseModel, Extra, Field, validator +from pydantic import BaseModel, Extra, Field, validator, root_validator +from typing import Any, Dict from langchain.chains.llm import LLMChain from langchain.prompts import PromptTemplate @@ -81,28 +82,29 @@ class QueryCheckerTool(BaseSQLDatabaseTool, BaseTool): template: str = QUERY_CHECKER llm: BaseLLM - llm_chain: LLMChain = Field( - default_factory=lambda: LLMChain( - llm=QueryCheckerTool.llm, - prompt=PromptTemplate( - template=QueryCheckerTool.template, input_variables=["query", "dialect"] - ), - ) - ) + llm_chain: LLMChain = Field(init=False) name = "query_checker_sql_db" description = """ Use this tool to double check if your query is correct before executing it. Always use this tool before executing a query with query_sql_db! """ - @validator("llm_chain") - def validate_llm_chain_input_variables(cls, llm_chain: LLMChain) -> LLMChain: - """Make sure the LLM chain has the correct input variables.""" - if llm_chain.prompt.input_variables != ["query", "dialect"]: + @root_validator(pre=True) + def initialize_llm_chain(cls, values: Dict[str, Any]) -> Dict[str, Any]: + if "llm_chain" not in values: + values["llm_chain"] = LLMChain( + llm=values.get("llm"), + prompt=PromptTemplate( + template=QUERY_CHECKER, input_variables=["query", "dialect"] + ), + ) + + if values["llm_chain"].prompt.input_variables != ["query", "dialect"]: raise ValueError( "LLM chain for QueryCheckerTool must have input variables ['query', 'dialect']" ) - return llm_chain + + return values def _run(self, query: str) -> str: """Use the LLM to check the query.""" diff --git a/tests/unit_tests/agents/test_sql.py b/tests/unit_tests/agents/test_sql.py new file mode 100644 index 00000000000..89b8f90df2c --- /dev/null +++ b/tests/unit_tests/agents/test_sql.py @@ -0,0 +1,18 @@ +from langchain.agents import create_sql_agent +from langchain.agents.agent_toolkits import SQLDatabaseToolkit +from langchain.sql_database import SQLDatabase +from tests.unit_tests.llms.fake_llm import FakeLLM + + +def test_create_sql_agent() -> None: + db = SQLDatabase.from_uri("sqlite:///:memory:") + queries = {"foo": "Final Answer: baz"} + llm = FakeLLM(queries=queries, sequential_responses=True) + toolkit = SQLDatabaseToolkit(db=db, llm=llm) + + agent_executor = create_sql_agent( + llm=llm, + toolkit=toolkit, + ) + + assert agent_executor.run("hello") == "baz" diff --git a/tests/unit_tests/llms/fake_llm.py b/tests/unit_tests/llms/fake_llm.py index 263bc2b6308..cc12a7cab7b 100644 --- a/tests/unit_tests/llms/fake_llm.py +++ b/tests/unit_tests/llms/fake_llm.py @@ -1,5 +1,7 @@ """Fake LLM wrapper for testing purposes.""" -from typing import Any, List, Mapping, Optional +from typing import Any, List, Mapping, Optional, cast + +from pydantic import validator from langchain.llms.base import LLM @@ -8,6 +10,18 @@ class FakeLLM(LLM): """Fake LLM wrapper for testing purposes.""" queries: Optional[Mapping] = None + sequential_responses: Optional[bool] = False + response_index: int = 0 + + @validator("queries", always=True) + def check_queries_required( + cls, queries: Optional[Mapping], values: Mapping[str, Any] + ) -> Optional[Mapping]: + if values.get("sequential_response") and not queries: + raise ValueError( + "queries is required when sequential_response is set to True" + ) + return queries @property def _llm_type(self) -> str: @@ -15,7 +29,9 @@ class FakeLLM(LLM): return "fake" def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str: - """First try to lookup in queries, else return 'foo' or 'bar'.""" + if self.sequential_responses: + return self._get_next_response_in_sequence + if self.queries is not None: return self.queries[prompt] if stop is None: @@ -26,3 +42,10 @@ class FakeLLM(LLM): @property def _identifying_params(self) -> Mapping[str, Any]: return {} + + @property + def _get_next_response_in_sequence(self) -> str: + queries = cast(Mapping, self.queries) + response = queries[list(queries.keys())[self.response_index]] + self.response_index = self.response_index + 1 + return response From 1cc7ea333c0a9899241eb9e3247d5e333eb8948d Mon Sep 17 00:00:00 2001 From: rafael Date: Fri, 14 Apr 2023 07:08:46 +0200 Subject: [PATCH 19/49] chat_models.openai: Set tenacity timeout to openai's recommendation (#2768) [OpenAI's cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/How_to_handle_rate_limits.ipynb) suggest a tenacity backoff between 1 and 60 seconds. Currently langchain's backoff is between 4 and 10 seconds, which causes frequent timeout errors on my end. This PR changes the timeout to the suggested values. --- langchain/chat_models/openai.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/langchain/chat_models/openai.py b/langchain/chat_models/openai.py index 69e3f9fd2a6..644ed261dcb 100644 --- a/langchain/chat_models/openai.py +++ b/langchain/chat_models/openai.py @@ -32,8 +32,8 @@ logger = logging.getLogger(__file__) def _create_retry_decorator(llm: ChatOpenAI) -> Callable[[Any], Any]: import openai - min_seconds = 4 - max_seconds = 10 + min_seconds = 1 + max_seconds = 60 # Wait 2^x * 1 second between each retry starting with # 4 seconds, then up to 10 seconds, then 10 seconds afterwards return retry( @@ -199,8 +199,8 @@ class ChatOpenAI(BaseChatModel): def _create_retry_decorator(self) -> Callable[[Any], Any]: import openai - min_seconds = 4 - max_seconds = 10 + min_seconds = 1 + max_seconds = 60 # Wait 2^x * 1 second between each retry starting with # 4 seconds, then up to 10 seconds, then 10 seconds afterwards return retry( From 9907cb0485a577a2e041d227a9359f8102fcb3a2 Mon Sep 17 00:00:00 2001 From: drod Date: Fri, 14 Apr 2023 07:09:00 +0200 Subject: [PATCH 20/49] Refactor similarity_search function in elastic_vector_search.py (#2761) Optimization :Limit search results when k < 10 Fix issue when k > 10: Elasticsearch will return only 10 docs [default-search-result](https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html) By default, searches return the top 10 matching hits Add size parameter to the search request to limit the number of returned results from Elasticsearch. Remove slicing of the hits list, since the response will already contain the desired number of results. --- langchain/vectorstores/elastic_vector_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langchain/vectorstores/elastic_vector_search.py b/langchain/vectorstores/elastic_vector_search.py index cb238563553..17af42c66ad 100644 --- a/langchain/vectorstores/elastic_vector_search.py +++ b/langchain/vectorstores/elastic_vector_search.py @@ -200,8 +200,8 @@ class ElasticVectorSearch(VectorStore, ABC): """ embedding = self.embedding.embed_query(query) script_query = _default_script_query(embedding) - response = self.client.search(index=self.index_name, query=script_query) - hits = [hit["_source"] for hit in response["hits"]["hits"][:k]] + response = self.client.search(index=self.index_name, query=script_query, size=k) + hits = [hit["_source"] for hit in response["hits"]["hits"]] documents = [ Document(page_content=hit["text"], metadata=hit["metadata"]) for hit in hits ] From 5565f56273a39c7c1926912ccd7a87bfc051869c Mon Sep 17 00:00:00 2001 From: Jon Luo <20971593+jzluo@users.noreply.github.com> Date: Fri, 14 Apr 2023 01:10:49 -0400 Subject: [PATCH 21/49] Use SQL dialect-specific prompts for SQLDatabaseChain (#2748) Mentioned the idea here initially: https://github.com/hwchase17/langchain/pull/2106#issuecomment-1487509106 Since there have been dialect-specific issues, we should use dialect-specific prompts. This way, each prompt can be separately modified to best suit each dialect as needed. This adds a prompt for each dialect supported in sqlalchemy (mssql, mysql, mariadb, postgres, oracle, sqlite). For this initial implementation, the only differencse between the prompts is the instruction for the clause to use to limit the number of rows queried for, and the instruction for wrapping column names using each dialect's identifier quote character. --- langchain/chains/sql_database/base.py | 15 ++- langchain/chains/sql_database/prompt.py | 147 ++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 5 deletions(-) diff --git a/langchain/chains/sql_database/base.py b/langchain/chains/sql_database/base.py index 1885ad3eb8e..56ce0d27a04 100644 --- a/langchain/chains/sql_database/base.py +++ b/langchain/chains/sql_database/base.py @@ -1,13 +1,13 @@ """Chain for interacting with SQL Database.""" from __future__ import annotations -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from pydantic import Extra, Field from langchain.chains.base import Chain from langchain.chains.llm import LLMChain -from langchain.chains.sql_database.prompt import DECIDER_PROMPT, PROMPT +from langchain.chains.sql_database.prompt import DECIDER_PROMPT, PROMPT, SQL_PROMPTS from langchain.prompts.base import BasePromptTemplate from langchain.schema import BaseLanguageModel from langchain.sql_database import SQLDatabase @@ -28,7 +28,7 @@ class SQLDatabaseChain(Chain): """LLM wrapper to use.""" database: SQLDatabase = Field(exclude=True) """SQL Database to connect to.""" - prompt: BasePromptTemplate = PROMPT + prompt: Optional[BasePromptTemplate] = None """Prompt to use to translate natural language to SQL.""" top_k: int = 5 """Number of results to return from the query""" @@ -65,8 +65,13 @@ class SQLDatabaseChain(Chain): return [self.output_key, "intermediate_steps"] def _call(self, inputs: Dict[str, Any]) -> Dict[str, Any]: - llm_chain = LLMChain(llm=self.llm, prompt=self.prompt) - input_text = f"{inputs[self.input_key]} \nSQLQuery:" + try: + prompt = self.prompt or SQL_PROMPTS[self.database.dialect] + except KeyError: + # fallback to generic prompt if dialect-specific prompt doesn't exist yet + prompt = PROMPT + llm_chain = LLMChain(llm=self.llm, prompt=prompt) + input_text = f"{inputs[self.input_key]}\nSQLQuery:" self.callback_manager.on_text(input_text, verbose=self.verbose) # If not present, then defaults to None which is all tables. table_names_to_use = inputs.get("table_names_to_use") diff --git a/langchain/chains/sql_database/prompt.py b/langchain/chains/sql_database/prompt.py index 730c5a23743..e0eabf6edac 100644 --- a/langchain/chains/sql_database/prompt.py +++ b/langchain/chains/sql_database/prompt.py @@ -2,6 +2,7 @@ from langchain.output_parsers.list import CommaSeparatedListOutputParser from langchain.prompts.prompt import PromptTemplate + _DEFAULT_TEMPLATE = """Given an input question, first create a syntactically correct {dialect} query to run, then look at the results of the query and return the answer. Unless the user specifies in his question a specific number of examples he wishes to obtain, always limit your query to at most {top_k} results. You can order the results by a relevant column to return the most interesting examples in the database. Never query for all the columns from a specific table, only ask for a the few relevant columns given the question. @@ -38,3 +39,149 @@ DECIDER_PROMPT = PromptTemplate( template=_DECIDER_TEMPLATE, output_parser=CommaSeparatedListOutputParser(), ) + + +_mssql_prompt = """You are an MS SQL expert. Given an input question, first create a syntactically correct MS SQL query to run, then look at the results of the query and return the answer to the input question. +Unless the user specifies in the question a specific number of examples to obtain, query for at most {top_k} results using the TOP clause as per MS SQL. You can order the results to return the most informative data in the database. +Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in square brackets ([]) to denote them as delimited identifiers. +Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table. + +Use the following format: + +Question: "Question here" +SQLQuery: "SQL Query to run" +SQLResult: "Result of the SQLQuery" +Answer: "Final answer here" + +Only use the following tables: +{table_info} + +Question: {input}""" + +MSSQL_PROMPT = PromptTemplate( + input_variables=["input", "table_info", "top_k"], template=_mssql_prompt +) + + +_mysql_prompt = """You are a MySQL expert. Given an input question, first create a syntactically correct MySQL query to run, then look at the results of the query and return the answer to the input question. +Unless the user specifies in the question a specific number of examples to obtain, query for at most {top_k} results using the LIMIT clause as per MySQL. You can order the results to return the most informative data in the database. +Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in backticks (`) to denote them as delimited identifiers. +Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table. + +Use the following format: + +Question: "Question here" +SQLQuery: "SQL Query to run" +SQLResult: "Result of the SQLQuery" +Answer: "Final answer here" + +Only use the following tables: +{table_info} + +Question: {input}""" + +MYSQL_PROMPT = PromptTemplate( + input_variables=["input", "table_info", "top_k"], + template=_mysql_prompt, +) + + +_mariadb_prompt = """You are a MariaDB expert. Given an input question, first create a syntactically correct MariaDB query to run, then look at the results of the query and return the answer to the input question. +Unless the user specifies in the question a specific number of examples to obtain, query for at most {top_k} results using the LIMIT clause as per MariaDB. You can order the results to return the most informative data in the database. +Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in backticks (`) to denote them as delimited identifiers. +Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table. + +Use the following format: + +Question: "Question here" +SQLQuery: "SQL Query to run" +SQLResult: "Result of the SQLQuery" +Answer: "Final answer here" + +Only use the following tables: +{table_info} + +Question: {input}""" + +MARIADB_PROMPT = PromptTemplate( + input_variables=["input", "table_info", "top_k"], + template=_mariadb_prompt, +) + + +_oracle_prompt = """You are an Oracle SQL expert. Given an input question, first create a syntactically correct Oracle SQL query to run, then look at the results of the query and return the answer to the input question. +Unless the user specifies in the question a specific number of examples to obtain, query for at most {top_k} results using the FETCH FIRST n ROWS ONLY clause as per Oracle SQL. You can order the results to return the most informative data in the database. +Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in double quotes (") to denote them as delimited identifiers. +Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table. + +Use the following format: + +Question: "Question here" +SQLQuery: "SQL Query to run" +SQLResult: "Result of the SQLQuery" +Answer: "Final answer here" + +Only use the following tables: +{table_info} + +Question: {input}""" + +ORACLE_PROMPT = PromptTemplate( + input_variables=["input", "table_info", "top_k"], + template=_oracle_prompt, +) + + +_postgres_prompt = """You are a PostgreSQL expert. Given an input question, first create a syntactically correct PostgreSQL query to run, then look at the results of the query and return the answer to the input question. +Unless the user specifies in the question a specific number of examples to obtain, query for at most {top_k} results using the LIMIT clause as per PostgreSQL. You can order the results to return the most informative data in the database. +Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in double quotes (") to denote them as delimited identifiers. +Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table. + +Use the following format: + +Question: "Question here" +SQLQuery: "SQL Query to run" +SQLResult: "Result of the SQLQuery" +Answer: "Final answer here" + +Only use the following tables: +{table_info} + +Question: {input}""" + +POSTGRES_PROMPT = PromptTemplate( + input_variables=["input", "table_info", "top_k"], template=_postgres_prompt +) + + +_sqlite_prompt = """You are a SQLite expert. Given an input question, first create a syntactically correct SQLite query to run, then look at the results of the query and return the answer to the input question. +Unless the user specifies in the question a specific number of examples to obtain, query for at most {top_k} results using the LIMIT clause as per SQLite. You can order the results to return the most informative data in the database. +Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in double quotes (") to denote them as delimited identifiers. +Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table. + +Use the following format: + +Question: "Question here" +SQLQuery: "SQL Query to run" +SQLResult: "Result of the SQLQuery" +Answer: "Final answer here" + +Only use the following tables: +{table_info} + +Question: {input}""" + +SQLITE_PROMPT = PromptTemplate( + input_variables=["input", "table_info", "top_k"], + template=_sqlite_prompt, +) + + +SQL_PROMPTS = { + "mssql": MSSQL_PROMPT, + "mysql": MYSQL_PROMPT, + "mariadb": MARIADB_PROMPT, + "oracle": ORACLE_PROMPT, + "postgresql": POSTGRES_PROMPT, + "sqlite": SQLITE_PROMPT, +} From 07d7096de684395e615ef502fd48933862923b76 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Thu, 13 Apr 2023 22:15:03 -0700 Subject: [PATCH 22/49] Harrison/playwright (#2871) Co-authored-by: Manuel Saelices --- .../document_loaders/examples/url.ipynb | 75 +++++++++++++++- langchain/document_loaders/__init__.py | 2 + langchain/document_loaders/url_playwright.py | 87 +++++++++++++++++++ 3 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 langchain/document_loaders/url_playwright.py diff --git a/docs/modules/indexes/document_loaders/examples/url.ipynb b/docs/modules/indexes/document_loaders/examples/url.ipynb index 581f1a33f7b..517f2f66426 100644 --- a/docs/modules/indexes/document_loaders/examples/url.ipynb +++ b/docs/modules/indexes/document_loaders/examples/url.ipynb @@ -112,6 +112,79 @@ "source": [ "data = loader.load()" ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "a2c1c79f", + "metadata": {}, + "source": [ + "# Playwright URL Loader\n", + "\n", + "This covers how to load HTML documents from a list of URLs using the `PlaywrightURLLoader`.\n", + "\n", + "As in the Selenium case, Playwright allows us to load pages that need JavaScript to render.\n", + "\n", + "## Setup\n", + "\n", + "To use the `PlaywrightURLLoader`, you will need to install `playwright` and `unstructured`. Additionally, you will need to install the Playwright Chromium browser:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53158417", + "metadata": {}, + "outputs": [], + "source": [ + "# Install playwright\n", + "!pip install \"playwright\"\n", + "!pip install \"unstructured\"\n", + "!playwright install" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ab4e115", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.document_loaders import PlaywrightURLLoader" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce5a9a0a", + "metadata": {}, + "outputs": [], + "source": [ + "urls = [\n", + " \"https://www.youtube.com/watch?v=dQw4w9WgXcQ\",\n", + " \"https://goo.gl/maps/NDSHwePEyaHMFGwh8\"\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2dc3e0bc", + "metadata": {}, + "outputs": [], + "source": [ + "loader = PlaywrightURLLoader(urls=urls, remove_selectors=[\"header\", \"footer\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10b79f80", + "metadata": {}, + "outputs": [], + "source": [ + "data = loader.load()" + ] } ], "metadata": { @@ -130,7 +203,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.10.6" } }, "nbformat": 4, diff --git a/langchain/document_loaders/__init__.py b/langchain/document_loaders/__init__.py index c2ea430abfc..17d2cde9b8a 100644 --- a/langchain/document_loaders/__init__.py +++ b/langchain/document_loaders/__init__.py @@ -64,6 +64,7 @@ from langchain.document_loaders.unstructured import ( UnstructuredFileLoader, ) from langchain.document_loaders.url import UnstructuredURLLoader +from langchain.document_loaders.url_playwright import PlaywrightURLLoader from langchain.document_loaders.url_selenium import SeleniumURLLoader from langchain.document_loaders.web_base import WebBaseLoader from langchain.document_loaders.whatsapp_chat import WhatsAppChatLoader @@ -82,6 +83,7 @@ __all__ = [ "UnstructuredFileIOLoader", "UnstructuredURLLoader", "SeleniumURLLoader", + "PlaywrightURLLoader", "DirectoryLoader", "NotionDirectoryLoader", "NotionDBLoader", diff --git a/langchain/document_loaders/url_playwright.py b/langchain/document_loaders/url_playwright.py new file mode 100644 index 00000000000..15a5dbd77ef --- /dev/null +++ b/langchain/document_loaders/url_playwright.py @@ -0,0 +1,87 @@ +"""Loader that uses Playwright to load a page, then uses unstructured to load the html. +""" +import logging +from typing import List, Optional + +from langchain.docstore.document import Document +from langchain.document_loaders.base import BaseLoader + +logger = logging.getLogger(__file__) + + +class PlaywrightURLLoader(BaseLoader): + """Loader that uses Playwright and to load a page and unstructured to load the html. + This is useful for loading pages that require javascript to render. + + Attributes: + urls (List[str]): List of URLs to load. + continue_on_failure (bool): If True, continue loading other URLs on failure. + headless (bool): If True, the browser will run in headless mode. + """ + + def __init__( + self, + urls: List[str], + continue_on_failure: bool = True, + headless: bool = True, + remove_selectors: Optional[List[str]] = None, + ): + """Load a list of URLs using Playwright and unstructured.""" + try: + import playwright # noqa:F401 + except ImportError: + raise ValueError( + "playwright package not found, please install it with " + "`pip install playwright`" + ) + + try: + import unstructured # noqa:F401 + except ImportError: + raise ValueError( + "unstructured package not found, please install it with " + "`pip install unstructured`" + ) + + self.urls = urls + self.continue_on_failure = continue_on_failure + self.headless = headless + self.remove_selectors = remove_selectors + + def load(self) -> List[Document]: + """Load the specified URLs using Playwright and create Document instances. + + Returns: + List[Document]: A list of Document instances with loaded content. + """ + from playwright.sync_api import sync_playwright + from unstructured.partition.html import partition_html + + docs: List[Document] = list() + + with sync_playwright() as p: + browser = p.chromium.launch(headless=self.headless) + for url in self.urls: + try: + page = browser.new_page() + page.goto(url) + + for selector in self.remove_selectors or []: + element = page.locator(selector) + if element.is_visible(): + element.evaluate("element => element.remove()") + + page_source = page.content() + elements = partition_html(text=page_source) + text = "\n\n".join([str(el) for el in elements]) + metadata = {"source": url} + docs.append(Document(page_content=text, metadata=metadata)) + except Exception as e: + if self.continue_on_failure: + logger.error( + f"Error fetching or processing {url}, exception: {e}" + ) + else: + raise e + browser.close() + return docs From 1e9378d0a873abd0849f645a3cd9788f57602183 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Thu, 13 Apr 2023 22:37:34 -0700 Subject: [PATCH 23/49] Harrison/weaviate fixes (#2872) Co-authored-by: cs0lar Co-authored-by: cs0lar --- langchain/vectorstores/weaviate.py | 100 +++++++++++++++++- poetry.lock | 16 +-- pyproject.toml | 1 + .../vectorstores/docker-compose/weaviate.yml | 22 ++++ .../vectorstores/test_weaviate.py | 51 +++++++++ 5 files changed, 178 insertions(+), 12 deletions(-) create mode 100644 tests/integration_tests/vectorstores/docker-compose/weaviate.yml create mode 100644 tests/integration_tests/vectorstores/test_weaviate.py diff --git a/langchain/vectorstores/weaviate.py b/langchain/vectorstores/weaviate.py index 011dc470e66..d5d8c13419a 100644 --- a/langchain/vectorstores/weaviate.py +++ b/langchain/vectorstores/weaviate.py @@ -6,9 +6,22 @@ from uuid import uuid4 from langchain.docstore.document import Document from langchain.embeddings.base import Embeddings +from langchain.utils import get_from_dict_or_env from langchain.vectorstores.base import VectorStore +def _default_schema(index_name: str) -> Dict: + return { + "class": index_name, + "properties": [ + { + "name": "text", + "dataType": ["text"], + } + ], + } + + class Weaviate(VectorStore): """Wrapper around Weaviate vector database. @@ -70,14 +83,24 @@ class Weaviate(VectorStore): data_properties[key] = metadatas[i][key] _id = get_valid_uuid(uuid4()) - batch.add_data_object(data_properties, self._index_name, _id) + batch.add_data_object( + data_object=data_properties, class_name=self._index_name, uuid=_id + ) ids.append(_id) return ids def similarity_search( self, query: str, k: int = 4, **kwargs: Any ) -> List[Document]: - """Look up similar documents in weaviate.""" + """Return docs most similar to query. + + Args: + query: Text to look up documents similar to. + k: Number of Documents to return. Defaults to 4. + + Returns: + List of Documents most similar to the query. + """ content: Dict[str, Any] = {"concepts": [query]} if kwargs.get("search_distance"): content["certainty"] = kwargs.get("search_distance") @@ -114,5 +137,74 @@ class Weaviate(VectorStore): metadatas: Optional[List[dict]] = None, **kwargs: Any, ) -> Weaviate: - """Not implemented for Weaviate yet.""" - raise NotImplementedError("weaviate does not currently support `from_texts`.") + """Construct Weaviate wrapper from raw documents. + + This is a user-friendly interface that: + 1. Embeds documents. + 2. Creates a new index for the embeddings in the Weaviate instance. + 3. Adds the documents to the newly created Weaviate index. + + This is intended to be a quick way to get started. + + Example: + .. code-block:: python + + from langchain.vectorstores.weaviate import Weaviate + from langchain.embeddings import OpenAIEmbeddings + embeddings = OpenAIEmbeddings() + weaviate = Weaviate.from_texts( + texts, + embeddings, + weaviate_url="http://localhost:8080" + ) + """ + weaviate_url = get_from_dict_or_env(kwargs, "weaviate_url", "WEAVIATE_URL") + + try: + from weaviate import Client + from weaviate.util import get_valid_uuid + except ImportError: + raise ValueError( + "Could not import weaviate python package. " + "Please install it with `pip instal weaviate-client`" + ) + + client = Client(weaviate_url) + index_name = kwargs.get("index_name", f"LangChain_{uuid4().hex}") + embeddings = embedding.embed_documents(texts) if embedding else None + text_key = "text" + schema = _default_schema(index_name) + attributes = list(metadatas[0].keys()) if metadatas else None + + # check whether the index already exists + if not client.schema.contains(schema): + client.schema.create_class(schema) + + with client.batch as batch: + for i, text in enumerate(texts): + data_properties = { + text_key: text, + } + if metadatas is not None: + for key in metadatas[i].keys(): + data_properties[key] = metadatas[i][key] + + _id = get_valid_uuid(uuid4()) + + # if an embedding strategy is not provided, we let + # weaviate create the embedding. Note that this will only + # work if weaviate has been installed with a vectorizer module + # like text2vec-contextionary for example + params = { + "uuid": _id, + "data_object": data_properties, + "class_name": index_name, + } + if embeddings is not None: + params["vector"] = (embeddings[i],) + + batch.add_data_object(**params) + + batch.flush() + + return cls(client, index_name, text_key, attributes) diff --git a/poetry.lock b/poetry.lock index 434a6bccba5..7a88ee2a9f7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "absl-py" @@ -499,7 +499,7 @@ name = "authlib" version = "1.2.0" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." category = "main" -optional = true +optional = false python-versions = "*" files = [ {file = "Authlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:4ddf4fd6cfa75c9a460b361d4bd9dac71ffda0be879dbe4292a02e92349ad55a"}, @@ -7258,7 +7258,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\""} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] @@ -8360,7 +8360,7 @@ name = "validators" version = "0.20.0" description = "Python Data Validation for Humans™." category = "main" -optional = true +optional = false python-versions = ">=3.4" files = [ {file = "validators-0.20.0.tar.gz", hash = "sha256:24148ce4e64100a2d5e267233e23e7afeb55316b47d30faae7eb6e7292bc226a"}, @@ -8497,7 +8497,7 @@ name = "weaviate-client" version = "3.15.5" description = "A python native weaviate client" category = "main" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "weaviate-client-3.15.5.tar.gz", hash = "sha256:6da7e5d08dc9bb8b7879661d1a457c50af7d73e621a5305efe131160e83da69e"}, @@ -9026,13 +9026,13 @@ cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\ cffi = ["cffi (>=1.11)"] [extras] -all = ["aleph-alpha-client", "anthropic", "beautifulsoup4", "cohere", "deeplake", "elasticsearch", "faiss-cpu", "google-api-python-client", "google-search-results", "huggingface_hub", "jina", "jinja2", "manifest-ml", "networkx", "nlpcloud", "nltk", "nomic", "openai", "opensearch-py", "pgvector", "pinecone-client", "pinecone-text", "psycopg2-binary", "pyowm", "pypdf", "qdrant-client", "redis", "sentence-transformers", "spacy", "tensorflow-text", "tiktoken", "torch", "transformers", "weaviate-client", "wikipedia", "wolframalpha"] +all = ["anthropic", "cohere", "openai", "nlpcloud", "huggingface_hub", "jina", "manifest-ml", "elasticsearch", "opensearch-py", "google-search-results", "faiss-cpu", "sentence-transformers", "transformers", "spacy", "nltk", "wikipedia", "beautifulsoup4", "tiktoken", "torch", "jinja2", "pinecone-client", "pinecone-text", "weaviate-client", "redis", "google-api-python-client", "wolframalpha", "qdrant-client", "tensorflow-text", "pypdf", "networkx", "nomic", "aleph-alpha-client", "deeplake", "pgvector", "psycopg2-binary", "pyowm"] cohere = ["cohere"] -llms = ["anthropic", "cohere", "huggingface_hub", "manifest-ml", "nlpcloud", "openai", "torch", "transformers"] +llms = ["anthropic", "cohere", "openai", "nlpcloud", "huggingface_hub", "manifest-ml", "torch", "transformers"] openai = ["openai"] qdrant = ["qdrant-client"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "3c8488864a754852fdec3e56dd5630ed73852ec2120a94cfe22537c075901b24" +content-hash = "373f68ef16e7f3d5d9cde8b81c5f261096cc537ddca4f6a36711d7215b63f226" diff --git a/pyproject.toml b/pyproject.toml index d3f34c3ce7f..3cc2d497615 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,6 +101,7 @@ pgvector = "^0.1.6" transformers = "^4.27.4" pandas = "^2.0.0" deeplake = "^3.2.21" +weaviate-client = "^3.15.5" torch = "^1.0.0" chromadb = "^0.3.21" tiktoken = "^0.3.3" diff --git a/tests/integration_tests/vectorstores/docker-compose/weaviate.yml b/tests/integration_tests/vectorstores/docker-compose/weaviate.yml new file mode 100644 index 00000000000..a1911480a0f --- /dev/null +++ b/tests/integration_tests/vectorstores/docker-compose/weaviate.yml @@ -0,0 +1,22 @@ +version: '3.4' + +services: + weaviate: + command: + - --host + - 0.0.0.0 + - --port + - '8080' + - --scheme + - http + image: semitechnologies/weaviate:1.18.2 + ports: + - 8080:8080 + restart: on-failure:0 + environment: + QUERY_DEFAULTS_LIMIT: 25 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: '/var/lib/weaviate' + DEFAULT_VECTORIZER_MODULE: 'none' + ENABLE_MODULES: '' + CLUSTER_HOSTNAME: 'node1' diff --git a/tests/integration_tests/vectorstores/test_weaviate.py b/tests/integration_tests/vectorstores/test_weaviate.py new file mode 100644 index 00000000000..5699ecea33a --- /dev/null +++ b/tests/integration_tests/vectorstores/test_weaviate.py @@ -0,0 +1,51 @@ +"""Test Weaviate functionality.""" +import logging +from typing import Generator, Union + +import pytest +from weaviate import Client + +from langchain.docstore.document import Document +from langchain.embeddings.openai import OpenAIEmbeddings +from langchain.vectorstores.weaviate import Weaviate + +logging.basicConfig(level=logging.DEBUG) + +""" +cd tests/integration_tests/vectorstores/docker-compose +docker compose -f weaviate.yml up +""" + + +class TestWeaviate: + @pytest.fixture(scope="class", autouse=True) + def weaviate_url(self) -> Union[str, Generator[str, None, None]]: + """Return the weaviate url.""" + url = "http://localhost:8080" + yield url + + # Clear the test index + client = Client(url) + client.schema.delete_all() + + def test_similarity_search_without_metadata(self, weaviate_url: str) -> None: + """Test end to end construction and search without metadata.""" + texts = ["foo", "bar", "baz"] + docsearch = Weaviate.from_texts( + texts, + OpenAIEmbeddings(), + weaviate_url=weaviate_url, + ) + + output = docsearch.similarity_search("foo", k=1) + assert output == [Document(page_content="foo")] + + def test_similarity_search_with_metadata(self, weaviate_url: str) -> None: + """Test end to end construction and search with metadata.""" + texts = ["foo", "bar", "baz"] + metadatas = [{"page": i} for i in range(len(texts))] + docsearch = Weaviate.from_texts( + texts, OpenAIEmbeddings(), metadatas=metadatas, weaviate_url=weaviate_url + ) + output = docsearch.similarity_search("foo", k=1) + assert output == [Document(page_content="foo", metadata={"page": 0})] From 3c7204d604fe3700f37d406e1f112da710a35864 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 14 Apr 2023 02:15:57 -0400 Subject: [PATCH 24/49] docs: Quick fix to Mendable Search (#2876) Fixed a small issue on the icon UI when using in Safari. --- docs/_static/js/mendablesearch.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/_static/js/mendablesearch.js b/docs/_static/js/mendablesearch.js index e3d2c1369aa..0dd1bc4c00a 100644 --- a/docs/_static/js/mendablesearch.js +++ b/docs/_static/js/mendablesearch.js @@ -21,9 +21,15 @@ document.addEventListener('DOMContentLoaded', () => { const { MendableFloatingButton } = Mendable; + const iconSpan1 = React.createElement('span', { + }, '🦜'); + + const iconSpan2 = React.createElement('span', { + }, '🔗'); + const icon = React.createElement('p', { - style: { color: '#ffffff', fontSize: '22px',width: '48px', height: '48px', margin: '0px', padding: '0px', display: 'flex', alignItems: 'center', justifyContent: 'center' }, - }, '🦜🔗'); + style: { color: '#ffffff', fontSize: '22px',width: '48px', height: '48px', margin: '0px', padding: '0px', display: 'flex', alignItems: 'center', justifyContent: 'center', textAlign: 'center' }, + }, [iconSpan1, iconSpan2]); From 1a44b71ddfe36a27cf51712b4d2b7b5a47c0023b Mon Sep 17 00:00:00 2001 From: ecneladis Date: Fri, 14 Apr 2023 16:40:04 +0200 Subject: [PATCH 25/49] Fix Baby AGI notebooks (#2882) - fix broken notebook cell in https://github.com/hwchase17/langchain/commit/ae485b623d29f086e2b939986b05783e3f355445 - Python Black formatting --- docs/use_cases/agents/baby_agi.ipynb | 95 +++++++++----- .../agents/baby_agi_with_agent.ipynb | 124 +++++++++++------- 2 files changed, 140 insertions(+), 79 deletions(-) diff --git a/docs/use_cases/agents/baby_agi.ipynb b/docs/use_cases/agents/baby_agi.ipynb index 7005fb709ec..7cfc761fc00 100644 --- a/docs/use_cases/agents/baby_agi.ipynb +++ b/docs/use_cases/agents/baby_agi.ipynb @@ -38,7 +38,7 @@ "from langchain.llms import BaseLLM\n", "from langchain.vectorstores.base import VectorStore\n", "from pydantic import BaseModel, Field\n", - "from langchain.chains.base import Chain\n" + "from langchain.chains.base import Chain" ] }, { @@ -73,6 +73,7 @@ "embeddings_model = OpenAIEmbeddings()\n", "# Initialize the vectorstore as empty\n", "import faiss\n", + "\n", "embedding_size = 1536\n", "index = faiss.IndexFlatL2(embedding_size)\n", "vectorstore = FAISS(embeddings_model.embed_query, index, InMemoryDocstore({}), {})" @@ -116,7 +117,12 @@ " )\n", " prompt = PromptTemplate(\n", " template=task_creation_template,\n", - " input_variables=[\"result\", \"task_description\", \"incomplete_tasks\", \"objective\"],\n", + " input_variables=[\n", + " \"result\",\n", + " \"task_description\",\n", + " \"incomplete_tasks\",\n", + " \"objective\",\n", + " ],\n", " )\n", " return cls(prompt=prompt, llm=llm, verbose=verbose)" ] @@ -147,7 +153,7 @@ " template=task_prioritization_template,\n", " input_variables=[\"task_names\", \"next_task_id\", \"objective\"],\n", " )\n", - " return cls(prompt=prompt, llm=llm, verbose=verbose)\n" + " return cls(prompt=prompt, llm=llm, verbose=verbose)" ] }, { @@ -173,7 +179,7 @@ " template=execution_template,\n", " input_variables=[\"objective\", \"context\", \"task\"],\n", " )\n", - " return cls(prompt=prompt, llm=llm, verbose=verbose)\n" + " return cls(prompt=prompt, llm=llm, verbose=verbose)" ] }, { @@ -193,11 +199,22 @@ "metadata": {}, "outputs": [], "source": [ - "def get_next_task(task_creation_chain: LLMChain, result: Dict, task_description: str, task_list: List[str], objective: str) -> List[Dict]:\n", + "def get_next_task(\n", + " task_creation_chain: LLMChain,\n", + " result: Dict,\n", + " task_description: str,\n", + " task_list: List[str],\n", + " objective: str,\n", + ") -> List[Dict]:\n", " \"\"\"Get the next task.\"\"\"\n", " incomplete_tasks = \", \".join(task_list)\n", - " response = task_creation_chain.run(result=result, task_description=task_description, incomplete_tasks=incomplete_tasks, objective=objective)\n", - " new_tasks = response.split('\\n')\n", + " response = task_creation_chain.run(\n", + " result=result,\n", + " task_description=task_description,\n", + " incomplete_tasks=incomplete_tasks,\n", + " objective=objective,\n", + " )\n", + " new_tasks = response.split(\"\\n\")\n", " return [{\"task_name\": task_name} for task_name in new_tasks if task_name.strip()]" ] }, @@ -208,12 +225,19 @@ "metadata": {}, "outputs": [], "source": [ - "def prioritize_tasks(task_prioritization_chain: LLMChain, this_task_id: int, task_list: List[Dict], objective: str) -> List[Dict]:\n", + "def prioritize_tasks(\n", + " task_prioritization_chain: LLMChain,\n", + " this_task_id: int,\n", + " task_list: List[Dict],\n", + " objective: str,\n", + ") -> List[Dict]:\n", " \"\"\"Prioritize tasks.\"\"\"\n", " task_names = [t[\"task_name\"] for t in task_list]\n", " next_task_id = int(this_task_id) + 1\n", - " response = task_prioritization_chain.run(task_names=task_names, next_task_id=next_task_id, objective=objective)\n", - " new_tasks = response.split('\\n')\n", + " response = task_prioritization_chain.run(\n", + " task_names=task_names, next_task_id=next_task_id, objective=objective\n", + " )\n", + " new_tasks = response.split(\"\\n\")\n", " prioritized_task_list = []\n", " for task_string in new_tasks:\n", " if not task_string.strip():\n", @@ -239,9 +263,12 @@ " if not results:\n", " return []\n", " sorted_results, _ = zip(*sorted(results, key=lambda x: x[1], reverse=True))\n", - " return [str(item.metadata['task']) for item in sorted_results]\n", + " return [str(item.metadata[\"task\"]) for item in sorted_results]\n", "\n", - "def execute_task(vectorstore, execution_chain: LLMChain, objective: str, task: str, k: int = 5) -> str:\n", + "\n", + "def execute_task(\n", + " vectorstore, execution_chain: LLMChain, objective: str, task: str, k: int = 5\n", + ") -> str:\n", " \"\"\"Execute a task.\"\"\"\n", " context = _get_top_tasks(vectorstore, query=objective, k=k)\n", " return execution_chain.run(objective=objective, context=context, task=task)" @@ -254,7 +281,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "class BabyAGI(Chain, BaseModel):\n", " \"\"\"Controller model for the BabyAGI agent.\"\"\"\n", "\n", @@ -265,9 +291,10 @@ " task_id_counter: int = Field(1)\n", " vectorstore: VectorStore = Field(init=False)\n", " max_iterations: Optional[int] = None\n", - " \n", + "\n", " class Config:\n", " \"\"\"Configuration for this pydantic object.\"\"\"\n", + "\n", " arbitrary_types_allowed = True\n", "\n", " def add_task(self, task: Dict):\n", @@ -285,18 +312,18 @@ " def print_task_result(self, result: str):\n", " print(\"\\033[93m\\033[1m\" + \"\\n*****TASK RESULT*****\\n\" + \"\\033[0m\\033[0m\")\n", " print(result)\n", - " \n", + "\n", " @property\n", " def input_keys(self) -> List[str]:\n", " return [\"objective\"]\n", - " \n", + "\n", " @property\n", " def output_keys(self) -> List[str]:\n", " return []\n", "\n", " def _call(self, inputs: Dict[str, Any]) -> Dict[str, Any]:\n", " \"\"\"Run the agent.\"\"\"\n", - " objective = inputs['objective']\n", + " objective = inputs[\"objective\"]\n", " first_task = inputs.get(\"first_task\", \"Make a todo list\")\n", " self.add_task({\"task_id\": 1, \"task_name\": first_task})\n", " num_iters = 0\n", @@ -325,7 +352,11 @@ "\n", " # Step 4: Create new tasks and reprioritize task list\n", " new_tasks = get_next_task(\n", - " self.task_creation_chain, result, task[\"task_name\"], [t[\"task_name\"] for t in self.task_list], objective\n", + " self.task_creation_chain,\n", + " result,\n", + " task[\"task_name\"],\n", + " [t[\"task_name\"] for t in self.task_list],\n", + " objective,\n", " )\n", " for new_task in new_tasks:\n", " self.task_id_counter += 1\n", @@ -333,27 +364,26 @@ " self.add_task(new_task)\n", " self.task_list = deque(\n", " prioritize_tasks(\n", - " self.task_prioritization_chain, this_task_id, list(self.task_list), objective\n", + " self.task_prioritization_chain,\n", + " this_task_id,\n", + " list(self.task_list),\n", + " objective,\n", " )\n", " )\n", " num_iters += 1\n", " if self.max_iterations is not None and num_iters == self.max_iterations:\n", - " print(\"\\033[91m\\033[1m\" + \"\\n*****TASK ENDING*****\\n\" + \"\\033[0m\\033[0m\")\n", + " print(\n", + " \"\\033[91m\\033[1m\" + \"\\n*****TASK ENDING*****\\n\" + \"\\033[0m\\033[0m\"\n", + " )\n", " break\n", " return {}\n", "\n", " @classmethod\n", " def from_llm(\n", - " cls,\n", - " llm: BaseLLM,\n", - " vectorstore: VectorStore,\n", - " verbose: bool = False,\n", - " **kwargs\n", + " cls, llm: BaseLLM, vectorstore: VectorStore, verbose: bool = False, **kwargs\n", " ) -> \"BabyAGI\":\n", " \"\"\"Initialize the BabyAGI Controller.\"\"\"\n", - " task_creation_chain = TaskCreationChain.from_llm(\n", - " llm, verbose=verbose\n", - " )\n", + " task_creation_chain = TaskCreationChain.from_llm(llm, verbose=verbose)\n", " task_prioritization_chain = TaskPrioritizationChain.from_llm(\n", " llm, verbose=verbose\n", " )\n", @@ -363,7 +393,7 @@ " task_prioritization_chain=task_prioritization_chain,\n", " execution_chain=execution_chain,\n", " vectorstore=vectorstore,\n", - " **kwargs\n", + " **kwargs,\n", " )" ] }, @@ -405,14 +435,11 @@ "outputs": [], "source": [ "# Logging of LLMChains\n", - "verbose=False\n", + "verbose = False\n", "# If None, will keep on going forever\n", "max_iterations: Optional[int] = 3\n", "baby_agi = BabyAGI.from_llm(\n", - " llm=llm,\n", - " vectorstore=vectorstore,\n", - " verbose=verbose,\n", - " max_iterations=max_iterations\n", + " llm=llm, vectorstore=vectorstore, verbose=verbose, max_iterations=max_iterations\n", ")" ] }, diff --git a/docs/use_cases/agents/baby_agi_with_agent.ipynb b/docs/use_cases/agents/baby_agi_with_agent.ipynb index 7244f7fda92..bf71533ae5f 100644 --- a/docs/use_cases/agents/baby_agi_with_agent.ipynb +++ b/docs/use_cases/agents/baby_agi_with_agent.ipynb @@ -34,7 +34,7 @@ "from langchain.llms import BaseLLM\n", "from langchain.vectorstores.base import VectorStore\n", "from pydantic import BaseModel, Field\n", - "from langchain.chains.base import Chain\n" + "from langchain.chains.base import Chain" ] }, { @@ -54,7 +54,9 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install faiss-cpu > /dev/null%pip install google-search-results > /dev/nullfrom langchain.vectorstores import FAISS\n", + "%pip install faiss-cpu > /dev/null\n", + "%pip install google-search-results > /dev/null\n", + "from langchain.vectorstores import FAISS\n", "from langchain.docstore import InMemoryDocstore" ] }, @@ -69,6 +71,7 @@ "embeddings_model = OpenAIEmbeddings()\n", "# Initialize the vectorstore as empty\n", "import faiss\n", + "\n", "embedding_size = 1536\n", "index = faiss.IndexFlatL2(embedding_size)\n", "vectorstore = FAISS(embeddings_model.embed_query, index, InMemoryDocstore({}), {})" @@ -115,7 +118,12 @@ " )\n", " prompt = PromptTemplate(\n", " template=task_creation_template,\n", - " input_variables=[\"result\", \"task_description\", \"incomplete_tasks\", \"objective\"],\n", + " input_variables=[\n", + " \"result\",\n", + " \"task_description\",\n", + " \"incomplete_tasks\",\n", + " \"objective\",\n", + " ],\n", " )\n", " return cls(prompt=prompt, llm=llm, verbose=verbose)" ] @@ -146,7 +154,7 @@ " template=task_prioritization_template,\n", " input_variables=[\"task_names\", \"next_task_id\", \"objective\"],\n", " )\n", - " return cls(prompt=prompt, llm=llm, verbose=verbose)\n" + " return cls(prompt=prompt, llm=llm, verbose=verbose)" ] }, { @@ -158,20 +166,23 @@ "source": [ "from langchain.agents import ZeroShotAgent, Tool, AgentExecutor\n", "from langchain import OpenAI, SerpAPIWrapper, LLMChain\n", - "todo_prompt = PromptTemplate.from_template(\"You are a planner who is an expert at coming up with a todo list for a given objective. Come up with a todo list for this objective: {objective}\")\n", + "\n", + "todo_prompt = PromptTemplate.from_template(\n", + " \"You are a planner who is an expert at coming up with a todo list for a given objective. Come up with a todo list for this objective: {objective}\"\n", + ")\n", "todo_chain = LLMChain(llm=OpenAI(temperature=0), prompt=todo_prompt)\n", "search = SerpAPIWrapper()\n", "tools = [\n", " Tool(\n", - " name = \"Search\",\n", + " name=\"Search\",\n", " func=search.run,\n", - " description=\"useful for when you need to answer questions about current events\"\n", + " description=\"useful for when you need to answer questions about current events\",\n", " ),\n", " Tool(\n", - " name = \"TODO\",\n", + " name=\"TODO\",\n", " func=todo_chain.run,\n", - " description=\"useful for when you need to come up with todo lists. Input: an objective to create a todo list for. Output: a todo list for that objective. Please be very clear what the objective is!\"\n", - " )\n", + " description=\"useful for when you need to come up with todo lists. Input: an objective to create a todo list for. Output: a todo list for that objective. Please be very clear what the objective is!\",\n", + " ),\n", "]\n", "\n", "\n", @@ -179,10 +190,10 @@ "suffix = \"\"\"Question: {task}\n", "{agent_scratchpad}\"\"\"\n", "prompt = ZeroShotAgent.create_prompt(\n", - " tools, \n", - " prefix=prefix, \n", - " suffix=suffix, \n", - " input_variables=[\"objective\", \"task\", \"context\",\"agent_scratchpad\"]\n", + " tools,\n", + " prefix=prefix,\n", + " suffix=suffix,\n", + " input_variables=[\"objective\", \"task\", \"context\", \"agent_scratchpad\"],\n", ")" ] }, @@ -203,11 +214,22 @@ "metadata": {}, "outputs": [], "source": [ - "def get_next_task(task_creation_chain: LLMChain, result: Dict, task_description: str, task_list: List[str], objective: str) -> List[Dict]:\n", + "def get_next_task(\n", + " task_creation_chain: LLMChain,\n", + " result: Dict,\n", + " task_description: str,\n", + " task_list: List[str],\n", + " objective: str,\n", + ") -> List[Dict]:\n", " \"\"\"Get the next task.\"\"\"\n", " incomplete_tasks = \", \".join(task_list)\n", - " response = task_creation_chain.run(result=result, task_description=task_description, incomplete_tasks=incomplete_tasks, objective=objective)\n", - " new_tasks = response.split('\\n')\n", + " response = task_creation_chain.run(\n", + " result=result,\n", + " task_description=task_description,\n", + " incomplete_tasks=incomplete_tasks,\n", + " objective=objective,\n", + " )\n", + " new_tasks = response.split(\"\\n\")\n", " return [{\"task_name\": task_name} for task_name in new_tasks if task_name.strip()]" ] }, @@ -218,12 +240,19 @@ "metadata": {}, "outputs": [], "source": [ - "def prioritize_tasks(task_prioritization_chain: LLMChain, this_task_id: int, task_list: List[Dict], objective: str) -> List[Dict]:\n", + "def prioritize_tasks(\n", + " task_prioritization_chain: LLMChain,\n", + " this_task_id: int,\n", + " task_list: List[Dict],\n", + " objective: str,\n", + ") -> List[Dict]:\n", " \"\"\"Prioritize tasks.\"\"\"\n", " task_names = [t[\"task_name\"] for t in task_list]\n", " next_task_id = int(this_task_id) + 1\n", - " response = task_prioritization_chain.run(task_names=task_names, next_task_id=next_task_id, objective=objective)\n", - " new_tasks = response.split('\\n')\n", + " response = task_prioritization_chain.run(\n", + " task_names=task_names, next_task_id=next_task_id, objective=objective\n", + " )\n", + " new_tasks = response.split(\"\\n\")\n", " prioritized_task_list = []\n", " for task_string in new_tasks:\n", " if not task_string.strip():\n", @@ -249,9 +278,12 @@ " if not results:\n", " return []\n", " sorted_results, _ = zip(*sorted(results, key=lambda x: x[1], reverse=True))\n", - " return [str(item.metadata['task']) for item in sorted_results]\n", + " return [str(item.metadata[\"task\"]) for item in sorted_results]\n", "\n", - "def execute_task(vectorstore, execution_chain: LLMChain, objective: str, task: str, k: int = 5) -> str:\n", + "\n", + "def execute_task(\n", + " vectorstore, execution_chain: LLMChain, objective: str, task: str, k: int = 5\n", + ") -> str:\n", " \"\"\"Execute a task.\"\"\"\n", " context = _get_top_tasks(vectorstore, query=objective, k=k)\n", " return execution_chain.run(objective=objective, context=context, task=task)" @@ -264,7 +296,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "class BabyAGI(Chain, BaseModel):\n", " \"\"\"Controller model for the BabyAGI agent.\"\"\"\n", "\n", @@ -275,9 +306,10 @@ " task_id_counter: int = Field(1)\n", " vectorstore: VectorStore = Field(init=False)\n", " max_iterations: Optional[int] = None\n", - " \n", + "\n", " class Config:\n", " \"\"\"Configuration for this pydantic object.\"\"\"\n", + "\n", " arbitrary_types_allowed = True\n", "\n", " def add_task(self, task: Dict):\n", @@ -295,18 +327,18 @@ " def print_task_result(self, result: str):\n", " print(\"\\033[93m\\033[1m\" + \"\\n*****TASK RESULT*****\\n\" + \"\\033[0m\\033[0m\")\n", " print(result)\n", - " \n", + "\n", " @property\n", " def input_keys(self) -> List[str]:\n", " return [\"objective\"]\n", - " \n", + "\n", " @property\n", " def output_keys(self) -> List[str]:\n", " return []\n", "\n", " def _call(self, inputs: Dict[str, Any]) -> Dict[str, Any]:\n", " \"\"\"Run the agent.\"\"\"\n", - " objective = inputs['objective']\n", + " objective = inputs[\"objective\"]\n", " first_task = inputs.get(\"first_task\", \"Make a todo list\")\n", " self.add_task({\"task_id\": 1, \"task_name\": first_task})\n", " num_iters = 0\n", @@ -335,7 +367,11 @@ "\n", " # Step 4: Create new tasks and reprioritize task list\n", " new_tasks = get_next_task(\n", - " self.task_creation_chain, result, task[\"task_name\"], [t[\"task_name\"] for t in self.task_list], objective\n", + " self.task_creation_chain,\n", + " result,\n", + " task[\"task_name\"],\n", + " [t[\"task_name\"] for t in self.task_list],\n", + " objective,\n", " )\n", " for new_task in new_tasks:\n", " self.task_id_counter += 1\n", @@ -343,40 +379,41 @@ " self.add_task(new_task)\n", " self.task_list = deque(\n", " prioritize_tasks(\n", - " self.task_prioritization_chain, this_task_id, list(self.task_list), objective\n", + " self.task_prioritization_chain,\n", + " this_task_id,\n", + " list(self.task_list),\n", + " objective,\n", " )\n", " )\n", " num_iters += 1\n", " if self.max_iterations is not None and num_iters == self.max_iterations:\n", - " print(\"\\033[91m\\033[1m\" + \"\\n*****TASK ENDING*****\\n\" + \"\\033[0m\\033[0m\")\n", + " print(\n", + " \"\\033[91m\\033[1m\" + \"\\n*****TASK ENDING*****\\n\" + \"\\033[0m\\033[0m\"\n", + " )\n", " break\n", " return {}\n", "\n", " @classmethod\n", " def from_llm(\n", - " cls,\n", - " llm: BaseLLM,\n", - " vectorstore: VectorStore,\n", - " verbose: bool = False,\n", - " **kwargs\n", + " cls, llm: BaseLLM, vectorstore: VectorStore, verbose: bool = False, **kwargs\n", " ) -> \"BabyAGI\":\n", " \"\"\"Initialize the BabyAGI Controller.\"\"\"\n", - " task_creation_chain = TaskCreationChain.from_llm(\n", - " llm, verbose=verbose\n", - " )\n", + " task_creation_chain = TaskCreationChain.from_llm(llm, verbose=verbose)\n", " task_prioritization_chain = TaskPrioritizationChain.from_llm(\n", " llm, verbose=verbose\n", " )\n", " llm_chain = LLMChain(llm=llm, prompt=prompt)\n", " tool_names = [tool.name for tool in tools]\n", " agent = ZeroShotAgent(llm_chain=llm_chain, allowed_tools=tool_names)\n", - " agent_executor = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True)\n", + " agent_executor = AgentExecutor.from_agent_and_tools(\n", + " agent=agent, tools=tools, verbose=True\n", + " )\n", " return cls(\n", " task_creation_chain=task_creation_chain,\n", " task_prioritization_chain=task_prioritization_chain,\n", " execution_chain=agent_executor,\n", " vectorstore=vectorstore,\n", - " **kwargs\n", + " **kwargs,\n", " )" ] }, @@ -418,14 +455,11 @@ "outputs": [], "source": [ "# Logging of LLMChains\n", - "verbose=False\n", + "verbose = False\n", "# If None, will keep on going forever\n", "max_iterations: Optional[int] = 3\n", "baby_agi = BabyAGI.from_llm(\n", - " llm=llm,\n", - " vectorstore=vectorstore,\n", - " verbose=verbose,\n", - " max_iterations=max_iterations\n", + " llm=llm, vectorstore=vectorstore, verbose=verbose, max_iterations=max_iterations\n", ")" ] }, From 203c0eb2aea5a60ae2f5f060c9f13529bb7dcf75 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Fri, 14 Apr 2023 23:40:26 +0900 Subject: [PATCH 26/49] docs: update getting_started.ipynb (#2883) HuggingFace -> Hugging Face --- docs/modules/models/llms/getting_started.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/models/llms/getting_started.ipynb b/docs/modules/models/llms/getting_started.ipynb index 64908b622c6..bd031ac9edf 100644 --- a/docs/modules/models/llms/getting_started.ipynb +++ b/docs/modules/models/llms/getting_started.ipynb @@ -186,7 +186,7 @@ "source": [ "**Number of Tokens:** You can also estimate how many tokens a piece of text will be in that model. This is useful because models have a context length (and cost more for more tokens), which means you need to be aware of how long the text you are passing in is.\n", "\n", - "Notice that by default the tokens are estimated using [tiktoken](https://github.com/openai/tiktoken) (except for legacy version <3.8, where a HuggingFace tokenizer is used)" + "Notice that by default the tokens are estimated using [tiktoken](https://github.com/openai/tiktoken) (except for legacy version <3.8, where a Hugging Face tokenizer is used)" ] }, { From 0a38bbc750dad086e111237a3ee500d0eaf0074e Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Fri, 14 Apr 2023 07:54:57 -0700 Subject: [PATCH 27/49] updates to vectorstore memory (#2875) --- .../types/vectorstore_retriever_memory.ipynb | 117 ++++++++++++------ 1 file changed, 78 insertions(+), 39 deletions(-) diff --git a/docs/modules/memory/types/vectorstore_retriever_memory.ipynb b/docs/modules/memory/types/vectorstore_retriever_memory.ipynb index a7d54810fd4..27fdc82f84b 100644 --- a/docs/modules/memory/types/vectorstore_retriever_memory.ipynb +++ b/docs/modules/memory/types/vectorstore_retriever_memory.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "id": "ff4be5f3", "metadata": {}, @@ -10,7 +9,9 @@ "\n", "`VectorStoreRetrieverMemory` stores memories in a VectorDB and queries the top-K most \"salient\" docs every time it is called.\n", "\n", - "This differs from most of the other Memory classes in that it doesn't explicitly track the order of interactions." + "This differs from most of the other Memory classes in that it doesn't explicitly track the order of interactions.\n", + "\n", + "In this case, the \"docs\" are previous conversation snippets. This can be useful to refer to relevant pieces of information that the AI was told earlier in the conversation." ] }, { @@ -26,7 +27,8 @@ "from langchain.embeddings.openai import OpenAIEmbeddings\n", "from langchain.llms import OpenAI\n", "from langchain.memory import VectorStoreRetrieverMemory\n", - "from langchain.chains import ConversationChain" + "from langchain.chains import ConversationChain\n", + "from langchain.prompts import PromptTemplate" ] }, { @@ -41,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 29, "id": "eef56f65", "metadata": { "tags": [] @@ -61,7 +63,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "8f4bdf92", "metadata": {}, @@ -73,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 30, "id": "e00d4938", "metadata": { "tags": [] @@ -86,14 +87,14 @@ "memory = VectorStoreRetrieverMemory(retriever=retriever)\n", "\n", "# When added to an agent, the memory object can save pertinent information from conversations or used tools\n", - "memory.save_context({\"input\": \"check the latest scores of the Warriors game\"}, {\"output\": \"the Warriors are up against the Astros 88 to 84\"})\n", - "memory.save_context({\"input\": \"I need help doing my taxes - what's the standard deduction this year?\"}, {\"output\": \"...\"})\n", - "memory.save_context({\"input\": \"What's the the time?\"}, {\"output\": f\"It's {datetime.now()}\"}) # " + "memory.save_context({\"input\": \"My favorite food is pizza\"}, {\"output\": \"thats good to know\"})\n", + "memory.save_context({\"input\": \"My favorite sport is soccer\"}, {\"output\": \"...\"})\n", + "memory.save_context({\"input\": \"I don't the Celtics\"}, {\"output\": \"ok\"}) # " ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 31, "id": "2fe28a28", "metadata": { "tags": [] @@ -103,7 +104,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "input: I need help doing my taxes - what's the standard deduction this year?\n", + "input: My favorite sport is soccer\n", "output: ...\n" ] } @@ -111,7 +112,7 @@ "source": [ "# Notice the first result returned is the memory pertaining to tax help, which the language model deems more semantically relevant\n", "# to a 1099 than the other documents, despite them both containing numbers.\n", - "print(memory.load_memory_variables({\"prompt\": \"What's a 1099?\"})[\"history\"])" + "print(memory.load_memory_variables({\"prompt\": \"what sport should i watch?\"})[\"history\"])" ] }, { @@ -125,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 32, "id": "ebd68c10", "metadata": { "tags": [] @@ -141,9 +142,13 @@ "Prompt after formatting:\n", "\u001b[32;1m\u001b[1;3mThe following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n", "\n", + "Relevant pieces of previous conversation:\n", + "input: My favorite food is pizza\n", + "output: thats good to know\n", + "\n", + "(You do not need to use these pieces of information if not relevant)\n", + "\n", "Current conversation:\n", - "input: I need help doing my taxes - what's the standard deduction this year?\n", - "output: ...\n", "Human: Hi, my name is Perry, what's up?\n", "AI:\u001b[0m\n", "\n", @@ -153,18 +158,32 @@ { "data": { "text/plain": [ - "\" Hi Perry, my name is AI. I'm doing great, how about you? I understand you need help with your taxes. What specifically do you need help with?\"" + "\" Hi Perry, I'm doing well. How about you?\"" ] }, - "execution_count": 5, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "llm = OpenAI(temperature=0) # Can be any valid LLM\n", + "_DEFAULT_TEMPLATE = \"\"\"The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n", + "\n", + "Relevant pieces of previous conversation:\n", + "{history}\n", + "\n", + "(You do not need to use these pieces of information if not relevant)\n", + "\n", + "Current conversation:\n", + "Human: {input}\n", + "AI:\"\"\"\n", + "PROMPT = PromptTemplate(\n", + " input_variables=[\"history\", \"input\"], template=_DEFAULT_TEMPLATE\n", + ")\n", "conversation_with_summary = ConversationChain(\n", " llm=llm, \n", + " prompt=PROMPT,\n", " # We set a very low max_token_limit for the purposes of testing.\n", " memory=memory,\n", " verbose=True\n", @@ -174,7 +193,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 33, "id": "86207a61", "metadata": { "tags": [] @@ -190,10 +209,14 @@ "Prompt after formatting:\n", "\u001b[32;1m\u001b[1;3mThe following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n", "\n", + "Relevant pieces of previous conversation:\n", + "input: My favorite sport is soccer\n", + "output: ...\n", + "\n", + "(You do not need to use these pieces of information if not relevant)\n", + "\n", "Current conversation:\n", - "input: check the latest scores of the Warriors game\n", - "output: the Warriors are up against the Astros 88 to 84\n", - "Human: If the Cavaliers were to face off against the Warriers or the Astros, who would they most stand a chance to beat?\n", + "Human: what's my favorite sport?\n", "AI:\u001b[0m\n", "\n", "\u001b[1m> Finished chain.\u001b[0m\n" @@ -202,22 +225,22 @@ { "data": { "text/plain": [ - "\" It's hard to say without knowing the current form of the teams. However, based on the current scores, it looks like the Cavaliers would have a better chance of beating the Astros than the Warriors.\"" + "' You told me earlier that your favorite sport is soccer.'" ] }, - "execution_count": 6, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Here, the basketball related content is surfaced\n", - "conversation_with_summary.predict(input=\"If the Cavaliers were to face off against the Warriers or the Astros, who would they most stand a chance to beat?\")" + "conversation_with_summary.predict(input=\"what's my favorite sport?\")" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 34, "id": "8c669db1", "metadata": { "tags": [] @@ -233,10 +256,14 @@ "Prompt after formatting:\n", "\u001b[32;1m\u001b[1;3mThe following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n", "\n", + "Relevant pieces of previous conversation:\n", + "input: My favorite food is pizza\n", + "output: thats good to know\n", + "\n", + "(You do not need to use these pieces of information if not relevant)\n", + "\n", "Current conversation:\n", - "input: What's the the time?\n", - "output: It's 2023-04-13 09:18:55.623736\n", - "Human: What day is it tomorrow?\n", + "Human: Whats my favorite food\n", "AI:\u001b[0m\n", "\n", "\u001b[1m> Finished chain.\u001b[0m\n" @@ -245,10 +272,10 @@ { "data": { "text/plain": [ - "' Tomorrow is 2023-04-14.'" + "' You said your favorite food is pizza.'" ] }, - "execution_count": 7, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -256,12 +283,12 @@ "source": [ "# Even though the language model is stateless, since relavent memory is fetched, it can \"reason\" about the time.\n", "# Timestamping memories and data is useful in general to let the agent determine temporal relevance\n", - "conversation_with_summary.predict(input=\"What day is it tomorrow?\")" + "conversation_with_summary.predict(input=\"Whats my favorite food\")" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 35, "id": "8c09a239", "metadata": { "tags": [] @@ -277,10 +304,14 @@ "Prompt after formatting:\n", "\u001b[32;1m\u001b[1;3mThe following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n", "\n", - "Current conversation:\n", + "Relevant pieces of previous conversation:\n", "input: Hi, my name is Perry, what's up?\n", - "response: Hi Perry, my name is AI. I'm doing great, how about you? I understand you need help with your taxes. What specifically do you need help with?\n", - "Human: What's your name?\n", + "response: Hi Perry, I'm doing well. How about you?\n", + "\n", + "(You do not need to use these pieces of information if not relevant)\n", + "\n", + "Current conversation:\n", + "Human: What's my name?\n", "AI:\u001b[0m\n", "\n", "\u001b[1m> Finished chain.\u001b[0m\n" @@ -289,10 +320,10 @@ { "data": { "text/plain": [ - "\" My name is AI. It's nice to meet you, Perry.\"" + "' Your name is Perry.'" ] }, - "execution_count": 8, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -301,8 +332,16 @@ "# The memories from the conversation are automatically stored,\n", "# since this query best matches the introduction chat above,\n", "# the agent is able to 'remember' the user's name.\n", - "conversation_with_summary.predict(input=\"What's your name?\")" + "conversation_with_summary.predict(input=\"What's my name?\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df27c7dc", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -321,7 +360,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.2" + "version": "3.9.1" } }, "nbformat": 4, From 8fef69296db61a705123d9f3d13f94edbec3bd80 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Fri, 14 Apr 2023 07:55:12 -0700 Subject: [PATCH 28/49] nits (#2873) --- .../indexes/document_loaders/examples/git.ipynb | 13 +++++-------- langchain/chains/sql_database/base.py | 6 +----- langchain/document_loaders/__init__.py | 2 ++ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/docs/modules/indexes/document_loaders/examples/git.ipynb b/docs/modules/indexes/document_loaders/examples/git.ipynb index 87a6feb8fd7..306a68fa467 100644 --- a/docs/modules/indexes/document_loaders/examples/git.ipynb +++ b/docs/modules/indexes/document_loaders/examples/git.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -11,7 +10,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -38,7 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain.document_loaders.git import GitLoader" + "from langchain.document_loaders import GitLoader" ] }, { @@ -109,7 +107,7 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain.document_loaders.git import GitLoader" + "from langchain.document_loaders import GitLoader" ] }, { @@ -164,7 +162,7 @@ ], "metadata": { "kernelspec": { - "display_name": "ai", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -178,9 +176,8 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.6" - }, - "orig_nbformat": 4 + "version": "3.9.1" + } }, "nbformat": 4, "nbformat_minor": 2 diff --git a/langchain/chains/sql_database/base.py b/langchain/chains/sql_database/base.py index 56ce0d27a04..aa1a2e6dc56 100644 --- a/langchain/chains/sql_database/base.py +++ b/langchain/chains/sql_database/base.py @@ -65,11 +65,7 @@ class SQLDatabaseChain(Chain): return [self.output_key, "intermediate_steps"] def _call(self, inputs: Dict[str, Any]) -> Dict[str, Any]: - try: - prompt = self.prompt or SQL_PROMPTS[self.database.dialect] - except KeyError: - # fallback to generic prompt if dialect-specific prompt doesn't exist yet - prompt = PROMPT + prompt = self.prompt or SQL_PROMPTS.get(self.database.dialect, PROMPT) llm_chain = LLMChain(llm=self.llm, prompt=prompt) input_text = f"{inputs[self.input_key]}\nSQLQuery:" self.callback_manager.on_text(input_text, verbose=self.verbose) diff --git a/langchain/document_loaders/__init__.py b/langchain/document_loaders/__init__.py index 17d2cde9b8a..b5924cefed2 100644 --- a/langchain/document_loaders/__init__.py +++ b/langchain/document_loaders/__init__.py @@ -27,6 +27,7 @@ from langchain.document_loaders.evernote import EverNoteLoader from langchain.document_loaders.facebook_chat import FacebookChatLoader from langchain.document_loaders.gcs_directory import GCSDirectoryLoader from langchain.document_loaders.gcs_file import GCSFileLoader +from langchain.document_loaders.git import GitLoader from langchain.document_loaders.gitbook import GitbookLoader from langchain.document_loaders.googledrive import GoogleDriveLoader from langchain.document_loaders.gutenberg import GutenbergLoader @@ -142,4 +143,5 @@ __all__ = [ "BigQueryLoader", "BiliBiliLoader", "SlackDirectoryLoader", + "GitLoader", ] From 86189cdcf9085c569fc94f012e5359d0d5d1f736 Mon Sep 17 00:00:00 2001 From: Francis Felici Date: Fri, 14 Apr 2023 12:51:30 -0300 Subject: [PATCH 29/49] Update load_qa_chain() docstring (#2900) Seems to be missing `map_rerank` as a potential argument of `chain_type` --- langchain/chains/question_answering/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain/chains/question_answering/__init__.py b/langchain/chains/question_answering/__init__.py index 522f7906088..2ba684f07af 100644 --- a/langchain/chains/question_answering/__init__.py +++ b/langchain/chains/question_answering/__init__.py @@ -196,7 +196,7 @@ def load_qa_chain( Args: llm: Language Model to use in the chain. chain_type: Type of document combining chain to use. Should be one of "stuff", - "map_reduce", and "refine". + "map_reduce", "map_rerank", and "refine". verbose: Whether chains should be run in verbose mode or not. Note that this applies to all chains that make up the final chain. callback_manager: Callback manager to use for the chain. From ccacf804a84548a50502b7d64d683a41e329b2ef Mon Sep 17 00:00:00 2001 From: Peter Stolz <50801264+PeterStolz@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:53:02 +0200 Subject: [PATCH 30/49] Fix format string in pinecone error handling (#2897) --- langchain/vectorstores/pinecone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain/vectorstores/pinecone.py b/langchain/vectorstores/pinecone.py index f06c5b9f05a..cc2169ac16a 100644 --- a/langchain/vectorstores/pinecone.py +++ b/langchain/vectorstores/pinecone.py @@ -218,7 +218,7 @@ class Pinecone(VectorStore): else: raise ValueError( f"Index '{index_name}' not found in your Pinecone project. " - "Did you mean one of the following indexes: {', '.join(indexes)}" + f"Did you mean one of the following indexes: {', '.join(indexes)}" ) for i in range(0, len(texts), batch_size): From 7e525a3b91ddf2cc0d9b3add5928460449c1d600 Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Fri, 14 Apr 2023 17:55:21 +0200 Subject: [PATCH 31/49] Add link to repo for deploying LangChain to Digitalocean App Platform (#2894) This PR adds a link to a minimal example of deploying `LangChain` to `Digitalocean App Platform`. --- docs/deployments.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/deployments.md b/docs/deployments.md index eef491e5da5..753b6ced43f 100644 --- a/docs/deployments.md +++ b/docs/deployments.md @@ -33,12 +33,17 @@ It implements a Question Answering app and contains instructions for deploying t A minimal example on how to run LangChain on Vercel using Flask. +## [Digitalocean App Platform](https://github.com/homanp/digitalocean-langchain) + +A minimal example on how to deploy LangChain to DigitalOcean App Platform. ## [SteamShip](https://github.com/steamship-core/steamship-langchain/) + This repository contains LangChain adapters for Steamship, enabling LangChain developers to rapidly deploy their apps on Steamship. This includes: production ready endpoints, horizontal scaling across dependencies, persistant storage of app state, multi-tenancy support, etc. ## [Langchain-serve](https://github.com/jina-ai/langchain-serve) + This repository allows users to serve local chains and agents as RESTful, gRPC, or Websocket APIs thanks to [Jina](https://docs.jina.ai/). Deploy your chains & agents with ease and enjoy independent scaling, serverless and autoscaling APIs, as well as a Streamlit playground on Jina AI Cloud. ## [BentoML](https://github.com/ssheng/BentoChain) From a508afa91c0bc9d9e84a8418a8ae378cbaf89571 Mon Sep 17 00:00:00 2001 From: Kwuang Tang <10319942+cktang88@users.noreply.github.com> Date: Fri, 14 Apr 2023 13:45:54 -0400 Subject: [PATCH 32/49] Add file filter param to Git loader (#2904) Allows users to specify what files should be loaded instead of indiscriminately loading the entire repo. extends #2851 NOTE: for reviewers, `hide whitespace` option recommended since I changed the indentation of an if-block to use `continue` instead so it looks less like a Christmas tree :) --- .../examples/example_data/test_repo1 | 1 + .../document_loaders/examples/git.ipynb | 58 +++++++++++-------- langchain/document_loaders/git.py | 53 ++++++++++------- 3 files changed, 65 insertions(+), 47 deletions(-) create mode 160000 docs/modules/indexes/document_loaders/examples/example_data/test_repo1 diff --git a/docs/modules/indexes/document_loaders/examples/example_data/test_repo1 b/docs/modules/indexes/document_loaders/examples/example_data/test_repo1 new file mode 160000 index 00000000000..7e525a3b91d --- /dev/null +++ b/docs/modules/indexes/document_loaders/examples/example_data/test_repo1 @@ -0,0 +1 @@ +Subproject commit 7e525a3b91ddf2cc0d9b3add5928460449c1d600 diff --git a/docs/modules/indexes/document_loaders/examples/git.ipynb b/docs/modules/indexes/document_loaders/examples/git.ipynb index 306a68fa467..d2ac89fbf0c 100644 --- a/docs/modules/indexes/document_loaders/examples/git.ipynb +++ b/docs/modules/indexes/document_loaders/examples/git.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -59,27 +59,16 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1040" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "len(data)" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -103,7 +92,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -112,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -125,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -134,16 +123,16 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "1040" + "1074" ] }, - "execution_count": 30, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -152,6 +141,25 @@ "len(data)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Filtering files to load" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.document_loaders import GitLoader\n", + "\n", + "# eg. loading only python files\n", + "loader = GitLoader(repo_path=\"./example_data/test_repo1/\", file_filter=lambda file_path: file_path.endswith(\".py\"))" + ] + }, { "cell_type": "code", "execution_count": null, @@ -176,7 +184,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.1" + "version": "3.11.3" } }, "nbformat": 4, diff --git a/langchain/document_loaders/git.py b/langchain/document_loaders/git.py index 155767629ec..eb10dcde8fb 100644 --- a/langchain/document_loaders/git.py +++ b/langchain/document_loaders/git.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional +from typing import Callable, List, Optional from langchain.docstore.document import Document from langchain.document_loaders.base import BaseLoader @@ -21,10 +21,12 @@ class GitLoader(BaseLoader): repo_path: str, clone_url: Optional[str] = None, branch: Optional[str] = "main", + file_filter: Optional[Callable[[str], bool]] = None, ): self.repo_path = repo_path self.clone_url = clone_url self.branch = branch + self.file_filter = file_filter def load(self) -> List[Document]: try: @@ -47,28 +49,35 @@ class GitLoader(BaseLoader): docs: List[Document] = [] for item in repo.tree().traverse(): - if isinstance(item, Blob): - file_path = os.path.join(self.repo_path, item.path) - rel_file_path = os.path.relpath(file_path, self.repo_path) - try: - with open(file_path, "rb") as f: - content = f.read() - file_type = os.path.splitext(item.name)[1] + if not isinstance(item, Blob): + continue - # loads only text files - try: - text_content = content.decode("utf-8") - except UnicodeDecodeError: - continue + file_path = os.path.join(self.repo_path, item.path) - metadata = { - "file_path": rel_file_path, - "file_name": item.name, - "file_type": file_type, - } - doc = Document(page_content=text_content, metadata=metadata) - docs.append(doc) - except Exception as e: - print(f"Error reading file {file_path}: {e}") + # uses filter to skip files + if self.file_filter and not self.file_filter(file_path): + continue + + rel_file_path = os.path.relpath(file_path, self.repo_path) + try: + with open(file_path, "rb") as f: + content = f.read() + file_type = os.path.splitext(item.name)[1] + + # loads only text files + try: + text_content = content.decode("utf-8") + except UnicodeDecodeError: + continue + + metadata = { + "file_path": rel_file_path, + "file_name": item.name, + "file_type": file_type, + } + doc = Document(page_content=text_content, metadata=metadata) + docs.append(doc) + except Exception as e: + print(f"Error reading file {file_path}: {e}") return docs From 30573b2e30551d93e10b8814d826db6b229e8870 Mon Sep 17 00:00:00 2001 From: pranjaldoshi96 Date: Fri, 14 Apr 2023 23:16:20 +0530 Subject: [PATCH 33/49] Correct instruction to use openweathermap utility in docstring (#2906) Co-authored-by: Pranjal Doshi --- langchain/utilities/openweathermap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langchain/utilities/openweathermap.py b/langchain/utilities/openweathermap.py index eac6109cb73..54737bdf835 100644 --- a/langchain/utilities/openweathermap.py +++ b/langchain/utilities/openweathermap.py @@ -13,8 +13,8 @@ class OpenWeatherMapAPIWrapper(BaseModel): Docs for using: 1. Go to OpenWeatherMap and sign up for an API key - 3. Save your API KEY into OPENWEATHERMAP_API_KEY env variable - 4. pip install wolframalpha + 2. Save your API KEY into OPENWEATHERMAP_API_KEY env variable + 3. pip install pyowm """ owm: Any From 634358db5e9d0f091c66c82b8ed1379ec6531f88 Mon Sep 17 00:00:00 2001 From: dev2049 <130488702+dev2049@users.noreply.github.com> Date: Fri, 14 Apr 2023 11:09:36 -0700 Subject: [PATCH 34/49] Fix OpenAI LLM docstring (#2910) --- langchain/llms/openai.py | 45 ++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/langchain/llms/openai.py b/langchain/llms/openai.py index ff99ee48bf0..8a54c5d3703 100644 --- a/langchain/llms/openai.py +++ b/langchain/llms/openai.py @@ -114,20 +114,7 @@ async def acompletion_with_retry( class BaseOpenAI(BaseLLM): - """Wrapper around OpenAI large language models. - - To use, you should have the ``openai`` python package installed, and the - environment variable ``OPENAI_API_KEY`` set with your API key. - - Any parameters that are valid to be passed to the openai.create call can be passed - in, even if not explicitly saved on this class. - - Example: - .. code-block:: python - - from langchain.llms import OpenAI - openai = OpenAI(model_name="text-davinci-003") - """ + """Wrapper around OpenAI large language models.""" client: Any #: :meta private: model_name: str = "text-davinci-003" @@ -541,7 +528,20 @@ class BaseOpenAI(BaseLLM): class OpenAI(BaseOpenAI): - """Generic OpenAI class that uses model name.""" + """Wrapper around OpenAI large language models. + + To use, you should have the ``openai`` python package installed, and the + environment variable ``OPENAI_API_KEY`` set with your API key. + + Any parameters that are valid to be passed to the openai.create call can be passed + in, even if not explicitly saved on this class. + + Example: + .. code-block:: python + + from langchain.llms import OpenAI + openai = OpenAI(model_name="text-davinci-003") + """ @property def _invocation_params(self) -> Dict[str, Any]: @@ -549,7 +549,20 @@ class OpenAI(BaseOpenAI): class AzureOpenAI(BaseOpenAI): - """Azure specific OpenAI class that uses deployment name.""" + """Wrapper around Azure-specific OpenAI large language models. + + To use, you should have the ``openai`` python package installed, and the + environment variable ``OPENAI_API_KEY`` set with your API key. + + Any parameters that are valid to be passed to the openai.create call can be passed + in, even if not explicitly saved on this class. + + Example: + .. code-block:: python + + from langchain.llms import AzureOpenAI + openai = AzureOpenAI(model_name="text-davinci-003") + """ deployment_name: str = "" """Deployment name to use.""" From 7ee87eb0c8df10315b45ebbddcad36a72b7fe7b9 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Fri, 14 Apr 2023 22:19:58 +0200 Subject: [PATCH 35/49] Comet callback updates (#2889) I'm working with @DN6 and I made some small fixes and improvements after playing with the integration. --- docs/ecosystem/comet_tracking.ipynb | 52 ++++++++++++------------ langchain/callbacks/comet_ml_callback.py | 12 +++--- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/docs/ecosystem/comet_tracking.ipynb b/docs/ecosystem/comet_tracking.ipynb index fa8b1217dbd..4d33bd00ab5 100644 --- a/docs/ecosystem/comet_tracking.ipynb +++ b/docs/ecosystem/comet_tracking.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -9,7 +8,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -17,7 +15,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -31,7 +28,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -39,7 +35,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -52,14 +47,13 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install comet_ml\n", - "!pip install langchain\n", - "!pip install openai\n", - "!pip install google-search-results" + "%pip install comet_ml langchain openai google-search-results spacy textstat pandas\n", + "\n", + "import sys\n", + "!{sys.executable} -m spacy download en_core_web_sm" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -67,7 +61,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -86,7 +79,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -94,7 +86,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -109,12 +100,12 @@ "source": [ "import os\n", "\n", - "%env OPENAI_API_KEY=\"...\"\n", - "%env SERPAPI_API_KEY=\"...\"" + "os.environ[\"OPENAI_API_KEY\"] = \"...\"\n", + "#os.environ[\"OPENAI_ORGANIZATION\"] = \"...\"\n", + "os.environ[\"SERPAPI_API_KEY\"] = \"...\"" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -149,7 +140,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -185,12 +175,11 @@ "synopsis_chain = LLMChain(llm=llm, prompt=prompt_template, callback_manager=manager)\n", "\n", "test_prompts = [{\"title\": \"Documentary about Bigfoot in Paris\"}]\n", - "synopsis_chain.apply(test_prompts)\n", + "print(synopsis_chain.apply(test_prompts))\n", "comet_callback.flush_tracker(synopsis_chain, finish=True)" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -232,7 +221,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -240,7 +228,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -256,7 +243,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install rouge-score" + "%pip install rouge-score" ] }, { @@ -336,16 +323,29 @@ " \"\"\"\n", " }\n", "]\n", - "synopsis_chain.apply(test_prompts)\n", + "print(synopsis_chain.apply(test_prompts))\n", "comet_callback.flush_tracker(synopsis_chain, finish=True)" ] } ], "metadata": { - "language_info": { - "name": "python" + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" }, - "orig_nbformat": 4 + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.15" + } }, "nbformat": 4, "nbformat_minor": 2 diff --git a/langchain/callbacks/comet_ml_callback.py b/langchain/callbacks/comet_ml_callback.py index c716d43d3f4..6f061f14aa4 100644 --- a/langchain/callbacks/comet_ml_callback.py +++ b/langchain/callbacks/comet_ml_callback.py @@ -34,12 +34,10 @@ def _get_experiment( ) -> Any: comet_ml = import_comet_ml() - experiment = comet_ml.config.get_global_experiment() - if experiment is None: - experiment = comet_ml.Experiment( # type: ignore - workspace=workspace, - project_name=project_name, - ) + experiment = comet_ml.Experiment( # type: ignore + workspace=workspace, + project_name=project_name, + ) return experiment @@ -132,7 +130,7 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): warning = ( "The comet_ml callback is currently in beta and is subject to change " "based on updates to `langchain`. Please report any issues to " - "https://github.com/comet_ml/issue_tracking/issues with the tag " + "https://github.com/comet-ml/issue_tracking/issues with the tag " "`langchain`." ) comet_ml.LOGGER.warning(warning) From 66bef1d7ed17f00e7b554ca5413e336970489253 Mon Sep 17 00:00:00 2001 From: Kwuang Tang <10319942+cktang88@users.noreply.github.com> Date: Fri, 14 Apr 2023 18:02:21 -0400 Subject: [PATCH 36/49] Ignore files from .gitignore in Git loader (#2909) fixes #2905 extends #2851 --- langchain/document_loaders/git.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/langchain/document_loaders/git.py b/langchain/document_loaders/git.py index eb10dcde8fb..a862e8f0312 100644 --- a/langchain/document_loaders/git.py +++ b/langchain/document_loaders/git.py @@ -54,6 +54,10 @@ class GitLoader(BaseLoader): file_path = os.path.join(self.repo_path, item.path) + ignored_files = repo.ignored([file_path]) + if len(ignored_files): + continue + # uses filter to skip files if self.file_filter and not self.file_filter(file_path): continue From 392f1b32188d40e45adabb85a1641780eedd006b Mon Sep 17 00:00:00 2001 From: Mike Lambert Date: Fri, 14 Apr 2023 18:09:07 -0400 Subject: [PATCH 37/49] Add Anthropic ChatModel to langchain (#2293) * Adds an Anthropic ChatModel * Factors out common code in our LLMModel and ChatModel * Supports streaming llm-tokens to the callbacks on a delta basis (until a future V2 API does that for us) * Some fixes --- langchain/chat_models/anthropic.py | 145 +++++++++++++++ langchain/llms/anthropic.py | 172 +++++++++--------- .../chat_models/test_anthropic.py | 81 +++++++++ .../integration_tests/llms/test_anthropic.py | 2 - 4 files changed, 316 insertions(+), 84 deletions(-) create mode 100644 langchain/chat_models/anthropic.py create mode 100644 tests/integration_tests/chat_models/test_anthropic.py diff --git a/langchain/chat_models/anthropic.py b/langchain/chat_models/anthropic.py new file mode 100644 index 00000000000..b63fbf052d9 --- /dev/null +++ b/langchain/chat_models/anthropic.py @@ -0,0 +1,145 @@ +from typing import List, Optional + +from pydantic import Extra + +from langchain.chat_models.base import BaseChatModel +from langchain.llms.anthropic import _AnthropicCommon +from langchain.schema import ( + AIMessage, + BaseMessage, + ChatGeneration, + ChatMessage, + ChatResult, + HumanMessage, + SystemMessage, +) + + +class ChatAnthropic(BaseChatModel, _AnthropicCommon): + r"""Wrapper around Anthropic's large language model. + + To use, you should have the ``anthropic`` python package installed, and the + environment variable ``ANTHROPIC_API_KEY`` set with your API key, or pass + it as a named parameter to the constructor. + + Example: + .. code-block:: python + import anthropic + from langchain.llms import Anthropic + model = Anthropic(model="", anthropic_api_key="my-api-key") + + # Simplest invocation, automatically wrapped with HUMAN_PROMPT + # and AI_PROMPT. + response = model("What are the biggest risks facing humanity?") + + # Or if you want to use the chat mode, build a few-shot-prompt, or + # put words in the Assistant's mouth, use HUMAN_PROMPT and AI_PROMPT: + raw_prompt = "What are the biggest risks facing humanity?" + prompt = f"{anthropic.HUMAN_PROMPT} {prompt}{anthropic.AI_PROMPT}" + response = model(prompt) + """ + + class Config: + """Configuration for this pydantic object.""" + + extra = Extra.forbid + + @property + def _llm_type(self) -> str: + """Return type of chat model.""" + return "anthropic-chat" + + def _convert_one_message_to_text(self, message: BaseMessage) -> str: + if isinstance(message, ChatMessage): + message_text = f"\n\n{message.role.capitalize()}: {message.content}" + elif isinstance(message, HumanMessage): + message_text = f"{self.HUMAN_PROMPT} {message.content}" + elif isinstance(message, AIMessage): + message_text = f"{self.AI_PROMPT} {message.content}" + elif isinstance(message, SystemMessage): + message_text = f"{self.HUMAN_PROMPT} {message.content}" + else: + raise ValueError(f"Got unknown type {message}") + return message_text + + def _convert_messages_to_text(self, messages: List[BaseMessage]) -> str: + """Format a list of strings into a single string with necessary newlines. + + Args: + messages (List[BaseMessage]): List of BaseMessage to combine. + + Returns: + str: Combined string with necessary newlines. + """ + return "".join( + self._convert_one_message_to_text(message) for message in messages + ) + + def _convert_messages_to_prompt(self, messages: List[BaseMessage]) -> str: + """Format a list of messages into a full prompt for the Anthropic model + + Args: + messages (List[BaseMessage]): List of BaseMessage to combine. + + Returns: + str: Combined string with necessary HUMAN_PROMPT and AI_PROMPT tags. + """ + if not self.AI_PROMPT: + raise NameError("Please ensure the anthropic package is loaded") + + if not isinstance(messages[-1], AIMessage): + messages.append(AIMessage(content="")) + text = self._convert_messages_to_text(messages) + return ( + text.rstrip() + ) # trim off the trailing ' ' that might come from the "Assistant: " + + def _generate( + self, messages: List[BaseMessage], stop: Optional[List[str]] = None + ) -> ChatResult: + prompt = self._convert_messages_to_prompt(messages) + params = {"prompt": prompt, "stop_sequences": stop, **self._default_params} + + if self.streaming: + completion = "" + stream_resp = self.client.completion_stream(**params) + for data in stream_resp: + delta = data["completion"][len(completion) :] + completion = data["completion"] + self.callback_manager.on_llm_new_token( + delta, + verbose=self.verbose, + ) + else: + response = self.client.completion(**params) + completion = response["completion"] + message = AIMessage(content=completion) + return ChatResult(generations=[ChatGeneration(message=message)]) + + async def _agenerate( + self, messages: List[BaseMessage], stop: Optional[List[str]] = None + ) -> ChatResult: + prompt = self._convert_messages_to_prompt(messages) + params = {"prompt": prompt, "stop_sequences": stop, **self._default_params} + + if self.streaming: + completion = "" + stream_resp = await self.client.acompletion_stream(**params) + async for data in stream_resp: + delta = data["completion"][len(completion) :] + completion = data["completion"] + if self.callback_manager.is_async: + await self.callback_manager.on_llm_new_token( + delta, + verbose=self.verbose, + ) + else: + self.callback_manager.on_llm_new_token( + delta, + verbose=self.verbose, + ) + else: + response = await self.client.acompletion(**params) + completion = response["completion"] + message = AIMessage(content=completion) + return ChatResult(generations=[ChatGeneration(message=message)]) diff --git a/langchain/llms/anthropic.py b/langchain/llms/anthropic.py index bc4cfd42032..24d9def1eb7 100644 --- a/langchain/llms/anthropic.py +++ b/langchain/llms/anthropic.py @@ -1,15 +1,100 @@ """Wrapper around Anthropic APIs.""" import re -from typing import Any, Dict, Generator, List, Mapping, Optional +from typing import Any, Callable, Dict, Generator, List, Mapping, Optional -from pydantic import Extra, root_validator +from pydantic import BaseModel, Extra, root_validator from langchain.llms.base import LLM from langchain.utils import get_from_dict_or_env -class Anthropic(LLM): - r"""Wrapper around Anthropic large language models. +class _AnthropicCommon(BaseModel): + client: Any = None #: :meta private: + model: str = "claude-latest" + """Model name to use.""" + + max_tokens_to_sample: int = 256 + """Denotes the number of tokens to predict per generation.""" + + temperature: Optional[float] = None + """A non-negative float that tunes the degree of randomness in generation.""" + + top_k: Optional[int] = None + """Number of most likely tokens to consider at each step.""" + + top_p: Optional[float] = None + """Total probability mass of tokens to consider at each step.""" + + streaming: bool = False + """Whether to stream the results.""" + + anthropic_api_key: Optional[str] = None + + HUMAN_PROMPT: Optional[str] = None + AI_PROMPT: Optional[str] = None + count_tokens: Optional[Callable[[str], int]] = None + + @root_validator() + def validate_environment(cls, values: Dict) -> Dict: + """Validate that api key and python package exists in environment.""" + anthropic_api_key = get_from_dict_or_env( + values, "anthropic_api_key", "ANTHROPIC_API_KEY" + ) + try: + import anthropic + + values["client"] = anthropic.Client(anthropic_api_key) + values["HUMAN_PROMPT"] = anthropic.HUMAN_PROMPT + values["AI_PROMPT"] = anthropic.AI_PROMPT + values["count_tokens"] = anthropic.count_tokens + except ImportError: + raise ValueError( + "Could not import anthropic python package. " + "Please it install it with `pip install anthropic`." + ) + return values + + @property + def _default_params(self) -> Mapping[str, Any]: + """Get the default parameters for calling Anthropic API.""" + d = { + "max_tokens_to_sample": self.max_tokens_to_sample, + "model": self.model, + } + if self.temperature is not None: + d["temperature"] = self.temperature + if self.top_k is not None: + d["top_k"] = self.top_k + if self.top_p is not None: + d["top_p"] = self.top_p + return d + + @property + def _identifying_params(self) -> Mapping[str, Any]: + """Get the identifying parameters.""" + return {**{}, **self._default_params} + + def _get_anthropic_stop(self, stop: Optional[List[str]] = None) -> List[str]: + if not self.HUMAN_PROMPT or not self.AI_PROMPT: + raise NameError("Please ensure the anthropic package is loaded") + + if stop is None: + stop = [] + + # Never want model to invent new turns of Human / Assistant dialog. + stop.extend([self.HUMAN_PROMPT]) + + return stop + + def get_num_tokens(self, text: str) -> int: + """Calculate number of tokens.""" + if not self.count_tokens: + raise NameError("Please ensure the anthropic package is loaded") + return self.count_tokens(text) + + +class Anthropic(LLM, _AnthropicCommon): + r"""Wrapper around Anthropic's large language models. To use, you should have the ``anthropic`` python package installed, and the environment variable ``ANTHROPIC_API_KEY`` set with your API key, or pass @@ -32,73 +117,15 @@ class Anthropic(LLM): response = model(prompt) """ - client: Any #: :meta private: - model: str = "claude-v1" - """Model name to use.""" - - max_tokens_to_sample: int = 256 - """Denotes the number of tokens to predict per generation.""" - - temperature: float = 1.0 - """A non-negative float that tunes the degree of randomness in generation.""" - - top_k: int = 0 - """Number of most likely tokens to consider at each step.""" - - top_p: float = 1 - """Total probability mass of tokens to consider at each step.""" - - streaming: bool = False - """Whether to stream the results.""" - - anthropic_api_key: Optional[str] = None - - HUMAN_PROMPT: Optional[str] = None - AI_PROMPT: Optional[str] = None - class Config: """Configuration for this pydantic object.""" extra = Extra.forbid - @root_validator() - def validate_environment(cls, values: Dict) -> Dict: - """Validate that api key and python package exists in environment.""" - anthropic_api_key = get_from_dict_or_env( - values, "anthropic_api_key", "ANTHROPIC_API_KEY" - ) - try: - import anthropic - - values["client"] = anthropic.Client(anthropic_api_key) - values["HUMAN_PROMPT"] = anthropic.HUMAN_PROMPT - values["AI_PROMPT"] = anthropic.AI_PROMPT - except ImportError: - raise ValueError( - "Could not import anthropic python package. " - "Please install it with `pip install anthropic`." - ) - return values - - @property - def _default_params(self) -> Mapping[str, Any]: - """Get the default parameters for calling Anthropic API.""" - return { - "max_tokens_to_sample": self.max_tokens_to_sample, - "temperature": self.temperature, - "top_k": self.top_k, - "top_p": self.top_p, - } - - @property - def _identifying_params(self) -> Mapping[str, Any]: - """Get the identifying parameters.""" - return {**{"model": self.model}, **self._default_params} - @property def _llm_type(self) -> str: """Return type of llm.""" - return "anthropic" + return "anthropic-llm" def _wrap_prompt(self, prompt: str) -> str: if not self.HUMAN_PROMPT or not self.AI_PROMPT: @@ -115,18 +142,6 @@ class Anthropic(LLM): # As a last resort, wrap the prompt ourselves to emulate instruct-style. return f"{self.HUMAN_PROMPT} {prompt}{self.AI_PROMPT} Sure, here you go:\n" - def _get_anthropic_stop(self, stop: Optional[List[str]] = None) -> List[str]: - if not self.HUMAN_PROMPT or not self.AI_PROMPT: - raise NameError("Please ensure the anthropic package is loaded") - - if stop is None: - stop = [] - - # Never want model to invent new turns of Human / Assistant dialog. - stop.extend([self.HUMAN_PROMPT, self.AI_PROMPT]) - - return stop - def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str: r"""Call out to Anthropic's completion endpoint. @@ -148,10 +163,8 @@ class Anthropic(LLM): stop = self._get_anthropic_stop(stop) if self.streaming: stream_resp = self.client.completion_stream( - model=self.model, prompt=self._wrap_prompt(prompt), stop_sequences=stop, - stream=True, **self._default_params, ) current_completion = "" @@ -163,7 +176,6 @@ class Anthropic(LLM): ) return current_completion response = self.client.completion( - model=self.model, prompt=self._wrap_prompt(prompt), stop_sequences=stop, **self._default_params, @@ -175,10 +187,8 @@ class Anthropic(LLM): stop = self._get_anthropic_stop(stop) if self.streaming: stream_resp = await self.client.acompletion_stream( - model=self.model, prompt=self._wrap_prompt(prompt), stop_sequences=stop, - stream=True, **self._default_params, ) current_completion = "" @@ -195,7 +205,6 @@ class Anthropic(LLM): ) return current_completion response = await self.client.acompletion( - model=self.model, prompt=self._wrap_prompt(prompt), stop_sequences=stop, **self._default_params, @@ -227,7 +236,6 @@ class Anthropic(LLM): """ stop = self._get_anthropic_stop(stop) return self.client.completion_stream( - model=self.model, prompt=self._wrap_prompt(prompt), stop_sequences=stop, **self._default_params, diff --git a/tests/integration_tests/chat_models/test_anthropic.py b/tests/integration_tests/chat_models/test_anthropic.py new file mode 100644 index 00000000000..f04b30e2514 --- /dev/null +++ b/tests/integration_tests/chat_models/test_anthropic.py @@ -0,0 +1,81 @@ +"""Test Anthropic API wrapper.""" +from typing import List + +import pytest + +from langchain.callbacks.base import CallbackManager +from langchain.chat_models.anthropic import ChatAnthropic +from langchain.schema import ( + AIMessage, + BaseMessage, + ChatGeneration, + HumanMessage, + LLMResult, +) +from tests.unit_tests.callbacks.fake_callback_handler import FakeCallbackHandler + + +def test_anthropic_call() -> None: + """Test valid call to anthropic.""" + chat = ChatAnthropic(model="bare-nano-0") + message = HumanMessage(content="Hello") + response = chat([message]) + assert isinstance(response, AIMessage) + assert isinstance(response.content, str) + + +def test_anthropic_streaming() -> None: + """Test streaming tokens from anthropic.""" + chat = ChatAnthropic(model="bare-nano-0", streaming=True) + message = HumanMessage(content="Hello") + response = chat([message]) + assert isinstance(response, AIMessage) + assert isinstance(response.content, str) + + +def test_anthropic_streaming_callback() -> None: + """Test that streaming correctly invokes on_llm_new_token callback.""" + callback_handler = FakeCallbackHandler() + callback_manager = CallbackManager([callback_handler]) + chat = ChatAnthropic( + streaming=True, + callback_manager=callback_manager, + verbose=True, + ) + message = HumanMessage(content="Write me a sentence with 100 words.") + chat([message]) + assert callback_handler.llm_streams > 1 + + +@pytest.mark.asyncio +async def test_anthropic_async_streaming_callback() -> None: + """Test that streaming correctly invokes on_llm_new_token callback.""" + callback_handler = FakeCallbackHandler() + callback_manager = CallbackManager([callback_handler]) + chat = ChatAnthropic( + streaming=True, + callback_manager=callback_manager, + verbose=True, + ) + chat_messages: List[BaseMessage] = [ + HumanMessage(content="How many toes do dogs have?") + ] + result: LLMResult = await chat.agenerate([chat_messages]) + assert callback_handler.llm_streams > 1 + assert isinstance(result, LLMResult) + for response in result.generations[0]: + assert isinstance(response, ChatGeneration) + assert isinstance(response.text, str) + assert response.text == response.message.content + + +def test_formatting() -> None: + chat = ChatAnthropic() + + chat_messages: List[BaseMessage] = [HumanMessage(content="Hello")] + result = chat._convert_messages_to_prompt(chat_messages) + assert result == "\n\nHuman: Hello\n\nAssistant:" + + chat_messages = [HumanMessage(content="Hello"), AIMessage(content="Answer:")] + result = chat._convert_messages_to_prompt(chat_messages) + assert result == "\n\nHuman: Hello\n\nAssistant: Answer:" diff --git a/tests/integration_tests/llms/test_anthropic.py b/tests/integration_tests/llms/test_anthropic.py index eaa509bf644..8c7717cfc7d 100644 --- a/tests/integration_tests/llms/test_anthropic.py +++ b/tests/integration_tests/llms/test_anthropic.py @@ -32,7 +32,6 @@ def test_anthropic_streaming_callback() -> None: callback_handler = FakeCallbackHandler() callback_manager = CallbackManager([callback_handler]) llm = Anthropic( - model="claude-v1", streaming=True, callback_manager=callback_manager, verbose=True, @@ -55,7 +54,6 @@ async def test_anthropic_async_streaming_callback() -> None: callback_handler = FakeCallbackHandler() callback_manager = CallbackManager([callback_handler]) llm = Anthropic( - model="claude-v1", streaming=True, callback_manager=callback_manager, verbose=True, From 13a0ed064b4c4c4b5978d61cc6ec0950dcdcf870 Mon Sep 17 00:00:00 2001 From: Akash NP <91617769+9akashnp8@users.noreply.github.com> Date: Sat, 15 Apr 2023 05:06:03 +0530 Subject: [PATCH 38/49] add encoding to avoid UnicodeDecodeError (#2908) **About** Specify encoding to avoid UnicodeDecodeError when reading .txt for users who are following the tutorial. **Reference** ``` return codecs.charmap_decode(input,self.errors,decoding_table)[0] UnicodeDecodeError: 'charmap' codec can't decode byte 0x9d in position 1205: character maps to ``` **Environment** OS: Win 11 Python: 3.8 --- docs/modules/indexes/getting_started.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/indexes/getting_started.ipynb b/docs/modules/indexes/getting_started.ipynb index 58a133ada0f..0c6a9d593aa 100644 --- a/docs/modules/indexes/getting_started.ipynb +++ b/docs/modules/indexes/getting_started.ipynb @@ -99,7 +99,7 @@ "outputs": [], "source": [ "from langchain.document_loaders import TextLoader\n", - "loader = TextLoader('../state_of_the_union.txt')" + "loader = TextLoader('../state_of_the_union.txt', encoding='utf8')" ] }, { From ec59e9d886904e27e6436a50a2cb755a621382d6 Mon Sep 17 00:00:00 2001 From: Ankush Gola <9536492+agola11@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:22:01 -0700 Subject: [PATCH 39/49] Fix ChatAnthropic stop_sequences error (#2919) (#2920) Note to self: Always run integration tests, even on "that last minute change you thought would be safe" :) --------- Co-authored-by: Mike Lambert --- .../models/chat/integrations/anthropic.ipynb | 171 ++++++++++++++++++ langchain/chat_models/__init__.py | 3 +- langchain/chat_models/anthropic.py | 22 +-- langchain/llms/anthropic.py | 2 +- poetry.lock | 2 +- pyproject.toml | 2 +- .../chat_models/test_anthropic.py | 8 +- 7 files changed, 189 insertions(+), 21 deletions(-) create mode 100644 docs/modules/models/chat/integrations/anthropic.ipynb diff --git a/docs/modules/models/chat/integrations/anthropic.ipynb b/docs/modules/models/chat/integrations/anthropic.ipynb new file mode 100644 index 00000000000..57cbf48821d --- /dev/null +++ b/docs/modules/models/chat/integrations/anthropic.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "bf733a38-db84-4363-89e2-de6735c37230", + "metadata": {}, + "source": [ + "# Anthropic\n", + "\n", + "This notebook covers how to get started with Anthropic chat models." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "d4a7c55d-b235-4ca4-a579-c90cc9570da9", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain.chat_models import ChatAnthropic\n", + "from langchain.prompts.chat import (\n", + " ChatPromptTemplate,\n", + " SystemMessagePromptTemplate,\n", + " AIMessagePromptTemplate,\n", + " HumanMessagePromptTemplate,\n", + ")\n", + "from langchain.schema import (\n", + " AIMessage,\n", + " HumanMessage,\n", + " SystemMessage\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "70cf04e8-423a-4ff6-8b09-f11fb711c817", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "chat = ChatAnthropic()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "8199ef8f-eb8b-4253-9ea0-6c24a013ca4c", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "AIMessage(content=\" J'adore programmer.\", additional_kwargs={})" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "messages = [\n", + " HumanMessage(content=\"Translate this sentence from English to French. I love programming.\")\n", + "]\n", + "chat(messages)" + ] + }, + { + "cell_type": "markdown", + "id": "c361ab1e-8c0c-4206-9e3c-9d1424a12b9c", + "metadata": {}, + "source": [ + "## `ChatAnthropic` also supports async and streaming functionality:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "93a21c5c-6ef9-4688-be60-b2e1f94842fb", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain.callbacks.base import CallbackManager\n", + "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "c5fac0e9-05a4-4fc1-a3b3-e5bbb24b971b", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "LLMResult(generations=[[ChatGeneration(text=\" J'aime programmer.\", generation_info=None, message=AIMessage(content=\" J'aime programmer.\", additional_kwargs={}))]], llm_output={})" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "await chat.agenerate([messages])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "025be980-e50d-4a68-93dc-c9c7b500ce34", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " J'aime la programmation." + ] + }, + { + "data": { + "text/plain": [ + "AIMessage(content=\" J'aime la programmation.\", additional_kwargs={})" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "chat = ChatAnthropic(streaming=True, verbose=True, callback_manager=CallbackManager([StreamingStdOutCallbackHandler()]))\n", + "chat(messages)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/langchain/chat_models/__init__.py b/langchain/chat_models/__init__.py index 88bafc7e7a2..fdfe5e7d355 100644 --- a/langchain/chat_models/__init__.py +++ b/langchain/chat_models/__init__.py @@ -1,5 +1,6 @@ +from langchain.chat_models.anthropic import ChatAnthropic from langchain.chat_models.azure_openai import AzureChatOpenAI from langchain.chat_models.openai import ChatOpenAI from langchain.chat_models.promptlayer_openai import PromptLayerChatOpenAI -__all__ = ["ChatOpenAI", "AzureChatOpenAI", "PromptLayerChatOpenAI"] +__all__ = ["ChatOpenAI", "AzureChatOpenAI", "PromptLayerChatOpenAI", "ChatAnthropic"] diff --git a/langchain/chat_models/anthropic.py b/langchain/chat_models/anthropic.py index b63fbf052d9..f56c606361e 100644 --- a/langchain/chat_models/anthropic.py +++ b/langchain/chat_models/anthropic.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Any, Dict, List, Optional from pydantic import Extra @@ -26,17 +26,7 @@ class ChatAnthropic(BaseChatModel, _AnthropicCommon): .. code-block:: python import anthropic from langchain.llms import Anthropic - model = Anthropic(model="", anthropic_api_key="my-api-key") - - # Simplest invocation, automatically wrapped with HUMAN_PROMPT - # and AI_PROMPT. - response = model("What are the biggest risks facing humanity?") - - # Or if you want to use the chat mode, build a few-shot-prompt, or - # put words in the Assistant's mouth, use HUMAN_PROMPT and AI_PROMPT: - raw_prompt = "What are the biggest risks facing humanity?" - prompt = f"{anthropic.HUMAN_PROMPT} {prompt}{anthropic.AI_PROMPT}" - response = model(prompt) + model = ChatAnthropic(model="", anthropic_api_key="my-api-key") """ class Config: @@ -98,7 +88,9 @@ class ChatAnthropic(BaseChatModel, _AnthropicCommon): self, messages: List[BaseMessage], stop: Optional[List[str]] = None ) -> ChatResult: prompt = self._convert_messages_to_prompt(messages) - params = {"prompt": prompt, "stop_sequences": stop, **self._default_params} + params: Dict[str, Any] = {"prompt": prompt, **self._default_params} + if stop: + params["stop_sequences"] = stop if self.streaming: completion = "" @@ -120,7 +112,9 @@ class ChatAnthropic(BaseChatModel, _AnthropicCommon): self, messages: List[BaseMessage], stop: Optional[List[str]] = None ) -> ChatResult: prompt = self._convert_messages_to_prompt(messages) - params = {"prompt": prompt, "stop_sequences": stop, **self._default_params} + params: Dict[str, Any] = {"prompt": prompt, **self._default_params} + if stop: + params["stop_sequences"] = stop if self.streaming: completion = "" diff --git a/langchain/llms/anthropic.py b/langchain/llms/anthropic.py index 24d9def1eb7..e609627967e 100644 --- a/langchain/llms/anthropic.py +++ b/langchain/llms/anthropic.py @@ -10,7 +10,7 @@ from langchain.utils import get_from_dict_or_env class _AnthropicCommon(BaseModel): client: Any = None #: :meta private: - model: str = "claude-latest" + model: str = "claude-v1" """Model name to use.""" max_tokens_to_sample: int = 256 diff --git a/poetry.lock b/poetry.lock index 7a88ee2a9f7..6b0e37aa1a5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -9035,4 +9035,4 @@ qdrant = ["qdrant-client"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "373f68ef16e7f3d5d9cde8b81c5f261096cc537ddca4f6a36711d7215b63f226" +content-hash = "7e343fa8e31d8fcf1023cbda592f64c05e80015c4e0e23c1d387d2e9671ce995" diff --git a/pyproject.toml b/pyproject.toml index 3cc2d497615..351d6e43b77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ pinecone-text = {version = "^0.4.2", optional = true} weaviate-client = {version = "^3", optional = true} google-api-python-client = {version = "2.70.0", optional = true} wolframalpha = {version = "5.0.0", optional = true} -anthropic = {version = "^0.2.4", optional = true} +anthropic = {version = "^0.2.6", optional = true} qdrant-client = {version = "^1.1.2", optional = true, python = ">=3.8.1,<3.12"} dataclasses-json = "^0.5.7" tensorflow-text = {version = "^2.11.0", optional = true, python = "^3.10, <3.12"} diff --git a/tests/integration_tests/chat_models/test_anthropic.py b/tests/integration_tests/chat_models/test_anthropic.py index f04b30e2514..60fe58f319f 100644 --- a/tests/integration_tests/chat_models/test_anthropic.py +++ b/tests/integration_tests/chat_models/test_anthropic.py @@ -17,7 +17,7 @@ from tests.unit_tests.callbacks.fake_callback_handler import FakeCallbackHandler def test_anthropic_call() -> None: """Test valid call to anthropic.""" - chat = ChatAnthropic(model="bare-nano-0") + chat = ChatAnthropic(model="test") message = HumanMessage(content="Hello") response = chat([message]) assert isinstance(response, AIMessage) @@ -26,7 +26,7 @@ def test_anthropic_call() -> None: def test_anthropic_streaming() -> None: """Test streaming tokens from anthropic.""" - chat = ChatAnthropic(model="bare-nano-0", streaming=True) + chat = ChatAnthropic(model="test", streaming=True) message = HumanMessage(content="Hello") response = chat([message]) assert isinstance(response, AIMessage) @@ -38,11 +38,12 @@ def test_anthropic_streaming_callback() -> None: callback_handler = FakeCallbackHandler() callback_manager = CallbackManager([callback_handler]) chat = ChatAnthropic( + model="test", streaming=True, callback_manager=callback_manager, verbose=True, ) - message = HumanMessage(content="Write me a sentence with 100 words.") + message = HumanMessage(content="Write me a sentence with 10 words.") chat([message]) assert callback_handler.llm_streams > 1 @@ -53,6 +54,7 @@ async def test_anthropic_async_streaming_callback() -> None: callback_handler = FakeCallbackHandler() callback_manager = CallbackManager([callback_handler]) chat = ChatAnthropic( + model="test", streaming=True, callback_manager=callback_manager, verbose=True, From 0aa828b1dc72b8e577500d3f1c9aaef28dee604f Mon Sep 17 00:00:00 2001 From: Hai Nguyen Mau Date: Sat, 15 Apr 2023 17:31:43 +0200 Subject: [PATCH 40/49] typo fix (#2937) missing w in link --- .../models/chat/integrations/promptlayer_chatopenai.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/models/chat/integrations/promptlayer_chatopenai.ipynb b/docs/modules/models/chat/integrations/promptlayer_chatopenai.ipynb index fbd23379c02..d75c3a0a3e4 100644 --- a/docs/modules/models/chat/integrations/promptlayer_chatopenai.ipynb +++ b/docs/modules/models/chat/integrations/promptlayer_chatopenai.ipynb @@ -115,7 +115,7 @@ "id": "a2d76826", "metadata": {}, "source": [ - "**The above request should now appear on your [PromptLayer dashboard](https://ww.promptlayer.com).**" + "**The above request should now appear on your [PromptLayer dashboard](https://www.promptlayer.com).**" ] }, { From cf2789d86d16cf33ff0cef03de40cdfdf6ec83af Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Sat, 15 Apr 2023 08:48:51 -0700 Subject: [PATCH 41/49] delete antropic chat notebook (#2945) --- .../models/chat/integrations/anthropic.ipynb | 171 ------------------ 1 file changed, 171 deletions(-) delete mode 100644 docs/modules/models/chat/integrations/anthropic.ipynb diff --git a/docs/modules/models/chat/integrations/anthropic.ipynb b/docs/modules/models/chat/integrations/anthropic.ipynb deleted file mode 100644 index 57cbf48821d..00000000000 --- a/docs/modules/models/chat/integrations/anthropic.ipynb +++ /dev/null @@ -1,171 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "bf733a38-db84-4363-89e2-de6735c37230", - "metadata": {}, - "source": [ - "# Anthropic\n", - "\n", - "This notebook covers how to get started with Anthropic chat models." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "d4a7c55d-b235-4ca4-a579-c90cc9570da9", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from langchain.chat_models import ChatAnthropic\n", - "from langchain.prompts.chat import (\n", - " ChatPromptTemplate,\n", - " SystemMessagePromptTemplate,\n", - " AIMessagePromptTemplate,\n", - " HumanMessagePromptTemplate,\n", - ")\n", - "from langchain.schema import (\n", - " AIMessage,\n", - " HumanMessage,\n", - " SystemMessage\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "70cf04e8-423a-4ff6-8b09-f11fb711c817", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "chat = ChatAnthropic()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "8199ef8f-eb8b-4253-9ea0-6c24a013ca4c", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "AIMessage(content=\" J'adore programmer.\", additional_kwargs={})" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "messages = [\n", - " HumanMessage(content=\"Translate this sentence from English to French. I love programming.\")\n", - "]\n", - "chat(messages)" - ] - }, - { - "cell_type": "markdown", - "id": "c361ab1e-8c0c-4206-9e3c-9d1424a12b9c", - "metadata": {}, - "source": [ - "## `ChatAnthropic` also supports async and streaming functionality:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "93a21c5c-6ef9-4688-be60-b2e1f94842fb", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from langchain.callbacks.base import CallbackManager\n", - "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "c5fac0e9-05a4-4fc1-a3b3-e5bbb24b971b", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "LLMResult(generations=[[ChatGeneration(text=\" J'aime programmer.\", generation_info=None, message=AIMessage(content=\" J'aime programmer.\", additional_kwargs={}))]], llm_output={})" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "await chat.agenerate([messages])" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "025be980-e50d-4a68-93dc-c9c7b500ce34", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " J'aime la programmation." - ] - }, - { - "data": { - "text/plain": [ - "AIMessage(content=\" J'aime la programmation.\", additional_kwargs={})" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "chat = ChatAnthropic(streaming=True, verbose=True, callback_manager=CallbackManager([StreamingStdOutCallbackHandler()]))\n", - "chat(messages)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From ad3973a3b8369315a5e780b57594cebe2b64aed5 Mon Sep 17 00:00:00 2001 From: Nahin Khan Date: Sat, 15 Apr 2023 18:53:25 +0300 Subject: [PATCH 42/49] Fix typo (#2942) --- docs/modules/memory/examples/agent_with_memory_in_db.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/memory/examples/agent_with_memory_in_db.ipynb b/docs/modules/memory/examples/agent_with_memory_in_db.ipynb index 201a6533d84..b96760bbbe2 100644 --- a/docs/modules/memory/examples/agent_with_memory_in_db.ipynb +++ b/docs/modules/memory/examples/agent_with_memory_in_db.ipynb @@ -16,7 +16,7 @@ "In order to add a memory with an external message store to an agent we are going to do the following steps:\n", "\n", "1. We are going to create a `RedisChatMessageHistory` to connect to an external database to store the messages in.\n", - "2. We are going to create an `LLMChain` useing that chat history as memory.\n", + "2. We are going to create an `LLMChain` using that chat history as memory.\n", "3. We are going to use that `LLMChain` to create a custom Agent.\n", "\n", "For the purposes of this exercise, we are going to create a simple custom Agent that has access to a search tool and utilizes the `ConversationBufferMemory` class." From c4ae8c1d243a30c0e7cb5aceb53a232b7559c869 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Sat, 15 Apr 2023 09:23:19 -0700 Subject: [PATCH 43/49] bump ver to 140 (#2895) --- docs/use_cases/code.md | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/use_cases/code.md b/docs/use_cases/code.md index 01d79f3d80b..0977b7269a4 100644 --- a/docs/use_cases/code.md +++ b/docs/use_cases/code.md @@ -23,3 +23,4 @@ Query Understanding: GPT-4 processes user queries, grasping the context and extr The full tutorial is available below. - [Twitter the-algorithm codebase analysis with Deep Lake](code/twitter-the-algorithm-analysis-deeplake.ipynb): A notebook walking through how to parse github source code and run queries conversation. +- [LangChain codebase analysis with Deep Lake](code/code-analysis-deeplake.ipynb): A notebook walking through how to analyze and do question answering over THIS code base. diff --git a/pyproject.toml b/pyproject.toml index 351d6e43b77..c7e292649c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langchain" -version = "0.0.139" +version = "0.0.140" description = "Building applications with LLMs through composability" authors = [] license = "MIT" From b3a5b51728a0a69b508bed42bbea5cba4714ff8e Mon Sep 17 00:00:00 2001 From: Davit Buniatyan Date: Sat, 15 Apr 2023 10:49:16 -0700 Subject: [PATCH 44/49] [minor] Deep Lake auth improvements in docs, kwargs pass, faster tests (#2927) Minor cosmetic changes - Activeloop environment cred authentication in notebooks with `getpass.getpass` (instead of CLI which not always works) - much faster tests with Deep Lake pytest mode on - Deep Lake kwargs pass Notes - I put pytest environment creds inside `vectorstores/conftest.py`, but feel free to suggest a better location. For context, if I put in `test_deeplake.py`, `ruff` doesn't let me to set them before import deeplake --------- Co-authored-by: Davit Buniatyan --- .../vectorstores/examples/deeplake.ipynb | 320 ++++++++++++++++-- .../code/code-analysis-deeplake.ipynb | 39 +-- ...tter-the-algorithm-analysis-deeplake.ipynb | 71 ++-- langchain/vectorstores/deeplake.py | 12 +- .../vectorstores/conftest.py | 7 + 5 files changed, 348 insertions(+), 101 deletions(-) diff --git a/docs/modules/indexes/vectorstores/examples/deeplake.ipynb b/docs/modules/indexes/vectorstores/examples/deeplake.ipynb index 3ec489ac83f..ff0a5a487d6 100644 --- a/docs/modules/indexes/vectorstores/examples/deeplake.ipynb +++ b/docs/modules/indexes/vectorstores/examples/deeplake.ipynb @@ -17,34 +17,36 @@ "metadata": {}, "outputs": [], "source": [ - "!python3 -m pip install openai deeplake" + "!python3 -m pip install openai deeplake tiktoken" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from langchain.embeddings.openai import OpenAIEmbeddings\n", "from langchain.text_splitter import CharacterTextSplitter\n", - "from langchain.vectorstores import DeepLake\n", - "from langchain.document_loaders import TextLoader" + "from langchain.vectorstores import DeepLake" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import os\n", - "os.environ['OPENAI_API_KEY'] = 'sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'" + "import getpass\n", + "\n", + "os.environ['OPENAI_API_KEY'] = getpass.getpass('OpenAI API Key:')\n", + "embeddings = OpenAIEmbeddings()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -60,9 +62,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mem://langchain loaded successfully.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating ingest: 100%|██████████| 1/1 [00:04<00:00\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset(path='mem://langchain', tensors=['embedding', 'ids', 'metadata', 'text'])\n", + "\n", + " tensor htype shape dtype compression\n", + " ------- ------- ------- ------- ------- \n", + " embedding generic (4, 1536) float32 None \n", + " ids text (4, 1) str None \n", + " metadata json (4, 1) str None \n", + " text text (4, 1) str None \n" + ] + } + ], "source": [ "db = DeepLake.from_documents(docs, embeddings)\n", "\n", @@ -72,9 +103,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \n", + "\n", + "Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \n", + "\n", + "One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \n", + "\n", + "And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.\n" + ] + } + ], "source": [ "print(docs[0].page_content)" ] @@ -89,9 +134,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/sdb/davit/.local/lib/python3.10/site-packages/langchain/llms/openai.py:624: UserWarning: You are trying to use a chat model. This way of initializing it is no longer supported. Instead, please use: `from langchain.chat_models import ChatOpenAI`\n", + " warnings.warn(\n" + ] + } + ], "source": [ "from langchain.chains import RetrievalQA\n", "from langchain.llms import OpenAIChat\n", @@ -101,9 +155,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'The president nominated Circuit Court of Appeals Judge Ketanji Brown Jackson for the United States Supreme Court and praised her qualifications and broad support from both Democrats and Republicans.'" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "query = 'What did the president say about Ketanji Brown Jackson'\n", "qa.run(query)" @@ -119,9 +184,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mem://langchain loaded successfully.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating ingest: 100%|██████████| 1/1 [00:04<00:00\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset(path='mem://langchain', tensors=['embedding', 'ids', 'metadata', 'text'])\n", + "\n", + " tensor htype shape dtype compression\n", + " ------- ------- ------- ------- ------- \n", + " embedding generic (42, 1536) float32 None \n", + " ids text (42, 1) str None \n", + " metadata json (42, 1) str None \n", + " text text (42, 1) str None \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + } + ], "source": [ "import random\n", "\n", @@ -133,9 +232,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 42/42 [00:00<00:00, 3456.17it/s]\n" + ] + }, + { + "data": { + "text/plain": [ + "[Document(page_content='A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder. Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. \\n\\nAnd if we are to advance liberty and justice, we need to secure the Border and fix the immigration system. \\n\\nWe can do both. At our border, we’ve installed new technology like cutting-edge scanners to better detect drug smuggling. \\n\\nWe’ve set up joint patrols with Mexico and Guatemala to catch more human traffickers. \\n\\nWe’re putting in place dedicated immigration judges so families fleeing persecution and violence can have their cases heard faster. \\n\\nWe’re securing commitments and supporting partners in South and Central America to host more refugees and secure their own borders.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2013}),\n", + " Document(page_content='And for our LGBTQ+ Americans, let’s finally get the bipartisan Equality Act to my desk. The onslaught of state laws targeting transgender Americans and their families is wrong. \\n\\nAs I said last year, especially to our younger transgender Americans, I will always have your back as your President, so you can be yourself and reach your God-given potential. \\n\\nWhile it often appears that we never agree, that isn’t true. I signed 80 bipartisan bills into law last year. From preventing government shutdowns to protecting Asian-Americans from still-too-common hate crimes to reforming military justice. \\n\\nAnd soon, we’ll strengthen the Violence Against Women Act that I first wrote three decades ago. It is important for us to show the nation that we can come together and do big things. \\n\\nSo tonight I’m offering a Unity Agenda for the Nation. Four big things we can do together. \\n\\nFirst, beat the opioid epidemic.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2013}),\n", + " Document(page_content='Vice President Harris and I ran for office with a new economic vision for America. \\n\\nInvest in America. Educate Americans. Grow the workforce. Build the economy from the bottom up \\nand the middle out, not from the top down. \\n\\nBecause we know that when the middle class grows, the poor have a ladder up and the wealthy do very well. \\n\\nAmerica used to have the best roads, bridges, and airports on Earth. \\n\\nNow our infrastructure is ranked 13th in the world. \\n\\nWe won’t be able to compete for the jobs of the 21st Century if we don’t fix that. \\n\\nThat’s why it was so important to pass the Bipartisan Infrastructure Law—the most sweeping investment to rebuild America in history. \\n\\nThis was a bipartisan effort, and I want to thank the members of both parties who worked to make it happen. \\n\\nWe’re done talking about infrastructure weeks. \\n\\nWe’re going to have an infrastructure decade.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2013}),\n", + " Document(page_content='It is going to transform America and put us on a path to win the economic competition of the 21st Century that we face with the rest of the world—particularly with China. \\n\\nAs I’ve told Xi Jinping, it is never a good bet to bet against the American people. \\n\\nWe’ll create good jobs for millions of Americans, modernizing roads, airports, ports, and waterways all across America. \\n\\nAnd we’ll do it all to withstand the devastating effects of the climate crisis and promote environmental justice. \\n\\nWe’ll build a national network of 500,000 electric vehicle charging stations, begin to replace poisonous lead pipes—so every child—and every American—has clean water to drink at home and at school, provide affordable high-speed internet for every American—urban, suburban, rural, and tribal communities. \\n\\n4,000 projects have already been announced. \\n\\nAnd tonight, I’m announcing that this year we will start fixing over 65,000 miles of highway and 1,500 bridges in disrepair.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2013})]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "db.similarity_search('What did the president say about Ketanji Brown Jackson', filter={'year': 2013})" ] @@ -151,9 +271,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(page_content='Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \\n\\nTonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \\n\\nOne of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \\n\\nAnd I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2012}),\n", + " Document(page_content='A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder. Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. \\n\\nAnd if we are to advance liberty and justice, we need to secure the Border and fix the immigration system. \\n\\nWe can do both. At our border, we’ve installed new technology like cutting-edge scanners to better detect drug smuggling. \\n\\nWe’ve set up joint patrols with Mexico and Guatemala to catch more human traffickers. \\n\\nWe’re putting in place dedicated immigration judges so families fleeing persecution and violence can have their cases heard faster. \\n\\nWe’re securing commitments and supporting partners in South and Central America to host more refugees and secure their own borders.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2013}),\n", + " Document(page_content='And for our LGBTQ+ Americans, let’s finally get the bipartisan Equality Act to my desk. The onslaught of state laws targeting transgender Americans and their families is wrong. \\n\\nAs I said last year, especially to our younger transgender Americans, I will always have your back as your President, so you can be yourself and reach your God-given potential. \\n\\nWhile it often appears that we never agree, that isn’t true. I signed 80 bipartisan bills into law last year. From preventing government shutdowns to protecting Asian-Americans from still-too-common hate crimes to reforming military justice. \\n\\nAnd soon, we’ll strengthen the Violence Against Women Act that I first wrote three decades ago. It is important for us to show the nation that we can come together and do big things. \\n\\nSo tonight I’m offering a Unity Agenda for the Nation. Four big things we can do together. \\n\\nFirst, beat the opioid epidemic.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2013}),\n", + " Document(page_content='Tonight, I’m announcing a crackdown on these companies overcharging American businesses and consumers. \\n\\nAnd as Wall Street firms take over more nursing homes, quality in those homes has gone down and costs have gone up. \\n\\nThat ends on my watch. \\n\\nMedicare is going to set higher standards for nursing homes and make sure your loved ones get the care they deserve and expect. \\n\\nWe’ll also cut costs and keep the economy going strong by giving workers a fair shot, provide more training and apprenticeships, hire them based on their skills not degrees. \\n\\nLet’s pass the Paycheck Fairness Act and paid leave. \\n\\nRaise the minimum wage to $15 an hour and extend the Child Tax Credit, so no one has to raise a family in poverty. \\n\\nLet’s increase Pell Grants and increase our historic support of HBCUs, and invest in what Jill—our First Lady who teaches full-time—calls America’s best-kept secret: community colleges.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2014})]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "db.similarity_search('What did the president say about Ketanji Brown Jackson?', distance_metric='cos')" ] @@ -169,9 +303,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(page_content='Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \\n\\nTonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \\n\\nOne of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \\n\\nAnd I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2012}),\n", + " Document(page_content='One was stationed at bases and breathing in toxic smoke from “burn pits” that incinerated wastes of war—medical and hazard material, jet fuel, and more. \\n\\nWhen they came home, many of the world’s fittest and best trained warriors were never the same. \\n\\nHeadaches. Numbness. Dizziness. \\n\\nA cancer that would put them in a flag-draped coffin. \\n\\nI know. \\n\\nOne of those soldiers was my son Major Beau Biden. \\n\\nWe don’t know for sure if a burn pit was the cause of his brain cancer, or the diseases of so many of our troops. \\n\\nBut I’m committed to finding out everything we can. \\n\\nCommitted to military families like Danielle Robinson from Ohio. \\n\\nThe widow of Sergeant First Class Heath Robinson. \\n\\nHe was born a soldier. Army National Guard. Combat medic in Kosovo and Iraq. \\n\\nStationed near Baghdad, just yards from burn pits the size of football fields. \\n\\nHeath’s widow Danielle is here with us tonight. They loved going to Ohio State football games. He loved building Legos with their daughter.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2014}),\n", + " Document(page_content='As Ohio Senator Sherrod Brown says, “It’s time to bury the label “Rust Belt.” \\n\\nIt’s time. \\n\\nBut with all the bright spots in our economy, record job growth and higher wages, too many families are struggling to keep up with the bills. \\n\\nInflation is robbing them of the gains they might otherwise feel. \\n\\nI get it. That’s why my top priority is getting prices under control. \\n\\nLook, our economy roared back faster than most predicted, but the pandemic meant that businesses had a hard time hiring enough workers to keep up production in their factories. \\n\\nThe pandemic also disrupted global supply chains. \\n\\nWhen factories close, it takes longer to make goods and get them from the warehouse to the store, and prices go up. \\n\\nLook at cars. \\n\\nLast year, there weren’t enough semiconductors to make all the cars that people wanted to buy. \\n\\nAnd guess what, prices of automobiles went up. \\n\\nSo—we have a choice. \\n\\nOne way to fight inflation is to drive down wages and make Americans poorer.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2012}),\n", + " Document(page_content='We can’t change how divided we’ve been. But we can change how we move forward—on COVID-19 and other issues we must face together. \\n\\nI recently visited the New York City Police Department days after the funerals of Officer Wilbert Mora and his partner, Officer Jason Rivera. \\n\\nThey were responding to a 9-1-1 call when a man shot and killed them with a stolen gun. \\n\\nOfficer Mora was 27 years old. \\n\\nOfficer Rivera was 22. \\n\\nBoth Dominican Americans who’d grown up on the same streets they later chose to patrol as police officers. \\n\\nI spoke with their families and told them that we are forever in debt for their sacrifice, and we will carry on their mission to restore the trust and safety every community deserves. \\n\\nI’ve worked on these issues a long time. \\n\\nI know what works: Investing in crime preventionand community police officers who’ll walk the beat, who’ll know the neighborhood, and who can restore trust and safety.', metadata={'source': '../../../state_of_the_union.txt', 'year': 2012})]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "db.max_marginal_relevance_search('What did the president say about Ketanji Brown Jackson?')" ] @@ -187,21 +335,87 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ - "!activeloop login -t " + "os.environ['ACTIVELOOP_TOKEN'] = getpass.getpass('Activeloop Token:')" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Your Deep Lake dataset has been successfully created!\n", + "The dataset is private so make sure you are logged in!\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\\" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This dataset can be visualized in Jupyter Notebook by ds.visualize() or at https://app.activeloop.ai/davitbun/linkedin\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hub://davitbun/linkedin loaded successfully.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating ingest: 100%|██████████| 1/1 [00:23<00:00\n", + "/" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset(path='hub://davitbun/linkedin', tensors=['embedding', 'ids', 'metadata', 'text'])\n", + "\n", + " tensor htype shape dtype compression\n", + " ------- ------- ------- ------- ------- \n", + " embedding generic (42, 1536) float32 None \n", + " ids text (42, 1) str None \n", + " metadata json (42, 1) str None \n", + " text text (42, 1) str None \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], "source": [ "# Embed and store the texts\n", - "dataset_path = \"hub://{username}/{dataset_name}\" # could be also ./local/path (much faster locally), s3://bucket/path/to/dataset, gcs://path/to/dataset, etc.\n", + "dataset_path = f\"hub://{USERNAME}/{DATASET_NAME}\" # could be also ./local/path (much faster locally), s3://bucket/path/to/dataset, gcs://path/to/dataset, etc.\n", "\n", "embedding = OpenAIEmbeddings()\n", "vectordb = DeepLake.from_documents(documents=docs, embedding=embedding, dataset_path=dataset_path)" @@ -209,9 +423,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \n", + "\n", + "Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \n", + "\n", + "One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \n", + "\n", + "And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.\n" + ] + } + ], "source": [ "query = \"What did the president say about Ketanji Brown Jackson\"\n", "docs = db.similarity_search(query)\n", @@ -220,11 +448,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset(path='hub://davitbun/linkedin', tensors=['embedding', 'ids', 'metadata', 'text'])\n", + "\n", + " tensor htype shape dtype compression\n", + " ------- ------- ------- ------- ------- \n", + " embedding generic (42, 1536) float32 None \n", + " ids text (42, 1) str None \n", + " metadata json (42, 1) str None \n", + " text text (42, 1) str None \n" + ] + } + ], + "source": [ + "vectordb.ds.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ - "vectordb.ds.summary()" + "embeddings = vectordb.ds.embedding.numpy()" ] }, { @@ -232,9 +484,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "embeddings = vectordb.ds.embedding.numpy()" - ] + "source": [] } ], "metadata": { diff --git a/docs/use_cases/code/code-analysis-deeplake.ipynb b/docs/use_cases/code/code-analysis-deeplake.ipynb index d946f2778ca..19bafcf1b59 100644 --- a/docs/use_cases/code/code-analysis-deeplake.ipynb +++ b/docs/use_cases/code/code-analysis-deeplake.ipynb @@ -80,7 +80,7 @@ }, "outputs": [ { - "name": "stdin", + "name": "stdout", "output_type": "stream", "text": [ " ········\n" @@ -96,10 +96,11 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "Authenticate into Deep Lake if you want to create your own dataset and publish it. You can get an API key from the platform at https://app.activeloop.ai" + "Authenticate into Deep Lake if you want to create your own dataset and publish it. You can get an API key from the platform at [app.activeloop.ai](https://app.activeloop.ai)" ] }, { @@ -110,7 +111,7 @@ }, "outputs": [ { - "name": "stdin", + "name": "stdout", "output_type": "stream", "text": [ " ········\n" @@ -118,37 +119,7 @@ } ], "source": [ - "DEEPLAKE_ACCOUNT_NAME = getpass()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdin", - "output_type": "stream", - "text": [ - " ········\n" - ] - } - ], - "source": [ - "os.environ['DEEPLAKE_KEY'] = getpass()" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "!activeloop login -t $DEEPLAKE_KEY" + "os.environ['ACTIVELOOP_TOKEN'] = getpass.getpass('Activeloop Token:')" ] }, { diff --git a/docs/use_cases/code/twitter-the-algorithm-analysis-deeplake.ipynb b/docs/use_cases/code/twitter-the-algorithm-analysis-deeplake.ipynb index c6b0919aac7..cbfc09a3df0 100644 --- a/docs/use_cases/code/twitter-the-algorithm-analysis-deeplake.ipynb +++ b/docs/use_cases/code/twitter-the-algorithm-analysis-deeplake.ipynb @@ -18,31 +18,13 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "Define OpenAI embeddings, Deep Lake multi-modal vector store api and authenticate. For full documentation of Deep Lake please follow https://docs.activeloop.ai/ and API reference https://docs.deeplake.ai/en/latest/" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from langchain.embeddings.openai import OpenAIEmbeddings\n", - "from langchain.vectorstores import DeepLake\n", + "Define OpenAI embeddings, Deep Lake multi-modal vector store api and authenticate. For full documentation of Deep Lake please follow [docs](https://docs.activeloop.ai/) and [API reference](https://docs.deeplake.ai/en/latest/).\n", "\n", - "os.environ['OPENAI_API_KEY']='sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'\n", - "embeddings = OpenAIEmbeddings()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Authenticate into Deep Lake if you want to create your own dataset and publish it. You can get an API key from the platform at https://app.activeloop.ai" + "Authenticate into Deep Lake if you want to create your own dataset and publish it. You can get an API key from the [platform](https://app.activeloop.ai)" ] }, { @@ -51,7 +33,15 @@ "metadata": {}, "outputs": [], "source": [ - "!activeloop login -t " + "import os\n", + "import getpass\n", + "\n", + "from langchain.embeddings.openai import OpenAIEmbeddings\n", + "from langchain.vectorstores import DeepLake\n", + "\n", + "os.environ['OPENAI_API_KEY'] = getpass.getpass('OpenAI API Key:')\n", + "os.environ['ACTIVELOOP_TOKEN'] = getpass.getpass('Activeloop Token:')\n", + "embeddings = OpenAIEmbeddings()" ] }, { @@ -143,15 +133,35 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 7, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "-" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ "This dataset can be visualized in Jupyter Notebook by ds.visualize() or at https://app.activeloop.ai/davitbun/twitter-algorithm\n", - "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "-" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "hub://davitbun/twitter-algorithm loaded successfully.\n", "\n" ] @@ -184,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -205,7 +215,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -224,7 +234,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -267,9 +277,14 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ + "-> **Question**: What does favCountParams do? \n", + "\n", + "**Answer**: `favCountParams` is an optional ThriftLinearFeatureRankingParams instance that represents the parameters related to the \"favorite count\" feature in the ranking process. It is used to control the weight of the favorite count feature while ranking tweets. The favorite count is the number of times a tweet has been marked as a favorite by users, and it is considered an important signal in the ranking of tweets. By using `favCountParams`, the system can adjust the importance of the favorite count while calculating the final ranking score of a tweet. \n", + "\n", "-> **Question**: is it Likes + Bookmarks, or not clear from the code?\n", "\n", "**Answer**: From the provided code, it is not clear if the favorite count metric is determined by the sum of likes and bookmarks. The favorite count is mentioned in the code, but there is no explicit reference to how it is calculated in terms of likes and bookmarks. \n", @@ -423,7 +438,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.1" + "version": "3.10.0" } }, "nbformat": 4, diff --git a/langchain/vectorstores/deeplake.py b/langchain/vectorstores/deeplake.py index 92aef77fc6a..8e2e3e5e868 100644 --- a/langchain/vectorstores/deeplake.py +++ b/langchain/vectorstores/deeplake.py @@ -97,6 +97,7 @@ class DeepLake(VectorStore): read_only: Optional[bool] = False, ingestion_batch_size: int = 1024, num_workers: int = 4, + **kwargs: Any, ) -> None: """Initialize with Deep Lake client.""" self.ingestion_batch_size = ingestion_batch_size @@ -113,14 +114,18 @@ class DeepLake(VectorStore): self._deeplake = deeplake if deeplake.exists(dataset_path, token=token): - self.ds = deeplake.load(dataset_path, token=token, read_only=read_only) + self.ds = deeplake.load( + dataset_path, token=token, read_only=read_only, **kwargs + ) logger.warning( f"Deep Lake Dataset in {dataset_path} already exists, " f"loading from the storage" ) self.ds.summary() else: - self.ds = deeplake.empty(dataset_path, token=token, overwrite=True) + self.ds = deeplake.empty( + dataset_path, token=token, overwrite=True, **kwargs + ) with self.ds: self.ds.create_tensor( @@ -466,8 +471,7 @@ class DeepLake(VectorStore): DeepLake: Deep Lake dataset. """ deeplake_dataset = cls( - dataset_path=dataset_path, - embedding_function=embedding, + dataset_path=dataset_path, embedding_function=embedding, **kwargs ) deeplake_dataset.add_texts(texts=texts, metadatas=metadatas, ids=ids) return deeplake_dataset diff --git a/tests/integration_tests/vectorstores/conftest.py b/tests/integration_tests/vectorstores/conftest.py index 9687af92ced..507e0e1eead 100644 --- a/tests/integration_tests/vectorstores/conftest.py +++ b/tests/integration_tests/vectorstores/conftest.py @@ -9,6 +9,13 @@ from langchain.embeddings import OpenAIEmbeddings from langchain.schema import Document from langchain.text_splitter import CharacterTextSplitter +# Those environment variables turn on Deep Lake pytest mode. +# It significantly makes tests run much faster. +# Need to run before `import deeplake` +os.environ["BUGGER_OFF"] = "true" +os.environ["DEEPLAKE_DOWNLOAD_PATH"] = "./testing/local_storage" +os.environ["DEEPLAKE_PYTEST_ENABLED"] = "true" + # This fixture returns a dictionary containing filter_headers options # for replacing certain headers with dummy values during cassette playback From 7c73e9df5d1e24f5b7d34c4de7bebdaa4bfa962a Mon Sep 17 00:00:00 2001 From: dev2049 <130488702+dev2049@users.noreply.github.com> Date: Sat, 15 Apr 2023 10:49:49 -0700 Subject: [PATCH 45/49] Add kwargs to VectorStore.maximum_marginal_relevance (#2921) Same as similarity_search, allows child classes to add vector store-specific args (this was technically already happening in couple places but now typing is correct). --- langchain/vectorstores/base.py | 10 +++++----- langchain/vectorstores/chroma.py | 2 ++ langchain/vectorstores/deeplake.py | 4 ++-- langchain/vectorstores/faiss.py | 8 ++++++-- langchain/vectorstores/qdrant.py | 6 +++++- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/langchain/vectorstores/base.py b/langchain/vectorstores/base.py index f995d3dda31..f6a37286d21 100644 --- a/langchain/vectorstores/base.py +++ b/langchain/vectorstores/base.py @@ -118,7 +118,7 @@ class VectorStore(ABC): return await asyncio.get_event_loop().run_in_executor(None, func) def max_marginal_relevance_search( - self, query: str, k: int = 4, fetch_k: int = 20 + self, query: str, k: int = 4, fetch_k: int = 20, **kwargs: Any ) -> List[Document]: """Return docs selected using the maximal marginal relevance. @@ -136,18 +136,18 @@ class VectorStore(ABC): raise NotImplementedError async def amax_marginal_relevance_search( - self, query: str, k: int = 4, fetch_k: int = 20 + self, query: str, k: int = 4, fetch_k: int = 20, **kwargs: Any ) -> List[Document]: """Return docs selected using the maximal marginal relevance.""" # This is a temporary workaround to make the similarity search # asynchronous. The proper solution is to make the similarity search # asynchronous in the vector store implementations. - func = partial(self.max_marginal_relevance_search, query, k, fetch_k) + func = partial(self.max_marginal_relevance_search, query, k, fetch_k, **kwargs) return await asyncio.get_event_loop().run_in_executor(None, func) def max_marginal_relevance_search_by_vector( - self, embedding: List[float], k: int = 4, fetch_k: int = 20 + self, embedding: List[float], k: int = 4, fetch_k: int = 20, **kwargs: Any ) -> List[Document]: """Return docs selected using the maximal marginal relevance. @@ -165,7 +165,7 @@ class VectorStore(ABC): raise NotImplementedError async def amax_marginal_relevance_search_by_vector( - self, embedding: List[float], k: int = 4, fetch_k: int = 20 + self, embedding: List[float], k: int = 4, fetch_k: int = 20, **kwargs: Any ) -> List[Document]: """Return docs selected using the maximal marginal relevance.""" raise NotImplementedError diff --git a/langchain/vectorstores/chroma.py b/langchain/vectorstores/chroma.py index 60ecf3de3c4..64d34efb25e 100644 --- a/langchain/vectorstores/chroma.py +++ b/langchain/vectorstores/chroma.py @@ -193,6 +193,7 @@ class Chroma(VectorStore): k: int = 4, fetch_k: int = 20, filter: Optional[Dict[str, str]] = None, + **kwargs: Any, ) -> List[Document]: """Return docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity @@ -227,6 +228,7 @@ class Chroma(VectorStore): k: int = 4, fetch_k: int = 20, filter: Optional[Dict[str, str]] = None, + **kwargs: Any, ) -> List[Document]: """Return docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity diff --git a/langchain/vectorstores/deeplake.py b/langchain/vectorstores/deeplake.py index 8e2e3e5e868..71d51cf41b4 100644 --- a/langchain/vectorstores/deeplake.py +++ b/langchain/vectorstores/deeplake.py @@ -391,7 +391,7 @@ class DeepLake(VectorStore): ) def max_marginal_relevance_search_by_vector( - self, embedding: List[float], k: int = 4, fetch_k: int = 20 + self, embedding: List[float], k: int = 4, fetch_k: int = 20, **kwargs: Any ) -> List[Document]: """Return docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity @@ -411,7 +411,7 @@ class DeepLake(VectorStore): ) def max_marginal_relevance_search( - self, query: str, k: int = 4, fetch_k: int = 20 + self, query: str, k: int = 4, fetch_k: int = 20, **kwargs: Any ) -> List[Document]: """Return docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity diff --git a/langchain/vectorstores/faiss.py b/langchain/vectorstores/faiss.py index 4157fa9db68..4727a555903 100644 --- a/langchain/vectorstores/faiss.py +++ b/langchain/vectorstores/faiss.py @@ -208,7 +208,7 @@ class FAISS(VectorStore): return [doc for doc, _ in docs_and_scores] def max_marginal_relevance_search_by_vector( - self, embedding: List[float], k: int = 4, fetch_k: int = 20 + self, embedding: List[float], k: int = 4, fetch_k: int = 20, **kwargs: Any ) -> List[Document]: """Return docs selected using the maximal marginal relevance. @@ -243,7 +243,11 @@ class FAISS(VectorStore): return docs def max_marginal_relevance_search( - self, query: str, k: int = 4, fetch_k: int = 20 + self, + query: str, + k: int = 4, + fetch_k: int = 20, + **kwargs: Any, ) -> List[Document]: """Return docs selected using the maximal marginal relevance. diff --git a/langchain/vectorstores/qdrant.py b/langchain/vectorstores/qdrant.py index 1526379e885..b46f38cdb4e 100644 --- a/langchain/vectorstores/qdrant.py +++ b/langchain/vectorstores/qdrant.py @@ -147,7 +147,11 @@ class Qdrant(VectorStore): ] def max_marginal_relevance_search( - self, query: str, k: int = 4, fetch_k: int = 20 + self, + query: str, + k: int = 4, + fetch_k: int = 20, + **kwargs: Any, ) -> List[Document]: """Return docs selected using the maximal marginal relevance. From 36aa7f30e4294a9ca8db1293e38a89409930a5ef Mon Sep 17 00:00:00 2001 From: dev2049 <130488702+dev2049@users.noreply.github.com> Date: Sat, 15 Apr 2023 10:50:25 -0700 Subject: [PATCH 46/49] Move PythonRepl -> langchain.utilities (#2917) --- langchain/chains/llm_math/base.py | 2 +- langchain/chains/pal/base.py | 2 +- langchain/python.py | 28 +++------------------------- langchain/tools/python/tool.py | 2 +- langchain/utilities/__init__.py | 4 ++-- langchain/utilities/python.py | 25 +++++++++++++++++++++++++ tests/unit_tests/test_python.py | 2 +- 7 files changed, 34 insertions(+), 31 deletions(-) create mode 100644 langchain/utilities/python.py diff --git a/langchain/chains/llm_math/base.py b/langchain/chains/llm_math/base.py index 65a8adf4942..5c7ef09efc6 100644 --- a/langchain/chains/llm_math/base.py +++ b/langchain/chains/llm_math/base.py @@ -7,8 +7,8 @@ from langchain.chains.base import Chain from langchain.chains.llm import LLMChain from langchain.chains.llm_math.prompt import PROMPT from langchain.prompts.base import BasePromptTemplate -from langchain.python import PythonREPL from langchain.schema import BaseLanguageModel +from langchain.utilities import PythonREPL class LLMMathChain(Chain): diff --git a/langchain/chains/pal/base.py b/langchain/chains/pal/base.py index 1bcdef3ae7f..0d15b90be76 100644 --- a/langchain/chains/pal/base.py +++ b/langchain/chains/pal/base.py @@ -13,8 +13,8 @@ from langchain.chains.llm import LLMChain from langchain.chains.pal.colored_object_prompt import COLORED_OBJECT_PROMPT from langchain.chains.pal.math_prompt import MATH_PROMPT from langchain.prompts.base import BasePromptTemplate -from langchain.python import PythonREPL from langchain.schema import BaseLanguageModel +from langchain.utilities import PythonREPL class PALChain(Chain): diff --git a/langchain/python.py b/langchain/python.py index a14bc2e13e9..28b9c306968 100644 --- a/langchain/python.py +++ b/langchain/python.py @@ -1,26 +1,4 @@ -"""Mock Python REPL.""" -import sys -from io import StringIO -from typing import Dict, Optional +"""For backwards compatibility.""" +from langchain.utilities.python import PythonREPL -from pydantic import BaseModel, Field - - -class PythonREPL(BaseModel): - """Simulates a standalone Python REPL.""" - - globals: Optional[Dict] = Field(default_factory=dict, alias="_globals") - locals: Optional[Dict] = Field(default_factory=dict, alias="_locals") - - def run(self, command: str) -> str: - """Run command with own globals/locals and returns anything printed.""" - old_stdout = sys.stdout - sys.stdout = mystdout = StringIO() - try: - exec(command, self.globals, self.locals) - sys.stdout = old_stdout - output = mystdout.getvalue() - except Exception as e: - sys.stdout = old_stdout - output = str(e) - return output +__all__ = ["PythonREPL"] diff --git a/langchain/tools/python/tool.py b/langchain/tools/python/tool.py index 8f32643d40d..81339cedc34 100644 --- a/langchain/tools/python/tool.py +++ b/langchain/tools/python/tool.py @@ -7,8 +7,8 @@ from typing import Dict, Optional from pydantic import Field, root_validator -from langchain.python import PythonREPL from langchain.tools.base import BaseTool +from langchain.utilities import PythonREPL def _get_default_python_repl() -> PythonREPL: diff --git a/langchain/utilities/__init__.py b/langchain/utilities/__init__.py index 1d83373b5d3..5b82f2a9dcf 100644 --- a/langchain/utilities/__init__.py +++ b/langchain/utilities/__init__.py @@ -1,5 +1,4 @@ """General utilities.""" -from langchain.python import PythonREPL from langchain.requests import TextRequestsWrapper from langchain.utilities.apify import ApifyWrapper from langchain.utilities.bash import BashProcess @@ -7,6 +6,7 @@ from langchain.utilities.bing_search import BingSearchAPIWrapper from langchain.utilities.google_search import GoogleSearchAPIWrapper from langchain.utilities.google_serper import GoogleSerperAPIWrapper from langchain.utilities.openweathermap import OpenWeatherMapAPIWrapper +from langchain.utilities.python import PythonREPL from langchain.utilities.searx_search import SearxSearchWrapper from langchain.utilities.serpapi import SerpAPIWrapper from langchain.utilities.wikipedia import WikipediaAPIWrapper @@ -16,7 +16,6 @@ __all__ = [ "ApifyWrapper", "BashProcess", "TextRequestsWrapper", - "PythonREPL", "GoogleSearchAPIWrapper", "GoogleSerperAPIWrapper", "WolframAlphaAPIWrapper", @@ -25,4 +24,5 @@ __all__ = [ "BingSearchAPIWrapper", "WikipediaAPIWrapper", "OpenWeatherMapAPIWrapper", + "PythonREPL", ] diff --git a/langchain/utilities/python.py b/langchain/utilities/python.py new file mode 100644 index 00000000000..4abb22a1ebc --- /dev/null +++ b/langchain/utilities/python.py @@ -0,0 +1,25 @@ +import sys +from io import StringIO +from typing import Dict, Optional + +from pydantic import BaseModel, Field + + +class PythonREPL(BaseModel): + """Simulates a standalone Python REPL.""" + + globals: Optional[Dict] = Field(default_factory=dict, alias="_globals") + locals: Optional[Dict] = Field(default_factory=dict, alias="_locals") + + def run(self, command: str) -> str: + """Run command with own globals/locals and returns anything printed.""" + old_stdout = sys.stdout + sys.stdout = mystdout = StringIO() + try: + exec(command, self.globals, self.locals) + sys.stdout = old_stdout + output = mystdout.getvalue() + except Exception as e: + sys.stdout = old_stdout + output = str(e) + return output diff --git a/tests/unit_tests/test_python.py b/tests/unit_tests/test_python.py index e048bcb2dcb..28319542723 100644 --- a/tests/unit_tests/test_python.py +++ b/tests/unit_tests/test_python.py @@ -3,8 +3,8 @@ import sys import pytest -from langchain.python import PythonREPL from langchain.tools.python.tool import PythonAstREPLTool, PythonREPLTool +from langchain.utilities import PythonREPL _SAMPLE_CODE = """ ``` From baf350e32bdc85f61efde736eeb61c243b8437b2 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Sat, 15 Apr 2023 12:47:36 -0700 Subject: [PATCH 47/49] parametrize redis (#2946) --- langchain/vectorstores/redis.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/langchain/vectorstores/redis.py b/langchain/vectorstores/redis.py index 02a5d19d1af..30b926518b8 100644 --- a/langchain/vectorstores/redis.py +++ b/langchain/vectorstores/redis.py @@ -70,6 +70,9 @@ class Redis(VectorStore): redis_url: str, index_name: str, embedding_function: Callable, + content_key: str = "content", + metadata_key: str = "metadata", + vector_key: str = "content_vector", **kwargs: Any, ): """Initialize with necessary components.""" @@ -92,6 +95,9 @@ class Redis(VectorStore): raise ValueError(f"Redis failed to connect: {e}") self.client = redis_client + self.content_key = content_key + self.metadata_key = metadata_key + self.vector_key = vector_key def add_texts( self, @@ -112,11 +118,11 @@ class Redis(VectorStore): pipeline.hset( key, mapping={ - "content": text, - "content_vector": np.array( + self.content_key: text, + self.vector_key: np.array( self.embedding_function(text), dtype=np.float32 ).tobytes(), - "metadata": json.dumps(metadata), + self.metadata_key: json.dumps(metadata), }, ) ids.append(key) @@ -191,8 +197,8 @@ class Redis(VectorStore): embedding = self.embedding_function(query) # Prepare the Query - return_fields = ["metadata", "content", "vector_score"] - vector_field = "content_vector" + return_fields = [self.metadata_key, self.content_key, "vector_score"] + vector_field = self.vector_key hybrid_fields = "*" base_query = ( f"{hybrid_fields}=>[KNN {k} @{vector_field} $vector AS vector_score]" @@ -232,6 +238,9 @@ class Redis(VectorStore): embedding: Embeddings, metadatas: Optional[List[dict]] = None, index_name: Optional[str] = None, + content_key: str = "content", + metadata_key: str = "metadata", + vector_key: str = "content_vector", **kwargs: Any, ) -> Redis: """Construct RediSearch wrapper from raw documents. @@ -287,10 +296,10 @@ class Redis(VectorStore): "COSINE" # distance metric for the vectors (ex. COSINE, IP, L2) ) schema = ( - TextField(name="content"), - TextField(name="metadata"), + TextField(name=content_key), + TextField(name=metadata_key), VectorField( - "content_vector", + vector_key, "FLAT", { "TYPE": "FLOAT32", @@ -313,11 +322,9 @@ class Redis(VectorStore): pipeline.hset( key, mapping={ - "content": text, - "content_vector": np.array( - embeddings[i], dtype=np.float32 - ).tobytes(), - "metadata": json.dumps(metadata), + content_key: text, + vector_key: np.array(embeddings[i], dtype=np.float32).tobytes(), + metadata_key: json.dumps(metadata), }, ) pipeline.execute() From 274b25c010743c4b1450bde5ed7fce4f95824f22 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Sat, 15 Apr 2023 12:49:59 -0700 Subject: [PATCH 48/49] SVM retriever (#2947) (#2949) Add SVM retriever class, based on https://github.com/karpathy/randomfun/blob/master/knn_vs_svm.ipynb. Testing still WIP, but the logic is correct (I have a local implementation outside of Langchain working). --------- Co-authored-by: Lance Martin <122662504+PineappleExpress808@users.noreply.github.com> Co-authored-by: rlm <31treehaus@31s-MacBook-Pro.local> --- .../retrievers/examples/svm_retriever.ipynb | 128 ++++++++++++++++++ langchain/retrievers/__init__.py | 2 + langchain/retrievers/svm.py | 61 +++++++++ 3 files changed, 191 insertions(+) create mode 100644 docs/modules/indexes/retrievers/examples/svm_retriever.ipynb create mode 100644 langchain/retrievers/svm.py diff --git a/docs/modules/indexes/retrievers/examples/svm_retriever.ipynb b/docs/modules/indexes/retrievers/examples/svm_retriever.ipynb new file mode 100644 index 00000000000..ad14b33d1ff --- /dev/null +++ b/docs/modules/indexes/retrievers/examples/svm_retriever.ipynb @@ -0,0 +1,128 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ab66dd43", + "metadata": {}, + "source": [ + "# SVM Retriever\n", + "\n", + "This notebook goes over how to use a retriever that under the hood uses an SVM using scikit-learn.\n", + "\n", + "Largely based on https://github.com/karpathy/randomfun/blob/master/knn_vs_svm.ipynb" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "393ac030", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.retrievers import SVMRetriever\n", + "from langchain.embeddings import OpenAIEmbeddings" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a801b57c", + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install scikit-learn" + ] + }, + { + "cell_type": "markdown", + "id": "aaf80e7f", + "metadata": {}, + "source": [ + "## Create New Retriever with Texts" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "98b1c017", + "metadata": {}, + "outputs": [], + "source": [ + "retriever = SVMRetriever.from_texts([\"foo\", \"bar\", \"world\", \"hello\", \"foo bar\"], OpenAIEmbeddings())" + ] + }, + { + "cell_type": "markdown", + "id": "08437fa2", + "metadata": {}, + "source": [ + "## Use Retriever\n", + "\n", + "We can now use the retriever!" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "c0455218", + "metadata": {}, + "outputs": [], + "source": [ + "result = retriever.get_relevant_documents(\"foo\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7dfa5c29", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(page_content='foo', metadata={}),\n", + " Document(page_content='foo bar', metadata={}),\n", + " Document(page_content='hello', metadata={}),\n", + " Document(page_content='world', metadata={})]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74bd9256", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/langchain/retrievers/__init__.py b/langchain/retrievers/__init__.py index cb1ed4ce3a5..9ca44072267 100644 --- a/langchain/retrievers/__init__.py +++ b/langchain/retrievers/__init__.py @@ -4,6 +4,7 @@ from langchain.retrievers.elastic_search_bm25 import ElasticSearchBM25Retriever from langchain.retrievers.metal import MetalRetriever from langchain.retrievers.pinecone_hybrid_search import PineconeHybridSearchRetriever from langchain.retrievers.remote_retriever import RemoteLangChainRetriever +from langchain.retrievers.svm import SVMRetriever from langchain.retrievers.tfidf import TFIDFRetriever from langchain.retrievers.weaviate_hybrid_search import WeaviateHybridSearchRetriever @@ -16,4 +17,5 @@ __all__ = [ "TFIDFRetriever", "WeaviateHybridSearchRetriever", "DataberryRetriever", + "SVMRetriever", ] diff --git a/langchain/retrievers/svm.py b/langchain/retrievers/svm.py new file mode 100644 index 00000000000..d69abc4b500 --- /dev/null +++ b/langchain/retrievers/svm.py @@ -0,0 +1,61 @@ +"""SMV Retriever. +Largely based on +https://github.com/karpathy/randomfun/blob/master/knn_vs_svm.ipynb""" + +from __future__ import annotations + +from typing import Any, List + +import numpy as np +from pydantic import BaseModel + +from langchain.embeddings.base import Embeddings +from langchain.schema import BaseRetriever, Document + + +def create_index(contexts: List[str], embeddings: Embeddings) -> np.ndarray: + return np.array([embeddings.embed_query(split) for split in contexts]) + + +class SVMRetriever(BaseRetriever, BaseModel): + embeddings: Embeddings + index: Any + texts: List[str] + k: int = 4 + + class Config: + + """Configuration for this pydantic object.""" + + arbitrary_types_allowed = True + + @classmethod + def from_texts( + cls, texts: List[str], embeddings: Embeddings, **kwargs: Any + ) -> SVMRetriever: + index = create_index(texts, embeddings) + return cls(embeddings=embeddings, index=index, texts=texts, **kwargs) + + def get_relevant_documents(self, query: str) -> List[Document]: + from sklearn import svm + + query_embeds = np.array(self.embeddings.embed_query(query)) + x = np.concatenate([query_embeds[None, ...], self.index]) + y = np.zeros(x.shape[0]) + y[0] = 1 + + clf = svm.LinearSVC( + class_weight="balanced", verbose=False, max_iter=10000, tol=1e-6, C=0.1 + ) + clf.fit(x, y) + + similarities = clf.decision_function(x) + sorted_ix = np.argsort(-similarities) + + top_k_results = [] + for row in sorted_ix[1 : self.k + 1]: + top_k_results.append(Document(page_content=self.texts[row - 1])) + return top_k_results + + async def aget_relevant_documents(self, query: str) -> List[Document]: + raise NotImplementedError From b634489b2e8951b880c2ec467cdcf00f11830705 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Sat, 15 Apr 2023 12:56:39 -0700 Subject: [PATCH 49/49] bump version to 141 (#2950) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c7e292649c9..5ab29855f8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langchain" -version = "0.0.140" +version = "0.0.141" description = "Building applications with LLMs through composability" authors = [] license = "MIT"