diff --git a/docs/Makefile b/docs/Makefile index 85f756cb940..ef646744712 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -42,6 +42,8 @@ generate-files: $(PYTHON) scripts/document_loader_feat_table.py $(INTERMEDIATE_DIR) + $(PYTHON) scripts/kv_store_feat_table.py $(INTERMEDIATE_DIR) + $(PYTHON) scripts/partner_pkg_table.py $(INTERMEDIATE_DIR) $(PYTHON) scripts/copy_templates.py $(INTERMEDIATE_DIR) diff --git a/docs/docs/concepts.mdx b/docs/docs/concepts.mdx index 3120b6f2f00..19ba44f6133 100644 --- a/docs/docs/concepts.mdx +++ b/docs/docs/concepts.mdx @@ -500,7 +500,8 @@ For specifics on how to use retrievers, see the [relevant how-to guides here](/d ### Key-value stores -For some techniques, such as [indexing and retrieval with multiple vectors per document](/docs/how_to/multi_vector/), having some sort of key-value (KV) storage is helpful. +For some techniques, such as [indexing and retrieval with multiple vectors per document](/docs/how_to/multi_vector/) or +[caching embeddings](/docs/how_to/caching_embeddings/), having a form of key-value (KV) storage is helpful. LangChain includes a [`BaseStore`](https://api.python.langchain.com/en/latest/stores/langchain_core.stores.BaseStore.html) interface, which allows for storage of arbitrary data. However, LangChain components that require KV-storage accept a diff --git a/docs/docs/integrations/stores/astradb.ipynb b/docs/docs/integrations/stores/astradb.ipynb index 87bb7e67aed..e187821d1d3 100644 --- a/docs/docs/integrations/stores/astradb.ipynb +++ b/docs/docs/integrations/stores/astradb.ipynb @@ -2,10 +2,14 @@ "cells": [ { "cell_type": "raw", - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "raw" + } + }, "source": [ "---\n", - "sidebar_label: Astra DB\n", + "sidebar_label: AstraDB\n", "---" ] }, @@ -13,130 +17,121 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Astra DB\n", + "# AstraDBByteStore\n", + "\n", + "This will help you get started with Astra DB [key-value stores](/docs/concepts/#key-value-stores). For detailed documentation of all `AstraDBByteStore` features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/storage/langchain_astradb.storage.AstraDBByteStore.html).\n", + "\n", + "## Overview\n", "\n", "DataStax [Astra DB](https://docs.datastax.com/en/astra/home/astra.html) is a serverless vector-capable database built on Cassandra and made conveniently available through an easy-to-use JSON API.\n", "\n", - "`AstraDBStore` and `AstraDBByteStore` need the `astrapy` package to be installed:" + "### Integration details\n", + "\n", + "| Class | Package | Local | JS support | Package downloads | Package latest |\n", + "| :--- | :--- | :---: | :---: | :---: | :---: |\n", + "| [AstraDBByteStore](https://api.python.langchain.com/en/latest/storage/langchain_astradb.storage.AstraDBByteStore.html) | [langchain_astradb](https://api.python.langchain.com/en/latest/astradb_api_reference.html) | ❌ | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_astradb?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_astradb?style=flat-square&label=%20) |\n", + "\n", + "## Setup\n", + "\n", + "To create an `AstraDBByteStore` byte store, you'll need to [create a DataStax account](https://www.datastax.com/products/datastax-astra).\n", + "\n", + "### Credentials\n", + "\n", + "After signing up, set the following credentials:" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, - "outputs": [], - "source": [ - "%pip install --upgrade --quiet astrapy" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The Store takes the following parameters:\n", - "\n", - "* `api_endpoint`: Astra DB API endpoint. Looks like `https://01234567-89ab-cdef-0123-456789abcdef-us-east1.apps.astra.datastax.com`\n", - "* `token`: Astra DB token. Looks like `AstraCS:6gBhNmsk135....`\n", - "* `collection_name` : Astra DB collection name\n", - "* `namespace`: (Optional) Astra DB namespace" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## AstraDBStore\n", - "\n", - "The `AstraDBStore` is an implementation of `BaseStore` that stores everything in your DataStax Astra DB instance.\n", - "The store keys must be strings and will be mapped to the `_id` field of the Astra DB document.\n", - "The store values can be any object that can be serialized by `json.dumps`.\n", - "In the database, entries will have the form:\n", - "\n", - "```json\n", - "{\n", - " \"_id\": \"\",\n", - " \"value\": \n", - "}\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from langchain_community.storage import AstraDBStore" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "from getpass import getpass\n", "\n", - "ASTRA_DB_API_ENDPOINT = input(\"ASTRA_DB_API_ENDPOINT = \")\n", + "ASTRA_DB_API_ENDPOINT = getpass(\"ASTRA_DB_API_ENDPOINT = \")\n", "ASTRA_DB_APPLICATION_TOKEN = getpass(\"ASTRA_DB_APPLICATION_TOKEN = \")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Installation\n", + "\n", + "The LangChain AstraDB integration lives in the `langchain_astradb` package:" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "store = AstraDBStore(\n", + "%pip install -qU langchain_astradb" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Instantiation\n", + "\n", + "Now we can instantiate our byte store:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_astradb import AstraDBByteStore\n", + "\n", + "kv_store = AstraDBByteStore(\n", " api_endpoint=ASTRA_DB_API_ENDPOINT,\n", " token=ASTRA_DB_APPLICATION_TOKEN,\n", " collection_name=\"my_store\",\n", ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['v1', [0.1, 0.2, 0.3]]\n" - ] - } - ], - "source": [ - "store.mset([(\"k1\", \"v1\"), (\"k2\", [0.1, 0.2, 0.3])])\n", - "print(store.mget([\"k1\", \"k2\"]))" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Usage with CacheBackedEmbeddings\n", + "## Usage\n", "\n", - "You may use the `AstraDBStore` in conjunction with a [`CacheBackedEmbeddings`](/docs/how_to/caching_embeddings) to cache the result of embeddings computations.\n", - "Note that `AstraDBStore` stores the embeddings as a list of floats without converting them first to bytes so we don't use `fromByteStore` there." + "You can set data under keys like this using the `mset` method:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[b'value1', b'value2']" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "from langchain.embeddings import CacheBackedEmbeddings\n", - "from langchain_openai import OpenAIEmbeddings\n", + "kv_store.mset(\n", + " [\n", + " [\"key1\", b\"value1\"],\n", + " [\"key2\", b\"value2\"],\n", + " ]\n", + ")\n", "\n", - "embeddings = CacheBackedEmbeddings(\n", - " underlying_embeddings=OpenAIEmbeddings(), document_embedding_store=store\n", + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", ")" ] }, @@ -144,96 +139,67 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## AstraDBByteStore\n", - "\n", - "The `AstraDBByteStore` is an implementation of `ByteStore` that stores everything in your DataStax Astra DB instance.\n", - "The store keys must be strings and will be mapped to the `_id` field of the Astra DB document.\n", - "The store `bytes` values are converted to base64 strings for storage into Astra DB.\n", - "In the database, entries will have the form:\n", - "\n", - "```json\n", - "{\n", - " \"_id\": \"\",\n", - " \"value\": \"bytes encoded in base 64\"\n", - "}\n", - "```" + "And you can delete data using the `mdelete` method:" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from langchain_community.storage import AstraDBByteStore" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from getpass import getpass\n", - "\n", - "ASTRA_DB_API_ENDPOINT = input(\"ASTRA_DB_API_ENDPOINT = \")\n", - "ASTRA_DB_APPLICATION_TOKEN = getpass(\"ASTRA_DB_APPLICATION_TOKEN = \")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "store = AstraDBByteStore(\n", - " api_endpoint=ASTRA_DB_API_ENDPOINT,\n", - " token=ASTRA_DB_APPLICATION_TOKEN,\n", - " collection_name=\"my_store\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "[b'v1', b'v2']\n" - ] + "data": { + "text/plain": [ + "[None, None]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "store.mset([(\"k1\", b\"v1\"), (\"k2\", b\"v2\")])\n", - "print(store.mget([\"k1\", \"k2\"]))" + "kv_store.mdelete(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")\n", + "\n", + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")" ] }, { "cell_type": "markdown", "metadata": {}, - "source": [] + "source": [ + "You can use an `AstraDBByteStore` anywhere you'd use other ByteStores, including as a [cache for embeddings](/docs/how_to/caching_embeddings)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API reference\n", + "\n", + "For detailed documentation of all `AstraDBByteStore` features and configurations, head to the API reference: https://api.python.langchain.com/en/latest/storage/langchain_astradb.storage.AstraDBByteStore.html" + ] } ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.10.5" } }, "nbformat": 4, diff --git a/docs/docs/integrations/stores/cassandra.ipynb b/docs/docs/integrations/stores/cassandra.ipynb index bd9413da773..a1766d627f8 100644 --- a/docs/docs/integrations/stores/cassandra.ipynb +++ b/docs/docs/integrations/stores/cassandra.ipynb @@ -2,7 +2,11 @@ "cells": [ { "cell_type": "raw", - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "raw" + } + }, "source": [ "---\n", "sidebar_label: Cassandra\n", @@ -13,68 +17,62 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Cassandra\n", + "# CassandraByteStore\n", + "\n", + "This will help you get started with Cassandra [key-value stores](/docs/concepts/#key-value-stores). For detailed documentation of all `CassandraByteStore` features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/storage/langchain_community.storage.cassandra.CassandraByteStore.html).\n", + "\n", + "## Overview\n", "\n", "[Cassandra](https://cassandra.apache.org/) is a NoSQL, row-oriented, highly scalable and highly available database.\n", "\n", - "`CassandraByteStore` needs the `cassio` package to be installed:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, - "outputs": [], - "source": [ - "%pip install --upgrade --quiet cassio" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The Store takes the following parameters:\n", + "### Integration details\n", "\n", - "* table: The table where to store the data.\n", - "* session: (Optional) The cassandra driver session. If not provided, the cassio resolved session will be used.\n", - "* keyspace: (Optional) The keyspace of the table. If not provided, the cassio resolved keyspace will be used.\n", - "* setup_mode: (Optional) The mode used to create the Cassandra table (SYNC, ASYNC or OFF). Defaults to SYNC." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## CassandraByteStore\n", + "| Class | Package | Local | [JS support](https://js.langchain.com/v0.2/docs/integrations/stores/cassandra_storage) | Package downloads | Package latest |\n", + "| :--- | :--- | :---: | :---: | :---: | :---: |\n", + "| [CassandraByteStore](https://api.python.langchain.com/en/latest/storage/langchain_community.storage.cassandra.CassandraByteStore.html) | [langchain_community](https://api.python.langchain.com/en/latest/community_api_reference.html) | ✅ | ✅ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_community?style=flat-square&label=%20) |\n", + "\n", + "## Setup\n", "\n", "The `CassandraByteStore` is an implementation of `ByteStore` that stores the data in your Cassandra instance.\n", "The store keys must be strings and will be mapped to the `row_id` column of the Cassandra table.\n", "The store `bytes` values are mapped to the `body_blob` column of the Cassandra table." ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Installation\n", + "\n", + "The LangChain `CassandraByteStore` integration lives in the `langchain_community` package. You'll also need to install the `cassio` package or the `cassandra-driver` package as a peer dependency depending on which initialization method you're using:" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "from langchain_community.storage import CassandraByteStore" + "%pip install -qU langchain_community\n", + "%pip install -qU cassandra-driver\n", + "%pip install -qU cassio" ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ - "### Init from a cassandra driver Session\n", + "You'll also need to create a `cassandra.cluster.Session` object, as described in the [Cassandra driver documentation](https://docs.datastax.com/en/developer/python-driver/latest/api/cassandra/cluster/#module-cassandra.cluster). The details vary (e.g. with network settings and authentication), but this might be something like:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Instantiation\n", "\n", - "You need to create a `cassandra.cluster.Session` object, as described in the [Cassandra driver documentation](https://docs.datastax.com/en/developer/python-driver/latest/api/cassandra/cluster/#module-cassandra.cluster). The details vary (e.g. with network settings and authentication), but this might be something like:" - ], - "metadata": { - "collapsed": false - } + "You'll first need to create a `cassandra.cluster.Session` object, as described in the [Cassandra driver documentation](https://docs.datastax.com/en/developer/python-driver/latest/api/cassandra/cluster/#module-cassandra.cluster). The details vary (e.g. with network settings and authentication), but this might be something like:" + ] }, { "cell_type": "code", @@ -90,12 +88,10 @@ }, { "cell_type": "markdown", + "metadata": {}, "source": [ - "You need to provide the name of an existing keyspace of the Cassandra instance:" - ], - "metadata": { - "collapsed": false - } + "Then you can create your store! You'll also need to provide the name of an existing keyspace of the Cassandra instance:" + ] }, { "cell_type": "code", @@ -103,61 +99,91 @@ "metadata": {}, "outputs": [], "source": [ - "CASSANDRA_KEYSPACE = input(\"CASSANDRA_KEYSPACE = \")" + "from langchain_community.storage import CassandraByteStore\n", + "\n", + "kv_store = CassandraByteStore(\n", + " table=\"my_store\",\n", + " session=session,\n", + " keyspace=\"\",\n", + ")" ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ - "Creating the store:" - ], - "metadata": { - "collapsed": false - } + "## Usage\n", + "\n", + "You can set data under keys like this using the `mset` method:" + ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[b'v1', b'v2']\n" - ] - } - ], + "outputs": [], "source": [ - "store = CassandraByteStore(\n", - " table=\"my_store\",\n", - " session=session,\n", - " keyspace=CASSANDRA_KEYSPACE,\n", + "kv_store.mset(\n", + " [\n", + " [\"key1\", b\"value1\"],\n", + " [\"key2\", b\"value2\"],\n", + " ]\n", ")\n", "\n", - "store.mset([(\"k1\", b\"v1\"), (\"k2\", b\"v2\")])\n", - "print(store.mget([\"k1\", \"k2\"]))" + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")" ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ - "### Init from cassio\n", - "\n", - "It's also possible to use cassio to configure the session and keyspace." - ], - "metadata": { - "collapsed": false - } + "And you can delete data using the `mdelete` method:" + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kv_store.mdelete(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")\n", + "\n", + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Init using `cassio`\n", + "\n", + "It's also possible to use cassio to configure the session and keyspace." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "import cassio\n", "\n", - "cassio.init(contact_points=\"127.0.0.1\", keyspace=CASSANDRA_KEYSPACE)\n", + "cassio.init(contact_points=\"127.0.0.1\", keyspace=\"\")\n", "\n", "store = CassandraByteStore(\n", " table=\"my_store\",\n", @@ -165,62 +191,27 @@ "\n", "store.mset([(\"k1\", b\"v1\"), (\"k2\", b\"v2\")])\n", "print(store.mget([\"k1\", \"k2\"]))" - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ - "### Usage with CacheBackedEmbeddings\n", + "## API reference\n", "\n", - "You may use the `CassandraByteStore` in conjunction with a [`CacheBackedEmbeddings`](/docs/how_to/caching_embeddings) to cache the result of embeddings computations.\n" - ], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "from langchain.embeddings import CacheBackedEmbeddings\n", - "from langchain_openai import OpenAIEmbeddings\n", - "\n", - "cassio.init(contact_points=\"127.0.0.1\", keyspace=CASSANDRA_KEYSPACE)\n", - "\n", - "store = CassandraByteStore(\n", - " table=\"my_store\",\n", - ")\n", - "\n", - "embeddings = CacheBackedEmbeddings.from_bytes_store(\n", - " underlying_embeddings=OpenAIEmbeddings(), document_embedding_cache=store\n", - ")" - ], - "metadata": { - "collapsed": false - } + "For detailed documentation of all `CassandraByteStore` features and configurations, head to the API reference: https://api.python.langchain.com/en/latest/storage/langchain_community.storage.cassandra.CassandraByteStore.html" + ] } ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.10.5" } }, "nbformat": 4, diff --git a/docs/docs/integrations/stores/elasticsearch.ipynb b/docs/docs/integrations/stores/elasticsearch.ipynb index f55919c23d6..1015073a989 100644 --- a/docs/docs/integrations/stores/elasticsearch.ipynb +++ b/docs/docs/integrations/stores/elasticsearch.ipynb @@ -2,10 +2,14 @@ "cells": [ { "cell_type": "raw", - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "raw" + } + }, "source": [ "---\n", - "sidebar_label: Elasticsearch \n", + "sidebar_label: Elasticsearch\n", "---" ] }, @@ -15,25 +19,31 @@ "source": [ "# ElasticsearchEmbeddingsCache\n", "\n", + "This will help you get started with Elasticsearch [key-value stores](/docs/concepts/#key-value-stores). For detailed documentation of all `ElasticsearchEmbeddingsCache` features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/cache/langchain_elasticsearch.cache.ElasticsearchEmbeddingsCache.html).\n", + "\n", + "## Overview\n", + "\n", "The `ElasticsearchEmbeddingsCache` is a `ByteStore` implementation that uses your Elasticsearch instance for efficient storage and retrieval of embeddings.\n", "\n", + "### Integration details\n", "\n", - "First install the LangChain integration with Elasticsearch." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U langchain-elasticsearch" + "| Class | Package | Local | JS support | Package downloads | Package latest |\n", + "| :--- | :--- | :---: | :---: | :---: | :---: |\n", + "| [ElasticsearchEmbeddingsCache](https://api.python.langchain.com/en/latest/cache/langchain_elasticsearch.cache.ElasticsearchEmbeddingsCache.html) | [langchain_elasticsearch](https://api.python.langchain.com/en/latest/elasticsearch_api_reference.html) | ✅ | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_elasticsearch?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_elasticsearch?style=flat-square&label=%20) |\n", + "\n", + "## Setup\n", + "\n", + "To create a `ElasticsearchEmbeddingsCache` byte store, you'll need an Elasticsearch cluster. You can [set one up locally](https://www.elastic.co/downloads/elasticsearch) or create an [Elastic account](https://www.elastic.co/elasticsearch)." ] }, { "cell_type": "markdown", "metadata": {}, - "source": "it can be instantiated using `CacheBackedEmbeddings.from_bytes_store` method." + "source": [ + "### Installation\n", + "\n", + "The LangChain `ElasticsearchEmbeddingsCache` integration lives in the `__package_name__` package:" + ] }, { "cell_type": "code", @@ -41,23 +51,37 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain.embeddings import CacheBackedEmbeddings\n", + "%pip install -qU langchain_elasticsearch" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Instantiation\n", + "\n", + "Now we can instantiate our byte store:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ "from langchain_elasticsearch import ElasticsearchEmbeddingsCache\n", - "from langchain_openai import OpenAIEmbeddings\n", "\n", - "underlying_embeddings = OpenAIEmbeddings(model=\"text-embedding-3-small\")\n", - "\n", - "store = ElasticsearchEmbeddingsCache(\n", - " es_url=\"http://localhost:9200\",\n", + "# Example config for a locally running Elasticsearch instance\n", + "kv_store = ElasticsearchEmbeddingsCache(\n", + " es_url=\"https://localhost:9200\",\n", " index_name=\"llm-chat-cache\",\n", " metadata={\"project\": \"my_chatgpt_project\"},\n", " namespace=\"my_chatgpt_project\",\n", - ")\n", - "\n", - "embeddings = CacheBackedEmbeddings.from_bytes_store(\n", - " underlying_embeddings=OpenAIEmbeddings(),\n", - " document_embedding_cache=store,\n", - " query_embedding_cache=store,\n", + " es_user=\"elastic\",\n", + " es_password=\"\",\n", + " es_params={\n", + " \"ca_certs\": \"~/http_ca.crt\",\n", + " },\n", ")" ] }, @@ -65,19 +89,93 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The index_name parameter can also accept aliases. This allows to use the ILM: Manage the index lifecycle that we suggest to consider for managing retention and controlling cache growth.\n", + "## Usage\n", "\n", - "Look at the class docstring for all parameters." + "You can set data under keys like this using the `mset` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[b'value1', b'value2']" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kv_store.mset(\n", + " [\n", + " [\"key1\", b\"value1\"],\n", + " [\"key2\", b\"value2\"],\n", + " ]\n", + ")\n", + "\n", + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Index the generated vectors\n", - "The cached vectors won't be searchable by default. The developer can customize the building of the Elasticsearch document in order to add indexed vector field.\n", + "And you can delete data using the `mdelete` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[None, None]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kv_store.mdelete(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")\n", "\n", - "This can be done by subclassing end overriding methods. " + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Use as an embeddings cache\n", + "\n", + "Like other `ByteStores`, you can use an `ElasticsearchEmbeddingsCache` instance for [persistent caching in document ingestion](/docs/how_to/caching_embeddings/) for RAG.\n", + "\n", + "However, cached vectors won't be searchable by default. The developer can customize the building of the Elasticsearch document in order to add indexed vector field.\n", + "\n", + "This can be done by subclassing and overriding methods:" ] }, { @@ -88,8 +186,6 @@ "source": [ "from typing import Any, Dict, List\n", "\n", - "from langchain_elasticsearch import ElasticsearchEmbeddingsCache\n", - "\n", "\n", "class SearchableElasticsearchStore(ElasticsearchEmbeddingsCache):\n", " @property\n", @@ -112,26 +208,29 @@ { "cell_type": "markdown", "metadata": {}, - "source": "When overriding the mapping and the document building, please only make additive modifications, keeping the base mapping intact." + "source": [ + "When overriding the mapping and the document building, please only make additive modifications, keeping the base mapping intact." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API reference\n", + "\n", + "For detailed documentation of all `ElasticsearchEmbeddingsCache` features and configurations, head to the API reference: https://api.python.langchain.com/en/latest/cache/langchain_elasticsearch.cache.ElasticsearchEmbeddingsCache.html" + ] } ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.10.5" } }, "nbformat": 4, diff --git a/docs/docs/integrations/stores/file_system.ipynb b/docs/docs/integrations/stores/file_system.ipynb index b16e5f40515..677b754c3d9 100644 --- a/docs/docs/integrations/stores/file_system.ipynb +++ b/docs/docs/integrations/stores/file_system.ipynb @@ -2,11 +2,14 @@ "cells": [ { "cell_type": "raw", - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "raw" + } + }, "source": [ "---\n", "sidebar_label: Local Filesystem\n", - "sidebar_position: 3\n", "---" ] }, @@ -16,51 +19,119 @@ "source": [ "# LocalFileStore\n", "\n", - "The `LocalFileStore` is a persistent implementation of `ByteStore` that stores everything in a folder of your choosing." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[b'v1', b'v2']\n" - ] - } - ], - "source": [ - "from pathlib import Path\n", + "This will help you get started with local filesystem [key-value stores](/docs/concepts/#key-value-stores). For detailed documentation of all LocalFileStore features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/storage/langchain.storage.file_system.LocalFileStore.html).\n", "\n", - "from langchain.storage import LocalFileStore\n", + "## Overview\n", "\n", - "root_path = Path.cwd() / \"data\" # can also be a path set by a string\n", - "store = LocalFileStore(root_path)\n", + "The `LocalFileStore` is a persistent implementation of `ByteStore` that stores everything in a folder of your choosing. It's useful if you're using a single machine and are tolerant of files being added or deleted.\n", "\n", - "store.mset([(\"k1\", b\"v1\"), (\"k2\", b\"v2\")])\n", - "print(store.mget([\"k1\", \"k2\"]))" + "### Integration details\n", + "\n", + "| Class | Package | Local | [JS support](https://js.langchain.com/v0.2/docs/integrations/stores/file_system) | Package downloads | Package latest |\n", + "| :--- | :--- | :---: | :---: | :---: | :---: |\n", + "| [LocalFileStore](https://api.python.langchain.com/en/latest/storage/langchain.storage.file_system.LocalFileStore.html) | [langchain](https://api.python.langchain.com/en/latest/langchain_api_reference.html) | ✅ | ✅ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain?style=flat-square&label=%20) |" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Now let's see which files exist in our `data` folder:" + "### Installation\n", + "\n", + "The LangChain `LocalFileStore` integration lives in the `langchain` package:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -qU langchain" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Instantiation\n", + "\n", + "Now we can instantiate our byte store:" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "from langchain.storage import LocalFileStore\n", + "\n", + "root_path = Path.cwd() / \"data\" # can also be a path set by a string\n", + "\n", + "kv_store = LocalFileStore(root_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Usage\n", + "\n", + "You can set data under keys like this using the `mset` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[b'value1', b'value2']" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kv_store.mset(\n", + " [\n", + " [\"key1\", b\"value1\"],\n", + " [\"key2\", b\"value2\"],\n", + " ]\n", + ")\n", + "\n", + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can see the created files in your `data` folder:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "k1 k2\n" + "key1 key2\n" ] } ], @@ -69,16 +140,57 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], - "source": [] + "source": [ + "And you can delete data using the `mdelete` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[None, None]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kv_store.mdelete(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")\n", + "\n", + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API reference\n", + "\n", + "For detailed documentation of all `LocalFileStore` features and configurations, head to the API reference: https://api.python.langchain.com/en/latest/storage/langchain.storage.file_system.LocalFileStore.html" + ] } ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -92,7 +204,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.10.5" } }, "nbformat": 4, diff --git a/docs/docs/integrations/stores/in_memory.ipynb b/docs/docs/integrations/stores/in_memory.ipynb index 6288a1dabdb..55445763035 100644 --- a/docs/docs/integrations/stores/in_memory.ipynb +++ b/docs/docs/integrations/stores/in_memory.ipynb @@ -9,7 +9,7 @@ }, "source": [ "---\n", - "sidebar_label: InMemoryByteStore\n", + "sidebar_label: In-memory\n", "---" ] }, @@ -28,7 +28,7 @@ "### Integration details\n", "\n", "| Class | Package | Local | [JS support](https://js.langchain.com/v0.2/docs/integrations/stores/in_memory/) | Package downloads | Package latest |\n", - "| :--- | :--- | :---: | :---: | :---: | :---: | :---: |\n", + "| :--- | :--- | :---: | :---: | :---: | :---: |\n", "| [InMemoryByteStore](https://api.python.langchain.com/en/latest/stores/langchain_core.stores.InMemoryByteStore.html) | [langchain_core](https://api.python.langchain.com/en/latest/core_api_reference.html) | ✅ | ✅ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_core?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_core?style=flat-square&label=%20) |" ] }, diff --git a/docs/docs/integrations/stores/index.mdx b/docs/docs/integrations/stores/index.mdx deleted file mode 100644 index aa8706d43a3..00000000000 --- a/docs/docs/integrations/stores/index.mdx +++ /dev/null @@ -1,12 +0,0 @@ ---- -sidebar_position: 1 -sidebar_class_name: hidden ---- - -# Key-value stores - -[Key-value stores](/docs/concepts/#key-value-stores) are used by other LangChain components to store and retrieve data. - -import DocCardList from "@theme/DocCardList"; - - diff --git a/docs/docs/integrations/stores/redis.ipynb b/docs/docs/integrations/stores/redis.ipynb index 9f0da976350..d0300211e08 100644 --- a/docs/docs/integrations/stores/redis.ipynb +++ b/docs/docs/integrations/stores/redis.ipynb @@ -2,7 +2,11 @@ "cells": [ { "cell_type": "raw", - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "raw" + } + }, "source": [ "---\n", "sidebar_label: Redis\n", @@ -15,9 +19,30 @@ "source": [ "# RedisStore\n", "\n", + "This will help you get started with Redis [key-value stores](/docs/concepts/#key-value-stores). For detailed documentation of all `RedisStore` features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/storage/langchain_community.storage.redis.RedisStore.html).\n", + "\n", + "## Overview\n", + "\n", "The `RedisStore` is an implementation of `ByteStore` that stores everything in your Redis instance.\n", "\n", - "To configure Redis, follow our [Redis guide](/docs/integrations/providers/redis)." + "### Integration details\n", + "\n", + "| Class | Package | Local | [JS support](https://js.langchain.com/v0.2/docs/integrations/stores/ioredis_storage) | Package downloads | Package latest |\n", + "| :--- | :--- | :---: | :---: | :---: | :---: |\n", + "| [RedisStore](https://api.python.langchain.com/en/latest/storage/langchain_community.storage.redis.RedisStore.html) | [langchain_community](https://api.python.langchain.com/en/latest/community_api_reference.html) | ✅ | ✅ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_community?style=flat-square&label=%20) |\n", + "\n", + "## Setup\n", + "\n", + "To create a Redis byte store, you'll need to set up a Redis instance. You can do this locally or via a provider - see our [Redis guide](/docs/integrations/providers/redis) for an overview of options." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Installation\n", + "\n", + "The LangChain `RedisStore` integration lives in the `langchain_community` package:" ] }, { @@ -26,56 +51,128 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install --upgrade --quiet redis" + "%pip install -qU langchain_community redis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Instantiation\n", + "\n", + "Now we can instantiate our byte store:" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[b'v1', b'v2']\n" - ] - } - ], + "outputs": [], "source": [ "from langchain_community.storage import RedisStore\n", "\n", - "store = RedisStore(redis_url=\"redis://localhost:6379\")\n", + "kv_store = RedisStore(redis_url=\"redis://localhost:6379\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Usage\n", "\n", - "store.mset([(\"k1\", b\"v1\"), (\"k2\", b\"v2\")])\n", - "print(store.mget([\"k1\", \"k2\"]))" + "You can set data under keys like this using the `mset` method:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "[b'value1', b'value2']" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kv_store.mset(\n", + " [\n", + " [\"key1\", b\"value1\"],\n", + " [\"key2\", b\"value2\"],\n", + " ]\n", + ")\n", + "\n", + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And you can delete data using the `mdelete` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[None, None]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kv_store.mdelete(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")\n", + "\n", + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API reference\n", + "\n", + "For detailed documentation of all `RedisStore` features and configurations, head to the API reference: https://api.python.langchain.com/en/latest/storage/langchain_community.storage.redis.RedisStore.html" + ] } ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.10.5" } }, "nbformat": 4, diff --git a/docs/docs/integrations/stores/upstash_redis.ipynb b/docs/docs/integrations/stores/upstash_redis.ipynb index 91df14f86c3..eeadb2cde3b 100644 --- a/docs/docs/integrations/stores/upstash_redis.ipynb +++ b/docs/docs/integrations/stores/upstash_redis.ipynb @@ -2,7 +2,11 @@ "cells": [ { "cell_type": "raw", - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "raw" + } + }, "source": [ "---\n", "sidebar_label: Upstash Redis\n", @@ -15,11 +19,48 @@ "source": [ "# UpstashRedisByteStore\n", "\n", - "The `UpstashRedisStore` is an implementation of `ByteStore` that stores everything in your Upstash-hosted Redis instance.\n", + "This will help you get started with Upstash redis [key-value stores](/docs/concepts/#key-value-stores). For detailed documentation of all `UpstashRedisByteStore` features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/storage/langchain_community.storage.upstash_redis.UpstashRedisByteStore.html).\n", "\n", - "To use the base `RedisStore` instead, see [this guide](/docs/integrations/stores/redis/)\n", + "## Overview\n", "\n", - "To configure Upstash Redis, follow our [Upstash guide](/docs/integrations/providers/upstash)." + "The `UpstashRedisStore` is an implementation of `ByteStore` that stores everything in your [Upstash](https://upstash.com/)-hosted Redis instance.\n", + "\n", + "To use the base `RedisStore` instead, see [this guide](/docs/integrations/stores/redis/).\n", + "\n", + "### Integration details\n", + "\n", + "| Class | Package | Local | [JS support](https://js.langchain.com/v0.2/docs/integrations/stores/upstash_redis_storage) | Package downloads | Package latest |\n", + "| :--- | :--- | :---: | :---: | :---: | :---: |\n", + "| [UpstashRedisByteStore](https://api.python.langchain.com/en/latest/storage/langchain_community.storage.upstash_redis.UpstashRedisByteStore.html) | [langchain_community](https://api.python.langchain.com/en/latest/community_api_reference.html) | ❌ | ✅ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_community?style=flat-square&label=%20) |\n", + "\n", + "## Setup\n", + "\n", + "You'll first need to [sign up for an Upstash account](https://upstash.com/docs/redis/overall/getstarted). Next, you'll need to create a Redis database to connect to.\n", + "\n", + "### Credentials\n", + "\n", + "Once you've created your database, get your database URL (don't forget the `https://`!) and token:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from getpass import getpass\n", + "\n", + "URL = getpass(\"Enter your Upstash URL\")\n", + "TOKEN = getpass(\"Enter your Upstash REST token\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Installation\n", + "\n", + "The LangChain Upstash integration lives in the `langchain_community` package. You'll also need to install the `upstash-redis` package as a peer dependency:" ] }, { @@ -28,61 +69,130 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install --upgrade --quiet upstash-redis" + "%pip install -qU langchain_community upstash-redis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Instantiation\n", + "\n", + "Now we can instantiate our byte store:" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[b'v1', b'v2']\n" - ] - } - ], + "outputs": [], "source": [ "from langchain_community.storage import UpstashRedisByteStore\n", "from upstash_redis import Redis\n", "\n", - "URL = \"\"\n", - "TOKEN = \"\"\n", - "\n", "redis_client = Redis(url=URL, token=TOKEN)\n", - "store = UpstashRedisByteStore(client=redis_client, ttl=None, namespace=\"test-ns\")\n", + "kv_store = UpstashRedisByteStore(client=redis_client, ttl=None, namespace=\"test-ns\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Usage\n", "\n", - "store.mset([(\"k1\", b\"v1\"), (\"k2\", b\"v2\")])\n", - "print(store.mget([\"k1\", \"k2\"]))" + "You can set data under keys like this using the `mset` method:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "[b'value1', b'value2']" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kv_store.mset(\n", + " [\n", + " [\"key1\", b\"value1\"],\n", + " [\"key2\", b\"value2\"],\n", + " ]\n", + ")\n", + "\n", + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And you can delete data using the `mdelete` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[None, None]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kv_store.mdelete(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")\n", + "\n", + "kv_store.mget(\n", + " [\n", + " \"key1\",\n", + " \"key2\",\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API reference\n", + "\n", + "For detailed documentation of all `UpstashRedisByteStore` features and configurations, head to the API reference: https://api.python.langchain.com/en/latest/storage/langchain_community.storage.upstash_redis.UpstashRedisByteStore.html" + ] } ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.10.5" } }, "nbformat": 4, diff --git a/docs/scripts/kv_store_feat_table.py b/docs/scripts/kv_store_feat_table.py new file mode 100644 index 00000000000..e6b04dc18e1 --- /dev/null +++ b/docs/scripts/kv_store_feat_table.py @@ -0,0 +1,107 @@ +import sys +from pathlib import Path + +from langchain_community import document_loaders +from langchain_core.document_loaders.base import BaseLoader + +KV_STORE_TEMPLATE = """\ +--- +sidebar_class_name: hidden +keywords: [compatibility] +custom_edit_url: +hide_table_of_contents: true +--- + +# Key-value stores + +[Key-value stores](/docs/concepts/#key-value-stores) are used by other LangChain components to store and retrieve data. + +:::info + +If you'd like to contribute an integration, see [Contributing integrations](/docs/contributing/integrations/). + +::: + + +## Features + +The following table shows information on all available key-value stores. + +{table} + +""" + +KV_STORE_FEAT_TABLE = { + "AstraDBByteStore": { + "class": "[AstraDBByteStore](https://api.python.langchain.com/en/latest/storage/langchain_astradb.storage.AstraDBByteStore.html)", + "local": False, + "package": "[langchain_astradb](https://api.python.langchain.com/en/latest/astradb_api_reference.html)", + "downloads": "![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_astradb?style=flat-square&label=%20)", + }, + "CassandraByteStore": { + "class": "[CassandraByteStore](https://api.python.langchain.com/en/latest/storage/langchain_community.storage.cassandra.CassandraByteStore.html)", + "local": False, + "package": "[langchain_community](https://api.python.langchain.com/en/latest/community_api_reference.html)", + "downloads": "![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20)", + }, + "ElasticsearchEmbeddingsCache": { + "class": "[ElasticsearchEmbeddingsCache](https://api.python.langchain.com/en/latest/cache/langchain_elasticsearch.cache.ElasticsearchEmbeddingsCache.html)", + "local": True, + "package": "[langchain_elasticsearch](https://api.python.langchain.com/en/latest/elasticsearch_api_reference.html)", + "downloads": "![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_elasticsearch?style=flat-square&label=%20)", + }, + "LocalFileStore": { + "class": "[LocalFileStore](https://api.python.langchain.com/en/latest/storage/langchain.storage.file_system.LocalFileStore.html)", + "local": True, + "package": "[langchain](https://api.python.langchain.com/en/latest/langchain_api_reference.html)", + "downloads": "![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain?style=flat-square&label=%20)", + }, + "InMemoryByteStore": { + "class": "[InMemoryByteStore](https://api.python.langchain.com/en/latest/stores/langchain_core.stores.InMemoryByteStore.html)", + "local": True, + "package": "[langchain_core](https://api.python.langchain.com/en/latest/core_api_reference.html)", + "downloads": "![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_core?style=flat-square&label=%20)", + }, + "RedisStore": { + "class": "[RedisStore](https://api.python.langchain.com/en/latest/storage/langchain_community.storage.redis.RedisStore.html)", + "local": True, + "package": "[langchain_community](https://api.python.langchain.com/en/latest/community_api_reference.html)", + "downloads": "![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20)", + }, + "UpstashRedisByteStore": { + "class": "[UpstashRedisByteStore](https://api.python.langchain.com/en/latest/storage/langchain_community.storage.upstash_redis.UpstashRedisByteStore.html)", + "local": False, + "package": "[langchain_community](https://api.python.langchain.com/en/latest/community_api_reference.html)", + "downloads": "![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20)", + }, +} + +DEPRECATED = [] + + +def get_kv_store_table() -> str: + """Get the table of KV stores.""" + + header = ["name", "package", "local", "downloads"] + title = ["Class", "Package", "Local", "Downloads"] + rows = [title, [":-"] + [":-:"] * (len(title) - 1)] + for loader, feats in sorted(KV_STORE_FEAT_TABLE.items()): + if not feats or loader in DEPRECATED: + continue + rows += [ + [feats["class"]] + + ["✅" if feats.get(h) else "❌" for h in header[1:2]] + + [feats["package"], feats["downloads"]] + ] + return "\n".join(["|".join(row) for row in rows]) + + +if __name__ == "__main__": + output_dir = Path(sys.argv[1]) + output_integrations_dir = output_dir / "integrations" + output_integrations_dir_kv_stores = output_integrations_dir / "stores" + output_integrations_dir_kv_stores.mkdir(parents=True, exist_ok=True) + + kv_stores_page = KV_STORE_TEMPLATE.format(table=get_kv_store_table()) + with open(output_integrations_dir / "stores" / "index.mdx", "w") as f: + f.write(kv_stores_page) diff --git a/libs/cli/langchain_cli/integration_template/docs/kv_store.ipynb b/libs/cli/langchain_cli/integration_template/docs/kv_store.ipynb index eed689016cf..c8a883dfdc6 100644 --- a/libs/cli/langchain_cli/integration_template/docs/kv_store.ipynb +++ b/libs/cli/langchain_cli/integration_template/docs/kv_store.ipynb @@ -27,7 +27,7 @@ "\n", "## Overview\n", "\n", - "- TODO: (Optional) A short introduciton to the underlying technology/API.\n", + "- TODO: (Optional) A short introduction to the underlying technology/API.\n", "\n", "### Integration details\n", "\n", @@ -36,7 +36,7 @@ "- TODO: Make sure API reference links are correct.\n", "\n", "| Class | Package | Local | [JS support](https://js.langchain.com/v0.2/docs/integrations/stores/_package_name_) | Package downloads | Package latest |\n", - "| :--- | :--- | :---: | :---: | :---: | :---: | :---: |\n", + "| :--- | :--- | :---: | :---: | :---: | :---: |\n", "| [__ModuleName__ByteStore](https://api.python.langchain.com/en/latest/stores/__module_name__.stores.__ModuleName__ByteStore.html) | [__package_name__](https://api.python.langchain.com/en/latest/__package_name_short_snake___api_reference.html) | ✅/❌ | ✅/❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/__package_name__?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/__package_name__?style=flat-square&label=%20) |\n", "\n", "## Setup\n",