diff --git a/docs/docs/integrations/vectorstores/astradb.ipynb b/docs/docs/integrations/vectorstores/astradb.ipynb
index 0e1e7b02743..5bac0f2d8f4 100644
--- a/docs/docs/integrations/vectorstores/astradb.ipynb
+++ b/docs/docs/integrations/vectorstores/astradb.ipynb
@@ -606,7 +606,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
- "- [Tutorials](/docs/tutorials/)\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
]
@@ -674,7 +674,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.12.0"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/docs/docs/integrations/vectorstores/chroma.ipynb b/docs/docs/integrations/vectorstores/chroma.ipynb
index 6814478605f..b0eca7eef34 100644
--- a/docs/docs/integrations/vectorstores/chroma.ipynb
+++ b/docs/docs/integrations/vectorstores/chroma.ipynb
@@ -670,7 +670,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.12.0"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/docs/docs/integrations/vectorstores/clickhouse.ipynb b/docs/docs/integrations/vectorstores/clickhouse.ipynb
index 8c47cbb64d6..b44b38280a5 100644
--- a/docs/docs/integrations/vectorstores/clickhouse.ipynb
+++ b/docs/docs/integrations/vectorstores/clickhouse.ipynb
@@ -358,7 +358,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
- "- [Tutorials](/docs/tutorials/)\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
]
@@ -398,7 +398,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.9"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/docs/docs/integrations/vectorstores/couchbase.ipynb b/docs/docs/integrations/vectorstores/couchbase.ipynb
index 01922fae4e3..5692967a784 100644
--- a/docs/docs/integrations/vectorstores/couchbase.ipynb
+++ b/docs/docs/integrations/vectorstores/couchbase.ipynb
@@ -918,7 +918,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
- "- [Tutorials](/docs/tutorials/)\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
]
@@ -1015,7 +1015,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.10.13"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/docs/docs/integrations/vectorstores/elasticsearch.ipynb b/docs/docs/integrations/vectorstores/elasticsearch.ipynb
index 6bc0c0c6ce1..a625ba414c8 100644
--- a/docs/docs/integrations/vectorstores/elasticsearch.ipynb
+++ b/docs/docs/integrations/vectorstores/elasticsearch.ipynb
@@ -940,7 +940,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
- "- [Tutorials](/docs/tutorials/)\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
]
@@ -1091,7 +1091,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.9"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/docs/docs/integrations/vectorstores/faiss.ipynb b/docs/docs/integrations/vectorstores/faiss.ipynb
index f931db9ba80..9bd7e9893f0 100644
--- a/docs/docs/integrations/vectorstores/faiss.ipynb
+++ b/docs/docs/integrations/vectorstores/faiss.ipynb
@@ -410,7 +410,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
- "- [Tutorials](/docs/tutorials/)\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
]
@@ -574,7 +574,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.10.12"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/docs/docs/integrations/vectorstores/milvus.ipynb b/docs/docs/integrations/vectorstores/milvus.ipynb
index 8745e7d0f60..4cd7768f405 100644
--- a/docs/docs/integrations/vectorstores/milvus.ipynb
+++ b/docs/docs/integrations/vectorstores/milvus.ipynb
@@ -671,7 +671,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
- "- [Tutorials](/docs/tutorials/)\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
]
@@ -834,7 +834,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.13.2"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/docs/docs/integrations/vectorstores/mongodb_atlas.ipynb b/docs/docs/integrations/vectorstores/mongodb_atlas.ipynb
index d44367f4c07..f55438b37da 100644
--- a/docs/docs/integrations/vectorstores/mongodb_atlas.ipynb
+++ b/docs/docs/integrations/vectorstores/mongodb_atlas.ipynb
@@ -488,7 +488,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
- "- [Tutorials](/docs/tutorials/)\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
]
@@ -532,10 +532,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.9"
+ "version": "3.9.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
-
diff --git a/docs/docs/integrations/vectorstores/opengauss.ipynb b/docs/docs/integrations/vectorstores/opengauss.ipynb
index 28e9b1183d3..de01da259f0 100644
--- a/docs/docs/integrations/vectorstores/opengauss.ipynb
+++ b/docs/docs/integrations/vectorstores/opengauss.ipynb
@@ -32,8 +32,9 @@
]
},
{
- "metadata": {},
"cell_type": "markdown",
+ "id": "e006fdc593107ef5",
+ "metadata": {},
"source": [
"```bash\n",
"docker run --name opengauss \\\n",
@@ -42,14 +43,15 @@
" -p 8888:5432 \\\n",
" opengauss/opengauss-server:latest\n",
"```"
- ],
- "id": "e006fdc593107ef5"
+ ]
},
{
"cell_type": "markdown",
"id": "a51b3f07b83b8a1d",
"metadata": {},
- "source": "### Install langchain-opengauss"
+ "source": [
+ "### Install langchain-opengauss"
+ ]
},
{
"cell_type": "raw",
@@ -149,7 +151,9 @@
"cell_type": "markdown",
"id": "c738c3e0",
"metadata": {},
- "source": "### Update items in vector store\n"
+ "source": [
+ "### Update items in vector store\n"
+ ]
},
{
"cell_type": "code",
@@ -170,7 +174,9 @@
"cell_type": "markdown",
"id": "dcf1b905",
"metadata": {},
- "source": "### Delete items from vector store\n"
+ "source": [
+ "### Delete items from vector store\n"
+ ]
},
{
"cell_type": "code",
@@ -216,7 +222,9 @@
"cell_type": "markdown",
"id": "3ed9d733",
"metadata": {},
- "source": "If you want to execute a similarity search and receive the corresponding scores you can run:\n"
+ "source": [
+ "If you want to execute a similarity search and receive the corresponding scores you can run:\n"
+ ]
},
{
"cell_type": "code",
@@ -264,7 +272,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
- "- [Tutorials](/docs/tutorials/)\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval/)"
]
@@ -365,7 +373,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.10.12"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/docs/docs/integrations/vectorstores/pgvector.ipynb b/docs/docs/integrations/vectorstores/pgvector.ipynb
index f02700abb06..a44157f3f93 100644
--- a/docs/docs/integrations/vectorstores/pgvector.ipynb
+++ b/docs/docs/integrations/vectorstores/pgvector.ipynb
@@ -435,7 +435,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
- "- [Tutorials](/docs/tutorials/)\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
]
@@ -467,7 +467,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.9"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/docs/docs/integrations/vectorstores/pgvectorstore.ipynb b/docs/docs/integrations/vectorstores/pgvectorstore.ipynb
index f770b4fe5ea..01a453f1f5b 100644
--- a/docs/docs/integrations/vectorstores/pgvectorstore.ipynb
+++ b/docs/docs/integrations/vectorstores/pgvectorstore.ipynb
@@ -15,7 +15,6 @@
},
{
"cell_type": "markdown",
- "id": "f8f2830ee9ca1e01",
"metadata": {
"id": "f8f2830ee9ca1e01"
},
@@ -76,7 +75,6 @@
{
"cell_type": "code",
"execution_count": 2,
- "id": "irl7eMFnSPZr",
"metadata": {
"id": "irl7eMFnSPZr"
},
@@ -101,7 +99,6 @@
},
{
"cell_type": "markdown",
- "id": "QuQigs4UoFQ2",
"metadata": {
"id": "QuQigs4UoFQ2"
},
@@ -741,6 +738,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
]
@@ -761,7 +759,8 @@
"toc_visible": true
},
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
"name": "python3"
},
"language_info": {
@@ -774,9 +773,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.12.8"
+ "version": "3.9.6"
}
},
"nbformat": 4,
- "nbformat_minor": 0
+ "nbformat_minor": 4
}
diff --git a/docs/docs/integrations/vectorstores/pinecone.ipynb b/docs/docs/integrations/vectorstores/pinecone.ipynb
index 7f8b4dad393..230830e29bf 100644
--- a/docs/docs/integrations/vectorstores/pinecone.ipynb
+++ b/docs/docs/integrations/vectorstores/pinecone.ipynb
@@ -342,7 +342,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
- "- [Tutorials](/docs/tutorials/)\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
]
@@ -360,7 +360,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": ".venv",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -374,7 +374,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.10.15"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/docs/docs/integrations/vectorstores/qdrant.ipynb b/docs/docs/integrations/vectorstores/qdrant.ipynb
index d4f8acaaedf..c8fe60c80f1 100644
--- a/docs/docs/integrations/vectorstores/qdrant.ipynb
+++ b/docs/docs/integrations/vectorstores/qdrant.ipynb
@@ -749,7 +749,7 @@
"\n",
"For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
"\n",
- "- [Tutorials](/docs/tutorials/)\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
"- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
"- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
]
@@ -774,7 +774,10 @@
"execution_count": null,
"id": "1f11adf8",
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [],
"source": [
@@ -796,7 +799,10 @@
"cell_type": "markdown",
"id": "b2350093",
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"source": [
"### Metadata\n",
@@ -866,7 +872,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.9"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/docs/docs/integrations/vectorstores/redis.ipynb b/docs/docs/integrations/vectorstores/redis.ipynb
index ae92648b38b..3cd77195ba6 100644
--- a/docs/docs/integrations/vectorstores/redis.ipynb
+++ b/docs/docs/integrations/vectorstores/redis.ipynb
@@ -1,1134 +1,1134 @@
{
- "cells": [
- {
- "cell_type": "raw",
- "id": "1957f5cb",
- "metadata": {},
- "source": [
- "---\n",
- "sidebar_label: Redis\n",
- "---"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "ef1f0986",
- "metadata": {},
- "source": [
- "# Redis Vector Store\n",
- "\n",
- "This notebook covers how to get started with the Redis vector store.\n",
- "\n",
- ">[Redis](https://redis.io/docs/stack/vectorsearch/) is a popular open-source, in-memory data structure store that can be used as a database, cache, message broker, and queue. It now includes vector similarity search capabilities, making it suitable for use as a vector store."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "a717e10e-c8d7-41bc-9dd0-447d11f90b68",
- "metadata": {},
- "source": [
- "### What is Redis?\n",
- "\n",
- "Most developers are familiar with `Redis`. At its core, `Redis` is a NoSQL Database in the key-value family that can used as a cache, message broker, stream processing and a primary database. Developers choose `Redis` because it is fast, has a large ecosystem of client libraries, and has been deployed by major enterprises for years.\n",
- "\n",
- "On top of these traditional use cases, `Redis` provides additional capabilities like the Search and Query capability that allows users to create secondary index structures within `Redis`. This allows `Redis` to be a Vector Database, at the speed of a cache.\n",
- "\n",
- "\n",
- "### Redis as a Vector Database\n",
- "\n",
- "`Redis` uses compressed, inverted indexes for fast indexing with a low memory footprint. It also supports a number of advanced features such as:\n",
- "\n",
- "* Indexing of multiple fields in Redis hashes and `JSON`\n",
- "* Vector similarity search (with `HNSW` (ANN) or `FLAT` (KNN))\n",
- "* Vector Range Search (e.g. find all vectors within a radius of a query vector)\n",
- "* Incremental indexing without performance loss\n",
- "* Document ranking (using [tf-idf](https://en.wikipedia.org/wiki/Tf%E2%80%93idf), with optional user-provided weights)\n",
- "* Field weighting\n",
- "* Complex boolean queries with `AND`, `OR`, and `NOT` operators\n",
- "* Prefix matching, fuzzy matching, and exact-phrase queries\n",
- "* Support for [double-metaphone phonetic matching](https://redis.io/docs/stack/search/reference/phonetic_matching/)\n",
- "* Auto-complete suggestions (with fuzzy prefix suggestions)\n",
- "* Stemming-based query expansion in [many languages](https://redis.io/docs/stack/search/reference/stemming/) (using [Snowball](http://snowballstem.org/))\n",
- "* Support for Chinese-language tokenization and querying (using [Friso](https://github.com/lionsoul2014/friso))\n",
- "* Numeric filters and ranges\n",
- "* Geospatial searches using Redis geospatial indexing\n",
- "* A powerful aggregations engine\n",
- "* Supports for all `utf-8` encoded text\n",
- "* Retrieve full documents, selected fields, or only the document IDs\n",
- "* Sorting results (for example, by creation date)\n",
- "\n",
- "### Clients\n",
- "\n",
- "Since `Redis` is much more than just a vector database, there are often use cases that demand the usage of a `Redis` client besides just the `LangChain` integration. You can use any standard `Redis` client library to run Search and Query commands, but it's easiest to use a library that wraps the Search and Query API. Below are a few examples, but you can find more client libraries [here](https://redis.io/resources/clients/).\n",
- "\n",
- "| Project | Language | License | Author | Stars |\n",
- "|----------|---------|--------|---------|-------|\n",
- "| [jedis][jedis-url] | Java | MIT | [Redis][redis-url] | ![Stars][jedis-stars] |\n",
- "| [redisvl][redisvl-url] | Python | MIT | [Redis][redis-url] | ![Stars][redisvl-stars] |\n",
- "| [redis-py][redis-py-url] | Python | MIT | [Redis][redis-url] | ![Stars][redis-py-stars] |\n",
- "| [node-redis][node-redis-url] | Node.js | MIT | [Redis][redis-url] | ![Stars][node-redis-stars] |\n",
- "| [nredisstack][nredisstack-url] | .NET | MIT | [Redis][redis-url] | ![Stars][nredisstack-stars] |\n",
- "\n",
- "[redis-url]: https://redis.com\n",
- "\n",
- "[redisvl-url]: https://github.com/redis/redis-vl-python\n",
- "[redisvl-stars]: https://img.shields.io/github/stars/redis/redisvl.svg?style=social&label=Star&maxAge=2592000\n",
- "[redisvl-package]: https://pypi.python.org/pypi/redisvl\n",
- "\n",
- "[redis-py-url]: https://github.com/redis/redis-py\n",
- "[redis-py-stars]: https://img.shields.io/github/stars/redis/redis-py.svg?style=social&label=Star&maxAge=2592000\n",
- "[redis-py-package]: https://pypi.python.org/pypi/redis\n",
- "\n",
- "[jedis-url]: https://github.com/redis/jedis\n",
- "[jedis-stars]: https://img.shields.io/github/stars/redis/jedis.svg?style=social&label=Star&maxAge=2592000\n",
- "[Jedis-package]: https://search.maven.org/artifact/redis.clients/jedis\n",
- "\n",
- "[nredisstack-url]: https://github.com/redis/nredisstack\n",
- "[nredisstack-stars]: https://img.shields.io/github/stars/redis/nredisstack.svg?style=social&label=Star&maxAge=2592000\n",
- "[nredisstack-package]: https://www.nuget.org/packages/nredisstack/\n",
- "\n",
- "[node-redis-url]: https://github.com/redis/node-redis\n",
- "[node-redis-stars]: https://img.shields.io/github/stars/redis/node-redis.svg?style=social&label=Star&maxAge=2592000\n",
- "[node-redis-package]: https://www.npmjs.com/package/redis\n",
- "\n",
- "[redis-om-python-url]: https://github.com/redis/redis-om-python\n",
- "[redis-om-python-author]: https://redis.com\n",
- "[redis-om-python-stars]: https://img.shields.io/github/stars/redis/redis-om-python.svg?style=social&label=Star&maxAge=2592000\n",
- "\n",
- "[redisearch-go-url]: https://github.com/RediSearch/redisearch-go\n",
- "[redisearch-go-author]: https://redis.com\n",
- "[redisearch-go-stars]: https://img.shields.io/github/stars/RediSearch/redisearch-go.svg?style=social&label=Star&maxAge=2592000\n",
- "\n",
- "[redisearch-api-rs-url]: https://github.com/RediSearch/redisearch-api-rs\n",
- "[redisearch-api-rs-author]: https://redis.com\n",
- "[redisearch-api-rs-stars]: https://img.shields.io/github/stars/RediSearch/redisearch-api-rs.svg?style=social&label=Star&maxAge=2592000\n",
- "\n",
- "\n",
- "### Deployment options\n",
- "\n",
- "There are many ways to deploy Redis with RediSearch. The easiest way to get started is to use Docker, but there are are many potential options for deployment such as\n",
- "\n",
- "- [Redis Cloud](https://redis.com/redis-enterprise-cloud/overview/)\n",
- "- [Docker (Redis Stack)](https://hub.docker.com/r/redis/redis-stack)\n",
- "- Cloud marketplaces: [AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-e6y7ork67pjwg?sr=0-2&ref_=beagle&applicationId=AWSMPContessa), [Google Marketplace](https://console.cloud.google.com/marketplace/details/redislabs-public/redis-enterprise?pli=1), or [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/garantiadata.redis_enterprise_1sp_public_preview?tab=Overview)\n",
- "- On-premise: [Redis Enterprise Software](https://redis.com/redis-enterprise-software/overview/)\n",
- "- Kubernetes: [Redis Enterprise Software on Kubernetes](https://docs.redis.com/latest/kubernetes/)\n",
- "\n",
- "### Redis connection Url schemas\n",
- "\n",
- "Valid Redis Url schemas are:\n",
- "1. `redis://` - Connection to Redis standalone, unencrypted\n",
- "2. `rediss://` - Connection to Redis standalone, with TLS encryption\n",
- "3. `redis+sentinel://` - Connection to Redis server via Redis Sentinel, unencrypted\n",
- "4. `rediss+sentinel://` - Connection to Redis server via Redis Sentinel, both connections with TLS encryption\n",
- "\n",
- "More information about additional connection parameters can be found in the [redis-py documentation](https://redis-py.readthedocs.io/en/stable/connections.html)."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "36fdc060",
- "metadata": {},
- "source": [
- "## Setup\n",
- "\n",
- "To use the RedisVectorStore, you'll need to install the `langchain-redis` partner package, as well as the other packages used throughout this notebook."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "id": "64e28aa6",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Note: you may need to restart the kernel to use updated packages.\n"
- ]
- }
- ],
- "source": [
- "%pip install -qU langchain-redis langchain-huggingface sentence-transformers scikit-learn"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "37d388a3-1a56-453e-8f84-e24a72d682eb",
- "metadata": {},
- "source": [
- "### Credentials\n",
- "\n",
- "Redis connection credentials are passed as part of the Redis Connection URL. Redis Connection URLs are versatile and can accommodate various Redis server topologies and authentication methods. These URLs follow a specific format that includes the connection protocol, authentication details, host, port, and database information.\n",
- "The basic structure of a Redis Connection URL is:\n",
- "\n",
- "```\n",
- "[protocol]://[auth]@[host]:[port]/[database]\n",
- "```\n",
- "\n",
- "Where:\n",
- "\n",
- "* protocol can be redis for standard connections, rediss for SSL/TLS connections, or redis+sentinel for Sentinel connections.\n",
- "* auth includes username and password (if applicable).\n",
- "* host is the Redis server hostname or IP address.\n",
- "* port is the Redis server port.\n",
- "* database is the Redis database number.\n",
- "\n",
- "Redis Connection URLs support various configurations, including:\n",
- "\n",
- "* Standalone Redis servers (with or without authentication)\n",
- "* Redis Sentinel setups\n",
- "* SSL/TLS encrypted connections\n",
- "* Different authentication methods (password-only or username-password)\n",
- "\n",
- "Below are examples of Redis Connection URLs for different configurations:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b1b1eb90-5155-44ca-a8a7-b04b02d5e77c",
- "metadata": {},
- "outputs": [],
- "source": [
- "# connection to redis standalone at localhost, db 0, no password\n",
- "redis_url = \"redis://localhost:6379\"\n",
- "# connection to host \"redis\" port 7379 with db 2 and password \"secret\" (old style authentication scheme without username / pre 6.x)\n",
- "redis_url = \"redis://:secret@redis:7379/2\"\n",
- "# connection to host redis on default port with user \"joe\", pass \"secret\" using redis version 6+ ACLs\n",
- "redis_url = \"redis://joe:secret@redis/0\"\n",
- "\n",
- "# connection to sentinel at localhost with default group mymaster and db 0, no password\n",
- "redis_url = \"redis+sentinel://localhost:26379\"\n",
- "# connection to sentinel at host redis with default port 26379 and user \"joe\" with password \"secret\" with default group mymaster and db 0\n",
- "redis_url = \"redis+sentinel://joe:secret@redis\"\n",
- "# connection to sentinel, no auth with sentinel monitoring group \"zone-1\" and database 2\n",
- "redis_url = \"redis+sentinel://redis:26379/zone-1/2\"\n",
- "\n",
- "# connection to redis standalone at localhost, db 0, no password but with TLS support\n",
- "redis_url = \"rediss://localhost:6379\"\n",
- "# connection to redis sentinel at localhost and default port, db 0, no password\n",
- "# but with TLS support for both Sentinel and Redis server\n",
- "redis_url = \"rediss+sentinel://localhost\""
- ]
- },
- {
- "cell_type": "markdown",
- "id": "9695dee7",
- "metadata": {},
- "source": [
- "### Launching a Redis Instance with Docker\n",
- "\n",
- "To use Redis with LangChain, you need a running Redis instance. You can start one using Docker with:\n",
- "\n",
- "```bash\n",
- "docker run -d -p 6379:6379 redis/redis-stack:latest\n",
- "```\n",
- "\n",
- "For this example, we'll use a local Redis instance. If you're using a remote instance, you'll need to modify the Redis URL accordingly."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "id": "894c30e4",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Connecting to Redis at: redis://redis:6379\n"
- ]
- }
- ],
- "source": [
- "import os\n",
- "\n",
- "REDIS_URL = os.getenv(\"REDIS_URL\", \"redis://localhost:6379\")\n",
- "print(f\"Connecting to Redis at: {REDIS_URL}\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "7f98392b",
- "metadata": {},
- "source": [
- "To enable automated tracing of your model calls, set your [LangSmith](https://docs.smith.langchain.com/) API key:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "id": "e7b6a6e0",
- "metadata": {},
- "outputs": [],
- "source": [
- "# os.environ[\"LANGSMITH_API_KEY\"] = getpass.getpass(\"Enter your LangSmith API key: \")\n",
- "# os.environ[\"LANGSMITH_TRACING\"] = \"true\""
- ]
- },
- {
- "cell_type": "markdown",
- "id": "63dd7f8d-df13-45d8-8e13-892b29803e96",
- "metadata": {},
- "source": [
- "Let's check that Redis is up an running by pinging it:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "id": "37fc2d36-f5bf-465f-9774-510bdc134b62",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "True"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "import redis\n",
- "\n",
- "redis_client = redis.from_url(REDIS_URL)\n",
- "redis_client.ping()"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "4e388814-4188-4f29-8f24-cc67d4048ebe",
- "metadata": {},
- "source": [
- "### Sample Data\n",
- "\n",
- "The 20 newsgroups dataset comprises around 18000 newsgroups posts on 20 topics. We'll use a subset for this demonstration and focus on two categories: 'alt.atheism' and 'sci.space':"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "id": "869a4726-1e24-48fd-9ffd-c62a589d0bb1",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "250"
- ]
- },
- "execution_count": 6,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "from langchain.docstore.document import Document\n",
- "from sklearn.datasets import fetch_20newsgroups\n",
- "\n",
- "categories = [\"alt.atheism\", \"sci.space\"]\n",
- "newsgroups = fetch_20newsgroups(\n",
- " subset=\"train\", categories=categories, shuffle=True, random_state=42\n",
- ")\n",
- "\n",
- "# Use only the first 250 documents\n",
- "texts = newsgroups.data[:250]\n",
- "metadata = [\n",
- " {\"category\": newsgroups.target_names[target]} for target in newsgroups.target[:250]\n",
- "]\n",
- "\n",
- "len(texts)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "93df377e",
- "metadata": {},
- "source": [
- "## Initialization\n",
- "\n",
- "The RedisVectorStore instance can be initialized in several ways:\n",
- "\n",
- "- `RedisVectorStore.__init__` - Initialize directly\n",
- "- `RedisVectorStore.from_texts` - Initialize from a list of texts (optionally with metadata)\n",
- "- `RedisVectorStore.from_documents` - Initialize from a list of `langchain_core.documents.Document` objects\n",
- "- `RedisVectorStore.from_existing_index` - Initialize from an existing Redis index\n",
- "\n",
- "Below we will use the `RedisVectorStore.__init__` method using a `RedisConfig` instance.\n",
- "\n",
- "import EmbeddingTabs from \"@theme/EmbeddingTabs\";\n",
- "\n",
- "\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "id": "7a95c110-015b-4300-93b7-c0100d55d024",
- "metadata": {},
- "outputs": [],
- "source": [
- "%%capture\n",
- "# | output: false\n",
- "# | echo: false\n",
- "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
- "from langchain_huggingface import HuggingFaceEmbeddings\n",
- "from tqdm.auto import tqdm\n",
- "\n",
- "embeddings = HuggingFaceEmbeddings(model_name=\"msmarco-distilbert-base-v4\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "ec110186-d5e4-4eaa-a4e3-899f405f719f",
- "metadata": {},
- "source": [
- "We'll use the SentenceTransformer model to create embeddings. This model runs locally and doesn't require an API key."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "id": "dc37144c-208d-4ab3-9f3a-0407a69fe052",
- "metadata": {
- "tags": []
- },
- "outputs": [],
- "source": [
- "from langchain_redis import RedisConfig, RedisVectorStore\n",
- "\n",
- "config = RedisConfig(\n",
- " index_name=\"newsgroups\",\n",
- " redis_url=REDIS_URL,\n",
- " metadata_schema=[\n",
- " {\"name\": \"category\", \"type\": \"tag\"},\n",
- " ],\n",
- ")\n",
- "\n",
- "vector_store = RedisVectorStore(embeddings, config=config)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "ac6071d4",
- "metadata": {},
- "source": [
- "## Manage vector store\n",
- "\n",
- "### Add items to vector store"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "id": "17f5efc0",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "['newsgroups:f1e788ee61fe410daa8ef941dd166223', 'newsgroups:80b39032181f4299a359a9aaed6e2401', 'newsgroups:99a3efc1883647afba53d115b49e6e92', 'newsgroups:503a6c07cd71418eb71e11b42589efd7', 'newsgroups:7351210e32d1427bbb3c7426cf93a44f', 'newsgroups:4e79fdf67abe471b8ee98ba0e8a1a055', 'newsgroups:03559a1d574e4f9ca0479d7b3891402e', 'newsgroups:9a1c2a7879b8409a805db72feac03580', 'newsgroups:3578a1e129f5435f9743cf803413f37a', 'newsgroups:9f68baf4d6b04f1683d6b871ce8ad92d']\n"
- ]
- }
- ],
- "source": [
- "ids = vector_store.add_texts(texts, metadata)\n",
- "\n",
- "print(ids[0:10])"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "f8822e55-40d5-48aa-8e29-79101feb645a",
- "metadata": {},
- "source": [
- "Let's inspect the first document:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "id": "ca27e394-ae1e-4fdb-b79a-4a6b45a953a8",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "('From: bil@okcforum.osrhe.edu (Bill Conner)\\nSubject: Re: Not the Omni!\\nNntp-Posting-Host: okcforum.osrhe.edu\\nOrganization: Okcforum Unix Users Group\\nX-Newsreader: TIN [version 1.1 PL6]\\nLines: 18\\n\\nCharley Wingate (mangoe@cs.umd.edu) wrote:\\n: \\n: >> Please enlighten me. How is omnipotence contradictory?\\n: \\n: >By definition, all that can occur in the universe is governed by the rules\\n: >of nature. Thus god cannot break them. Anything that god does must be allowed\\n: >in the rules somewhere. Therefore, omnipotence CANNOT exist! It contradicts\\n: >the rules of nature.\\n: \\n: Obviously, an omnipotent god can change the rules.\\n\\nWhen you say, \"By definition\", what exactly is being defined;\\ncertainly not omnipotence. You seem to be saying that the \"rules of\\nnature\" are pre-existent somehow, that they not only define nature but\\nactually cause it. If that\\'s what you mean I\\'d like to hear your\\nfurther thoughts on the question.\\n\\nBill\\n',\n",
- " {'category': 'alt.atheism'})"
- ]
- },
- "execution_count": 10,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "texts[0], metadata[0]"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "dcf1b905",
- "metadata": {},
- "source": [
- "### Delete items from vector store"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "id": "ef61e188",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "1"
- ]
- },
- "execution_count": 11,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "# Delete documents by passing one or more keys/ids\n",
- "vector_store.index.drop_keys(ids[0])"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "021e2e3a-8f87-4d62-a1f3-dc291c6b26be",
- "metadata": {},
- "source": [
- "### Inspecting the created Index\n",
- "\n",
- "Once the ``Redis`` VectorStore object has been constructed, an index will have been created in Redis if it did not already exist. The index can be inspected with both the ``rvl``and the ``redis-cli`` command line tool. If you installed ``redisvl`` above, you can use the ``rvl`` command line tool to inspect the index."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "id": "787d9cbf-8942-4e6f-b030-f404d4632972",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\u001b[32m17:54:50\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n",
- "\u001b[32m17:54:50\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n",
- "\u001b[32m17:54:50\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. newsgroups\n"
- ]
- }
- ],
- "source": [
- "# assumes you're running Redis locally (use --host, --port, --password, --username, to change this)\n",
- "!rvl index listall --port 6379"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "b869de6f-e3da-4bfc-a267-102df1165521",
- "metadata": {},
- "source": [
- "The ``Redis`` VectorStore implementation will attempt to generate index schema (fields for filtering) for any metadata passed through the ``from_texts``, ``from_texts_return_keys``, and ``from_documents`` methods. This way, whatever metadata is passed will be indexed into the Redis search index allowing\n",
- "for filtering on those fields.\n",
- "\n",
- "Below we show what fields were created from the metadata we defined above"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "id": "0eb45eb1-492f-487d-a8a7-7d2d301c7bdb",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\u001b[32m17:54:50\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n",
- "\n",
- "\n",
- "Index Information:\n",
- "╭──────────────┬────────────────┬────────────────┬─────────────────┬────────────╮\n",
- "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n",
- "├──────────────┼────────────────┼────────────────┼─────────────────┼────────────┤\n",
- "│ newsgroups │ HASH │ ['newsgroups'] │ [] │ 0 │\n",
- "╰──────────────┴────────────────┴────────────────┴─────────────────┴────────────╯\n",
- "Index Fields:\n",
- "╭───────────┬─────────────┬────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬─────────────────┬────────────────╮\n",
- "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n",
- "├───────────┼─────────────┼────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────────────────┼────────────────┤\n",
- "│ text │ text │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n",
- "│ embedding │ embedding │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 768 │ distance_metric │ COSINE │\n",
- "│ category │ category │ TAG │ SEPARATOR │ | │ │ │ │ │ │ │\n",
- "╰───────────┴─────────────┴────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴─────────────────┴────────────────╯\n"
- ]
- }
- ],
- "source": [
- "!rvl index info -i newsgroups --port 6379"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 14,
- "id": "84f9a77c-41b1-4515-97f4-2635998dc0dd",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\u001b[32m17:54:51\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n",
- "\n",
- "Statistics:\n",
- "╭─────────────────────────────┬────────────╮\n",
- "│ Stat Key │ Value │\n",
- "├─────────────────────────────┼────────────┤\n",
- "│ num_docs │ 249 │\n",
- "│ num_terms │ 16178 │\n",
- "│ max_doc_id │ 250 │\n",
- "│ num_records │ 50394 │\n",
- "│ percent_indexed │ 1 │\n",
- "│ hash_indexing_failures │ 0 │\n",
- "│ number_of_uses │ 2 │\n",
- "│ bytes_per_record_avg │ 38.2743 │\n",
- "│ doc_table_size_mb │ 0.0263586 │\n",
- "│ inverted_sz_mb │ 1.83944 │\n",
- "│ key_table_size_mb │ 0.00932026 │\n",
- "│ offset_bits_per_record_avg │ 10.6699 │\n",
- "│ offset_vectors_sz_mb │ 0.089057 │\n",
- "│ offsets_per_term_avg │ 1.38937 │\n",
- "│ records_per_doc_avg │ 202.386 │\n",
- "│ sortable_values_size_mb │ 0 │\n",
- "│ total_indexing_time │ 72.444 │\n",
- "│ total_inverted_index_blocks │ 16207 │\n",
- "│ vector_index_sz_mb │ 3.01776 │\n",
- "╰─────────────────────────────┴────────────╯\n"
- ]
- }
- ],
- "source": [
- "!rvl stats -i newsgroups --port 6379"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "c3620501",
- "metadata": {},
- "source": [
- "## Query vector store\n",
- "\n",
- "Once your vector store has been created and the relevant documents have been added you will most likely wish to query it during the running of your chain or agent.\n",
- "\n",
- "### Query directly\n",
- "\n",
- "Performing a simple similarity search can be done as follows:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 15,
- "id": "aa0a16fa",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Simple Similarity Search Results:\n",
- "Content: From: aa429@freenet.carleton.ca (Terry Ford)\n",
- "Subject: A flawed propulsion system: Space Shuttle\n",
- "X-Ad...\n",
- "Metadata: {'category': 'sci.space'}\n",
- "\n",
- "Content: From: nsmca@aurora.alaska.edu\n",
- "Subject: Space Design Movies?\n",
- "Article-I.D.: aurora.1993Apr23.124722.1\n",
- "...\n",
- "Metadata: {'category': 'sci.space'}\n",
- "\n"
- ]
- }
- ],
- "source": [
- "query = \"Tell me about space exploration\"\n",
- "results = vector_store.similarity_search(query, k=2)\n",
- "\n",
- "print(\"Simple Similarity Search Results:\")\n",
- "for doc in results:\n",
- " print(f\"Content: {doc.page_content[:100]}...\")\n",
- " print(f\"Metadata: {doc.metadata}\")\n",
- " print()"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "3ed9d733",
- "metadata": {},
- "source": [
- "If you want to execute a similarity search and receive the corresponding scores you can run:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 16,
- "id": "5efd2eaa",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Similarity Search with Score Results:\n",
- "Content: From: aa429@freenet.carleton.ca (Terry Ford)\n",
- "Subject: A flawed propulsion system: Space Shuttle\n",
- "X-Ad...\n",
- "Metadata: {'category': 'sci.space'}\n",
- "Score: 0.569670975208\n",
- "\n",
- "Content: From: nsmca@aurora.alaska.edu\n",
- "Subject: Space Design Movies?\n",
- "Article-I.D.: aurora.1993Apr23.124722.1\n",
- "...\n",
- "Metadata: {'category': 'sci.space'}\n",
- "Score: 0.590400338173\n",
- "\n"
- ]
- }
- ],
- "source": [
- "# Similarity search with score and filter\n",
- "scored_results = vector_store.similarity_search_with_score(query, k=2)\n",
- "\n",
- "print(\"Similarity Search with Score Results:\")\n",
- "for doc, score in scored_results:\n",
- " print(f\"Content: {doc.page_content[:100]}...\")\n",
- " print(f\"Metadata: {doc.metadata}\")\n",
- " print(f\"Score: {score}\")\n",
- " print()"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "0c235cdc",
- "metadata": {},
- "source": [
- "### Query by turning into retriever\n",
- "\n",
- "You can also transform the vector store into a retriever for easier usage in your chains."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 17,
- "id": "f3460093",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "[Document(metadata={'category': 'sci.space'}, page_content='Subject: Re: Comet in Temporary Orbit Around Jupiter?\\nFrom: Robert Coe \\nDistribution: world\\nOrganization: 1776 Enterprises, Sudbury MA\\nLines: 23\\n\\njgarland@kean.ucs.mun.ca writes:\\n\\n> >> Also, perihelions of Gehrels3 were:\\n> >> \\n> >> April 1973 83 jupiter radii\\n> >> August 1970 ~3 jupiter radii\\n> > \\n> > Where 1 Jupiter radius = 71,000 km = 44,000 mi = 0.0005 AU. So the\\n> > 1970 figure seems unlikely to actually be anything but a perijove.\\n> > Is that the case for the 1973 figure as well?\\n> > -- \\n> Sorry, _perijoves_...I\\'m not used to talking this language.\\n\\nHmmmm.... The prefix \"peri-\" is Greek, not Latin, so it\\'s usually used\\nwith the Greek form of the name of the body being orbited. (That\\'s why\\nit\\'s \"perihelion\" rather than \"perisol\", \"perigee\" rather than \"periterr\",\\nand \"pericynthion\" rather than \"perilune\".) So for Jupiter I\\'d expect it\\nto be something like \"perizeon\".) :^)\\n\\n ___ _ - Bob\\n /__) _ / / ) _ _\\n(_/__) (_)_(_) (___(_)_(/_______________________________________ bob@1776.COM\\nRobert K. Coe ** 14 Churchill St, Sudbury, Massachusetts 01776 ** 508-443-3265\\n'),\n",
- " Document(metadata={'category': 'sci.space'}, page_content='From: pyron@skndiv.dseg.ti.com (Dillon Pyron)\\nSubject: Re: Why not give $1 billion to first year-long moon residents?\\nLines: 42\\nNntp-Posting-Host: skndiv.dseg.ti.com\\nReply-To: pyron@skndiv.dseg.ti.com\\nOrganization: TI/DSEG VAX Support\\n\\n\\nIn article <1qve4kINNpas@sal-sun121.usc.edu>, schaefer@sal-sun121.usc.edu (Peter Schaefer) writes:\\n>In article <1993Apr19.130503.1@aurora.alaska.edu>, nsmca@aurora.alaska.edu writes:\\n>|> In article <6ZV82B2w165w@theporch.raider.net>, gene@theporch.raider.net (Gene Wright) writes:\\n>|> > With the continuin talk about the \"End of the Space Age\" and complaints \\n>|> > by government over the large cost, why not try something I read about \\n>|> > that might just work.\\n>|> > \\n>|> > Announce that a reward of $1 billion would go to the first corporation \\n>|> > who successfully keeps at least 1 person alive on the moon for a year. \\n>|> > Then you\\'d see some of the inexpensive but not popular technologies begin \\n>|> > to be developed. THere\\'d be a different kind of space race then!\\n>|> > \\n>|> > --\\n>|> > gene@theporch.raider.net (Gene Wright)\\n>|> > theporch.raider.net 615/297-7951 The MacInteresteds of Nashville\\n>|> ====\\n>|> If that were true, I\\'d go for it.. I have a few friends who we could pool our\\n>|> resources and do it.. Maybe make it a prize kind of liek the \"Solar Car Race\"\\n>|> in Australia..\\n>|> Anybody game for a contest!\\n>|> \\n>|> ==\\n>|> Michael Adams, nsmca@acad3.alaska.edu -- I\\'m not high, just jacked\\n>\\n>\\n>Oh gee, a billion dollars! That\\'d be just about enough to cover the cost of the\\n>feasability study! Happy, Happy, JOY! JOY!\\n>\\n\\nFeasability study?? What a wimp!! While you are studying, others would be\\ndoing. Too damn many engineers doing way too little engineering.\\n\\n\"He who sits on his arse sits on his fortune\" - Sir Richard Francis Burton\\n--\\nDillon Pyron | The opinions expressed are those of the\\nTI/DSEG Lewisville VAX Support | sender unless otherwise stated.\\n(214)462-3556 (when I\\'m here) |\\n(214)492-4656 (when I\\'m home) |Texans: Vote NO on Robin Hood. We need\\npyron@skndiv.dseg.ti.com |solutions, not gestures.\\nPADI DM-54909 |\\n\\n')]"
- ]
- },
- "execution_count": 17,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "retriever = vector_store.as_retriever(search_type=\"similarity\", search_kwargs={\"k\": 2})\n",
- "retriever.invoke(\"What planet in the solar system has the largest number of moons?\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "901c75dc",
- "metadata": {},
- "source": [
- "## Usage for retrieval-augmented generation\n",
- "\n",
- "For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
- "\n",
- "- [Tutorials](/docs/tutorials/)\n",
- "- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
- "- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "069f1b5f",
- "metadata": {},
- "source": [
- "## Redis-specific functionality\n",
- "\n",
- "Redis offers some unique features for vector search:"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "8a627d3a-af78-46e2-b314-007e641b4d1d",
- "metadata": {},
- "source": [
- "### Similarity search with metadata filtering\n",
- "We can filter our search results based on metadata:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 18,
- "id": "23d6e6fe-8aee-4cee-bf05-59cf3fba36ae",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Filtered Similarity Search Results:\n",
- "Content: From: aa429@freenet.carleton.ca (Terry Ford)\n",
- "Subject: A flawed propulsion system: Space Shuttle\n",
- "X-Ad...\n",
- "Metadata: {'category': 'sci.space'}\n",
- "\n",
- "Content: From: nsmca@aurora.alaska.edu\n",
- "Subject: Space Design Movies?\n",
- "Article-I.D.: aurora.1993Apr23.124722.1\n",
- "...\n",
- "Metadata: {'category': 'sci.space'}\n",
- "\n"
- ]
- }
- ],
- "source": [
- "from redisvl.query.filter import Tag\n",
- "\n",
- "query = \"Tell me about space exploration\"\n",
- "\n",
- "# Create a RedisVL filter expression\n",
- "filter_condition = Tag(\"category\") == \"sci.space\"\n",
- "\n",
- "filtered_results = vector_store.similarity_search(query, k=2, filter=filter_condition)\n",
- "\n",
- "print(\"Filtered Similarity Search Results:\")\n",
- "for doc in filtered_results:\n",
- " print(f\"Content: {doc.page_content[:100]}...\")\n",
- " print(f\"Metadata: {doc.metadata}\")\n",
- " print()"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "f35b9ebf-6419-4402-a066-c1d5287ed38e",
- "metadata": {},
- "source": [
- "### Maximum marginal relevance search\n",
- "Maximum marginal relevance search helps in getting diverse results:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 19,
- "id": "5be2afeb-d0a3-4075-bd3c-4cbe409dfb3a",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Maximum Marginal Relevance Search Results:\n",
- "Content: From: aa429@freenet.carleton.ca (Terry Ford)\n",
- "Subject: A flawed propulsion system: Space Shuttle\n",
- "X-Ad...\n",
- "Metadata: {'category': 'sci.space'}\n",
- "\n",
- "Content: From: moroney@world.std.com (Michael Moroney)\n",
- "Subject: Re: Vulcan? (No, not the guy with the ears!)\n",
- "...\n",
- "Metadata: {'category': 'sci.space'}\n",
- "\n"
- ]
- }
- ],
- "source": [
- "# Maximum marginal relevance search with filter\n",
- "mmr_results = vector_store.max_marginal_relevance_search(\n",
- " query, k=2, fetch_k=10, filter=filter_condition\n",
- ")\n",
- "\n",
- "print(\"Maximum Marginal Relevance Search Results:\")\n",
- "for doc in mmr_results:\n",
- " print(f\"Content: {doc.page_content[:100]}...\")\n",
- " print(f\"Metadata: {doc.metadata}\")\n",
- " print()"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "09c3343c-6af4-4151-ba0a-50800fc34855",
- "metadata": {},
- "source": [
- "## Chain usage\n",
- "The code below shows how to use the vector store as a retriever in a simple RAG chain:\n",
- "\n",
- "import ChatModelTabs from \"@theme/ChatModelTabs\";\n",
- "\n",
- "\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 20,
- "id": "9f6658f8-45b7-4004-a0b3-893bd23bff41",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "OpenAI API key not found in environment variables.\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Please enter your OpenAI API key: ········\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "OpenAI API key has been set for this session.\n"
- ]
- }
- ],
- "source": [
- "# | output: false\n",
- "# | echo: false\n",
- "from getpass import getpass\n",
- "\n",
- "from langchain_openai import ChatOpenAI\n",
- "\n",
- "# Check if OPENAI_API_KEY is already set in the environment\n",
- "openai_api_key = os.getenv(\"OPENAI_API_KEY\")\n",
- "\n",
- "if not openai_api_key:\n",
- " print(\"OpenAI API key not found in environment variables.\")\n",
- " openai_api_key = getpass(\"Please enter your OpenAI API key: \")\n",
- "\n",
- " # Set the API key for the current session\n",
- " os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
- " print(\"OpenAI API key has been set for this session.\")\n",
- "else:\n",
- " print(\"OpenAI API key found in environment variables.\")\n",
- "\n",
- "llm = ChatOpenAI(model=\"gpt-4o-mini\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 21,
- "id": "d0ac614c-3f80-4839-8451-d3322a870809",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "'The Space Shuttle program was a NASA initiative that enabled reusable spacecraft to transport astronauts and cargo to and from low Earth orbit. It conducted a variety of missions, including satellite deployment, scientific research, and assembly of the International Space Station, and typically carried a crew of five astronauts. Although it achieved many successes, the program faced criticism for its safety concerns and the complexity of its propulsion system.'"
- ]
- },
- "execution_count": 21,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "from langchain_core.output_parsers import StrOutputParser\n",
- "from langchain_core.prompts import ChatPromptTemplate\n",
- "from langchain_core.runnables import RunnablePassthrough\n",
- "\n",
- "# Prompt\n",
- "prompt = ChatPromptTemplate.from_messages(\n",
- " [\n",
- " (\n",
- " \"human\",\n",
- " \"\"\"You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum and keep the answer concise.\n",
- "Question: {question}\n",
- "Context: {context}\n",
- "Answer:\"\"\",\n",
- " ),\n",
- " ]\n",
- ")\n",
- "\n",
- "\n",
- "def format_docs(docs):\n",
- " return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
- "\n",
- "\n",
- "rag_chain = (\n",
- " {\"context\": retriever | format_docs, \"question\": RunnablePassthrough()}\n",
- " | prompt\n",
- " | llm\n",
- " | StrOutputParser()\n",
- ")\n",
- "\n",
- "rag_chain.invoke(\"Describe the Space Shuttle program?\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "8ad3e6e4-36ef-494a-be50-4bf8e374b077",
- "metadata": {},
- "source": [
- "## Connect to an existing Index\n",
- "\n",
- "In order to have the same metadata indexed when using the ``Redis`` VectorStore. You will need to have the same ``index_schema`` passed in either as a path to a yaml file or as a dictionary. The following shows how to obtain the schema from an index and connect to an existing index."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 24,
- "id": "6a0e7a49-8271-44b2-abb2-0ef499546b28",
- "metadata": {},
- "outputs": [],
- "source": [
- "# write the schema to a yaml file\n",
- "vector_store.index.schema.to_yaml(\"redis_schema.yaml\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 33,
- "id": "e3588805-b3d9-4af8-8786-b57fc640ebb0",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "18:19:58 redisvl.index.index INFO Index already exists, not overwriting.\n",
- "page_content='From: aa429@freenet.carleton.ca (Terry Ford)\n",
- "Subject: A flawed propulsion system: Space Shuttle\n",
- "X-Added: Forwarded by Space Digest\n",
- "Organization: [via International Space University]\n",
- "Original-Sender: isu@VACATION.VENARI.CS.CMU.EDU\n",
- "Distribution: sci\n",
- "Lines: 13\n",
- "\n",
- "\n",
- "\n",
- "For an essay, I am writing about the space shuttle and a need for a better\n",
- "propulsion system. Through research, I have found that it is rather clumsy \n",
- "(i.e. all the checks/tests before launch), the safety hazards (\"sitting\n",
- "on a hydrogen bomb\"), etc.. If you have any beefs about the current\n",
- "space shuttle program Re: propulsion, please send me your ideas.\n",
- "\n",
- "Thanks a lot.\n",
- "\n",
- "--\n",
- "Terry Ford [aa429@freenet.carleton.ca]\n",
- "Nepean, Ontario, Canada.\n",
- "' metadata={'category': 'sci.space'}\n"
- ]
- }
- ],
- "source": [
- "# now we can connect to our existing index as follows\n",
- "\n",
- "new_rdvs = RedisVectorStore(\n",
- " embeddings,\n",
- " redis_url=REDIS_URL,\n",
- " schema_path=\"redis_schema.yaml\",\n",
- ")\n",
- "\n",
- "results = new_rdvs.similarity_search(\"Space Shuttle Propulsion System\", k=3)\n",
- "print(results[0])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 35,
- "id": "4d7ff456-de2a-4c58-9a3f-a9a3cfdca492",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "True"
- ]
- },
- "execution_count": 35,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "# compare the two schemas to verify they are the same\n",
- "new_rdvs.index.schema == vector_store.index.schema"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "044a2a8c-cb25-453b-a439-38fcb06081ab",
- "metadata": {},
- "source": [
- "## Cleanup vector store"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 22,
- "id": "bb24ab8b-1040-489d-bef6-9137dd2215f3",
- "metadata": {},
- "outputs": [],
- "source": [
- "# Clear vector store\n",
- "vector_store.index.delete(drop=True)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "8a27244f",
- "metadata": {},
- "source": [
- "## API reference\n",
- "\n",
- "For detailed documentation of all RedisVectorStore features and configurations head to the API reference: https://python.langchain.com/api_reference/redis/vectorstores/langchain_redis.vectorstores.RedisVectorStore.html"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.11.9"
- }
+ "cells": [
+ {
+ "cell_type": "raw",
+ "id": "1957f5cb",
+ "metadata": {},
+ "source": [
+ "---\n",
+ "sidebar_label: Redis\n",
+ "---"
+ ]
},
- "nbformat": 4,
- "nbformat_minor": 5
+ {
+ "cell_type": "markdown",
+ "id": "ef1f0986",
+ "metadata": {},
+ "source": [
+ "# Redis Vector Store\n",
+ "\n",
+ "This notebook covers how to get started with the Redis vector store.\n",
+ "\n",
+ ">[Redis](https://redis.io/docs/stack/vectorsearch/) is a popular open-source, in-memory data structure store that can be used as a database, cache, message broker, and queue. It now includes vector similarity search capabilities, making it suitable for use as a vector store."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a717e10e-c8d7-41bc-9dd0-447d11f90b68",
+ "metadata": {},
+ "source": [
+ "### What is Redis?\n",
+ "\n",
+ "Most developers are familiar with `Redis`. At its core, `Redis` is a NoSQL Database in the key-value family that can used as a cache, message broker, stream processing and a primary database. Developers choose `Redis` because it is fast, has a large ecosystem of client libraries, and has been deployed by major enterprises for years.\n",
+ "\n",
+ "On top of these traditional use cases, `Redis` provides additional capabilities like the Search and Query capability that allows users to create secondary index structures within `Redis`. This allows `Redis` to be a Vector Database, at the speed of a cache.\n",
+ "\n",
+ "\n",
+ "### Redis as a Vector Database\n",
+ "\n",
+ "`Redis` uses compressed, inverted indexes for fast indexing with a low memory footprint. It also supports a number of advanced features such as:\n",
+ "\n",
+ "* Indexing of multiple fields in Redis hashes and `JSON`\n",
+ "* Vector similarity search (with `HNSW` (ANN) or `FLAT` (KNN))\n",
+ "* Vector Range Search (e.g. find all vectors within a radius of a query vector)\n",
+ "* Incremental indexing without performance loss\n",
+ "* Document ranking (using [tf-idf](https://en.wikipedia.org/wiki/Tf%E2%80%93idf), with optional user-provided weights)\n",
+ "* Field weighting\n",
+ "* Complex boolean queries with `AND`, `OR`, and `NOT` operators\n",
+ "* Prefix matching, fuzzy matching, and exact-phrase queries\n",
+ "* Support for [double-metaphone phonetic matching](https://redis.io/docs/stack/search/reference/phonetic_matching/)\n",
+ "* Auto-complete suggestions (with fuzzy prefix suggestions)\n",
+ "* Stemming-based query expansion in [many languages](https://redis.io/docs/stack/search/reference/stemming/) (using [Snowball](http://snowballstem.org/))\n",
+ "* Support for Chinese-language tokenization and querying (using [Friso](https://github.com/lionsoul2014/friso))\n",
+ "* Numeric filters and ranges\n",
+ "* Geospatial searches using Redis geospatial indexing\n",
+ "* A powerful aggregations engine\n",
+ "* Supports for all `utf-8` encoded text\n",
+ "* Retrieve full documents, selected fields, or only the document IDs\n",
+ "* Sorting results (for example, by creation date)\n",
+ "\n",
+ "### Clients\n",
+ "\n",
+ "Since `Redis` is much more than just a vector database, there are often use cases that demand the usage of a `Redis` client besides just the `LangChain` integration. You can use any standard `Redis` client library to run Search and Query commands, but it's easiest to use a library that wraps the Search and Query API. Below are a few examples, but you can find more client libraries [here](https://redis.io/resources/clients/).\n",
+ "\n",
+ "| Project | Language | License | Author | Stars |\n",
+ "|----------|---------|--------|---------|-------|\n",
+ "| [jedis][jedis-url] | Java | MIT | [Redis][redis-url] | ![Stars][jedis-stars] |\n",
+ "| [redisvl][redisvl-url] | Python | MIT | [Redis][redis-url] | ![Stars][redisvl-stars] |\n",
+ "| [redis-py][redis-py-url] | Python | MIT | [Redis][redis-url] | ![Stars][redis-py-stars] |\n",
+ "| [node-redis][node-redis-url] | Node.js | MIT | [Redis][redis-url] | ![Stars][node-redis-stars] |\n",
+ "| [nredisstack][nredisstack-url] | .NET | MIT | [Redis][redis-url] | ![Stars][nredisstack-stars] |\n",
+ "\n",
+ "[redis-url]: https://redis.com\n",
+ "\n",
+ "[redisvl-url]: https://github.com/redis/redis-vl-python\n",
+ "[redisvl-stars]: https://img.shields.io/github/stars/redis/redisvl.svg?style=social&label=Star&maxAge=2592000\n",
+ "[redisvl-package]: https://pypi.python.org/pypi/redisvl\n",
+ "\n",
+ "[redis-py-url]: https://github.com/redis/redis-py\n",
+ "[redis-py-stars]: https://img.shields.io/github/stars/redis/redis-py.svg?style=social&label=Star&maxAge=2592000\n",
+ "[redis-py-package]: https://pypi.python.org/pypi/redis\n",
+ "\n",
+ "[jedis-url]: https://github.com/redis/jedis\n",
+ "[jedis-stars]: https://img.shields.io/github/stars/redis/jedis.svg?style=social&label=Star&maxAge=2592000\n",
+ "[Jedis-package]: https://search.maven.org/artifact/redis.clients/jedis\n",
+ "\n",
+ "[nredisstack-url]: https://github.com/redis/nredisstack\n",
+ "[nredisstack-stars]: https://img.shields.io/github/stars/redis/nredisstack.svg?style=social&label=Star&maxAge=2592000\n",
+ "[nredisstack-package]: https://www.nuget.org/packages/nredisstack/\n",
+ "\n",
+ "[node-redis-url]: https://github.com/redis/node-redis\n",
+ "[node-redis-stars]: https://img.shields.io/github/stars/redis/node-redis.svg?style=social&label=Star&maxAge=2592000\n",
+ "[node-redis-package]: https://www.npmjs.com/package/redis\n",
+ "\n",
+ "[redis-om-python-url]: https://github.com/redis/redis-om-python\n",
+ "[redis-om-python-author]: https://redis.com\n",
+ "[redis-om-python-stars]: https://img.shields.io/github/stars/redis/redis-om-python.svg?style=social&label=Star&maxAge=2592000\n",
+ "\n",
+ "[redisearch-go-url]: https://github.com/RediSearch/redisearch-go\n",
+ "[redisearch-go-author]: https://redis.com\n",
+ "[redisearch-go-stars]: https://img.shields.io/github/stars/RediSearch/redisearch-go.svg?style=social&label=Star&maxAge=2592000\n",
+ "\n",
+ "[redisearch-api-rs-url]: https://github.com/RediSearch/redisearch-api-rs\n",
+ "[redisearch-api-rs-author]: https://redis.com\n",
+ "[redisearch-api-rs-stars]: https://img.shields.io/github/stars/RediSearch/redisearch-api-rs.svg?style=social&label=Star&maxAge=2592000\n",
+ "\n",
+ "\n",
+ "### Deployment options\n",
+ "\n",
+ "There are many ways to deploy Redis with RediSearch. The easiest way to get started is to use Docker, but there are are many potential options for deployment such as\n",
+ "\n",
+ "- [Redis Cloud](https://redis.com/redis-enterprise-cloud/overview/)\n",
+ "- [Docker (Redis Stack)](https://hub.docker.com/r/redis/redis-stack)\n",
+ "- Cloud marketplaces: [AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-e6y7ork67pjwg?sr=0-2&ref_=beagle&applicationId=AWSMPContessa), [Google Marketplace](https://console.cloud.google.com/marketplace/details/redislabs-public/redis-enterprise?pli=1), or [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/garantiadata.redis_enterprise_1sp_public_preview?tab=Overview)\n",
+ "- On-premise: [Redis Enterprise Software](https://redis.com/redis-enterprise-software/overview/)\n",
+ "- Kubernetes: [Redis Enterprise Software on Kubernetes](https://docs.redis.com/latest/kubernetes/)\n",
+ "\n",
+ "### Redis connection Url schemas\n",
+ "\n",
+ "Valid Redis Url schemas are:\n",
+ "1. `redis://` - Connection to Redis standalone, unencrypted\n",
+ "2. `rediss://` - Connection to Redis standalone, with TLS encryption\n",
+ "3. `redis+sentinel://` - Connection to Redis server via Redis Sentinel, unencrypted\n",
+ "4. `rediss+sentinel://` - Connection to Redis server via Redis Sentinel, both connections with TLS encryption\n",
+ "\n",
+ "More information about additional connection parameters can be found in the [redis-py documentation](https://redis-py.readthedocs.io/en/stable/connections.html)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "36fdc060",
+ "metadata": {},
+ "source": [
+ "## Setup\n",
+ "\n",
+ "To use the RedisVectorStore, you'll need to install the `langchain-redis` partner package, as well as the other packages used throughout this notebook."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "64e28aa6",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Note: you may need to restart the kernel to use updated packages.\n"
+ ]
+ }
+ ],
+ "source": [
+ "%pip install -qU langchain-redis langchain-huggingface sentence-transformers scikit-learn"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "37d388a3-1a56-453e-8f84-e24a72d682eb",
+ "metadata": {},
+ "source": [
+ "### Credentials\n",
+ "\n",
+ "Redis connection credentials are passed as part of the Redis Connection URL. Redis Connection URLs are versatile and can accommodate various Redis server topologies and authentication methods. These URLs follow a specific format that includes the connection protocol, authentication details, host, port, and database information.\n",
+ "The basic structure of a Redis Connection URL is:\n",
+ "\n",
+ "```\n",
+ "[protocol]://[auth]@[host]:[port]/[database]\n",
+ "```\n",
+ "\n",
+ "Where:\n",
+ "\n",
+ "* protocol can be redis for standard connections, rediss for SSL/TLS connections, or redis+sentinel for Sentinel connections.\n",
+ "* auth includes username and password (if applicable).\n",
+ "* host is the Redis server hostname or IP address.\n",
+ "* port is the Redis server port.\n",
+ "* database is the Redis database number.\n",
+ "\n",
+ "Redis Connection URLs support various configurations, including:\n",
+ "\n",
+ "* Standalone Redis servers (with or without authentication)\n",
+ "* Redis Sentinel setups\n",
+ "* SSL/TLS encrypted connections\n",
+ "* Different authentication methods (password-only or username-password)\n",
+ "\n",
+ "Below are examples of Redis Connection URLs for different configurations:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b1b1eb90-5155-44ca-a8a7-b04b02d5e77c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# connection to redis standalone at localhost, db 0, no password\n",
+ "redis_url = \"redis://localhost:6379\"\n",
+ "# connection to host \"redis\" port 7379 with db 2 and password \"secret\" (old style authentication scheme without username / pre 6.x)\n",
+ "redis_url = \"redis://:secret@redis:7379/2\"\n",
+ "# connection to host redis on default port with user \"joe\", pass \"secret\" using redis version 6+ ACLs\n",
+ "redis_url = \"redis://joe:secret@redis/0\"\n",
+ "\n",
+ "# connection to sentinel at localhost with default group mymaster and db 0, no password\n",
+ "redis_url = \"redis+sentinel://localhost:26379\"\n",
+ "# connection to sentinel at host redis with default port 26379 and user \"joe\" with password \"secret\" with default group mymaster and db 0\n",
+ "redis_url = \"redis+sentinel://joe:secret@redis\"\n",
+ "# connection to sentinel, no auth with sentinel monitoring group \"zone-1\" and database 2\n",
+ "redis_url = \"redis+sentinel://redis:26379/zone-1/2\"\n",
+ "\n",
+ "# connection to redis standalone at localhost, db 0, no password but with TLS support\n",
+ "redis_url = \"rediss://localhost:6379\"\n",
+ "# connection to redis sentinel at localhost and default port, db 0, no password\n",
+ "# but with TLS support for both Sentinel and Redis server\n",
+ "redis_url = \"rediss+sentinel://localhost\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9695dee7",
+ "metadata": {},
+ "source": [
+ "### Launching a Redis Instance with Docker\n",
+ "\n",
+ "To use Redis with LangChain, you need a running Redis instance. You can start one using Docker with:\n",
+ "\n",
+ "```bash\n",
+ "docker run -d -p 6379:6379 redis/redis-stack:latest\n",
+ "```\n",
+ "\n",
+ "For this example, we'll use a local Redis instance. If you're using a remote instance, you'll need to modify the Redis URL accordingly."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "894c30e4",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Connecting to Redis at: redis://redis:6379\n"
+ ]
+ }
+ ],
+ "source": [
+ "import os\n",
+ "\n",
+ "REDIS_URL = os.getenv(\"REDIS_URL\", \"redis://localhost:6379\")\n",
+ "print(f\"Connecting to Redis at: {REDIS_URL}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7f98392b",
+ "metadata": {},
+ "source": [
+ "To enable automated tracing of your model calls, set your [LangSmith](https://docs.smith.langchain.com/) API key:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "e7b6a6e0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# os.environ[\"LANGSMITH_API_KEY\"] = getpass.getpass(\"Enter your LangSmith API key: \")\n",
+ "# os.environ[\"LANGSMITH_TRACING\"] = \"true\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "63dd7f8d-df13-45d8-8e13-892b29803e96",
+ "metadata": {},
+ "source": [
+ "Let's check that Redis is up an running by pinging it:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "37fc2d36-f5bf-465f-9774-510bdc134b62",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "True"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import redis\n",
+ "\n",
+ "redis_client = redis.from_url(REDIS_URL)\n",
+ "redis_client.ping()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4e388814-4188-4f29-8f24-cc67d4048ebe",
+ "metadata": {},
+ "source": [
+ "### Sample Data\n",
+ "\n",
+ "The 20 newsgroups dataset comprises around 18000 newsgroups posts on 20 topics. We'll use a subset for this demonstration and focus on two categories: 'alt.atheism' and 'sci.space':"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "869a4726-1e24-48fd-9ffd-c62a589d0bb1",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "250"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from langchain.docstore.document import Document\n",
+ "from sklearn.datasets import fetch_20newsgroups\n",
+ "\n",
+ "categories = [\"alt.atheism\", \"sci.space\"]\n",
+ "newsgroups = fetch_20newsgroups(\n",
+ " subset=\"train\", categories=categories, shuffle=True, random_state=42\n",
+ ")\n",
+ "\n",
+ "# Use only the first 250 documents\n",
+ "texts = newsgroups.data[:250]\n",
+ "metadata = [\n",
+ " {\"category\": newsgroups.target_names[target]} for target in newsgroups.target[:250]\n",
+ "]\n",
+ "\n",
+ "len(texts)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "93df377e",
+ "metadata": {},
+ "source": [
+ "## Initialization\n",
+ "\n",
+ "The RedisVectorStore instance can be initialized in several ways:\n",
+ "\n",
+ "- `RedisVectorStore.__init__` - Initialize directly\n",
+ "- `RedisVectorStore.from_texts` - Initialize from a list of texts (optionally with metadata)\n",
+ "- `RedisVectorStore.from_documents` - Initialize from a list of `langchain_core.documents.Document` objects\n",
+ "- `RedisVectorStore.from_existing_index` - Initialize from an existing Redis index\n",
+ "\n",
+ "Below we will use the `RedisVectorStore.__init__` method using a `RedisConfig` instance.\n",
+ "\n",
+ "import EmbeddingTabs from \"@theme/EmbeddingTabs\";\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "7a95c110-015b-4300-93b7-c0100d55d024",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%capture\n",
+ "# | output: false\n",
+ "# | echo: false\n",
+ "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
+ "from langchain_huggingface import HuggingFaceEmbeddings\n",
+ "from tqdm.auto import tqdm\n",
+ "\n",
+ "embeddings = HuggingFaceEmbeddings(model_name=\"msmarco-distilbert-base-v4\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ec110186-d5e4-4eaa-a4e3-899f405f719f",
+ "metadata": {},
+ "source": [
+ "We'll use the SentenceTransformer model to create embeddings. This model runs locally and doesn't require an API key."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "dc37144c-208d-4ab3-9f3a-0407a69fe052",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "from langchain_redis import RedisConfig, RedisVectorStore\n",
+ "\n",
+ "config = RedisConfig(\n",
+ " index_name=\"newsgroups\",\n",
+ " redis_url=REDIS_URL,\n",
+ " metadata_schema=[\n",
+ " {\"name\": \"category\", \"type\": \"tag\"},\n",
+ " ],\n",
+ ")\n",
+ "\n",
+ "vector_store = RedisVectorStore(embeddings, config=config)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ac6071d4",
+ "metadata": {},
+ "source": [
+ "## Manage vector store\n",
+ "\n",
+ "### Add items to vector store"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "17f5efc0",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "['newsgroups:f1e788ee61fe410daa8ef941dd166223', 'newsgroups:80b39032181f4299a359a9aaed6e2401', 'newsgroups:99a3efc1883647afba53d115b49e6e92', 'newsgroups:503a6c07cd71418eb71e11b42589efd7', 'newsgroups:7351210e32d1427bbb3c7426cf93a44f', 'newsgroups:4e79fdf67abe471b8ee98ba0e8a1a055', 'newsgroups:03559a1d574e4f9ca0479d7b3891402e', 'newsgroups:9a1c2a7879b8409a805db72feac03580', 'newsgroups:3578a1e129f5435f9743cf803413f37a', 'newsgroups:9f68baf4d6b04f1683d6b871ce8ad92d']\n"
+ ]
+ }
+ ],
+ "source": [
+ "ids = vector_store.add_texts(texts, metadata)\n",
+ "\n",
+ "print(ids[0:10])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f8822e55-40d5-48aa-8e29-79101feb645a",
+ "metadata": {},
+ "source": [
+ "Let's inspect the first document:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "ca27e394-ae1e-4fdb-b79a-4a6b45a953a8",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "('From: bil@okcforum.osrhe.edu (Bill Conner)\\nSubject: Re: Not the Omni!\\nNntp-Posting-Host: okcforum.osrhe.edu\\nOrganization: Okcforum Unix Users Group\\nX-Newsreader: TIN [version 1.1 PL6]\\nLines: 18\\n\\nCharley Wingate (mangoe@cs.umd.edu) wrote:\\n: \\n: >> Please enlighten me. How is omnipotence contradictory?\\n: \\n: >By definition, all that can occur in the universe is governed by the rules\\n: >of nature. Thus god cannot break them. Anything that god does must be allowed\\n: >in the rules somewhere. Therefore, omnipotence CANNOT exist! It contradicts\\n: >the rules of nature.\\n: \\n: Obviously, an omnipotent god can change the rules.\\n\\nWhen you say, \"By definition\", what exactly is being defined;\\ncertainly not omnipotence. You seem to be saying that the \"rules of\\nnature\" are pre-existent somehow, that they not only define nature but\\nactually cause it. If that\\'s what you mean I\\'d like to hear your\\nfurther thoughts on the question.\\n\\nBill\\n',\n",
+ " {'category': 'alt.atheism'})"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "texts[0], metadata[0]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "dcf1b905",
+ "metadata": {},
+ "source": [
+ "### Delete items from vector store"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "ef61e188",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Delete documents by passing one or more keys/ids\n",
+ "vector_store.index.drop_keys(ids[0])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "021e2e3a-8f87-4d62-a1f3-dc291c6b26be",
+ "metadata": {},
+ "source": [
+ "### Inspecting the created Index\n",
+ "\n",
+ "Once the ``Redis`` VectorStore object has been constructed, an index will have been created in Redis if it did not already exist. The index can be inspected with both the ``rvl``and the ``redis-cli`` command line tool. If you installed ``redisvl`` above, you can use the ``rvl`` command line tool to inspect the index."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "787d9cbf-8942-4e6f-b030-f404d4632972",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u001b[32m17:54:50\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n",
+ "\u001b[32m17:54:50\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n",
+ "\u001b[32m17:54:50\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. newsgroups\n"
+ ]
+ }
+ ],
+ "source": [
+ "# assumes you're running Redis locally (use --host, --port, --password, --username, to change this)\n",
+ "!rvl index listall --port 6379"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b869de6f-e3da-4bfc-a267-102df1165521",
+ "metadata": {},
+ "source": [
+ "The ``Redis`` VectorStore implementation will attempt to generate index schema (fields for filtering) for any metadata passed through the ``from_texts``, ``from_texts_return_keys``, and ``from_documents`` methods. This way, whatever metadata is passed will be indexed into the Redis search index allowing\n",
+ "for filtering on those fields.\n",
+ "\n",
+ "Below we show what fields were created from the metadata we defined above"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "0eb45eb1-492f-487d-a8a7-7d2d301c7bdb",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u001b[32m17:54:50\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n",
+ "\n",
+ "\n",
+ "Index Information:\n",
+ "╭──────────────┬────────────────┬────────────────┬─────────────────┬────────────╮\n",
+ "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n",
+ "├──────────────┼────────────────┼────────────────┼─────────────────┼────────────┤\n",
+ "│ newsgroups │ HASH │ ['newsgroups'] │ [] │ 0 │\n",
+ "╰──────────────┴────────────────┴────────────────┴─────────────────┴────────────╯\n",
+ "Index Fields:\n",
+ "╭───────────┬─────────────┬────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬─────────────────┬────────────────╮\n",
+ "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n",
+ "├───────────┼─────────────┼────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────────────────┼────────────────┤\n",
+ "│ text │ text │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n",
+ "│ embedding │ embedding │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 768 │ distance_metric │ COSINE │\n",
+ "│ category │ category │ TAG │ SEPARATOR │ | │ │ │ │ │ │ │\n",
+ "╰───────────┴─────────────┴────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴─────────────────┴────────────────╯\n"
+ ]
+ }
+ ],
+ "source": [
+ "!rvl index info -i newsgroups --port 6379"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "id": "84f9a77c-41b1-4515-97f4-2635998dc0dd",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u001b[32m17:54:51\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n",
+ "\n",
+ "Statistics:\n",
+ "╭─────────────────────────────┬────────────╮\n",
+ "│ Stat Key │ Value │\n",
+ "├─────────────────────────────┼────────────┤\n",
+ "│ num_docs │ 249 │\n",
+ "│ num_terms │ 16178 │\n",
+ "│ max_doc_id │ 250 │\n",
+ "│ num_records │ 50394 │\n",
+ "│ percent_indexed │ 1 │\n",
+ "│ hash_indexing_failures │ 0 │\n",
+ "│ number_of_uses │ 2 │\n",
+ "│ bytes_per_record_avg │ 38.2743 │\n",
+ "│ doc_table_size_mb │ 0.0263586 │\n",
+ "│ inverted_sz_mb │ 1.83944 │\n",
+ "│ key_table_size_mb │ 0.00932026 │\n",
+ "│ offset_bits_per_record_avg │ 10.6699 │\n",
+ "│ offset_vectors_sz_mb │ 0.089057 │\n",
+ "│ offsets_per_term_avg │ 1.38937 │\n",
+ "│ records_per_doc_avg │ 202.386 │\n",
+ "│ sortable_values_size_mb │ 0 │\n",
+ "│ total_indexing_time │ 72.444 │\n",
+ "│ total_inverted_index_blocks │ 16207 │\n",
+ "│ vector_index_sz_mb │ 3.01776 │\n",
+ "╰─────────────────────────────┴────────────╯\n"
+ ]
+ }
+ ],
+ "source": [
+ "!rvl stats -i newsgroups --port 6379"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c3620501",
+ "metadata": {},
+ "source": [
+ "## Query vector store\n",
+ "\n",
+ "Once your vector store has been created and the relevant documents have been added you will most likely wish to query it during the running of your chain or agent.\n",
+ "\n",
+ "### Query directly\n",
+ "\n",
+ "Performing a simple similarity search can be done as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "id": "aa0a16fa",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Simple Similarity Search Results:\n",
+ "Content: From: aa429@freenet.carleton.ca (Terry Ford)\n",
+ "Subject: A flawed propulsion system: Space Shuttle\n",
+ "X-Ad...\n",
+ "Metadata: {'category': 'sci.space'}\n",
+ "\n",
+ "Content: From: nsmca@aurora.alaska.edu\n",
+ "Subject: Space Design Movies?\n",
+ "Article-I.D.: aurora.1993Apr23.124722.1\n",
+ "...\n",
+ "Metadata: {'category': 'sci.space'}\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "query = \"Tell me about space exploration\"\n",
+ "results = vector_store.similarity_search(query, k=2)\n",
+ "\n",
+ "print(\"Simple Similarity Search Results:\")\n",
+ "for doc in results:\n",
+ " print(f\"Content: {doc.page_content[:100]}...\")\n",
+ " print(f\"Metadata: {doc.metadata}\")\n",
+ " print()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3ed9d733",
+ "metadata": {},
+ "source": [
+ "If you want to execute a similarity search and receive the corresponding scores you can run:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "id": "5efd2eaa",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Similarity Search with Score Results:\n",
+ "Content: From: aa429@freenet.carleton.ca (Terry Ford)\n",
+ "Subject: A flawed propulsion system: Space Shuttle\n",
+ "X-Ad...\n",
+ "Metadata: {'category': 'sci.space'}\n",
+ "Score: 0.569670975208\n",
+ "\n",
+ "Content: From: nsmca@aurora.alaska.edu\n",
+ "Subject: Space Design Movies?\n",
+ "Article-I.D.: aurora.1993Apr23.124722.1\n",
+ "...\n",
+ "Metadata: {'category': 'sci.space'}\n",
+ "Score: 0.590400338173\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Similarity search with score and filter\n",
+ "scored_results = vector_store.similarity_search_with_score(query, k=2)\n",
+ "\n",
+ "print(\"Similarity Search with Score Results:\")\n",
+ "for doc, score in scored_results:\n",
+ " print(f\"Content: {doc.page_content[:100]}...\")\n",
+ " print(f\"Metadata: {doc.metadata}\")\n",
+ " print(f\"Score: {score}\")\n",
+ " print()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0c235cdc",
+ "metadata": {},
+ "source": [
+ "### Query by turning into retriever\n",
+ "\n",
+ "You can also transform the vector store into a retriever for easier usage in your chains."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "id": "f3460093",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[Document(metadata={'category': 'sci.space'}, page_content='Subject: Re: Comet in Temporary Orbit Around Jupiter?\\nFrom: Robert Coe \\nDistribution: world\\nOrganization: 1776 Enterprises, Sudbury MA\\nLines: 23\\n\\njgarland@kean.ucs.mun.ca writes:\\n\\n> >> Also, perihelions of Gehrels3 were:\\n> >> \\n> >> April 1973 83 jupiter radii\\n> >> August 1970 ~3 jupiter radii\\n> > \\n> > Where 1 Jupiter radius = 71,000 km = 44,000 mi = 0.0005 AU. So the\\n> > 1970 figure seems unlikely to actually be anything but a perijove.\\n> > Is that the case for the 1973 figure as well?\\n> > -- \\n> Sorry, _perijoves_...I\\'m not used to talking this language.\\n\\nHmmmm.... The prefix \"peri-\" is Greek, not Latin, so it\\'s usually used\\nwith the Greek form of the name of the body being orbited. (That\\'s why\\nit\\'s \"perihelion\" rather than \"perisol\", \"perigee\" rather than \"periterr\",\\nand \"pericynthion\" rather than \"perilune\".) So for Jupiter I\\'d expect it\\nto be something like \"perizeon\".) :^)\\n\\n ___ _ - Bob\\n /__) _ / / ) _ _\\n(_/__) (_)_(_) (___(_)_(/_______________________________________ bob@1776.COM\\nRobert K. Coe ** 14 Churchill St, Sudbury, Massachusetts 01776 ** 508-443-3265\\n'),\n",
+ " Document(metadata={'category': 'sci.space'}, page_content='From: pyron@skndiv.dseg.ti.com (Dillon Pyron)\\nSubject: Re: Why not give $1 billion to first year-long moon residents?\\nLines: 42\\nNntp-Posting-Host: skndiv.dseg.ti.com\\nReply-To: pyron@skndiv.dseg.ti.com\\nOrganization: TI/DSEG VAX Support\\n\\n\\nIn article <1qve4kINNpas@sal-sun121.usc.edu>, schaefer@sal-sun121.usc.edu (Peter Schaefer) writes:\\n>In article <1993Apr19.130503.1@aurora.alaska.edu>, nsmca@aurora.alaska.edu writes:\\n>|> In article <6ZV82B2w165w@theporch.raider.net>, gene@theporch.raider.net (Gene Wright) writes:\\n>|> > With the continuin talk about the \"End of the Space Age\" and complaints \\n>|> > by government over the large cost, why not try something I read about \\n>|> > that might just work.\\n>|> > \\n>|> > Announce that a reward of $1 billion would go to the first corporation \\n>|> > who successfully keeps at least 1 person alive on the moon for a year. \\n>|> > Then you\\'d see some of the inexpensive but not popular technologies begin \\n>|> > to be developed. THere\\'d be a different kind of space race then!\\n>|> > \\n>|> > --\\n>|> > gene@theporch.raider.net (Gene Wright)\\n>|> > theporch.raider.net 615/297-7951 The MacInteresteds of Nashville\\n>|> ====\\n>|> If that were true, I\\'d go for it.. I have a few friends who we could pool our\\n>|> resources and do it.. Maybe make it a prize kind of liek the \"Solar Car Race\"\\n>|> in Australia..\\n>|> Anybody game for a contest!\\n>|> \\n>|> ==\\n>|> Michael Adams, nsmca@acad3.alaska.edu -- I\\'m not high, just jacked\\n>\\n>\\n>Oh gee, a billion dollars! That\\'d be just about enough to cover the cost of the\\n>feasability study! Happy, Happy, JOY! JOY!\\n>\\n\\nFeasability study?? What a wimp!! While you are studying, others would be\\ndoing. Too damn many engineers doing way too little engineering.\\n\\n\"He who sits on his arse sits on his fortune\" - Sir Richard Francis Burton\\n--\\nDillon Pyron | The opinions expressed are those of the\\nTI/DSEG Lewisville VAX Support | sender unless otherwise stated.\\n(214)462-3556 (when I\\'m here) |\\n(214)492-4656 (when I\\'m home) |Texans: Vote NO on Robin Hood. We need\\npyron@skndiv.dseg.ti.com |solutions, not gestures.\\nPADI DM-54909 |\\n\\n')]"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "retriever = vector_store.as_retriever(search_type=\"similarity\", search_kwargs={\"k\": 2})\n",
+ "retriever.invoke(\"What planet in the solar system has the largest number of moons?\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "901c75dc",
+ "metadata": {},
+ "source": [
+ "## Usage for retrieval-augmented generation\n",
+ "\n",
+ "For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n",
+ "\n",
+ "- [Tutorials](/docs/tutorials/rag)\n",
+ "- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n",
+ "- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "069f1b5f",
+ "metadata": {},
+ "source": [
+ "## Redis-specific functionality\n",
+ "\n",
+ "Redis offers some unique features for vector search:"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8a627d3a-af78-46e2-b314-007e641b4d1d",
+ "metadata": {},
+ "source": [
+ "### Similarity search with metadata filtering\n",
+ "We can filter our search results based on metadata:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "id": "23d6e6fe-8aee-4cee-bf05-59cf3fba36ae",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Filtered Similarity Search Results:\n",
+ "Content: From: aa429@freenet.carleton.ca (Terry Ford)\n",
+ "Subject: A flawed propulsion system: Space Shuttle\n",
+ "X-Ad...\n",
+ "Metadata: {'category': 'sci.space'}\n",
+ "\n",
+ "Content: From: nsmca@aurora.alaska.edu\n",
+ "Subject: Space Design Movies?\n",
+ "Article-I.D.: aurora.1993Apr23.124722.1\n",
+ "...\n",
+ "Metadata: {'category': 'sci.space'}\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "from redisvl.query.filter import Tag\n",
+ "\n",
+ "query = \"Tell me about space exploration\"\n",
+ "\n",
+ "# Create a RedisVL filter expression\n",
+ "filter_condition = Tag(\"category\") == \"sci.space\"\n",
+ "\n",
+ "filtered_results = vector_store.similarity_search(query, k=2, filter=filter_condition)\n",
+ "\n",
+ "print(\"Filtered Similarity Search Results:\")\n",
+ "for doc in filtered_results:\n",
+ " print(f\"Content: {doc.page_content[:100]}...\")\n",
+ " print(f\"Metadata: {doc.metadata}\")\n",
+ " print()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f35b9ebf-6419-4402-a066-c1d5287ed38e",
+ "metadata": {},
+ "source": [
+ "### Maximum marginal relevance search\n",
+ "Maximum marginal relevance search helps in getting diverse results:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "id": "5be2afeb-d0a3-4075-bd3c-4cbe409dfb3a",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Maximum Marginal Relevance Search Results:\n",
+ "Content: From: aa429@freenet.carleton.ca (Terry Ford)\n",
+ "Subject: A flawed propulsion system: Space Shuttle\n",
+ "X-Ad...\n",
+ "Metadata: {'category': 'sci.space'}\n",
+ "\n",
+ "Content: From: moroney@world.std.com (Michael Moroney)\n",
+ "Subject: Re: Vulcan? (No, not the guy with the ears!)\n",
+ "...\n",
+ "Metadata: {'category': 'sci.space'}\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Maximum marginal relevance search with filter\n",
+ "mmr_results = vector_store.max_marginal_relevance_search(\n",
+ " query, k=2, fetch_k=10, filter=filter_condition\n",
+ ")\n",
+ "\n",
+ "print(\"Maximum Marginal Relevance Search Results:\")\n",
+ "for doc in mmr_results:\n",
+ " print(f\"Content: {doc.page_content[:100]}...\")\n",
+ " print(f\"Metadata: {doc.metadata}\")\n",
+ " print()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "09c3343c-6af4-4151-ba0a-50800fc34855",
+ "metadata": {},
+ "source": [
+ "## Chain usage\n",
+ "The code below shows how to use the vector store as a retriever in a simple RAG chain:\n",
+ "\n",
+ "import ChatModelTabs from \"@theme/ChatModelTabs\";\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "id": "9f6658f8-45b7-4004-a0b3-893bd23bff41",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "OpenAI API key not found in environment variables.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Please enter your OpenAI API key: ········\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "OpenAI API key has been set for this session.\n"
+ ]
+ }
+ ],
+ "source": [
+ "# | output: false\n",
+ "# | echo: false\n",
+ "from getpass import getpass\n",
+ "\n",
+ "from langchain_openai import ChatOpenAI\n",
+ "\n",
+ "# Check if OPENAI_API_KEY is already set in the environment\n",
+ "openai_api_key = os.getenv(\"OPENAI_API_KEY\")\n",
+ "\n",
+ "if not openai_api_key:\n",
+ " print(\"OpenAI API key not found in environment variables.\")\n",
+ " openai_api_key = getpass(\"Please enter your OpenAI API key: \")\n",
+ "\n",
+ " # Set the API key for the current session\n",
+ " os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
+ " print(\"OpenAI API key has been set for this session.\")\n",
+ "else:\n",
+ " print(\"OpenAI API key found in environment variables.\")\n",
+ "\n",
+ "llm = ChatOpenAI(model=\"gpt-4o-mini\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "id": "d0ac614c-3f80-4839-8451-d3322a870809",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'The Space Shuttle program was a NASA initiative that enabled reusable spacecraft to transport astronauts and cargo to and from low Earth orbit. It conducted a variety of missions, including satellite deployment, scientific research, and assembly of the International Space Station, and typically carried a crew of five astronauts. Although it achieved many successes, the program faced criticism for its safety concerns and the complexity of its propulsion system.'"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from langchain_core.output_parsers import StrOutputParser\n",
+ "from langchain_core.prompts import ChatPromptTemplate\n",
+ "from langchain_core.runnables import RunnablePassthrough\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\n",
+ " \"human\",\n",
+ " \"\"\"You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum and keep the answer concise.\n",
+ "Question: {question}\n",
+ "Context: {context}\n",
+ "Answer:\"\"\",\n",
+ " ),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "\n",
+ "def format_docs(docs):\n",
+ " return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+ "\n",
+ "\n",
+ "rag_chain = (\n",
+ " {\"context\": retriever | format_docs, \"question\": RunnablePassthrough()}\n",
+ " | prompt\n",
+ " | llm\n",
+ " | StrOutputParser()\n",
+ ")\n",
+ "\n",
+ "rag_chain.invoke(\"Describe the Space Shuttle program?\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8ad3e6e4-36ef-494a-be50-4bf8e374b077",
+ "metadata": {},
+ "source": [
+ "## Connect to an existing Index\n",
+ "\n",
+ "In order to have the same metadata indexed when using the ``Redis`` VectorStore. You will need to have the same ``index_schema`` passed in either as a path to a yaml file or as a dictionary. The following shows how to obtain the schema from an index and connect to an existing index."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "id": "6a0e7a49-8271-44b2-abb2-0ef499546b28",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# write the schema to a yaml file\n",
+ "vector_store.index.schema.to_yaml(\"redis_schema.yaml\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "id": "e3588805-b3d9-4af8-8786-b57fc640ebb0",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "18:19:58 redisvl.index.index INFO Index already exists, not overwriting.\n",
+ "page_content='From: aa429@freenet.carleton.ca (Terry Ford)\n",
+ "Subject: A flawed propulsion system: Space Shuttle\n",
+ "X-Added: Forwarded by Space Digest\n",
+ "Organization: [via International Space University]\n",
+ "Original-Sender: isu@VACATION.VENARI.CS.CMU.EDU\n",
+ "Distribution: sci\n",
+ "Lines: 13\n",
+ "\n",
+ "\n",
+ "\n",
+ "For an essay, I am writing about the space shuttle and a need for a better\n",
+ "propulsion system. Through research, I have found that it is rather clumsy \n",
+ "(i.e. all the checks/tests before launch), the safety hazards (\"sitting\n",
+ "on a hydrogen bomb\"), etc.. If you have any beefs about the current\n",
+ "space shuttle program Re: propulsion, please send me your ideas.\n",
+ "\n",
+ "Thanks a lot.\n",
+ "\n",
+ "--\n",
+ "Terry Ford [aa429@freenet.carleton.ca]\n",
+ "Nepean, Ontario, Canada.\n",
+ "' metadata={'category': 'sci.space'}\n"
+ ]
+ }
+ ],
+ "source": [
+ "# now we can connect to our existing index as follows\n",
+ "\n",
+ "new_rdvs = RedisVectorStore(\n",
+ " embeddings,\n",
+ " redis_url=REDIS_URL,\n",
+ " schema_path=\"redis_schema.yaml\",\n",
+ ")\n",
+ "\n",
+ "results = new_rdvs.similarity_search(\"Space Shuttle Propulsion System\", k=3)\n",
+ "print(results[0])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "id": "4d7ff456-de2a-4c58-9a3f-a9a3cfdca492",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "True"
+ ]
+ },
+ "execution_count": 35,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# compare the two schemas to verify they are the same\n",
+ "new_rdvs.index.schema == vector_store.index.schema"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "044a2a8c-cb25-453b-a439-38fcb06081ab",
+ "metadata": {},
+ "source": [
+ "## Cleanup vector store"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "id": "bb24ab8b-1040-489d-bef6-9137dd2215f3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Clear vector store\n",
+ "vector_store.index.delete(drop=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8a27244f",
+ "metadata": {},
+ "source": [
+ "## API reference\n",
+ "\n",
+ "For detailed documentation of all RedisVectorStore features and configurations head to the API reference: https://python.langchain.com/api_reference/redis/vectorstores/langchain_redis.vectorstores.RedisVectorStore.html"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.6"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
}
diff --git a/docs/docs/integrations/vectorstores/sqlserver.ipynb b/docs/docs/integrations/vectorstores/sqlserver.ipynb
index 2e6ee2a33c9..226a5720d1f 100644
--- a/docs/docs/integrations/vectorstores/sqlserver.ipynb
+++ b/docs/docs/integrations/vectorstores/sqlserver.ipynb
@@ -930,6 +930,7 @@
},
"source": [
"## Related\n",
+ "- Vector store [tutorials](/docs/tutorials/rag)\n",
"- Vector store [conceptual guide](https://python.langchain.com/docs/concepts/vectorstores/)\n",
"- Vector store [how-to guides](https://python.langchain.com/docs/how_to/#vector-stores)"
]
@@ -937,7 +938,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -951,9 +952,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.9"
+ "version": "3.9.6"
}
},
"nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
}