Supabase vector self querying retriever (#10304)

## Description
Adds Supabase Vector as a self-querying retriever.

- Designed to be backwards compatible with existing `filter` logic on
`SupabaseVectorStore`.
- Adds new filter `postgrest_filter` to `SupabaseVectorStore`
`similarity_search()` methods
- Supports entire PostgREST [filter query
language](https://postgrest.org/en/stable/references/api/tables_views.html#read)
(used by self-querying retriever, but also works as an escape hatch for
more query control)
- `SupabaseVectorTranslator` converts Langchain filter into the above
PostgREST query
- Adds Jupyter Notebook for the self-querying retriever
- Adds tests

## Tag maintainer
@hwchase17

## Twitter handle
[@ggrdson](https://twitter.com/ggrdson)
This commit is contained in:
Greg Richardson
2023-09-07 16:03:26 -06:00
committed by GitHub
parent 20c742d8a2
commit 300559695b
6 changed files with 839 additions and 43 deletions

View File

@@ -28,43 +28,41 @@
"The following function determines cosine similarity, but you can adjust to your needs.\n",
"\n",
"```sql\n",
" -- Enable the pgvector extension to work with embedding vectors\n",
" create extension vector;\n",
"-- Enable the pgvector extension to work with embedding vectors\n",
"create extension if not exists vector;\n",
"\n",
" -- Create a table to store your documents\n",
" create table documents (\n",
" id uuid primary key,\n",
" content text, -- corresponds to Document.pageContent\n",
" metadata jsonb, -- corresponds to Document.metadata\n",
" embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed\n",
" );\n",
"-- Create a table to store your documents\n",
"create table\n",
" documents (\n",
" id uuid primary key,\n",
" content text, -- corresponds to Document.pageContent\n",
" metadata jsonb, -- corresponds to Document.metadata\n",
" embedding vector (1536) -- 1536 works for OpenAI embeddings, change if needed\n",
" );\n",
"\n",
" CREATE FUNCTION match_documents(query_embedding vector(1536), match_count int)\n",
" RETURNS TABLE(\n",
" id uuid,\n",
" content text,\n",
" metadata jsonb,\n",
" -- we return matched vectors to enable maximal marginal relevance searches\n",
" embedding vector(1536),\n",
" similarity float)\n",
" LANGUAGE plpgsql\n",
" AS $$\n",
" # variable_conflict use_column\n",
" BEGIN\n",
" RETURN query\n",
" SELECT\n",
" id,\n",
" content,\n",
" metadata,\n",
" embedding,\n",
" 1 -(documents.embedding <=> query_embedding) AS similarity\n",
" FROM\n",
" documents\n",
" ORDER BY\n",
" documents.embedding <=> query_embedding\n",
" LIMIT match_count;\n",
" END;\n",
" $$;\n",
"-- Create a function to search for documents\n",
"create function match_documents (\n",
" query_embedding vector (1536),\n",
" filter jsonb default '{}'\n",
") returns table (\n",
" id uuid,\n",
" content text,\n",
" metadata jsonb,\n",
" similarity float\n",
") language plpgsql as $$\n",
"#variable_conflict use_column\n",
"begin\n",
" return query\n",
" select\n",
" id,\n",
" content,\n",
" metadata,\n",
" 1 - (documents.embedding <=> query_embedding) as similarity\n",
" from documents\n",
" where metadata @> filter\n",
" order by documents.embedding <=> query_embedding;\n",
"end;\n",
"$$;\n",
"```"
]
},