github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-05-31 03:59:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

langchain[patch], docs[patch]: use byte store in multivectorretriever (#14474)

Browse Source
This commit is contained in:
Erick Friis 2023-12-08 16:26:11 -08:00 committed by GitHub
parent 1ef13661b9
commit c24f277b7c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 26 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
docs/docs/modules/data_connection/retrievers/multi_vector.ipynb
View File

@ -38,7 +38,7 @@
"source": [
"from langchain.document_loaders import TextLoader\n",
"from langchain.embeddings import OpenAIEmbeddings\n",
"from langchain.storage import InMemoryStore\n",
"from langchain.storage import InMemoryByteStore\n",
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
"from langchain.vectorstores import Chroma"
]
@ -83,12 +83,12 @@
" collection_name=\"full_documents\", embedding_function=OpenAIEmbeddings()\n",
")\n",
"# The storage layer for the parent documents\n",
"store = InMemoryStore()\n",
"store = InMemoryByteStore()\n",
"id_key = \"doc_id\"\n",
"# The retriever (empty to start)\n",
"retriever = MultiVectorRetriever(\n",
" vectorstore=vectorstore,\n",
" base_store=store,\n",
" byte_store=store,\n",
" id_key=id_key,\n",
")\n",
"import uuid\n",
@ -143,7 +143,7 @@
{
"data": {
"text/plain": [
"Document(page_content='Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \\n\\nOne of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court.', metadata={'doc_id': '80a5dccb-606f-437a-927a-54090fb0247d', 'source': '../../state_of_the_union.txt'})"
"Document(page_content='Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \\n\\nOne of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court.', metadata={'doc_id': '3f826cfe-78bd-468d-adb8-f5c2719255df', 'source': '../../state_of_the_union.txt'})"
]
},
"execution_count": 8,
@ -271,12 +271,12 @@
"# The vectorstore to use to index the child chunks\n",
"vectorstore = Chroma(collection_name=\"summaries\", embedding_function=OpenAIEmbeddings())\n",
"# The storage layer for the parent documents\n",
"store = InMemoryStore()\n",
"store = InMemoryByteStore()\n",
"id_key = \"doc_id\"\n",
"# The retriever (empty to start)\n",
"retriever = MultiVectorRetriever(\n",
" vectorstore=vectorstore,\n",
" base_store=store,\n",
" byte_store=store,\n",
" id_key=id_key,\n",
")\n",
"doc_ids = [str(uuid.uuid4()) for _ in docs]"
@ -338,7 +338,7 @@
{
"data": {
"text/plain": [
"Document(page_content=\"The document summarizes President Biden's State of the Union address. It highlights his nominations for the Supreme Court, his plans for border security and immigration reform, his commitment to protecting women's rights and LGBTQ+ rights, his bipartisan achievements, and his agenda for addressing the opioid epidemic, mental health, supporting veterans, and ending cancer. The document concludes with a message of optimism and unity for the American people.\", metadata={'doc_id': 'aa42f0b8-5119-44f9-808d-58c2b6b76e7b'})"
"Document(page_content=\"The document is a speech given by the President of the United States, highlighting various issues and priorities. The President discusses the nomination of Judge Ketanji Brown Jackson for the Supreme Court and emphasizes the importance of securing the border and fixing the immigration system. The President also mentions the need to protect women's rights, support LGBTQ+ Americans, pass the Equality Act, and sign bipartisan bills into law. Additionally, the President addresses the opioid epidemic, mental health, support for veterans, and the fight against cancer. The speech concludes with a message of unity and optimism for the future of the United States.\", metadata={'doc_id': '1f0bb74d-4878-43ae-9a5d-4c63fb308ca1'})"
]
},
"execution_count": 19,
@ -447,9 +447,9 @@
{
"data": {
"text/plain": [
"[\"What was the author's initial reaction to the use of the IBM 1401 during his school years?\",\n",
" \"How did the author's interest in AI originate and evolve over time?\",\n",
" 'What led the author to switch his focus from AI to Lisp in grad school?']"
"[\"What was the author's initial career choice before deciding to switch to AI?\",\n",
" 'Why did the author become disillusioned with AI during his first year of grad school?',\n",
" 'What realization did the author have when visiting the Carnegie Institute?']"
]
},
"execution_count": 24,
@ -483,12 +483,12 @@
" collection_name=\"hypo-questions\", embedding_function=OpenAIEmbeddings()\n",
")\n",
"# The storage layer for the parent documents\n",
"store = InMemoryStore()\n",
"store = InMemoryByteStore()\n",
"id_key = \"doc_id\"\n",
"# The retriever (empty to start)\n",
"retriever = MultiVectorRetriever(\n",
" vectorstore=vectorstore,\n",
" base_store=store,\n",
" byte_store=store,\n",
" id_key=id_key,\n",
")\n",
"doc_ids = [str(uuid.uuid4()) for _ in docs]"
@ -538,10 +538,10 @@
{
"data": {
"text/plain": [
"[Document(page_content=\"How did Robert's advice influence the narrator's decision to step down from Y Combinator?\", metadata={'doc_id': 'ea931756-68b8-4cd1-8752-e98d7e3c499f'}),\n",
" Document(page_content='What factors led to the decision of handing over the leadership of Y Combinator to someone else?', metadata={'doc_id': 'ea931756-68b8-4cd1-8752-e98d7e3c499f'}),\n",
" Document(page_content=\"How does the Bipartisan Infrastructure Law aim to transform America's economic competitiveness in the 21st Century?\", metadata={'doc_id': '63d98582-bd93-4818-b729-e0933d3d4cde'}),\n",
" Document(page_content='What measures have been taken to secure the border and fix the immigration system?', metadata={'doc_id': '3d2b150f-dcd3-4277-8734-0a15888fdae4'})]"
"[Document(page_content='Who is the nominee for the United States Supreme Court, and what is their background?', metadata={'doc_id': 'd4a82bd9-9001-4bd7-bff1-d8ba2dca9692'}),\n",
" Document(page_content='Why did Robert Morris suggest the narrator to quit Y Combinator?', metadata={'doc_id': 'aba9b00d-860b-4b93-8e80-87dc08fa461d'}),\n",
" Document(page_content='What events led to the narrator deciding to hand over Y Combinator to someone else?', metadata={'doc_id': 'aba9b00d-860b-4b93-8e80-87dc08fa461d'}),\n",
" Document(page_content=\"How does the Bipartisan Infrastructure Law aim to improve America's infrastructure?\", metadata={'doc_id': '822c2ba8-0abe-4f28-a72e-7eb8f477cc3d'})]"
]
},
"execution_count": 30,
@ -572,7 +572,7 @@
{
"data": {
"text/plain": [
"9844"
"9194"
]
},
"execution_count": 32,
@ -601,7 +601,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
"version": "3.11.4"
}
},
"nbformat": 4,

16
libs/langchain/langchain/retrievers/multi_vector.py
View File

@ -1,8 +1,8 @@
from enum import Enum
from typing import Any, List, Optional
from typing import Dict, List, Optional
from langchain_core.documents import Document
from langchain_core.pydantic_v1 import Field, validator
from langchain_core.pydantic_v1 import Field, root_validator
from langchain_core.retrievers import BaseRetriever
from langchain_core.stores import BaseStore, ByteStore
from langchain_core.vectorstores import VectorStore
@ -26,7 +26,7 @@ class MultiVectorRetriever(BaseRetriever):
vectorstore: VectorStore
"""The underlying vectorstore to use to store small chunks
and their embedding vectors"""
byte_store: Optional[ByteStore]
byte_store: Optional[ByteStore] = None
"""The lower-level backing storage layer for the parent documents"""
docstore: BaseStore[str, Document]
"""The storage interface for the parent documents"""
@ -36,16 +36,16 @@ class MultiVectorRetriever(BaseRetriever):
search_type: SearchType = SearchType.similarity
"""Type of search to perform (similarity / mmr)"""
@validator("docstore", pre=True, always=True)
def shim_docstore(
cls, docstore: Optional[BaseStore[str, Document]], values: Any
) -> BaseStore[str, Document]:
@root_validator(pre=True)
def shim_docstore(cls, values: Dict) -> Dict:
byte_store = values.get("byte_store")
docstore = values.get("docstore")
if byte_store is not None:
docstore = create_kv_docstore(byte_store)
elif docstore is None:
raise Exception("You must pass a `byte_store` parameter.")
return docstore
values["docstore"] = docstore
return values
def _get_relevant_documents(
self, query: str, *, run_manager: CallbackManagerForRetrieverRun