mirror of
https://github.com/hwchase17/langchain.git
synced 2025-04-27 11:41:51 +00:00
partners: add langchain-vdms (#29857)
**Description:** Deprecate vdms in community, add integration langchain-vdms, and update any related files **Issue:** n/a **Dependencies:** langchain-vdms **Twitter handle:** n/a --------- Co-authored-by: Chester Curme <chester.curme@gmail.com>
This commit is contained in:
Chaunte W. Lacewell
GitHub
parent
8293142fa0
commit
d972c6d6ea
@ -21,40 +21,6 @@
|
||||
"* Passing raw images and text chunks to a multimodal LLM for answer synthesis "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6a6b6e73",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Start VDMS Server\n",
|
||||
"\n",
|
||||
"Let's start a VDMS docker using port 55559 instead of default 55555. \n",
|
||||
"Keep note of the port and hostname as this is needed for the vector store as it uses the VDMS Python client to connect to the server."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "5f483872",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"a1b9206b08ef626e15b356bf9e031171f7c7eb8f956a2733f196f0109246fe2b\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"! docker run --rm -d -p 55559:55555 --name vdms_rag_nb intellabs/vdms:latest\n",
|
||||
"\n",
|
||||
"# Connect to VDMS Vector Store\n",
|
||||
"from langchain_community.vectorstores.vdms import VDMS_Client\n",
|
||||
"\n",
|
||||
"vdms_client = VDMS_Client(port=55559)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2498a0a1",
|
||||
@ -67,20 +33,20 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 1,
|
||||
"id": "febbc459-ebba-4c1a-a52b-fed7731593f8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip install --quiet -U vdms langchain-experimental\n",
|
||||
"! pip install --quiet -U langchain-vdms langchain-experimental langchain-ollama\n",
|
||||
"\n",
|
||||
"# lock to 0.10.19 due to a persistent bug in more recent versions\n",
|
||||
"! pip install --quiet pdf2image \"unstructured[all-docs]==0.10.19\" pillow pydantic lxml open_clip_torch"
|
||||
"! pip install --quiet pdf2image \"unstructured[all-docs]==0.10.19\" \"onnxruntime==1.17.0\" pillow pydantic lxml open_clip_torch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 2,
|
||||
"id": "78ac6543",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -89,6 +55,40 @@
|
||||
"# load_dotenv(find_dotenv(), override=True);"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e5c8916e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Start VDMS Server\n",
|
||||
"\n",
|
||||
"Let's start a VDMS docker using port 55559 instead of default 55555. \n",
|
||||
"Keep note of the port and hostname as this is needed for the vector store as it uses the VDMS Python client to connect to the server."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "1e6e2c15",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"a701e5ac3523006e9540b5355e2d872d5d78383eab61562a675d5b9ac21fde65\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"! docker run --rm -d -p 55559:55555 --name vdms_rag_nb intellabs/vdms:latest\n",
|
||||
"\n",
|
||||
"# Connect to VDMS Vector Store\n",
|
||||
"from langchain_vdms.vectorstores import VDMS_Client\n",
|
||||
"\n",
|
||||
"vdms_client = VDMS_Client(port=55559)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1e94b3fb-8e3e-4736-be0a-ad881626c7bd",
|
||||
@ -115,11 +115,12 @@
|
||||
"import requests\n",
|
||||
"\n",
|
||||
"# Folder to store pdf and extracted images\n",
|
||||
"datapath = Path(\"./data/multimodal_files\").resolve()\n",
|
||||
"base_datapath = Path(\"./data/multimodal_files\").resolve()\n",
|
||||
"datapath = base_datapath / \"images\"\n",
|
||||
"datapath.mkdir(parents=True, exist_ok=True)\n",
|
||||
"\n",
|
||||
"pdf_url = \"https://www.loc.gov/lcm/pdf/LCM_2020_1112.pdf\"\n",
|
||||
"pdf_path = str(datapath / pdf_url.split(\"/\")[-1])\n",
|
||||
"pdf_path = str(base_datapath / pdf_url.split(\"/\")[-1])\n",
|
||||
"with open(pdf_path, \"wb\") as f:\n",
|
||||
" f.write(requests.get(pdf_url).content)"
|
||||
]
|
||||
@ -185,8 +186,8 @@
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"from langchain_community.vectorstores import VDMS\n",
|
||||
"from langchain_experimental.open_clip import OpenCLIPEmbeddings\n",
|
||||
"from langchain_vdms import VDMS\n",
|
||||
"\n",
|
||||
"# Create VDMS\n",
|
||||
"vectorstore = VDMS(\n",
|
||||
@ -312,10 +313,10 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_community.llms.ollama import Ollama\n",
|
||||
"from langchain_core.messages import HumanMessage\n",
|
||||
"from langchain_core.messages import HumanMessage, SystemMessage\n",
|
||||
"from langchain_core.output_parsers import StrOutputParser\n",
|
||||
"from langchain_core.runnables import RunnableLambda, RunnablePassthrough\n",
|
||||
"from langchain_ollama.llms import OllamaLLM\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def prompt_func(data_dict):\n",
|
||||
@ -340,8 +341,8 @@
|
||||
" \"As an expert art critic and historian, your task is to analyze and interpret images, \"\n",
|
||||
" \"considering their historical and cultural significance. Alongside the images, you will be \"\n",
|
||||
" \"provided with related text to offer context. Both will be retrieved from a vectorstore based \"\n",
|
||||
" \"on user-input keywords. Please convert answers to english and use your extensive knowledge \"\n",
|
||||
" \"and analytical skills to provide a comprehensive summary that includes:\\n\"\n",
|
||||
" \"on user-input keywords. Please use your extensive knowledge and analytical skills to provide a \"\n",
|
||||
" \"comprehensive summary that includes:\\n\"\n",
|
||||
" \"- A detailed description of the visual elements in the image.\\n\"\n",
|
||||
" \"- The historical and cultural context of the image.\\n\"\n",
|
||||
" \"- An interpretation of the image's symbolism and meaning.\\n\"\n",
|
||||
@ -359,7 +360,7 @@
|
||||
" \"\"\"Multi-modal RAG chain\"\"\"\n",
|
||||
"\n",
|
||||
" # Multi-modal LLM\n",
|
||||
" llm_model = Ollama(\n",
|
||||
" llm_model = OllamaLLM(\n",
|
||||
" verbose=True, temperature=0.5, model=\"llava\", base_url=\"http://localhost:11434\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
@ -419,6 +420,121 @@
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"© 2017 LARRY D. MOORE\n",
|
||||
"\n",
|
||||
"contemporary criticism of the less-than- thoughtful circumstances under which Lange photographed Thomson, the picture’s power to engage has not diminished. Artists in other countries have appropriated the image, changing the mother’s features into those of other ethnicities, but keeping her expression and the positions of her clinging children. Long after anyone could help the Thompson family, this picture has resonance in another time of national crisis, unemployment and food shortages.\n",
|
||||
"\n",
|
||||
"A striking, but very different picture is a 1900 portrait of the legendary Hin-mah-too-yah- lat-kekt (Chief Joseph) of the Nez Percé people. The Bureau of American Ethnology in Washington, D.C., regularly arranged for its photographer, De Lancey Gill, to photograph Native American delegations that came to the capital to confer with officials about tribal needs and concerns. Although Gill described Chief Joseph as having “an air of gentleness and quiet reserve,” the delegate skeptically appraises the photographer, which is not surprising given that the United States broke five treaties with Chief Joseph and his father between 1855 and 1885.\n",
|
||||
"\n",
|
||||
"More than a glance, second looks may reveal new knowledge into complex histories.\n",
|
||||
"\n",
|
||||
"Anne Wilkes Tucker is the photography curator emeritus of the Museum of Fine Arts, Houston and curator of the “Not an Ostrich” exhibition.\n",
|
||||
"\n",
|
||||
"28\n",
|
||||
"\n",
|
||||
"28 LIBRARY OF CONGRESS MAGAZINE\n",
|
||||
"\n",
|
||||
"LIBRARY OF CONGRESS MAGAZINE\n",
|
||||
"THEYRE WILLING TO HAVE MEENTERTAIN THEM DURING THE DAY,BUT AS SOON AS IT STARTSGETTING DARK, THEY ALLGO OFF, AND LEAVE ME! \n",
|
||||
"ROSA PARKS: IN HER OWN WORDS\n",
|
||||
"\n",
|
||||
"COMIC ART: 120 YEARS OF PANELS AND PAGES\n",
|
||||
"\n",
|
||||
"SHALL NOT BE DENIED: WOMEN FIGHT FOR THE VOTE\n",
|
||||
"\n",
|
||||
"More information loc.gov/exhibits\n",
|
||||
"Nuestra Sefiora de las Iguanas\n",
|
||||
"\n",
|
||||
"Graciela Iturbide’s 1979 portrait of Zobeida Díaz in the town of Juchitán in southeastern Mexico conveys the strength of women and reflects their important contributions to the economy. Díaz, a merchant, was selling iguanas to cook and eat, carrying them on her head, as is customary.\n",
|
||||
"\n",
|
||||
"GRACIELA ITURBIDE. “NUESTRA SEÑORA DE LAS IGUANAS.” 1979. GELATIN SILVER PRINT. © GRACIELA ITURBIDE, USED BY PERMISSION. PRINTS AND PHOTOGRAPHS DIVISION.\n",
|
||||
"\n",
|
||||
"Iturbide requested permission to take a photograph, but this proved challenging because the iguanas were constantly moving, causing Díaz to laugh. The result, however, was a brilliant portrait that the inhabitants of Juchitán claimed with pride. They have reproduced it on posters and erected a statue honoring Díaz and her iguanas. The photo now appears throughout the world, inspiring supporters of feminism, women’s rights and gender equality.\n",
|
||||
"\n",
|
||||
"—Adam Silvia is a curator in the Prints and Photographs Division.\n",
|
||||
"\n",
|
||||
"6\n",
|
||||
"\n",
|
||||
"6 LIBRARY OF CONGRESS MAGAZINE\n",
|
||||
"\n",
|
||||
"LIBRARY OF CONGRESS MAGAZINE\n",
|
||||
"\n",
|
||||
"‘Migrant Mother’ is Florence Owens Thompson\n",
|
||||
"\n",
|
||||
"The iconic portrait that became the face of the Great Depression is also the most famous photograph in the collections of the Library of Congress.\n",
|
||||
"\n",
|
||||
"The Library holds the original source of the photo — a nitrate negative measuring 4 by 5 inches. Do you see a faint thumb in the bottom right? The photographer, Dorothea Lange, found the thumb distracting and after a few years had the negative altered to make the thumb almost invisible. Lange’s boss at the Farm Security Administration, Roy Stryker, criticized her action because altering a negative undermines the credibility of a documentary photo.\n",
|
||||
"Shrimp Picker\n",
|
||||
"\n",
|
||||
"The photos and evocative captions of Lewis Hine served as source material for National Child Labor Committee reports and exhibits exposing abusive child labor practices in the United States in the first decades of the 20th century.\n",
|
||||
"\n",
|
||||
"LEWIS WICKES HINE. “MANUEL, THE YOUNG SHRIMP-PICKER, FIVE YEARS OLD, AND A MOUNTAIN OF CHILD-LABOR OYSTER SHELLS BEHIND HIM. HE WORKED LAST YEAR. UNDERSTANDS NOT A WORD OF ENGLISH. DUNBAR, LOPEZ, DUKATE COMPANY. LOCATION: BILOXI, MISSISSIPPI.” FEBRUARY 1911. NATIONAL CHILD LABOR COMMITTEE COLLECTION. PRINTS AND PHOTOGRAPHS DIVISION.\n",
|
||||
"\n",
|
||||
"For 15 years, Hine\n",
|
||||
"\n",
|
||||
"crisscrossed the country, documenting the practices of the worst offenders. His effective use of photography made him one of the committee's greatest publicists in the campaign for legislation to ban child labor.\n",
|
||||
"\n",
|
||||
"Hine was a master at taking photos that catch attention and convey a message and, in this photo, he framed Manuel in a setting that drove home the boy’s small size and unsafe environment.\n",
|
||||
"\n",
|
||||
"Captions on photos of other shrimp pickers emphasized their long working hours as well as one hazard of the job: The acid from the shrimp made pickers’ hands sore and “eats the shoes off your feet.”\n",
|
||||
"\n",
|
||||
"Such images alerted viewers to all that workers, their families and the nation sacrificed when children were part of the labor force. The Library holds paper records of the National Child Labor Committee as well as over 5,000 photographs.\n",
|
||||
"\n",
|
||||
"—Barbara Natanson is head of the Reference Section in the Prints and Photographs Division.\n",
|
||||
"\n",
|
||||
"8\n",
|
||||
"\n",
|
||||
"LIBRARY OF CONGRESS MAGAZINE\n",
|
||||
"\n",
|
||||
"LIBRARY OF CONGRESS MAGAZINE\n",
|
||||
"\n",
|
||||
"Intergenerational Portrait\n",
|
||||
"\n",
|
||||
"Raised on the Apsáalooke (Crow) reservation in Montana, photographer Wendy Red Star created her “Apsáalooke Feminist” self-portrait series with her daughter Beatrice. With a dash of wry humor, mother and daughter are their own first-person narrators.\n",
|
||||
"\n",
|
||||
"Red Star explains the significance of their appearance: “The dress has power: You feel strong and regal wearing it. In my art, the elk tooth dress specifically symbolizes Crow womanhood and the matrilineal line connecting me to my ancestors. As a mother, I spend hours searching for the perfect elk tooth dress materials to make a prized dress for my daughter.”\n",
|
||||
"\n",
|
||||
"In a world that struggles with cultural identities, this photograph shows us the power and beauty of blending traditional and contemporary styles.\n",
|
||||
"‘American Gothic’ Product #216040262 Price: $24\n",
|
||||
"\n",
|
||||
"U.S. Capitol at Night Product #216040052 Price: $24\n",
|
||||
"\n",
|
||||
"Good Reading Ahead Product #21606142 Price: $24\n",
|
||||
"\n",
|
||||
"Gordon Parks created an iconic image with this 1942 photograph of cleaning woman Ella Watson.\n",
|
||||
"\n",
|
||||
"Snow blankets the U.S. Capitol in this classic image by Ernest L. Crandall.\n",
|
||||
"\n",
|
||||
"Start your new year out right with a poster promising good reading for months to come.\n",
|
||||
"\n",
|
||||
"▪ Order online: loc.gov/shop ▪ Order by phone: 888.682.3557\n",
|
||||
"\n",
|
||||
"26\n",
|
||||
"\n",
|
||||
"LIBRARY OF CONGRESS MAGAZINE\n",
|
||||
"\n",
|
||||
"LIBRARY OF CONGRESS MAGAZINE\n",
|
||||
"\n",
|
||||
"SUPPORT\n",
|
||||
"\n",
|
||||
"A PICTURE OF PHILANTHROPY Annenberg Foundation Gives $1 Million and a Photographic Collection to the Library.\n",
|
||||
"\n",
|
||||
"A major gift by Wallis Annenberg and the Annenberg Foundation in Los Angeles will support the effort to reimagine the visitor experience at the Library of Congress. The foundation also is donating 1,000 photographic prints from its Annenberg Space for Photography exhibitions to the Library.\n",
|
||||
"\n",
|
||||
"The Library is pursuing a multiyear plan to transform the experience of its nearly 2 million annual visitors, share more of its treasures with the public and show how Library collections connect with visitors’ own creativity and research. The project is part of a strategic plan established by Librarian of Congress Carla Hayden to make the Library more user-centered for Congress, creators and learners of all ages.\n",
|
||||
"\n",
|
||||
"A 2018 exhibition at the Annenberg Space for Photography in Los Angeles featured over 400 photographs from the Library. The Library is planning a future photography exhibition, based on the Annenberg-curated show, along with a documentary film on the Library and its history, produced by the Annenberg Space for Photography.\n",
|
||||
"\n",
|
||||
"“The nation’s library is honored to have the strong support of Wallis Annenberg and the Annenberg Foundation as we enhance the experience for our visitors,” Hayden said. “We know that visitors will find new connections to the Library through the incredible photography collections and countless other treasures held here to document our nation’s history and creativity.”\n",
|
||||
"\n",
|
||||
"To enhance the Library’s holdings, the foundation is giving the Library photographic prints for long-term preservation from 10 other exhibitions hosted at the Annenberg Space for Photography. The Library holds one of the world’s largest photography collections, with about 14 million photos and over 1 million images digitized and available online.\n",
|
||||
"18 LIBRARY OF CONGRESS MAGAZINE\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
@ -461,10 +577,17 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" The image depicts a woman with several children. The woman appears to be of Cherokee heritage, as suggested by the text provided. The image is described as having been initially regretted by the subject, Florence Owens Thompson, due to her feeling that it did not accurately represent her leadership qualities.\n",
|
||||
"The historical and cultural context of the image is tied to the Great Depression and the Dust Bowl, both of which affected the Cherokee people in Oklahoma. The photograph was taken during this period, and its subject, Florence Owens Thompson, was a leader within her community who worked tirelessly to help those affected by these crises.\n",
|
||||
"The image's symbolism and meaning can be interpreted as a representation of resilience and strength in the face of adversity. The woman is depicted with multiple children, which could signify her role as a caregiver and protector during difficult times.\n",
|
||||
"Connections between the image and the related text include Florence Owens Thompson's leadership qualities and her regretted feelings about the photograph. Additionally, the mention of Dorothea Lange, the photographer who took this photo, ties the image to its historical context and the broader narrative of the Great Depression and Dust Bowl in Oklahoma. \n"
|
||||
" The image is a black and white photograph by Dorothea Lange titled \"Destitute Pea Pickers in California. Mother of Seven Children. Age Thirty-Two. Nipomo, California.\" It was taken in March 1936 as part of the Farm Security Administration-Office of War Information Collection.\n",
|
||||
"\n",
|
||||
"The photograph features a woman with seven children, who appear to be in a state of poverty and hardship. The woman is seated, looking directly at the camera, while three of her children are standing behind her. They all seem to be dressed in ragged clothing, indicative of their impoverished condition.\n",
|
||||
"\n",
|
||||
"The historical context of this image is related to the Great Depression, which was a period of economic hardship in the United States that lasted from 1929 to 1939. During this time, many people struggled to make ends meet, and poverty was widespread. This photograph captures the plight of one such family during this difficult period.\n",
|
||||
"\n",
|
||||
"The symbolism of the image is multifaceted. The woman's direct gaze at the camera can be seen as a plea for help or an expression of desperation. The ragged clothing of the children serves as a stark reminder of the poverty and hardship experienced by many during this time.\n",
|
||||
"\n",
|
||||
"In terms of connections to the related text, it is mentioned that Florence Owens Thompson, the woman in the photograph, initially regretted having her picture taken. However, she later came to appreciate the importance of the image as a representation of the struggles faced by many during the Great Depression. The mention of Helena Zinkham suggests that she may have played a role in the creation or distribution of this photograph.\n",
|
||||
"\n",
|
||||
"Overall, this image is a powerful depiction of poverty and hardship during the Great Depression, capturing the resilience and struggles of one family amidst difficult times. \n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -491,11 +614,17 @@
|
||||
"source": [
|
||||
"! docker kill vdms_rag_nb"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fe4a98ee",
|
||||
"metadata": {},
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": ".langchain-venv",
|
||||
"display_name": ".test-venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
@ -509,7 +638,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
"version": "3.11.10"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
@ -26,7 +26,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"2e44b44201c8778b462342ac97f5ccf05a4e02aa8a04505ecde97bf20dcc4cbb\n"
|
||||
"76e78b89cee4d6d31154823f93592315df79c28410dfbfc87c9f70cbfdfa648b\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -49,7 +49,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip install --quiet -U vdms langchain-experimental sentence-transformers opencv-python open_clip_torch torch accelerate"
|
||||
"! pip install --quiet -U langchain-vdms langchain-experimental sentence-transformers opencv-python open_clip_torch torch accelerate"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -63,7 +63,16 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/data1/cwlacewe/apps/cwlacewe_langchain/.langchain-venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||||
" from .autonotebook import tqdm as notebook_tqdm\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
@ -80,10 +89,10 @@
|
||||
"from langchain_community.embeddings.sentence_transformer import (\n",
|
||||
" SentenceTransformerEmbeddings,\n",
|
||||
")\n",
|
||||
"from langchain_community.vectorstores.vdms import VDMS, VDMS_Client\n",
|
||||
"from langchain_core.callbacks.manager import CallbackManagerForLLMRun\n",
|
||||
"from langchain_core.runnables import ConfigurableField\n",
|
||||
"from langchain_experimental.open_clip import OpenCLIPEmbeddings\n",
|
||||
"from langchain_vdms.vectorstores import VDMS, VDMS_Client\n",
|
||||
"from transformers import (\n",
|
||||
" AutoModelForCausalLM,\n",
|
||||
" AutoTokenizer,\n",
|
||||
@ -363,7 +372,7 @@
|
||||
"\t\tThere are 2 shoppers in this video. Shopper 1 is wearing a plaid shirt and a spectacle. Shopper 2 who is not completely captured in the frame seems to wear a black shirt and is moving away with his back turned towards the camera. There is a shelf towards the right of the camera frame. Shopper 2 is hanging an item back to a hanger and then quickly walks away in a similar fashion as shopper 2. Contents of the nearer side of the shelf with respect to camera seems to be camping lanterns and cleansing agents, arranged at the top. In the middle part of the shelf, various tools including grommets, a pocket saw, candles, and other helpful camping items can be observed. Midway through the shelf contains items which appear to be steel containers and items made up of plastic with red, green, orange, and yellow colors, while those at the bottom are packed in cardboard boxes. Contents at the farther part of the shelf are well stocked and organized but are not glaringly visible.\n",
|
||||
"\n",
|
||||
"\tMetadata:\n",
|
||||
"\t\t{'fps': 24.0, 'id': 'c6e5f894-b905-46f5-ac9e-4487a9235561', 'total_frames': 120.0, 'video': 'clip16.mp4'}\n",
|
||||
"\t\t{'fps': 24.0, 'total_frames': 120.0, 'video': 'clip16.mp4'}\n",
|
||||
"Retrieved Top matching video!\n",
|
||||
"\n",
|
||||
"\n"
|
||||
@ -392,18 +401,12 @@
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "3edf8783e114487ca490d8dec5c46884",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loading checkpoint shards: 100%|██████████| 2/2 [00:18<00:00, 9.01s/it]\n",
|
||||
"WARNING:accelerate.big_modeling:Some parameters are on the meta device because they were offloaded to the cpu.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
@ -555,7 +558,7 @@
|
||||
"\t\tA single shopper is seen in this video standing facing the shelf and in the bottom part of the frame. He's wearing a light-colored shirt and a spectacle. The shopper is carrying a red colored basket in his left hand. The entire basket is not clearly visible, but it does seem to contain something in a blue colored package which the shopper has just placed in the basket given his right hand was seen inside the basket. Then the shopper leans towards the shelf and checks out an item in orange package. He picks this single item with his right hand and proceeds to place the item in the basket. The entire shelf looks well stocked except for the top part of the shelf which is empty. The shopper has not picked any item from this part of the shelf. The rest of the shelf looks well stocked and does not need any restocking. The contents on the farther part of the shelf consists of items, majority of which are packed in black, yellow, and green packages. No other details are visible of these items.\n",
|
||||
"\n",
|
||||
"\tMetadata:\n",
|
||||
"\t\t{'fps': 24.0, 'id': '37ddc212-994e-4db0-877f-5ed09965ab90', 'total_frames': 162.0, 'video': 'clip10.mp4'}\n",
|
||||
"\t\t{'fps': 24.0, 'total_frames': 162.0, 'video': 'clip10.mp4'}\n",
|
||||
"Retrieved Top matching video!\n",
|
||||
"\n",
|
||||
"\n"
|
||||
@ -585,7 +588,7 @@
|
||||
"User : Find a man holding a red shopping basket\n",
|
||||
"Assistant : Most relevant retrieved video is **clip9.mp4** \n",
|
||||
"\n",
|
||||
"I see a person standing in front of a well-stocked shelf, they are wearing a light-colored shirt and glasses, and they have a red shopping basket in their left hand. They are leaning forward and picking up an item from the shelf with their right hand. The item is packaged in a blue-green box. Based on the scene description, I can confirm that the person is indeed holding a red shopping basket.</s>\n"
|
||||
"I see a person standing in front of a well-stocked shelf, they are wearing a light-colored shirt and glasses, and they have a red shopping basket in their left hand. They are leaning forward and picking up an item from the shelf with their right hand. The item is packaged in a blue-green box. Based on the available information, I cannot confirm whether the basket is empty or contains items. However, the rest of the\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -655,7 +658,7 @@
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": ".venv",
|
||||
"display_name": ".langchain-venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
@ -669,7 +672,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
"version": "3.11.10"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
@ -9,53 +9,35 @@
|
||||
### Install Client
|
||||
|
||||
```bash
|
||||
pip install vdms
|
||||
pip install langchain-vdms
|
||||
```
|
||||
|
||||
### Install Database
|
||||
|
||||
There are two ways to get started with VDMS:
|
||||
|
||||
#### Install VDMS on your local machine via docker
|
||||
```bash
|
||||
docker run -d -p 55555:55555 intellabs/vdms:latest
|
||||
```
|
||||
|
||||
#### Install VDMS directly on your local machine
|
||||
Please see [installation instructions](https://github.com/IntelLabs/vdms/blob/master/INSTALL.md).
|
||||
|
||||
1. Install VDMS on your local machine via docker
|
||||
```bash
|
||||
docker run -d -p 55555:55555 intellabs/vdms:latest
|
||||
```
|
||||
|
||||
2. Install VDMS directly on your local machine. Please see
|
||||
[installation instructions](https://github.com/IntelLabs/vdms/blob/master/INSTALL.md).
|
||||
|
||||
## VectorStore
|
||||
|
||||
The vector store is a simple wrapper around VDMS. It provides a simple interface to store and retrieve data.
|
||||
To import this vectorstore:
|
||||
|
||||
```python
|
||||
from langchain_community.document_loaders import TextLoader
|
||||
from langchain.text_splitter import CharacterTextSplitter
|
||||
|
||||
loader = TextLoader("./state_of_the_union.txt")
|
||||
documents = loader.load()
|
||||
text_splitter = CharacterTextSplitter(chunk_size=500, chunk_overlap=0)
|
||||
docs = text_splitter.split_documents(documents)
|
||||
|
||||
from langchain_community.vectorstores import VDMS
|
||||
from langchain_community.vectorstores.vdms import VDMS_Client
|
||||
from langchain_huggingface import HuggingFaceEmbeddings
|
||||
|
||||
client = VDMS_Client("localhost", 55555)
|
||||
model_name = "sentence-transformers/all-mpnet-base-v2"
|
||||
vectorstore = VDMS.from_documents(
|
||||
docs,
|
||||
client=client,
|
||||
collection_name="langchain-demo",
|
||||
embedding_function=HuggingFaceEmbeddings(model_name=model_name),
|
||||
engine="FaissFlat"
|
||||
distance_strategy="L2",
|
||||
)
|
||||
|
||||
query = "What did the president say about Ketanji Brown Jackson"
|
||||
results = vectorstore.similarity_search(query)
|
||||
from langchain_vdms import VDMS
|
||||
from langchain_vdms.vectorstores import VDMS
|
||||
```
|
||||
|
||||
For a more detailed walkthrough of the VDMS wrapper, see [this notebook](/docs/integrations/vectorstores/vdms)
|
||||
To import the VDMS Client connector:
|
||||
|
||||
```python
|
||||
from langchain_vdms.vectorstores import VDMS_Client
|
||||
```
|
||||
|
||||
For a more detailed walkthrough of the VDMS wrapper, see [this guide](/docs/integrations/vectorstores/vdms).
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -22,6 +22,7 @@ from typing import (
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
from langchain_core._api.deprecation import deprecated
|
||||
from langchain_core.documents import Document
|
||||
from langchain_core.embeddings import Embeddings
|
||||
from langchain_core.vectorstores import VectorStore
|
||||
@ -135,6 +136,7 @@ def VDMS_Client(host: str = "localhost", port: int = 55555) -> vdms.vdms:
|
||||
return client
|
||||
|
||||
|
||||
@deprecated(since="0.3.18", removal="1.0.0", alternative_import="langchain_vdms.VDMS")
|
||||
class VDMS(VectorStore):
|
||||
"""Intel Lab's VDMS for vector-store workloads.
|
||||
|
||||
|
||||
@ -445,6 +445,10 @@ packages:
|
||||
repo: Shikenso-Analytics/langchain-discord
|
||||
downloads: 1
|
||||
downloads_updated_at: '2025-02-15T16:00:00.000000+00:00'
|
||||
- name: langchain-vdms
|
||||
repo: IntelLabs/langchain-vdms
|
||||
path: .
|
||||
name_title: VDMS
|
||||
- name: langchain-deeplake
|
||||
path: .
|
||||
repo: activeloopai/langchain-deeplake
|
||||
|
||||
Loading…
Reference in New Issue
Block a user