github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-09-21 18:39:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bagatur/docs smith context (#13139)

Browse Source
This commit is contained in:
Bagatur
2023-11-09 10:22:49 -08:00
committed by GitHub
parent 58da6e0d47
commit 8b2a82b5ce
18 changed files with 386 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
cookbook/multi_modal_QA.ipynb
View File

@@ -7,7 +7,7 @@
"metadata": {},
"outputs": [],
"source": [
"! pip install \"openai>=1\" \"langchain>=0.0.331rc2\" matplotlib pillow "
"! pip install \"openai>=1\" \"langchain>=0.0.331rc2\" matplotlib pillow"
]
},
{
@@ -47,22 +47,24 @@
"from PIL import Image\n",
"from IPython.display import display, HTML\n",
"\n",
"\n",
"def encode_image(image_path):\n",
" ''' Getting the base64 string '''\n",
" \n",
" \"\"\"Getting the base64 string\"\"\"\n",
"\n",
" with open(image_path, \"rb\") as image_file:\n",
" return base64.b64encode(image_file.read()).decode('utf-8')\n",
" return base64.b64encode(image_file.read()).decode(\"utf-8\")\n",
"\n",
"\n",
"def plt_img_base64(img_base64):\n",
" ''' Display the base64 image '''\n",
" \"\"\"Display the base64 image\"\"\"\n",
"\n",
" # Create an HTML img tag with the base64 string as the source\n",
" image_html = f'<img src=\"data:image/jpeg;base64,{img_base64}\" />'\n",
" \n",
"\n",
" # Display the image by rendering the HTML\n",
" display(HTML(image_html))\n",
"\n",
"\n",
"# Image for QA\n",
"path = \"/Users/rlm/Desktop/Multimodal_Eval/qa/llm_strategies.jpeg\"\n",
"img_base64 = encode_image(path)\n",
@@ -99,19 +101,19 @@
"metadata": {},
"outputs": [],
"source": [
"chat = ChatOpenAI(model=\"gpt-4-vision-preview\",\n",
" max_tokens=1024)\n",
"chat = ChatOpenAI(model=\"gpt-4-vision-preview\", max_tokens=1024)\n",
"\n",
"msg = chat.invoke(\n",
" [\n",
" HumanMessage(\n",
" content=[\n",
" {\"type\": \"text\", \"text\":\"Based on the image, what is the difference in training strategy between a small and a large base model?\"},\n",
" {\n",
" \"type\": \"text\",\n",
" \"text\": \"Based on the image, what is the difference in training strategy between a small and a large base model?\",\n",
" },\n",
" {\n",
" \"type\": \"image_url\",\n",
" \"image_url\": {\n",
" \"url\": f\"data:image/jpeg;base64,{img_base64}\"\n",
" },\n",
" \"image_url\": {\"url\": f\"data:image/jpeg;base64,{img_base64}\"},\n",
" },\n",
" ]\n",
" )\n",