mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-01 02:50:47 +00:00
Support multimodal messages (#11320)
Co-authored-by: Bagatur <baskaryan@gmail.com>
This commit is contained in:
@@ -20,6 +20,64 @@
|
||||
"!pip install \"openai>=1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "c3e067ce-7a43-47a7-bc89-41f1de4cf136",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"from langchain.schema.messages import HumanMessage, SystemMessage"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fa7e7e95-90a1-4f73-98fe-10c4b4e0951b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## [Vision](https://platform.openai.com/docs/guides/vision)\n",
|
||||
"\n",
|
||||
"OpenAI released multi-modal models, which can take a sequence of text and images as input."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "1c8c3965-d3c9-4186-b5f3-5e67855ef916",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"AIMessage(content='The image appears to be a diagram illustrating the architecture or components of a software system')"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"chat = ChatOpenAI(model=\"gpt-4-vision-preview\")\n",
|
||||
"chat.invoke(\n",
|
||||
" [\n",
|
||||
" HumanMessage(\n",
|
||||
" content=[\n",
|
||||
" {\"type\": \"text\", \"text\": \"What is this image showing\"},\n",
|
||||
" {\n",
|
||||
" \"type\": \"image_url\",\n",
|
||||
" \"image_url\": {\n",
|
||||
" \"url\": \"https://python.langchain.com/assets/images/langchain_stack-da369071b058555da3d491a695651f15.jpg\",\n",
|
||||
" \"detail\": \"auto\",\n",
|
||||
" },\n",
|
||||
" },\n",
|
||||
" ]\n",
|
||||
" )\n",
|
||||
" ]\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "71c34763-d1e7-4b9a-a9d7-3e4cc0dfc2c4",
|
||||
@@ -32,17 +90,6 @@
|
||||
"Only works with certain models. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "c3e067ce-7a43-47a7-bc89-41f1de4cf136",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"from langchain.schema.messages import HumanMessage, SystemMessage"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
|
Reference in New Issue
Block a user