mirror of
https://github.com/hwchase17/langchain.git
synced 2025-08-27 21:45:08 +00:00
Add support Vertex AI Gemini uses a public image URL (#14949)
## What Since `langchain_google_genai.ChatGoogleGenerativeAI` supported A public image URL, we add to support it in `langchain.chat_models.ChatVertexAI` as well. ### Example ```py from langchain.chat_models.vertexai import ChatVertexAI from langchain_core.messages import HumanMessage llm = ChatVertexAI(model_name="gemini-pro-vision") image_message = { "type": "image_url", "image_url": { "url": "https://python.langchain.com/assets/images/cell-18-output-1-0c7fb8b94ff032d51bfe1880d8370104.png", }, } text_message = { "type": "text", "text": "What is shown in this image?", } message = HumanMessage(content=[text_message, image_message]) output = llm([message]) print(output.content) ``` ## Refs - https://python.langchain.com/docs/integrations/llms/google_vertex_ai_palm - https://python.langchain.com/docs/integrations/chat/google_generative_ai
This commit is contained in:
parent
affa3e755a
commit
6da2246215
@ -539,6 +539,35 @@
|
|||||||
"print(output2.content)"
|
"print(output2.content)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"You can also use the public image URL:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"image_message = {\n",
|
||||||
|
" \"type\": \"image_url\",\n",
|
||||||
|
" \"image_url\": {\n",
|
||||||
|
" \"url\": \"https://python.langchain.com/assets/images/cell-18-output-1-0c7fb8b94ff032d51bfe1880d8370104.png\",\n",
|
||||||
|
" },\n",
|
||||||
|
"}\n",
|
||||||
|
"text_message = {\n",
|
||||||
|
" \"type\": \"text\",\n",
|
||||||
|
" \"text\": \"What is shown in this image?\",\n",
|
||||||
|
"}\n",
|
||||||
|
"message = HumanMessage(content=[text_message, image_message])\n",
|
||||||
|
"\n",
|
||||||
|
"output = llm([message])\n",
|
||||||
|
"print(output.content)"
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {
|
"metadata": {
|
||||||
|
@ -6,7 +6,9 @@ import logging
|
|||||||
import re
|
import re
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union, cast
|
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union, cast
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
import requests
|
||||||
from langchain_core.callbacks import (
|
from langchain_core.callbacks import (
|
||||||
AsyncCallbackManagerForLLMRun,
|
AsyncCallbackManagerForLLMRun,
|
||||||
CallbackManagerForLLMRun,
|
CallbackManagerForLLMRun,
|
||||||
@ -87,6 +89,15 @@ def _parse_chat_history(history: List[BaseMessage]) -> _ChatHistory:
|
|||||||
return chat_history
|
return chat_history
|
||||||
|
|
||||||
|
|
||||||
|
def _is_url(s: str) -> bool:
|
||||||
|
try:
|
||||||
|
result = urlparse(s)
|
||||||
|
return all([result.scheme, result.netloc])
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"Unable to parse URL: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _parse_chat_history_gemini(
|
def _parse_chat_history_gemini(
|
||||||
history: List[BaseMessage], project: Optional[str]
|
history: List[BaseMessage], project: Optional[str]
|
||||||
) -> List["Content"]:
|
) -> List["Content"]:
|
||||||
@ -118,6 +129,10 @@ def _parse_chat_history_gemini(
|
|||||||
"data:image/<image_type>;base64,<base64_encoded_image>."
|
"data:image/<image_type>;base64,<base64_encoded_image>."
|
||||||
)
|
)
|
||||||
image = Image.from_bytes(base64.b64decode(encoded))
|
image = Image.from_bytes(base64.b64decode(encoded))
|
||||||
|
elif _is_url(path):
|
||||||
|
response = requests.get(path)
|
||||||
|
response.raise_for_status()
|
||||||
|
image = Image.from_bytes(response.content)
|
||||||
else:
|
else:
|
||||||
image = Image.load_from_file(path)
|
image = Image.load_from_file(path)
|
||||||
else:
|
else:
|
||||||
|
Loading…
Reference in New Issue
Block a user