mirror of
https://github.com/hwchase17/langchain.git
synced 2025-08-27 13:31:53 +00:00
Add support Vertex AI Gemini uses a public image URL (#14949)
## What Since `langchain_google_genai.ChatGoogleGenerativeAI` supported A public image URL, we add to support it in `langchain.chat_models.ChatVertexAI` as well. ### Example ```py from langchain.chat_models.vertexai import ChatVertexAI from langchain_core.messages import HumanMessage llm = ChatVertexAI(model_name="gemini-pro-vision") image_message = { "type": "image_url", "image_url": { "url": "https://python.langchain.com/assets/images/cell-18-output-1-0c7fb8b94ff032d51bfe1880d8370104.png", }, } text_message = { "type": "text", "text": "What is shown in this image?", } message = HumanMessage(content=[text_message, image_message]) output = llm([message]) print(output.content) ``` ## Refs - https://python.langchain.com/docs/integrations/llms/google_vertex_ai_palm - https://python.langchain.com/docs/integrations/chat/google_generative_ai
This commit is contained in:
parent
affa3e755a
commit
6da2246215
@ -539,6 +539,35 @@
|
||||
"print(output2.content)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"You can also use the public image URL:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"image_message = {\n",
|
||||
" \"type\": \"image_url\",\n",
|
||||
" \"image_url\": {\n",
|
||||
" \"url\": \"https://python.langchain.com/assets/images/cell-18-output-1-0c7fb8b94ff032d51bfe1880d8370104.png\",\n",
|
||||
" },\n",
|
||||
"}\n",
|
||||
"text_message = {\n",
|
||||
" \"type\": \"text\",\n",
|
||||
" \"text\": \"What is shown in this image?\",\n",
|
||||
"}\n",
|
||||
"message = HumanMessage(content=[text_message, image_message])\n",
|
||||
"\n",
|
||||
"output = llm([message])\n",
|
||||
"print(output.content)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
|
@ -6,7 +6,9 @@ import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
from langchain_core.callbacks import (
|
||||
AsyncCallbackManagerForLLMRun,
|
||||
CallbackManagerForLLMRun,
|
||||
@ -87,6 +89,15 @@ def _parse_chat_history(history: List[BaseMessage]) -> _ChatHistory:
|
||||
return chat_history
|
||||
|
||||
|
||||
def _is_url(s: str) -> bool:
|
||||
try:
|
||||
result = urlparse(s)
|
||||
return all([result.scheme, result.netloc])
|
||||
except Exception as e:
|
||||
logger.debug(f"Unable to parse URL: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def _parse_chat_history_gemini(
|
||||
history: List[BaseMessage], project: Optional[str]
|
||||
) -> List["Content"]:
|
||||
@ -118,6 +129,10 @@ def _parse_chat_history_gemini(
|
||||
"data:image/<image_type>;base64,<base64_encoded_image>."
|
||||
)
|
||||
image = Image.from_bytes(base64.b64decode(encoded))
|
||||
elif _is_url(path):
|
||||
response = requests.get(path)
|
||||
response.raise_for_status()
|
||||
image = Image.from_bytes(response.content)
|
||||
else:
|
||||
image = Image.load_from_file(path)
|
||||
else:
|
||||
|
Loading…
Reference in New Issue
Block a user